Upload 138 files

.gitattributes

@@ -32,3 +32,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+mlc-llm-main/site/gif/ios-demo.gif filter=lfs diff=lfs merge=lfs -text
+mlc-llm-main/site/img/android/android-studio.png filter=lfs diff=lfs merge=lfs -text

mlc-llm-main/.clang-format

@@ -0,0 +1,8 @@
+# Run the following command to reformat a file:
+# clang-format -i -style=Google <file>
+# Or use clang-format-diff to only reformat the changed lines:
+# https://clang.llvm.org/docs/ClangFormat.html
+BasedOnStyle: Google
+DerivePointerAlignment: false
+ColumnLimit:     100
+PointerAlignment: Left

mlc-llm-main/.gitignore

@@ -0,0 +1,323 @@
+dist/
+params/
+*.bak
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+.DS_Store
+
+*.S
+# C extensions
+*.so
+
+build*
+
+*.ll
+.npm
+# Distribution / packaging
+.Python
+env/
+build/
+build-*/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+.conda/
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Generated by python/gen_requirements.py
+python/requirements/*.txt
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+docs/_staging/
+
+# PyBuilder
+target/
+/target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+*~
+*.pyc
+*~
+config.mk
+config.cmake
+Win32
+*.dir
+perf
+*.wasm
+.emscripten
+
+## IOS
+DerivedData/
+
+## Java
+*.class
+jvm/*/target/
+jvm/*/*/target/
+jvm/native/*/generated
+jvm/native/src/main/native/org_apache_tvm_native_c_api.h
+*.worksheet
+*.idea
+*.iml
+*.classpath
+*.project
+*.settings
+*/node_modules/
+
+## Various settings
+*.pbxuser
+!default.pbxuser
+*.mode1v3
+!default.mode1v3
+*.mode2v3
+!default.mode2v3
+*.perspectivev3
+!default.perspectivev3
+xcuserdata/
+.pkl_memoize_*
+
+.emscripten*
+.m2
+
+# Compiled Dynamic libraries
+*.so
+*.dylib
+*.dll
+
+# Compiled Object files
+*.slo
+*.lo
+*.o
+*.obj
+
+# Precompiled Headers
+*.gch
+*.pch
+
+# Compiled Static libraries
+*.lai
+*.la
+*.a
+*.lib
+
+# Executables
+*.exe
+*.out
+*.app
+
+## Other
+*.moved-aside
+*.xccheckout
+*.xcscmblueprint
+.DS_Store
+tags
+cscope*
+*.lock
+
+# vim temporary files
+*.swp
+*.swo
+
+# TVM generated code
+perf
+.bash_history
+# *.json
+*.params
+*.ro
+*.onnx
+*.h5
+synset.txt
+cat.jpg
+cat.png
+docs.tgz
+cat.png
+*.mlmodel
+tvm_u.*
+tvm_t.*
+# Mac OS X
+.DS_Store
+
+# Jetbrain
+.idea
+.ipython
+.jupyter
+.nv
+.pylint.d
+.python_history
+.pytest_cache
+.local
+cmake-build-debug
+
+# Visual Studio
+.vs
+
+# Visual Studio Code
+.vscode
+
+# tmp file
+.nfs*
+
+# keys
+*.pem
+*.p12
+*.pfx
+*.cer
+*.crt
+*.der
+
+# patch sentinel
+patched.txt
+
+# Python type checking
+.mypy_cache/
+.pyre/
+
+# pipenv files
+Pipfile
+Pipfile.lock
+
+# conda package artifacts
+conda/Dockerfile.cuda*
+conda/pkg
+.node_repl_history
+# nix files
+.envrc
+*.nix
+
+# Docker files
+.sudo_as_admin_successful
+
+# Downloaded models/datasets
+.tvm_test_data
+.dgl
+.caffe2
+
+# Local docs build
+_docs/
+jvm/target
+.config/configstore/
+.ci-py-scripts/
+
+# Generated Hexagon files
+src/runtime/hexagon/rpc/hexagon_rpc.h
+src/runtime/hexagon/rpc/hexagon_rpc_skel.c
+src/runtime/hexagon/rpc/hexagon_rpc_stub.c
+
+# Local tvm-site checkout
+tvm-site/
+
+# Generated docs files
+gallery/how_to/work_with_microtvm/micro_tvmc.py
+
+# Test sample data files
+!tests/python/ci/sample_prs/*.json
+
+# Used in CI to communicate between Python and Jenkins
+.docker-image-names/
+
+# Printed TIR code on disk
+*.tir
+
+# GDB history file
+.gdb_history
+
+3rdparty
+dist

mlc-llm-main/.gitmodules

@@ -0,0 +1,6 @@
+[submodule "3rdparty/sentencepiece-js"]
+	path = 3rdparty/sentencepiece-js
+	url = https://github.com/tqchen/sentencepiece-js
+[submodule "3rdparty/argparse"]
+	path = 3rdparty/argparse
+	url = https://github.com/p-ranav/argparse

mlc-llm-main/3rdparty/tokenizers-cpp/CMakeLists.txt

@@ -0,0 +1,81 @@
+# update to contain more rust flags
+set(TOKENIZERS_CPP_RUST_FLAGS "")
+set(TOKENIZERS_CPP_CARGO_TARGET "")
+
+# extra link libraries
+set(TOKENIZERS_CPP_LINK_LIBS "")
+set(CARGO_EXTRA_ENVS "")
+
+if (CMAKE_SYSTEM_NAME STREQUAL "iOS")
+  set(TOKENIZERS_CPP_CARGO_TARGET aarch64-apple-ios)
+  # add extra dependency needed for rust tokenizer in iOS
+  find_library(FOUNDATION_LIB Foundation)
+  find_library(SECURITY_LIB Security)
+  list(APPEND TOKENIZERS_CPP_LINK_LIBS ${FOUNDATION_LIB} ${SECURITY_LIB})
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin")
+  if (CMAKE_SYSTEM_PROCESSOR STREQUAL "arm64")
+    set(TOKENIZERS_CPP_CARGO_TARGET aarch64-apple-darwin)
+  endif()
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Android")
+  set(TOKENIZERS_CPP_CARGO_TARGET aarch64-linux-android)
+  set(CARGO_EXTRA_ENVS
+    AR_aarch64_linux_android=${ANDROID_TOOLCHAIN_ROOT}/bin/llvm-ar
+    CC_aarch64_linux_android=${ANDROID_TOOLCHAIN_ROOT}/bin/aarch64-linux-android${ANDROID_NATIVE_API_LEVEL}-clang
+    CXX_aarch64_linux_android=${ANDROID_TOOLCHAIN_ROOT}/bin/aarch64-linux-android${ANDROID_NATIVE_API_LEVEL}-clang++
+  )
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Windows")
+  set(TOKENIZERS_CPP_CARGO_TARGET x86_64-pc-windows-msvc)
+endif()
+
+if(WIN32)
+  list(APPEND TOKENIZERS_CPP_LINK_LIBS
+    wsock32 ws2_32 Bcrypt
+    iphlpapi userenv psapi
+  )
+endif()
+
+set(TOKENIZERS_CPP_CARGO_FLAGS "")
+set(TOKENIZERS_CPP_CARGO_TARGET_DIR ${CMAKE_CURRENT_BINARY_DIR})
+set(TOKENIZERS_CPP_CARGO_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR})
+
+if (NOT TOKENIZERS_CPP_CARGO_TARGET STREQUAL "")
+    list(APPEND TOKENIZERS_CPP_CARGO_FLAGS --target ${TOKENIZERS_CPP_CARGO_TARGET})
+    set(TOKENIZERS_CPP_CARGO_BINARY_DIR
+        "${TOKENIZERS_CPP_CARGO_BINARY_DIR}/${TOKENIZERS_CPP_CARGO_TARGET}")
+endif()
+
+if (CMAKE_BUILD_TYPE STREQUAL "Debug")
+    set(TOKENIZERS_CPP_CARGO_BINARY_DIR "${TOKENIZERS_CPP_CARGO_BINARY_DIR}/debug")
+else ()
+    list(APPEND TOKENIZERS_CPP_CARGO_FLAGS --release)
+    set(TOKENIZERS_CPP_CARGO_BINARY_DIR "${TOKENIZERS_CPP_CARGO_BINARY_DIR}/release")
+endif ()
+
+get_filename_component(TOKENIZERS_CPP_CARGO_SOURCE_PATH ${CMAKE_CURRENT_LIST_FILE} DIRECTORY)
+
+if(MSVC)
+  set(TOKENIZERS_CPP_LIB "${TOKENIZERS_CPP_CARGO_BINARY_DIR}/tokenizers_cpp.lib")
+else()
+  set(TOKENIZERS_CPP_LIB "${TOKENIZERS_CPP_CARGO_BINARY_DIR}/libtokenizers_cpp.a")
+endif()
+set(TOKENIZERS_CPP_INCLUDE ${TOKENIZERS_CPP_CARGO_SOURCE_PATH})
+
+# NOTE: need to use cmake -E env to be portable in win
+
+add_custom_command(
+  OUTPUT ${TOKENIZERS_CPP_LIB}
+  COMMAND
+  ${CMAKE_COMMAND} -E env
+  CARGO_TARGET_DIR=${TOKENIZERS_CPP_CARGO_TARGET_DIR}
+  ${CARGO_EXTRA_ENVS}
+  RUSTFLAGS="${TOKENIZERS_CPP_RUST_FLAGS}"
+  cargo build ${TOKENIZERS_CPP_CARGO_FLAGS}
+  WORKING_DIRECTORY ${TOKENIZERS_CPP_CARGO_SOURCE_PATH}
+  POST_BUILD COMMAND
+  ${CMAKE_COMMAND} -E copy
+  ${TOKENIZERS_CPP_LIB} "${CMAKE_CURRENT_BINARY_DIR}"
+)
+
+add_library(tokenizers INTERFACE ${TOKENIZERS_CPP_LIB})
+target_link_libraries(tokenizers INTERFACE ${TOKENIZERS_CPP_LIB} ${TOKENIZERS_CPP_LINK_LIBS})
+target_include_directories(tokenizers INTERFACE ${TOKENIZERS_CPP_INCLUDE})

mlc-llm-main/3rdparty/tokenizers-cpp/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "tokenizers-cpp"
+version = "0.1.0"
+edition = "2018"
+
+[lib]
+crate-type = ["staticlib"]
+
+[dependencies]
+
+tokenizers = { version = "0.13.3", default-features = false, features = ["onig"] }
+serde = { version = "1.0", features = [ "derive" ] }
+serde_json = "1.0"

mlc-llm-main/3rdparty/tokenizers-cpp/src/lib.rs

@@ -0,0 +1,187 @@
+// A simple C wrapper of tokenzier library
+use serde_json::Value;
+use std::{collections::HashMap, str::FromStr};
+use tokenizers::models::bpe::BPE;
+use tokenizers::pre_tokenizers::byte_level::ByteLevel;
+use tokenizers::tokenizer::Tokenizer;
+
+pub struct TokenizerWrapper {
+    tokenizer: Tokenizer,
+    encode_ids: Vec<u32>,
+    decode_str: String,
+}
+
+pub type Vocab = HashMap<String, u32>;
+pub type Merges = Vec<(String, String)>;
+
+impl TokenizerWrapper {
+    pub fn from_str(json: &str) -> TokenizerWrapper {
+        TokenizerWrapper {
+            tokenizer: Tokenizer::from_str(json).unwrap().into(),
+            encode_ids: Vec::new(),
+            decode_str: String::new(),
+        }
+    }
+
+    pub fn byte_level_bpe_from_str(
+        vocab: &str,
+        merges: &str,
+        added_tokens: &str,
+    ) -> TokenizerWrapper {
+        let vocab_json: Value = serde_json::from_str(vocab).unwrap();
+        let added_tokens_json: Value = serde_json::from_str(added_tokens).unwrap();
+        let mut vocab = HashMap::new();
+        match vocab_json {
+            Value::Object(m) => {
+                for (token, id) in m {
+                    if let Value::Number(id) = id {
+                        let id = id.as_u64().unwrap() as u32;
+                        vocab.insert(token, id);
+                    }
+                }
+            }
+            _ => panic!("Invalid vocab.json file."),
+        };
+        match added_tokens_json {
+            Value::Object(m) => {
+                for (token, id) in m {
+                    if let Value::Number(id) = id {
+                        let id = id.as_u64().unwrap() as u32;
+                        vocab.insert(token, id);
+                    }
+                }
+            }
+            _ => panic!("Invalid added_tokens.json file."),
+        }
+
+        let merges = merges
+            .lines()
+            .filter(|line| !line.starts_with("#version"))
+            .map(|line| {
+                let parts = line.split(' ').collect::<Vec<_>>();
+                if parts.len() != 2 {
+                    panic!("Invalid merges.txt file.")
+                }
+                return (parts[0].to_string(), parts[1].to_string()); // Add the `return` keyword here
+            })
+            .collect::<Vec<(String, String)>>();
+        let byte_level = ByteLevel::new(
+            /*add_prefix_space=*/ false, /*trim_offsets=*/ false,
+            /*use_regex=*/ false,
+        );
+        let mut tokenizer = Tokenizer::new(BPE::new(vocab, merges));
+        tokenizer
+            .with_pre_tokenizer(byte_level)
+            .with_decoder(byte_level);
+        TokenizerWrapper {
+            tokenizer: tokenizer,
+            encode_ids: Vec::new(),
+            decode_str: String::new(),
+        }
+    }
+
+    pub fn encode(&mut self, text: &str, add_special_tokens: bool) {
+        self.encode_ids = Vec::from(
+            self.tokenizer
+                .encode(text, add_special_tokens)
+                .unwrap()
+                .get_ids(),
+        );
+    }
+
+    pub fn decode(&mut self, ids: Vec<u32>, skip_special_tokens: bool) {
+        self.decode_str = self.tokenizer.decode(ids, skip_special_tokens).unwrap();
+    }
+}
+
+#[no_mangle]
+extern "C" fn tokenizers_new_from_str(input_cstr: *const u8, len: usize) -> *mut TokenizerWrapper {
+    unsafe {
+        let json = std::str::from_utf8(std::slice::from_raw_parts(input_cstr, len)).unwrap();
+        return Box::into_raw(Box::new(TokenizerWrapper::from_str(json)));
+    }
+}
+
+#[no_mangle]
+extern "C" fn byte_level_bpe_tokenizers_new_from_str(
+    input_vocab_str: *const u8,
+    len_vocab: usize,
+    input_merges_str: *const u8,
+    len_merges: usize,
+    input_added_tokens_str: *const u8,
+    len_added_tokens: usize,
+) -> *mut TokenizerWrapper {
+    unsafe {
+        let vocab =
+            std::str::from_utf8(std::slice::from_raw_parts(input_vocab_str, len_vocab)).unwrap();
+        let merges =
+            std::str::from_utf8(std::slice::from_raw_parts(input_merges_str, len_merges)).unwrap();
+        let added_tokens = std::str::from_utf8(std::slice::from_raw_parts(
+            input_added_tokens_str,
+            len_added_tokens,
+        ))
+        .unwrap();
+        return Box::into_raw(Box::new(TokenizerWrapper::byte_level_bpe_from_str(
+            vocab,
+            merges,
+            added_tokens,
+        )));
+    }
+}
+
+#[no_mangle]
+extern "C" fn tokenizers_encode(
+    handle: *mut TokenizerWrapper,
+    input_cstr: *const u8,
+    len: usize,
+    add_special_tokens: i32,
+) {
+    unsafe {
+        let input_data = std::str::from_utf8(std::slice::from_raw_parts(input_cstr, len)).unwrap();
+        (*handle).encode(input_data, add_special_tokens != 0);
+    }
+}
+
+#[no_mangle]
+extern "C" fn tokenizers_get_encode_ids(
+    handle: *mut TokenizerWrapper,
+    out_data: *mut *mut u32,
+    out_len: *mut usize,
+) {
+    unsafe {
+        *out_data = (*handle).encode_ids.as_mut_ptr();
+        *out_len = (*handle).encode_ids.len()
+    }
+}
+
+#[no_mangle]
+extern "C" fn tokenizers_decode(
+    handle: *mut TokenizerWrapper,
+    input_ids: *const u32,
+    len: usize,
+    skip_special_tokens: i32,
+) {
+    unsafe {
+        let input_data = Vec::from(std::slice::from_raw_parts(input_ids, len));
+        (*handle).decode(input_data, skip_special_tokens != 0);
+    }
+}
+
+#[no_mangle]
+extern "C" fn tokenizers_get_decode_str(
+    handle: *mut TokenizerWrapper,
+    out_cstr: *mut *mut u8,
+    out_len: *mut usize,
+) {
+    unsafe {
+        *out_cstr = (*handle).decode_str.as_mut_ptr();
+        *out_len = (*handle).decode_str.len();
+    }
+}
+
+#[no_mangle]
+extern "C" fn tokenizers_free(wrapper: *mut TokenizerWrapper) {
+    unsafe {
+        drop(Box::from_raw(wrapper));
+    }
+}

mlc-llm-main/3rdparty/tokenizers-cpp/tokenizers.h

@@ -0,0 +1,101 @@
+/*!
+ *  Copyright (c) 2023 by Contributors
+ * \file tokenizers.cc
+ * \brief C binding to tokenizers
+ */
+#ifndef TOKENIZERS_H_
+#define TOKENIZERS_H_
+
+// The C API
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stddef.h>
+#include <stdint.h>
+
+typedef void* TokenizerHandle;
+
+TokenizerHandle tokenizers_new_from_str(const char* json, size_t len);
+
+TokenizerHandle byte_level_bpe_tokenizers_new_from_str(const char* vocab, size_t vocab_len,
+                                                       const char* merges, size_t merges_len,
+                                                       const char* added_tokens,
+                                                       size_t added_tokens_len);
+
+void tokenizers_encode(TokenizerHandle handle, const char* data, size_t len, int add_special_token);
+
+void tokenizers_decode(TokenizerHandle handle, const uint32_t* data, size_t len,
+                       int skip_special_token);
+
+void tokenizers_get_decode_str(TokenizerHandle handle, const char** data, size_t* len);
+
+void tokenizers_get_encode_ids(TokenizerHandle handle, const uint32_t** id_data, size_t* len);
+
+void tokenizers_free(TokenizerHandle handle);
+
+#ifdef __cplusplus
+}
+#endif
+
+#ifdef __cplusplus
+
+#include <string>
+#include <vector>
+
+// simple c++ binding
+namespace tokenizers {
+
+/*!
+ * \brief A simple c++ header of tokenizer via C API.
+ */
+class Tokenizer {
+ public:
+  Tokenizer(const Tokenizer&) = delete;
+  Tokenizer(Tokenizer&& other) { std::swap(other.handle_, handle_); }
+
+  ~Tokenizer() {
+    if (handle_ != nullptr) {
+      tokenizers_free(handle_);
+    }
+  }
+
+  // use i32 to be consistent with sentencepiece
+  std::vector<int32_t> Encode(const std::string& text, bool add_special_token) {
+    tokenizers_encode(handle_, text.data(), text.length(), static_cast<int>(add_special_token));
+    const uint32_t* data;
+    size_t len;
+    tokenizers_get_encode_ids(handle_, &data, &len);
+    return std::vector<int32_t>(data, data + len);
+  }
+
+  // use i32 to be consistent with sentencepiece
+  std::string Decode(const std::vector<int32_t>& ids, bool skip_special_token) {
+    tokenizers_decode(handle_, reinterpret_cast<const uint32_t*>(ids.data()), ids.size(),
+                      static_cast<int>(skip_special_token));
+    const char* data;
+    size_t len;
+    tokenizers_get_decode_str(handle_, &data, &len);
+    return std::string(data, len);
+  }
+
+  // avoid from_file so we don't have file system
+  static Tokenizer FromJSON(const std::string& json) {
+    return Tokenizer(tokenizers_new_from_str(json.data(), json.length()));
+  }
+
+  static Tokenizer FromBPE(const std::string& vocab, const std::string& merges,
+                           const std::string& added_tokens) {
+    return Tokenizer(byte_level_bpe_tokenizers_new_from_str(
+        vocab.data(), vocab.length(), merges.data(), merges.length(), added_tokens.data(),
+        added_tokens.length()));
+  }
+
+ private:
+  Tokenizer(TokenizerHandle handle) : handle_(handle) {}
+  // internal handle
+  TokenizerHandle handle_{nullptr};
+};
+}  // namespace tokenizers
+#endif
+#endif  // TOKENIZERS_H_

mlc-llm-main/CMakeLists.txt

@@ -0,0 +1,148 @@
+cmake_minimum_required(VERSION 3.18)
+project(mlc_llm C CXX)
+
+include(CheckCXXCompilerFlag)
+if(NOT MSVC)
+  check_cxx_compiler_flag("-std=c++17" SUPPORT_CXX17)
+  set(CMAKE_CXX_FLAGS "-std=c++17 ${CMAKE_CXX_FLAGS}")
+  set(CMAKE_CUDA_STANDARD 17)
+else()
+  check_cxx_compiler_flag("/std:c++17" SUPPORT_CXX17)
+  set(CMAKE_CXX_FLAGS "/std:c++17 ${CMAKE_CXX_FLAGS}")
+  set(CMAKE_CUDA_STANDARD 17)
+endif()
+
+if(EXISTS ${CMAKE_BINARY_DIR}/config.cmake)
+  include(${CMAKE_BINARY_DIR}/config.cmake)
+else()
+  if(EXISTS ${CMAKE_SOURCE_DIR}/config.cmake)
+    include(${CMAKE_SOURCE_DIR}/config.cmake)
+  endif()
+endif()
+
+if(NOT CMAKE_BUILD_TYPE)
+  set(
+    CMAKE_BUILD_TYPE RelWithDebInfo CACHE STRING "Build type" FORCE
+  )
+  message(STATUS "Setting default build type to " ${CMAKE_BUILD_TYPE})
+endif(NOT CMAKE_BUILD_TYPE)
+
+option(MLC_LLM_INSTALL_STATIC_LIB "Install static lib instead of cli" OFF)
+
+if (MLC_LLM_INSTALL_STATIC_LIB)
+  set(BUILD_STATIC_RUNTIME ON)
+endif()
+
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+
+# tvm runtime config: minimize runtime components
+set(USE_RPC OFF)
+set(USE_MICRO OFF)
+set(USE_GRAPH_EXECUTOR OFF)
+set(USE_GRAPH_EXECUTOR_DEBUG OFF)
+set(USE_AOT_EXECUTOR OFF)
+set(USE_PROFILER OFF)
+set(USE_GTEST OFF)
+set(USE_LIBBACKTRACE OFF)
+add_subdirectory($ENV{TVM_HOME} tvm EXCLUDE_FROM_ALL)
+
+# sentencepiece config
+option(SPM_ENABLE_SHARED "override sentence piece config" OFF)
+option(SPM_ENABLE_TCMALLOC "" OFF)
+
+
+set(MLC_LLM_RUNTIME_LINKER_LIB "")
+# provide macro if it does not exist in cmake system
+# it is OK to skip those since we do not provide these apps in the ios
+# instead just link to the sentencepiece directly
+if (CMAKE_SYSTEM_NAME STREQUAL "iOS")
+  macro (set_xcode_property TARGET XCODE_PROPERTY XCODE_VALUE)
+      set_property (TARGET ${TARGET} PROPERTY
+          XCODE_ATTRIBUTE_${XCODE_PROPERTY} ${XCODE_VALUE})
+  endmacro (set_xcode_property)
+endif()
+
+add_subdirectory(3rdparty/sentencepiece-js/sentencepiece sentencepiece EXCLUDE_FROM_ALL)
+add_subdirectory(3rdparty/tokenizers-cpp tokenizers EXCLUDE_FROM_ALL)
+
+
+tvm_file_glob(GLOB_RECURSE MLC_LLM_SRCS cpp/*.cc)
+tvm_file_glob(GLOB_RECURSE MLC_CLI_SRCS cpp/cli_main.cc)
+list(REMOVE_ITEM MLC_LLM_SRCS ${MLC_CLI_SRCS})
+
+add_library(mlc_llm_objs OBJECT ${MLC_LLM_SRCS})
+add_library(mlc_cli_objs OBJECT ${MLC_CLI_SRCS})
+
+set(
+  MLC_LLM_INCLUDES
+  $ENV{TVM_HOME}/include
+  $ENV{TVM_HOME}/3rdparty/dlpack/include
+  $ENV{TVM_HOME}/3rdparty/dmlc-core/include
+)
+
+set(MLC_LLM_COMPILE_DEFS DMLC_USE_LOGGING_LIBRARY=<tvm/runtime/logging.h>)
+
+target_include_directories(mlc_llm_objs PRIVATE ${MLC_LLM_INCLUDES})
+target_compile_definitions(mlc_llm_objs PRIVATE ${MLC_LLM_COMPILE_DEFS})
+target_include_directories(mlc_llm_objs PRIVATE 3rdparty/sentencepiece-js/sentencepiece/src)
+target_include_directories(mlc_llm_objs PRIVATE 3rdparty/tokenizers-cpp)
+target_compile_definitions(mlc_llm_objs PRIVATE -DMLC_LLM_EXPORTS)
+
+add_library(mlc_llm SHARED $<TARGET_OBJECTS:mlc_llm_objs>)
+add_library(mlc_llm_static STATIC $<TARGET_OBJECTS:mlc_llm_objs>)
+add_dependencies(mlc_llm_static sentencepiece-static tokenizers tvm_runtime)
+set_target_properties(mlc_llm_static PROPERTIES OUTPUT_NAME mlc_llm)
+
+target_link_libraries(mlc_llm PUBLIC tvm_runtime)
+target_link_libraries(mlc_llm PRIVATE sentencepiece)
+# note: somehow cmake needs to specify the dependency twice
+target_link_libraries(mlc_llm PRIVATE tokenizers)
+
+# Exmaple app that may depends on mlc_llm
+add_executable(mlc_chat_cli $<TARGET_OBJECTS:mlc_cli_objs>)
+target_include_directories(mlc_cli_objs PRIVATE ${MLC_LLM_INCLUDES})
+target_include_directories(mlc_cli_objs PRIVATE 3rdparty/argparse/include)
+target_compile_definitions(mlc_cli_objs PRIVATE ${MLC_LLM_COMPILE_DEFS})
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Android")
+  target_link_libraries(mlc_llm PRIVATE log)
+  target_link_libraries(mlc_chat_cli PRIVATE log)
+endif()
+
+if (MLC_LLM_INSTALL_STATIC_LIB)
+  target_link_libraries(
+    mlc_chat_cli PRIVATE mlc_llm_static sentencepiece-static tokenizers)
+  target_link_libraries(
+    mlc_chat_cli PRIVATE "$<LINK_LIBRARY:WHOLE_ARCHIVE,tvm_runtime>")
+else()
+  target_link_libraries(mlc_chat_cli PUBLIC mlc_llm)
+endif()
+
+# when this option is on,
+# we install all static lib deps into lib
+if (MLC_LLM_INSTALL_STATIC_LIB)
+  install(TARGETS
+    mlc_llm_static
+    sentencepiece-static
+    tvm_runtime
+    LIBRARY DESTINATION lib${LIB_SUFFIX}
+    )
+  # tokenizers need special handling as it builds from rust
+  if(MSVC)
+    install(FILES ${CMAKE_BINARY_DIR}/tokenizers/libtokenizers_cpp.lib
+      DESTINATION lib${LIB_SUFFIX}
+      )
+  else()
+    install(FILES ${CMAKE_BINARY_DIR}/tokenizers/libtokenizers_cpp.a
+      DESTINATION lib${LIB_SUFFIX}
+      )
+  endif()
+else()
+  install(TARGETS mlc_chat_cli tvm_runtime mlc_llm
+    mlc_llm_static
+    sentencepiece-static
+    RUNTIME_DEPENDENCY_SET tokenizers
+    RUNTIME DESTINATION bin
+    LIBRARY DESTINATION lib${LIB_SUFFIX}
+  )
+endif()

mlc-llm-main/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

mlc-llm-main/README.md

@@ -0,0 +1,65 @@
+# MLC LLM
+
+| [Project](https://mlc.ai/mlc-llm/) | [Blog](https://mlc.ai/blog/blog/2023/05/01/bringing-accelerated-llm-to-consumer-hardware) | [Demo: iOS](https://mlc.ai/mlc-llm/#iphone) | [Demo: Android](https://mlc.ai/mlc-llm/#android) | [Demo: CLI](https://mlc.ai/mlc-llm/#windows-linux-mac) | [WebLLM](https://mlc.ai/web-llm/) | [WebStableDiffusion](https://mlc.ai/web-stable-diffusion/) |
+
+MLC LLM is a **universal solution** that allows **any language models** to be **deployed natively** on a diverse set of hardware backends and native applications, plus a **productive framework** for everyone to further optimize model performance for their own use cases.
+
+Our mission is to **enable everyone to develop, optimize and deploy AI models natively on everyone's devices**.
+
+Everything runs locally  with no server support and accelerated with local GPUs on your phone and laptops.
+[Supported platforms](https://github.com/mlc-ai/mlc-llm/issues/15) include:
+* iPhone, iPad;
+* Android phones;
+* Metal GPUs and Intel/ARM MacBooks;
+* AMD, Intel and NVIDIA GPUs via Vulkan on Windows and Linux;
+* NVIDIA GPUs via CUDA on Windows and Linux;
+* WebGPU on browsers (through companion project [WebLLM](https://github.com/mlc-ai/web-llm/tree/main)).
+
+**[Check out our instruction page to try out!](https://mlc.ai/mlc-llm/)**
+
+<p align="center">
+  <img src="site/gif/ios-demo.gif" height="700">
+</p>
+
+## What is MLC LLM?
+
+In recent years, there has been remarkable progress in generative artificial intelligence (AI) and large language models (LLMs), which are becoming increasingly prevalent. Thanks to open-source initiatives, it is now possible to develop personal AI assistants using open-sourced models. However, LLMs tend to be resource-intensive and computationally demanding. To create a scalable service, developers may need to rely on powerful clusters and expensive hardware to run model inference. Additionally, deploying LLMs presents several challenges, such as their ever-evolving model innovation, memory constraints, and the need for potential optimization techniques.
+
+The goal of this project is to enable the development, optimization, and deployment of AI models for inference across a range of devices, including not just server-class hardware, but also users' browsers, laptops, and mobile apps. To achieve this, we need to address the diverse nature of compute devices and deployment environments. Some of the key challenges include:
+
+- Supporting different models of CPUs, GPUs, and potentially other co-processors and accelerators.
+- Deploying on the native environment of user devices, which may not have python or other necessary dependencies readily available.
+- Addressing memory constraints by carefully planning allocation and aggressively compressing model parameters.
+
+MLC LLM offers a repeatable, systematic, and customizable workflow that empowers developers and AI system researchers to implement models and optimizations in a productivity-focused, Python-first approach. This methodology enables quick experimentation with new models, new ideas and new compiler passes, followed by native deployment to the desired targets. Furthermore, we are continuously expanding LLM acceleration by broadening TVM backends to make model compilation more transparent and efficient.
+
+## How does MLC Enable Universal Native Deployment?
+
+The cornerstone of our solution is machine learning compilation ([MLC](https://mlc.ai/)), which we leverage to efficiently deploy AI models. We build on the shoulders of open-source ecosystems, including tokenizers from HuggingFace and Google, as well as open-source LLMs like Llama, Vicuna, Dolly, MOSS and more. Our primary workflow is based on [Apache TVM Unity](https://github.com/apache/tvm/tree/unity), an exciting ongoing development in the Apache TVM Community.
+
+- Dynamic shape: We bake a language model as a TVM IRModule with native dynamic shape support, avoiding the need for extra padding to the maximum length and reducing both computation amount and memory usage.
+- Composable ML compilation optimizations: we perform many model deployment optimizations, such as better compilation code transformation, fusion, memory planning, library offloading and manual code optimization can be easily incorporated as TVM's IRModule transformations exposed as Python APIs.
+- Quantization: We utilize low-bit quantizations to compress the model weights and leverage TVM's loop-level TensorIR to quickly customize code generations for different compression encoding schemes.
+- Runtime: The final generated libraries run on the native environment, with TVM runtime that comes with minimal dependencies, which supports various GPU driver APIs and native language bindings (C, JavaScript, etc).
+
+<img src="site/img/diag.svg" alt="Architecture Diagram" height=""/>
+
+Additionally, we also provide a lightweight C++-based example CLI app that showcases how to wrap up the compiled artifacts and necessary pre/post-processing, which will hopefully clarify the workflow to embed them into native applications.
+
+As a starting point, MLC generates GPU shaders for CUDA, Vulkan and Metal. It is possible to add more support, such as OpenCL, sycl, webgpu-native, through improvements to TVM compiler and runtime. MLC also supports various CPU targets including ARM and x86 via LLVM.
+
+We heavily rely on open-source ecosystem, more specifically, [TVM Unity](https://discuss.tvm.apache.org/t/establish-tvm-unity-connection-a-technical-strategy/13344), an exciting latest development in the TVM project that enables python-first interactive MLC development experiences that allows us to easily compose new optimizations all in Python, and incrementally bring our app to the environment of interest. We also leveraged optimizations such as fused quantization kernels, first class dynamic shape support and diverse GPU backends.
+
+## Links
+
+- You might also be interested in [WebLLM](https://github.com/mlc-ai/web-llm/tree/main), our companion derived project that focus on bringing LLM to browsers.
+- Project page for [instructions](site/index.md).
+- [Local build Instructions for ios App](ios/README.md).
+- You might want to check out our online public [Machine Learning Compilation course](https://mlc.ai) for a systematic
+walkthrough of our approaches.
+
+## Acknowledgements
+
+This project is initiated by members from CMU catalyst, UW SAMPL, SJTU, OctoML and the MLC community. We would love to continue developing and supporting the open-source ML community.
+
+This project is only possible thanks to the shoulders open-source ecosystems that we stand on. We want to thank the Apache TVM community and developers of the TVM Unity effort. The open-source ML community members made these models publicly available. PyTorch and Hugging Face communities that make these models accessible. We would like to thank the teams behind Vicuna, SentencePiece, LLaMA, Alpaca and MOSS. We also would like to thank the Vulkan, Swift, C++, Python Rust communities that enables this project.

mlc-llm-main/android/.gitignore

@@ -0,0 +1,3 @@
+MLCChat/app/src/main/jni/*.h
+MLCChat/app/src/main/jni/*.cc
+MLCChat/app/src/main/obj

mlc-llm-main/android/MLCChat/.gitignore

@@ -0,0 +1,15 @@
+*.iml
+.gradle
+/local.properties
+/.idea/caches
+/.idea/libraries
+/.idea/modules.xml
+/.idea/workspace.xml
+/.idea/navEditor.xml
+/.idea/assetWizardSettings.xml
+.DS_Store
+/build
+/captures
+.externalNativeBuild
+.cxx
+local.properties

mlc-llm-main/android/MLCChat/app/.gitignore

@@ -0,0 +1 @@
+/build

mlc-llm-main/android/MLCChat/app/build.gradle

@@ -0,0 +1,71 @@
+apply plugin: 'com.android.application'
+
+task generateJniHeaders(type: Exec, description: 'Generate JNI Headers') {
+    def headerPath = "${project.projectDir}/src/main/jni"
+    def classPath = "${project.projectDir}/../../build/tvm_home/jvm/core/target/*"
+    def filePath = "${project.projectDir}/../../build/tvm_home/jvm/core/src/main/java/org/apache/tvm/LibInfo.java"
+    commandLine "javac", "-h", headerPath, "-classpath", classPath, filePath
+    doLast {
+        file("${headerPath}/org_apache_tvm_LibInfo.h").renameTo(file("${headerPath}/org_apache_tvm_native_c_api.h"))
+    }
+}
+
+task copyFiles(type: Copy, description: 'Copy Sources for ndk-build') {
+    dependsOn "generateJniHeaders"
+    def ndkFilesPath = "${project.projectDir}/../../build/tvm_home/jvm/native/src/main/native"
+    def srcPath = "${project.projectDir}/src/main/jni/"
+
+    from "${ndkFilesPath}/org_apache_tvm_native_c_api.cc", "${ndkFilesPath}/jni_helper_func.h"
+    into srcPath
+}
+
+task deleteLibs(type: Delete, description: "Delete Compiled Libraries") {
+    dependsOn "copyFiles"
+    def libsPath = "${project.projectDir}/src/main/libs"
+    delete libsPath
+}
+
+task buildJni(type: Exec, description: 'Build JNI libs') {
+    dependsOn "deleteLibs"
+    def buildPath = "${project.projectDir}/src/main/jni"
+    commandLine "ndk-build", "--directory", buildPath
+}
+
+tasks.withType(JavaCompile) {
+    compileTask -> compileTask.dependsOn buildJni
+}
+
+android {
+    compileSdkVersion 33
+    defaultConfig {
+        applicationId "ai.mlc.mlcchat"
+        minSdkVersion 26
+        targetSdkVersion 33
+        versionCode 1
+        versionName "1.0"
+        testInstrumentationRunner 'androidx.test.runner.AndroidJUnitRunner'
+        ndk {
+            abiFilters 'arm64-v8a'
+        }
+    }
+    buildTypes {
+        release {
+            minifyEnabled false
+            proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.pro'
+        }
+    }
+    sourceSets {
+        main {
+            jni.srcDirs = []
+            jniLibs.srcDirs = ['src/main/libs']
+        }
+    }
+    namespace 'ai.mlc.mlcchat'
+    ndkVersion '25.2.9519653'
+}
+
+dependencies {
+    implementation fileTree(dir: 'libs', include: ['*.jar'])
+    implementation 'com.google.android.material:material:1.9.0'
+    implementation files("${project.projectDir}/../../build/tvm_home/jvm/core/target/tvm4j-core-0.0.1-SNAPSHOT.jar")
+}

mlc-llm-main/android/MLCChat/app/proguard-rules.pro

@@ -0,0 +1,21 @@
+# Add project specific ProGuard rules here.
+# You can control the set of applied configuration files using the
+# proguardFiles setting in build.gradle.
+#
+# For more details, see
+#   http://developer.android.com/guide/developing/tools/proguard.html
+
+# If your project uses WebView with JS, uncomment the following
+# and specify the fully qualified class name to the JavaScript interface
+# class:
+#-keepclassmembers class fqcn.of.javascript.interface.for.webview {
+#   public *;
+#}
+
+# Uncomment this to preserve the line number information for
+# debugging stack traces.
+#-keepattributes SourceFile,LineNumberTable
+
+# If you keep the line number information, uncomment this to
+# hide the original source file name.
+#-renamesourcefileattribute SourceFile

mlc-llm-main/android/MLCChat/app/src/main/AndroidManifest.xml

@@ -0,0 +1,36 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+    xmlns:tools="http://schemas.android.com/tools">
+
+    <uses-permission android:name="android.permission.INTERNET" />
+    <uses-permission
+        android:name="android.permission.WRITE_EXTERNAL_STORAGE"
+        android:maxSdkVersion="32"
+        tools:ignore="ScopedStorage" />
+
+    <application
+        android:allowBackup="true"
+        android:dataExtractionRules="@xml/data_extraction_rules"
+        android:fullBackupContent="@xml/backup_rules"
+        android:icon="@mipmap/ic_launcher"
+        android:label="@string/app_name"
+        android:roundIcon="@mipmap/ic_launcher_round"
+        android:supportsRtl="true"
+        android:theme="@style/Theme.MLCChat"
+        tools:targetApi="31">
+        <uses-native-library
+            android:name="libOpenCL.so"
+            android:required="false"/>
+        <activity
+            android:name=".MainActivity"
+            android:screenOrientation="portrait"
+            android:exported="true">
+            <intent-filter>
+                <action android:name="android.intent.action.MAIN" />
+
+                <category android:name="android.intent.category.LAUNCHER" />
+            </intent-filter>
+        </activity>
+    </application>
+
+</manifest>

mlc-llm-main/android/MLCChat/app/src/main/java/ai/mlc/mlcchat/ChatState.java

@@ -0,0 +1,117 @@
+package ai.mlc.mlcchat;
+
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import android.content.Context;
+import android.os.Handler;
+
+public class ChatState {
+    private boolean resetRequested;
+    private boolean stopRequested;
+    private final ExecutorService executorService;
+    private final Handler handler;
+
+    private final LLMChat backend;
+
+    public ChatState(Handler h, Context c) {
+        resetRequested = false;
+        stopRequested = false;
+        executorService = Executors.newSingleThreadExecutor();
+        handler = h;
+        backend = new LLMChat(c);
+        executorService.execute(() -> {
+            backend.Init();
+            Utils.sendInitDone(h);
+        });
+    }
+
+    public void stop() {
+        if (isStopRequested()) return;
+        requestStop();
+        executorService.execute(this::finishStop);
+    }
+
+    public void reset() {
+        stop();
+        if (isResetRequested()) return;
+        requestReset();
+        executorService.execute(() -> {
+            backend.ResetChat();
+            finishReset();
+            Utils.sendReset(handler);
+        });
+    }
+
+    public synchronized void requestReset() {
+        assert !resetRequested;
+        resetRequested = true;
+    }
+
+    public synchronized void finishReset() {
+        assert resetRequested;
+        resetRequested = false;
+    }
+
+    public synchronized boolean isResetRequested() {
+        return resetRequested;
+    }
+
+    public synchronized void requestStop() {
+        assert !stopRequested;
+        stopRequested = true;
+    }
+
+    public synchronized void finishStop() {
+        assert stopRequested;
+        stopRequested = false;
+    }
+
+    public synchronized boolean isStopRequested() {
+        return stopRequested;
+    }
+
+    void generate(String text) {
+        assert !isStopRequested() && !isResetRequested();
+        executorService.execute(() -> Generate(text, handler));
+    }
+
+
+    void Dummy(String text, Handler handler) {
+        String result = "";
+        for (int i = 0; i < 8; ++i) {
+            try {
+                Thread.sleep(1000);
+                if (isStopRequested()) {
+                    break;
+                }
+                result = result + " aba";
+                Utils.sendUpdateMessage(result, handler);
+            } catch (InterruptedException e) {
+                e.printStackTrace();
+            }
+        }
+        Utils.sendEnd("encode: 100.0 tok/s, decode: 100.0 tok/s", handler);
+    }
+
+    void Generate(String prompt, Handler handler) {
+        // System.err.println("Start generating");
+        backend.Encode(prompt);
+        // System.err.println("Encoding " + prompt);
+        while (!backend.Stopped()) {
+            backend.Decode();
+            // System.err.println("[INFO] " + backend.GetMessage());
+            Utils.sendUpdateMessage(backend.GetMessage(), handler);
+            if (isStopRequested()) {
+                break;
+            }
+        }
+        String stats = backend.RuntimeStatsText();
+        Utils.sendEnd(stats, handler);
+    }
+
+    public void terminate() {
+        Utils.terminate(executorService);
+    }
+
+}

mlc-llm-main/android/MLCChat/app/src/main/java/ai/mlc/mlcchat/Config.java

@@ -0,0 +1,12 @@
+package ai.mlc.mlcchat;
+
+public class Config {
+    public static final int UPD_MSG = 0;
+    public static final int APP_MSG = 1;
+    public static final int END = 2;
+    public static final int RESET = 3;
+    public static final int PARAMS_DONE = 4;
+    public static final int INIT_DONE = 5;
+    public static final String MSG_KEY = "message";
+    public static final String STATS_KEY = "stats";
+}

mlc-llm-main/android/MLCChat/app/src/main/java/ai/mlc/mlcchat/Downloader.java

@@ -0,0 +1,109 @@
+package ai.mlc.mlcchat;
+
+import android.content.Context;
+import android.os.Environment;
+import android.os.Handler;
+
+import java.io.DataInputStream;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.StandardCopyOption;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+public class Downloader {
+    private ExecutorService executorService;
+    private Handler handler;
+    private boolean isDownloading;
+
+    private Context context;
+
+    private HashMap<String, String> files;
+
+    public Downloader(Handler h, Context c) {
+        handler = h;
+        context = c;
+        executorService = Executors.newSingleThreadExecutor();
+        isDownloading = false;
+        files = new HashMap<>();
+    }
+
+    public synchronized void startDownload() {
+        assert !isDownloading;
+        isDownloading = true;
+    }
+
+    public synchronized void endDownload() {
+        assert isDownloading;
+        isDownloading = false;
+    }
+
+    public synchronized boolean inDownload() {
+        return isDownloading;
+    }
+
+    public void addFile(String url, String path) {
+        assert !inDownload();
+        File file = new File(context.getExternalFilesDir(Environment.DIRECTORY_DOWNLOADS), path);
+        if (!file.exists()) {
+            files.put(url, path);
+        }
+    }
+
+    public static String reportProgress(int done, int all) {
+        return String.format("[System] downloading [%d / %d]", done, all);
+    }
+
+    public void download() {
+        startDownload();
+        executorService.execute(() -> {
+            HashMap<String, String> filesToDownload = new HashMap<>(files);
+            files.clear();
+            int n = filesToDownload.size();
+            int i = 0;
+            if (n > 0) {
+                Utils.sendAppendMessage(reportProgress(i, n), handler);
+            }
+            for (Map.Entry<String, String> entry : filesToDownload.entrySet()) {
+                try {
+                    File temp = new File(context.getExternalFilesDir(Environment.DIRECTORY_DOWNLOADS), "temp");
+                    File file = new File(context.getExternalFilesDir(Environment.DIRECTORY_DOWNLOADS), entry.getValue());
+                    if (!file.exists()) {
+                        URL url = new URL(entry.getKey());
+                        InputStream inputStream = url.openStream();
+                        DataInputStream dataInputStream =
+                                new DataInputStream(inputStream);
+                        byte[] buffer = new byte[1024];
+                        int length;
+
+                        FileOutputStream fileOutputStream = new FileOutputStream(temp);
+                        while ((length = dataInputStream.read(buffer)) > 0) {
+                            fileOutputStream.write(buffer, 0, length);
+                        }
+
+                        Files.createDirectories(file.toPath().getParent());
+                        Files.move(temp.toPath(), file.toPath(), StandardCopyOption.ATOMIC_MOVE);
+                    }
+                    ++i;
+                    Utils.sendUpdateMessage(reportProgress(i, n), handler);
+                } catch (IOException e) {
+                    throw new RuntimeException(e);
+                }
+            }
+            endDownload();
+            Utils.sendParamsDone(handler);
+        });
+    }
+
+    public void terminate() {
+        Utils.terminate(executorService);
+    }
+}

mlc-llm-main/android/MLCChat/app/src/main/java/ai/mlc/mlcchat/LLMChat.java

@@ -0,0 +1,97 @@
+package ai.mlc.mlcchat;
+
+import android.content.Context;
+import android.os.Environment;
+
+import org.apache.tvm.Device;
+import org.apache.tvm.Function;
+import org.apache.tvm.Module;
+
+public class LLMChat {
+    private Function encode_func_;
+    private Function decode_func_;
+    private Function get_message_;
+    private Function stopped_func_;
+    private Function reset_chat_func_;
+    private Function runtime_stats_text_func_;
+    private Module llm_chat_;
+    private final Context context;
+
+    public LLMChat(Context c) {
+        this.context = c;
+    }
+
+    public void Init() {
+        Function systemlib_func = Function.getFunction("runtime.SystemLib");
+        assert systemlib_func != null;
+        Module lib = systemlib_func.invoke().asModule();
+        assert lib != null;
+        Function fcreate = Function.getFunction("mlc.llm_chat_create");
+        assert fcreate != null;
+        String dist_path = context.getExternalFilesDir(Environment.DIRECTORY_DOWNLOADS).getAbsolutePath();
+        String tokenizer_path = dist_path + "/vicuna-v1-7b/tokenizer.model";
+        String param_path = dist_path + "/vicuna-v1-7b/params";
+        System.err.println("[INFO] " + tokenizer_path);
+        System.err.println("[INFO] " + param_path);
+        System.err.println("[INFO] Before LLM Chat create");
+        llm_chat_ = fcreate.pushArg(lib).pushArg(tokenizer_path).pushArg(param_path).pushArg(Device.opencl().deviceType).pushArg(0).invoke().asModule();
+        System.err.println("[INFO] LLM Chat created!");
+        encode_func_ = llm_chat_.getFunction("encode");
+        decode_func_ = llm_chat_.getFunction("decode");
+        get_message_ = llm_chat_.getFunction("get_message");
+
+        stopped_func_ = llm_chat_.getFunction("stopped");
+        reset_chat_func_ = llm_chat_.getFunction("reset_chat");
+
+        runtime_stats_text_func_ = llm_chat_.getFunction("runtime_stats_text");
+
+        assert encode_func_ != null;
+        assert decode_func_ != null;
+        assert stopped_func_ != null;
+        assert runtime_stats_text_func_ != null;
+
+        String model = "vicuna";
+        String conv_template = "vicuna_v1.1";
+        int max_gen_len = 512 + 256;
+        double temperature = 0.7;
+        double top_p = 0.95;
+        int stream_interval = 1;
+        int max_window_size = 512 + 256;
+        int mean_gen_len = 128;
+        double shift_fill_factor = 0.2;
+        llm_chat_.getFunction("init_chat").pushArg(model).pushArg(conv_template).pushArg(max_gen_len).pushArg(temperature).pushArg(top_p).pushArg(stream_interval).pushArg(max_window_size).pushArg(mean_gen_len).pushArg(shift_fill_factor).invoke();
+
+        systemlib_func.release();
+        lib.release();
+        fcreate.release();
+
+        System.err.println("[INFO] Init done");
+    }
+    public void Evaluate() {
+        llm_chat_.getFunction("evaluate").invoke();
+    }
+
+    public String GetMessage() {
+        return get_message_.invoke().asString();
+    }
+
+    public void Encode(String prompt) {
+        encode_func_.pushArg(prompt).invoke();
+    }
+
+    public boolean Stopped() {
+        return stopped_func_.invoke().asLong() != 0;
+    }
+
+    public void Decode() {
+        decode_func_.invoke();
+    }
+
+    public String RuntimeStatsText() {
+        return runtime_stats_text_func_.invoke().asString();
+    }
+
+    public void ResetChat() {
+        reset_chat_func_.invoke();
+    }
+}

mlc-llm-main/android/MLCChat/app/src/main/java/ai/mlc/mlcchat/MainActivity.java

@@ -0,0 +1,178 @@
+package ai.mlc.mlcchat;
+
+import android.app.DownloadManager;
+import android.content.BroadcastReceiver;
+import android.content.Context;
+import android.content.Intent;
+import android.content.IntentFilter;
+import android.net.Uri;
+import android.os.Bundle;
+import android.os.Environment;
+import android.os.Handler;
+import android.os.Looper;
+import android.os.Message;
+import android.widget.Button;
+import android.widget.EditText;
+import android.widget.ListView;
+import android.widget.TextView;
+
+import androidx.annotation.NonNull;
+import androidx.appcompat.app.AppCompatActivity;
+
+import java.io.File;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.concurrent.ConcurrentHashMap;
+
+public class MainActivity extends AppCompatActivity {
+    private MessageAdapter messageAdapter;
+    private Button sendButton;
+    private Button resetButton;
+    private EditText editText;
+    private ListView listView;
+    private TextView speedText;
+    private Handler handler;
+    private ChatState chatState;
+    private Context context;
+
+    private Downloader downloader;
+
+    @Override
+    protected void onCreate(Bundle savedInstanceState) {
+        super.onCreate(savedInstanceState);
+        setContentView(R.layout.activity_main);
+        sendButton = findViewById(R.id.send);
+        resetButton = findViewById(R.id.reset);
+        editText = findViewById(R.id.input);
+        listView = findViewById(R.id.messages);
+        speedText = findViewById(R.id.speed);
+
+        context = getApplicationContext();
+
+        messageAdapter = new MessageAdapter(this);
+        listView.setAdapter(messageAdapter);
+        handler = new Handler(Looper.getMainLooper()) {
+            @Override
+            public void handleMessage(@NonNull Message msg) {
+                super.handleMessage(msg);
+                Bundle bundle = msg.getData();
+                switch (msg.what) {
+                    case Config.INIT_DONE:
+                        assert messageAdapter.isBotLast();
+                        updateMessage(new MessageData(MessageRole.BOT, "[System] Ready to chat"));
+                        resumeSend();
+                        resumeReset();
+                        break;
+                    case Config.UPD_MSG:
+                        assert messageAdapter.isBotLast();
+                        updateMessage(new MessageData(MessageRole.BOT, bundle.getString(Config.MSG_KEY)));
+                        break;
+                    case Config.APP_MSG:
+                        appendMessage(new MessageData(MessageRole.BOT, bundle.getString(Config.MSG_KEY)));
+                        break;
+                    case Config.END:
+                        speedText.setText(bundle.getString(Config.STATS_KEY));
+                        resumeSend();
+                        break;
+                    case Config.PARAMS_DONE:
+                        init();
+                        break;
+                    case Config.RESET:
+                        reset();
+                        break;
+                }
+            }
+        };
+        // System starting: freeze send and reset
+        freezeSend();
+        freezeReset();
+        // Prepare tokenizer and model params
+        downloader = new Downloader(handler, context);
+        prepareParams();
+    }
+
+    private void init() {
+        // Follow up after param is confirmed
+        // Chat state will initialize in another thread
+        appendMessage(new MessageData(MessageRole.BOT, "[System] Initializing..."));
+        chatState = new ChatState(handler, context);
+
+        sendButton.setOnClickListener(v -> {
+            String message = freezeSend();
+            appendMessage(new MessageData(MessageRole.USER, message));
+            appendMessage(new MessageData(MessageRole.BOT, ""));
+            chatState.generate(message);
+        });
+        resetButton.setOnClickListener(v -> {
+            freezeSend();
+            freezeReset();
+            chatState.reset();
+        });
+    }
+
+    private void prepareParams() {
+        appendMessage(new MessageData(MessageRole.BOT, "[System] Preparing Parameters..."));
+        String model_name = "vicuna-v1-7b";
+        String base_url = "https://huggingface.co/mlc-ai/demo-vicuna-v1-7b-int4/resolve/main";
+        downloader.addFile(base_url + "/tokenizer.model", model_name + "/tokenizer.model");
+        downloader.addFile(base_url + "/float16/ndarray-cache.json", model_name + "/params/ndarray-cache.json");
+        for (int i = 0; i <= 131; ++i) {
+            String param_name = "params_shard_" + i + ".bin";
+            downloader.addFile(base_url + "/float16/" + param_name, model_name + "/params/" + param_name);
+        }
+        downloader.download();
+    }
+
+
+    private String freezeSend() {
+        String text = editText.getText().toString();
+        editText.getText().clear();
+        editText.setEnabled(false);
+        sendButton.setEnabled(false);
+        sendButton.setAlpha((float) 0.1);
+        return text;
+    }
+
+    private void resumeSend() {
+        sendButton.setEnabled(true);
+        sendButton.setAlpha((float) 1.0);
+        editText.setEnabled(true);
+    }
+
+    private void freezeReset() {
+        resetButton.setEnabled(false);
+    }
+
+    private void resumeReset() {
+        resetButton.setEnabled(true);
+    }
+
+    @Override
+    protected void onRestart() {
+        super.onRestart();
+    }
+
+    @Override
+    protected void onDestroy() {
+        super.onDestroy();
+        chatState.terminate();
+        downloader.terminate();
+    }
+
+    private void appendMessage(MessageData messageData) {
+        messageAdapter.appendMessage(messageData);
+    }
+
+    private void updateMessage(MessageData messageData) {
+        messageAdapter.updateMessage(messageData);
+    }
+
+    private void reset() {
+        resumeSend();
+        editText.getText().clear();
+        messageAdapter.reset();
+        resumeReset();
+    }
+
+}

mlc-llm-main/android/MLCChat/app/src/main/java/ai/mlc/mlcchat/MessageAdapter.java

@@ -0,0 +1,83 @@
+package ai.mlc.mlcchat;
+
+import android.content.Context;
+import android.view.LayoutInflater;
+import android.view.View;
+import android.view.ViewGroup;
+import android.widget.BaseAdapter;
+import android.widget.TextView;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class MessageAdapter extends BaseAdapter {
+    private List<MessageData> messages;
+    private Context context;
+
+    public MessageAdapter(Context c) {
+        super();
+        context = c;
+        messages = new ArrayList<MessageData>();
+    }
+
+    public void appendMessage(MessageData m) {
+        messages.add(m);
+        notifyDataSetChanged();
+    }
+
+    public void updateMessage(MessageData m) {
+        messages.set(messages.size() - 1, m);
+        notifyDataSetChanged();
+    }
+
+    public boolean isBotLast() {
+        return messages.get(messages.size() - 1).isBot();
+    }
+
+    public int size() {
+        return messages.size();
+    }
+
+    public void reset() {
+        messages.clear();
+        notifyDataSetChanged();
+    }
+
+    @Override
+    public int getCount() {
+        return messages.size();
+    }
+
+    @Override
+    public Object getItem(int position) {
+        return messages.get(position);
+    }
+
+    @Override
+    public long getItemId(int position) {
+        return position;
+    }
+
+    @Override
+    public View getView(int position, View convertView, ViewGroup parent) {
+        MessageViewHolder holder = new MessageViewHolder();
+        LayoutInflater inflater = LayoutInflater.from(context);
+        MessageData chatMessage = messages.get(position);
+        if (chatMessage.isBot()) {
+            convertView = inflater.inflate(R.layout.bot_message, null);
+            holder.textView = convertView.findViewById(R.id.bot_message);
+            holder.textView.setText(chatMessage.getText());
+            convertView.setTag(holder);
+        } else {
+            convertView = inflater.inflate(R.layout.user_message, null);
+            holder.textView = convertView.findViewById(R.id.user_message);
+            holder.textView.setText(chatMessage.getText());
+            convertView.setTag(holder);
+        }
+        return convertView;
+    }
+
+    static class MessageViewHolder {
+        public TextView textView;
+    }
+}

mlc-llm-main/android/MLCChat/app/src/main/java/ai/mlc/mlcchat/MessageData.java

@@ -0,0 +1,26 @@
+package ai.mlc.mlcchat;
+
+import androidx.annotation.NonNull;
+
+enum MessageRole {
+    BOT, USER
+}
+
+public class MessageData {
+
+    private final MessageRole messageRole;
+    private final String text;
+
+    public MessageData(MessageRole r, String t) {
+        messageRole = r;
+        text = t;
+    }
+
+    public boolean isBot() {
+        return messageRole == MessageRole.BOT;
+    }
+
+    public String getText() {
+        return text;
+    }
+}

mlc-llm-main/android/MLCChat/app/src/main/java/ai/mlc/mlcchat/Utils.java

@@ -0,0 +1,73 @@
+package ai.mlc.mlcchat;
+
+import android.os.Bundle;
+import android.os.Handler;
+import android.os.Message;
+
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.TimeUnit;
+
+public class Utils {
+    public static void sendUpdateMessage(String text, Handler handler) {
+        Bundle bundle = new Bundle();
+        bundle.putString(Config.MSG_KEY, text);
+        Message message = new Message();
+        message.setData(bundle);
+        message.what = Config.UPD_MSG;
+        handler.sendMessage(message);
+    }
+
+    public static void sendAppendMessage(String text, Handler handler) {
+        Bundle bundle = new Bundle();
+        bundle.putString(Config.MSG_KEY, text);
+        Message message = new Message();
+        message.setData(bundle);
+        message.what = Config.APP_MSG;
+        handler.sendMessage(message);
+    }
+
+    public static void sendEnd(String stats, Handler handler) {
+        Bundle bundle = new Bundle();
+        bundle.putString(Config.STATS_KEY, stats);
+        Message message = new Message();
+        message.setData(bundle);
+        message.what = Config.END;
+        handler.sendMessage(message);
+    }
+
+    public static void sendReset(Handler handler) {
+        Message message = new Message();
+        message.what = Config.RESET;
+        handler.sendMessage(message);
+    }
+
+    public static void sendInitDone(Handler handler) {
+        Message message = new Message();
+        message.what = Config.INIT_DONE;
+        handler.sendMessage(message);
+    }
+
+    public static void sendParamsDone(Handler handler) {
+        Message message = new Message();
+        message.what = Config.PARAMS_DONE;
+        handler.sendMessage(message);
+    }
+
+    public static void terminate(ExecutorService executorService) {
+        executorService.shutdown(); // Disable new tasks from being submitted
+        try {
+            // Wait a while for existing tasks to terminate
+            if (!executorService.awaitTermination(60, TimeUnit.SECONDS)) {
+                executorService.shutdownNow(); // Cancel currently executing tasks
+                // Wait a while for tasks to respond to being cancelled
+                if (!executorService.awaitTermination(60, TimeUnit.SECONDS))
+                    System.err.println("Pool did not terminate");
+            }
+        } catch (InterruptedException ie) {
+            // (Re-)Cancel if current thread also interrupted
+            executorService.shutdownNow();
+            // Preserve interrupt status
+            Thread.currentThread().interrupt();
+        }
+    }
+}

mlc-llm-main/android/MLCChat/app/src/main/jni/Android.mk

@@ -0,0 +1,75 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+LOCAL_PATH := $(call my-dir)
+MY_PATH := $(abspath $(LOCAL_PATH))
+LIB_PATH := $(MY_PATH)/../../../../../build/lib
+TVM_HOME := $(MY_PATH)/../../../../../build/tvm_home
+
+#####################################################
+
+include $(CLEAR_VARS)
+LOCAL_MODULE := local_mlc_llm
+LOCAL_SRC_FILES := $(LIB_PATH)/libmlc_llm.a
+include $(PREBUILT_STATIC_LIBRARY)
+
+#####################################################
+
+include $(CLEAR_VARS)
+LOCAL_MODULE := local_sentencepiece
+LOCAL_SRC_FILES := $(LIB_PATH)/libsentencepiece.a
+include $(PREBUILT_STATIC_LIBRARY)
+
+#####################################################
+
+include $(CLEAR_VARS)
+LOCAL_MODULE := local_tokenizers_cpp
+LOCAL_SRC_FILES := $(LIB_PATH)/libtokenizers_cpp.a
+include $(PREBUILT_STATIC_LIBRARY)
+
+#####################################################
+
+include $(CLEAR_VARS)
+LOCAL_MODULE := local_tvm_runtime
+LOCAL_SRC_FILES := $(LIB_PATH)/libtvm_runtime.a
+include $(PREBUILT_STATIC_LIBRARY)
+
+#####################################################
+
+include $(CLEAR_VARS)
+LOCAL_MODULE := local_model_android
+LOCAL_SRC_FILES := $(LIB_PATH)/libmodel_android.a
+include $(PREBUILT_STATIC_LIBRARY)
+
+#####################################################
+
+include $(CLEAR_VARS)
+LOCAL_MODULE = tvm4j_runtime_packed
+LOCAL_SRC_FILES := org_apache_tvm_native_c_api.cc
+LOCAL_LDFLAGS := -L$(SYSROOT)/usr/lib/ -llog -pthread -ldl -lm
+LOCAL_C_INCLUDES := $(TVM_HOME)/include \
+                    $(TVM_HOME)/3rdparty/dlpack/include \
+                    $(TVM_HOME)/3rdparty/dmlc-core/include \
+                    $(MY_PATH)
+# LOCAL_C_FLAGS := -static
+
+LOCAL_WHOLE_STATIC_LIBRARIES := local_mlc_llm local_tvm_runtime local_model_android
+LOCAL_STATIC_LIBRARIES := local_sentencepiece local_tokenizers_cpp 
+LOCAL_CPP_FEATURES += exceptions
+LOCAL_ARM_MODE := arm
+
+include $(BUILD_SHARED_LIBRARY)

mlc-llm-main/android/MLCChat/app/src/main/jni/Application.mk

@@ -0,0 +1,5 @@
+APP_PLATFORM := android-24
+APP_ABI := arm64-v8a
+APP_STL := c++_static
+
+APP_CPPFLAGS += -DTVM_LOG_STACK_TRACE=0 -DTVM4J_ANDROID=1 -std=c++17 -Oz -frtti

mlc-llm-main/android/MLCChat/app/src/main/jni/tvm_runtime.h

@@ -0,0 +1,7 @@
+#define DMLC_USE_LOGGING_LIBRARY <tvm/runtime/logging.h>
+#define TVM_USE_LIBBACKTRACE 0
+
+#include <dlfcn.h>
+#include <dmlc/logging.h>
+#include <dmlc/thread_local.h>
+#include <tvm/runtime/c_runtime_api.h>

mlc-llm-main/android/MLCChat/app/src/main/res/drawable/bot_rounded_corner.xml

@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="utf-8"?>
+<shape xmlns:android="http://schemas.android.com/apk/res/android" >
+    <stroke
+        android:width="1dp"
+        android:color="@color/bot_background" />
+
+    <solid android:color="@color/bot_background" />
+
+    <padding
+        android:left="1dp"
+        android:right="1dp"
+        android:bottom="1dp"
+        android:top="1dp" />
+
+    <corners android:radius="10dp" />
+</shape>

mlc-llm-main/android/MLCChat/app/src/main/res/drawable/user_rounded_corner.xml

@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="utf-8"?>
+<shape xmlns:android="http://schemas.android.com/apk/res/android" >
+    <stroke
+        android:width="1dp"
+        android:color="@color/material_dynamic_primary80" />
+
+    <solid android:color="@color/material_dynamic_primary80" />
+
+    <padding
+        android:left="1dp"
+        android:right="1dp"
+        android:bottom="1dp"
+        android:top="1dp" />
+
+    <corners android:radius="10dp" />
+</shape>

mlc-llm-main/android/MLCChat/app/src/main/res/layout/activity_main.xml

@@ -0,0 +1,92 @@
+<?xml version="1.0" encoding="utf-8"?>
+<androidx.constraintlayout.widget.ConstraintLayout xmlns:android="http://schemas.android.com/apk/res/android"
+    xmlns:app="http://schemas.android.com/apk/res-auto"
+    xmlns:tools="http://schemas.android.com/tools"
+    android:layout_width="match_parent"
+    android:layout_height="match_parent"
+    tools:context=".MainActivity">
+
+    <com.google.android.material.appbar.MaterialToolbar
+        android:id="@+id/toolbar"
+        android:layout_width="match_parent"
+        android:layout_height="wrap_content"
+        android:background="?attr/colorPrimary"
+        android:minHeight="?attr/actionBarSize"
+        android:theme="?attr/actionBarTheme"
+        app:layout_constraintBottom_toTopOf="@+id/speed"
+        app:layout_constraintEnd_toEndOf="parent"
+        app:layout_constraintStart_toStartOf="parent"
+        app:layout_constraintTop_toTopOf="parent">
+
+        <androidx.appcompat.widget.AppCompatTextView
+            android:layout_width="wrap_content"
+            android:layout_height="match_parent"
+            android:text="MLC Chat"
+            android:textColor="@color/design_default_color_on_primary"
+            android:layout_gravity="center"
+            android:textAppearance="@style/TextAppearance.Material3.HeadlineMedium" />
+
+        <androidx.appcompat.widget.AppCompatButton
+            style="@style/Widget.AppCompat.Button.Borderless.Colored"
+            android:id="@+id/reset"
+            android:layout_width="wrap_content"
+            android:layout_height="match_parent"
+            android:text="reset"
+            android:textColor="@color/design_default_color_on_primary"
+            android:textAppearance="@style/TextAppearance.Material3.HeadlineSmall"
+            android:layout_marginEnd="5dp"
+            android:layout_gravity="end" />
+
+    </com.google.android.material.appbar.MaterialToolbar>
+
+    <androidx.appcompat.widget.AppCompatTextView
+        android:id="@+id/speed"
+        android:layout_width="match_parent"
+        android:layout_height="wrap_content"
+        android:textAlignment="center"
+        app:layout_constraintBottom_toTopOf="@id/messages"
+        app:layout_constraintEnd_toEndOf="parent"
+        app:layout_constraintStart_toStartOf="parent"
+        app:layout_constraintTop_toBottomOf="@id/toolbar" />
+
+
+    <androidx.appcompat.widget.AppCompatEditText
+        android:id="@+id/input"
+        android:layout_width="0dp"
+        android:layout_height="wrap_content"
+        android:layout_marginStart="5dp"
+        android:layout_marginEnd="5dp"
+        android:layout_marginBottom="5dp"
+        android:hint="Inputs..."
+        android:inputType="textMultiLine"
+        android:textAppearance="@style/TextAppearance.AppCompat.Small"
+        app:layout_constraintBottom_toBottomOf="parent"
+        app:layout_constraintEnd_toStartOf="@+id/send"
+        app:layout_constraintStart_toStartOf="parent" />
+
+    <androidx.appcompat.widget.AppCompatButton
+        android:id="@+id/send"
+        android:layout_width="wrap_content"
+        android:layout_height="wrap_content"
+        android:layout_marginTop="5dp"
+        android:layout_marginBottom="5dp"
+        android:layout_marginStart="5dp"
+        android:layout_marginEnd="5dp"
+        android:text="Send"
+        android:textAppearance="@style/TextAppearance.Material3.SearchBar"
+        app:layout_constraintBottom_toBottomOf="parent"
+        app:layout_constraintEnd_toEndOf="parent"
+        app:layout_constraintStart_toEndOf="@+id/input"
+        app:layout_constraintTop_toBottomOf="@+id/messages" />
+
+    <ListView
+        android:id="@+id/messages"
+        android:layout_width="match_parent"
+        android:layout_height="0dp"
+        android:divider="@null"
+        android:stackFromBottom="true"
+        android:transcriptMode="normal"
+        app:layout_constraintTop_toBottomOf="@id/speed"
+        app:layout_constraintBottom_toTopOf="@id/send" />
+
+</androidx.constraintlayout.widget.ConstraintLayout>

mlc-llm-main/android/MLCChat/app/src/main/res/layout/bot_message.xml

@@ -0,0 +1,30 @@
+<?xml version="1.0" encoding="utf-8"?>
+<androidx.constraintlayout.widget.ConstraintLayout xmlns:android="http://schemas.android.com/apk/res/android"
+    xmlns:app="http://schemas.android.com/apk/res-auto"
+    xmlns:tools="http://schemas.android.com/tools"
+    android:layout_width="match_parent"
+    android:layout_height="match_parent"
+    android:layout_marginEnd="50dp"
+    android:orientation="vertical">
+
+    <androidx.appcompat.widget.AppCompatTextView
+        android:id="@+id/bot_message"
+        android:layout_width="wrap_content"
+        android:layout_height="wrap_content"
+        android:layout_margin="5dp"
+        android:maxWidth="300dp"
+        android:background="@drawable/bot_rounded_corner"
+        android:gravity="start"
+        android:padding="10dp"
+        android:text="aaa"
+        android:textColor="#FFFFFF"
+        android:textAppearance="@style/TextAppearance.Material3.BodyMedium"
+        app:layout_constraintBottom_toBottomOf="parent"
+        app:layout_constraintEnd_toEndOf="parent"
+        app:layout_constraintHorizontal_bias="0.0"
+        app:layout_constraintStart_toStartOf="parent"
+        app:layout_constraintTop_toTopOf="parent"
+        app:layout_constraintVertical_bias="0.0" />
+
+
+</androidx.constraintlayout.widget.ConstraintLayout>

mlc-llm-main/android/MLCChat/app/src/main/res/layout/user_message.xml

@@ -0,0 +1,30 @@
+<?xml version="1.0" encoding="utf-8"?>
+<androidx.constraintlayout.widget.ConstraintLayout xmlns:android="http://schemas.android.com/apk/res/android"
+    xmlns:app="http://schemas.android.com/apk/res-auto"
+    xmlns:tools="http://schemas.android.com/tools"
+    android:layout_width="match_parent"
+    android:layout_height="match_parent"
+    android:layout_marginStart="50dp"
+    android:orientation="vertical">
+
+    <androidx.appcompat.widget.AppCompatTextView
+        android:id="@+id/user_message"
+        android:layout_width="wrap_content"
+        android:layout_height="wrap_content"
+        android:layout_margin="5dp"
+        android:maxWidth="300dp"
+        android:gravity="start"
+        android:padding="10dp"
+        android:text="aaa"
+        android:textAppearance="@style/TextAppearance.Material3.BodyMedium"
+        android:background="@drawable/user_rounded_corner"
+        android:textColor="@color/design_default_color_on_primary"
+        app:layout_constraintBottom_toBottomOf="parent"
+        app:layout_constraintEnd_toEndOf="parent"
+        app:layout_constraintHorizontal_bias="1.0"
+        app:layout_constraintStart_toStartOf="parent"
+        app:layout_constraintTop_toTopOf="parent"
+        app:layout_constraintVertical_bias="0.0" />
+
+
+</androidx.constraintlayout.widget.ConstraintLayout>

mlc-llm-main/android/MLCChat/app/src/main/res/mipmap-anydpi-v26/ic_launcher.xml

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="utf-8"?>
+<adaptive-icon xmlns:android="http://schemas.android.com/apk/res/android">
+    <background android:drawable="@color/ic_launcher_background"/>
+    <foreground android:drawable="@mipmap/ic_launcher_foreground"/>
+</adaptive-icon>

mlc-llm-main/android/MLCChat/app/src/main/res/mipmap-anydpi-v26/ic_launcher_round.xml

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="utf-8"?>
+<adaptive-icon xmlns:android="http://schemas.android.com/apk/res/android">
+    <background android:drawable="@color/ic_launcher_background"/>
+    <foreground android:drawable="@mipmap/ic_launcher_foreground"/>
+</adaptive-icon>