Upload 14 files

api/.dockerignore

@@ -0,0 +1,14 @@
+.env
+*.env.*
+
+storage/privkeys/*
+
+# Logs
+logs
+*.log*
+
+# jetbrains
+.idea
+
+# venv
+.venv

api/.env.example

@@ -0,0 +1,433 @@
+# Your App secret key will be used for securely signing the session cookie
+# Make sure you are changing this key for your deployment with a strong key.
+# You can generate a strong key using `openssl rand -base64 42`.
+# Alternatively you can set it with `SECRET_KEY` environment variable.
+SECRET_KEY=
+
+# Console API base URL
+CONSOLE_API_URL=http://127.0.0.1:5001
+CONSOLE_WEB_URL=http://127.0.0.1:3000
+
+# Service API base URL
+SERVICE_API_URL=http://127.0.0.1:5001
+
+# Web APP base URL
+APP_WEB_URL=http://127.0.0.1:3000
+
+# Files URL
+FILES_URL=http://127.0.0.1:5001
+
+# The time in seconds after the signature is rejected
+FILES_ACCESS_TIMEOUT=300
+
+# Access token expiration time in minutes
+ACCESS_TOKEN_EXPIRE_MINUTES=60
+
+# Refresh token expiration time in days
+REFRESH_TOKEN_EXPIRE_DAYS=30
+
+# celery configuration
+CELERY_BROKER_URL=redis://:difyai123456@localhost:6379/1
+
+# redis configuration
+REDIS_HOST=localhost
+REDIS_PORT=6379
+REDIS_USERNAME=
+REDIS_PASSWORD=difyai123456
+REDIS_USE_SSL=false
+REDIS_DB=0
+
+# redis Sentinel configuration.
+REDIS_USE_SENTINEL=false
+REDIS_SENTINELS=
+REDIS_SENTINEL_SERVICE_NAME=
+REDIS_SENTINEL_USERNAME=
+REDIS_SENTINEL_PASSWORD=
+REDIS_SENTINEL_SOCKET_TIMEOUT=0.1
+
+# redis Cluster configuration.
+REDIS_USE_CLUSTERS=false
+REDIS_CLUSTERS=
+REDIS_CLUSTERS_PASSWORD=
+
+# PostgreSQL database configuration
+DB_USERNAME=postgres
+DB_PASSWORD=difyai123456
+DB_HOST=localhost
+DB_PORT=5432
+DB_DATABASE=dify
+
+# Storage configuration
+# use for store upload files, private keys...
+# storage type: opendal, s3, aliyun-oss, azure-blob, baidu-obs, google-storage, huawei-obs, oci-storage, tencent-cos, volcengine-tos, supabase
+STORAGE_TYPE=opendal
+
+# Apache OpenDAL storage configuration, refer to https://github.com/apache/opendal
+OPENDAL_SCHEME=fs
+OPENDAL_FS_ROOT=storage
+
+# S3 Storage configuration
+S3_USE_AWS_MANAGED_IAM=false
+S3_ENDPOINT=https://your-bucket-name.storage.s3.cloudflare.com
+S3_BUCKET_NAME=your-bucket-name
+S3_ACCESS_KEY=your-access-key
+S3_SECRET_KEY=your-secret-key
+S3_REGION=your-region
+
+# Azure Blob Storage configuration
+AZURE_BLOB_ACCOUNT_NAME=your-account-name
+AZURE_BLOB_ACCOUNT_KEY=your-account-key
+AZURE_BLOB_CONTAINER_NAME=your-container-name
+AZURE_BLOB_ACCOUNT_URL=https://<your_account_name>.blob.core.windows.net
+
+# Aliyun oss Storage configuration
+ALIYUN_OSS_BUCKET_NAME=your-bucket-name
+ALIYUN_OSS_ACCESS_KEY=your-access-key
+ALIYUN_OSS_SECRET_KEY=your-secret-key
+ALIYUN_OSS_ENDPOINT=your-endpoint
+ALIYUN_OSS_AUTH_VERSION=v1
+ALIYUN_OSS_REGION=your-region
+# Don't start with '/'. OSS doesn't support leading slash in object names.
+ALIYUN_OSS_PATH=your-path
+
+# Google Storage configuration
+GOOGLE_STORAGE_BUCKET_NAME=your-bucket-name
+GOOGLE_STORAGE_SERVICE_ACCOUNT_JSON_BASE64=your-google-service-account-json-base64-string
+
+# Tencent COS Storage configuration
+TENCENT_COS_BUCKET_NAME=your-bucket-name
+TENCENT_COS_SECRET_KEY=your-secret-key
+TENCENT_COS_SECRET_ID=your-secret-id
+TENCENT_COS_REGION=your-region
+TENCENT_COS_SCHEME=your-scheme
+
+# Huawei OBS Storage Configuration
+HUAWEI_OBS_BUCKET_NAME=your-bucket-name
+HUAWEI_OBS_SECRET_KEY=your-secret-key
+HUAWEI_OBS_ACCESS_KEY=your-access-key
+HUAWEI_OBS_SERVER=your-server-url
+
+# Baidu OBS Storage Configuration
+BAIDU_OBS_BUCKET_NAME=your-bucket-name
+BAIDU_OBS_SECRET_KEY=your-secret-key
+BAIDU_OBS_ACCESS_KEY=your-access-key
+BAIDU_OBS_ENDPOINT=your-server-url
+
+# OCI Storage configuration
+OCI_ENDPOINT=your-endpoint
+OCI_BUCKET_NAME=your-bucket-name
+OCI_ACCESS_KEY=your-access-key
+OCI_SECRET_KEY=your-secret-key
+OCI_REGION=your-region
+
+# Volcengine tos Storage configuration
+VOLCENGINE_TOS_ENDPOINT=your-endpoint
+VOLCENGINE_TOS_BUCKET_NAME=your-bucket-name
+VOLCENGINE_TOS_ACCESS_KEY=your-access-key
+VOLCENGINE_TOS_SECRET_KEY=your-secret-key
+VOLCENGINE_TOS_REGION=your-region
+
+# Supabase Storage Configuration
+SUPABASE_BUCKET_NAME=your-bucket-name
+SUPABASE_API_KEY=your-access-key
+SUPABASE_URL=your-server-url
+
+# CORS configuration
+WEB_API_CORS_ALLOW_ORIGINS=http://127.0.0.1:3000,*
+CONSOLE_CORS_ALLOW_ORIGINS=http://127.0.0.1:3000,*
+
+# Vector database configuration
+# support: weaviate, qdrant, milvus, myscale, relyt, pgvecto_rs, pgvector, pgvector, chroma, opensearch, tidb_vector, couchbase, vikingdb, upstash, lindorm, oceanbase
+VECTOR_STORE=weaviate
+
+# Weaviate configuration
+WEAVIATE_ENDPOINT=http://localhost:8080
+WEAVIATE_API_KEY=WVF5YThaHlkYwhGUSmCRgsX3tD5ngdN8pkih
+WEAVIATE_GRPC_ENABLED=false
+WEAVIATE_BATCH_SIZE=100
+
+# Qdrant configuration, use `http://localhost:6333` for local mode or `https://your-qdrant-cluster-url.qdrant.io` for remote mode
+QDRANT_URL=http://localhost:6333
+QDRANT_API_KEY=difyai123456
+QDRANT_CLIENT_TIMEOUT=20
+QDRANT_GRPC_ENABLED=false
+QDRANT_GRPC_PORT=6334
+
+#Couchbase configuration
+COUCHBASE_CONNECTION_STRING=127.0.0.1
+COUCHBASE_USER=Administrator
+COUCHBASE_PASSWORD=password
+COUCHBASE_BUCKET_NAME=Embeddings
+COUCHBASE_SCOPE_NAME=_default
+
+# Milvus configuration
+MILVUS_URI=http://127.0.0.1:19530
+MILVUS_TOKEN=
+MILVUS_USER=root
+MILVUS_PASSWORD=Milvus
+
+# MyScale configuration
+MYSCALE_HOST=127.0.0.1
+MYSCALE_PORT=8123
+MYSCALE_USER=default
+MYSCALE_PASSWORD=
+MYSCALE_DATABASE=default
+MYSCALE_FTS_PARAMS=
+
+# Relyt configuration
+RELYT_HOST=127.0.0.1
+RELYT_PORT=5432
+RELYT_USER=postgres
+RELYT_PASSWORD=postgres
+RELYT_DATABASE=postgres
+
+# Tencent configuration
+TENCENT_VECTOR_DB_URL=http://127.0.0.1
+TENCENT_VECTOR_DB_API_KEY=dify
+TENCENT_VECTOR_DB_TIMEOUT=30
+TENCENT_VECTOR_DB_USERNAME=dify
+TENCENT_VECTOR_DB_DATABASE=dify
+TENCENT_VECTOR_DB_SHARD=1
+TENCENT_VECTOR_DB_REPLICAS=2
+
+# ElasticSearch configuration
+ELASTICSEARCH_HOST=127.0.0.1
+ELASTICSEARCH_PORT=9200
+ELASTICSEARCH_USERNAME=elastic
+ELASTICSEARCH_PASSWORD=elastic
+
+# PGVECTO_RS configuration
+PGVECTO_RS_HOST=localhost
+PGVECTO_RS_PORT=5431
+PGVECTO_RS_USER=postgres
+PGVECTO_RS_PASSWORD=difyai123456
+PGVECTO_RS_DATABASE=postgres
+
+# PGVector configuration
+PGVECTOR_HOST=127.0.0.1
+PGVECTOR_PORT=5433
+PGVECTOR_USER=postgres
+PGVECTOR_PASSWORD=postgres
+PGVECTOR_DATABASE=postgres
+PGVECTOR_MIN_CONNECTION=1
+PGVECTOR_MAX_CONNECTION=5
+
+# Tidb Vector configuration
+TIDB_VECTOR_HOST=xxx.eu-central-1.xxx.aws.tidbcloud.com
+TIDB_VECTOR_PORT=4000
+TIDB_VECTOR_USER=xxx.root
+TIDB_VECTOR_PASSWORD=xxxxxx
+TIDB_VECTOR_DATABASE=dify
+
+# Tidb on qdrant configuration
+TIDB_ON_QDRANT_URL=http://127.0.0.1
+TIDB_ON_QDRANT_API_KEY=dify
+TIDB_ON_QDRANT_CLIENT_TIMEOUT=20
+TIDB_ON_QDRANT_GRPC_ENABLED=false
+TIDB_ON_QDRANT_GRPC_PORT=6334
+TIDB_PUBLIC_KEY=dify
+TIDB_PRIVATE_KEY=dify
+TIDB_API_URL=http://127.0.0.1
+TIDB_IAM_API_URL=http://127.0.0.1
+TIDB_REGION=regions/aws-us-east-1
+TIDB_PROJECT_ID=dify
+TIDB_SPEND_LIMIT=100
+
+# Chroma configuration
+CHROMA_HOST=127.0.0.1
+CHROMA_PORT=8000
+CHROMA_TENANT=default_tenant
+CHROMA_DATABASE=default_database
+CHROMA_AUTH_PROVIDER=chromadb.auth.token_authn.TokenAuthenticationServerProvider
+CHROMA_AUTH_CREDENTIALS=difyai123456
+
+# AnalyticDB configuration
+ANALYTICDB_KEY_ID=your-ak
+ANALYTICDB_KEY_SECRET=your-sk
+ANALYTICDB_REGION_ID=cn-hangzhou
+ANALYTICDB_INSTANCE_ID=gp-ab123456
+ANALYTICDB_ACCOUNT=testaccount
+ANALYTICDB_PASSWORD=testpassword
+ANALYTICDB_NAMESPACE=dify
+ANALYTICDB_NAMESPACE_PASSWORD=difypassword
+ANALYTICDB_HOST=gp-test.aliyuncs.com
+ANALYTICDB_PORT=5432
+ANALYTICDB_MIN_CONNECTION=1
+ANALYTICDB_MAX_CONNECTION=5
+
+# OpenSearch configuration
+OPENSEARCH_HOST=127.0.0.1
+OPENSEARCH_PORT=9200
+OPENSEARCH_USER=admin
+OPENSEARCH_PASSWORD=admin
+OPENSEARCH_SECURE=true
+
+# Baidu configuration
+BAIDU_VECTOR_DB_ENDPOINT=http://127.0.0.1:5287
+BAIDU_VECTOR_DB_CONNECTION_TIMEOUT_MS=30000
+BAIDU_VECTOR_DB_ACCOUNT=root
+BAIDU_VECTOR_DB_API_KEY=dify
+BAIDU_VECTOR_DB_DATABASE=dify
+BAIDU_VECTOR_DB_SHARD=1
+BAIDU_VECTOR_DB_REPLICAS=3
+
+# Upstash configuration
+UPSTASH_VECTOR_URL=your-server-url
+UPSTASH_VECTOR_TOKEN=your-access-token
+
+# ViKingDB configuration
+VIKINGDB_ACCESS_KEY=your-ak
+VIKINGDB_SECRET_KEY=your-sk
+VIKINGDB_REGION=cn-shanghai
+VIKINGDB_HOST=api-vikingdb.xxx.volces.com
+VIKINGDB_SCHEMA=http
+VIKINGDB_CONNECTION_TIMEOUT=30
+VIKINGDB_SOCKET_TIMEOUT=30
+
+# Lindorm configuration
+LINDORM_URL=http://ld-*******************-proxy-search-pub.lindorm.aliyuncs.com:30070
+LINDORM_USERNAME=admin
+LINDORM_PASSWORD=admin
+USING_UGC_INDEX=False
+
+# OceanBase Vector configuration
+OCEANBASE_VECTOR_HOST=127.0.0.1
+OCEANBASE_VECTOR_PORT=2881
+OCEANBASE_VECTOR_USER=root@test
+OCEANBASE_VECTOR_PASSWORD=difyai123456
+OCEANBASE_VECTOR_DATABASE=test
+OCEANBASE_MEMORY_LIMIT=6G
+
+
+# Upload configuration
+UPLOAD_FILE_SIZE_LIMIT=15
+UPLOAD_FILE_BATCH_LIMIT=5
+UPLOAD_IMAGE_FILE_SIZE_LIMIT=10
+UPLOAD_VIDEO_FILE_SIZE_LIMIT=100
+UPLOAD_AUDIO_FILE_SIZE_LIMIT=50
+
+# Model configuration
+MULTIMODAL_SEND_FORMAT=base64
+PROMPT_GENERATION_MAX_TOKENS=512
+CODE_GENERATION_MAX_TOKENS=1024
+
+# Mail configuration, support: resend, smtp
+MAIL_TYPE=
+MAIL_DEFAULT_SEND_FROM=no-reply <no-reply@dify.ai>
+RESEND_API_KEY=
+RESEND_API_URL=https://api.resend.com
+# smtp configuration
+SMTP_SERVER=smtp.gmail.com
+SMTP_PORT=465
+SMTP_USERNAME=123
+SMTP_PASSWORD=abc
+SMTP_USE_TLS=true
+SMTP_OPPORTUNISTIC_TLS=false
+
+# Sentry configuration
+SENTRY_DSN=
+
+# DEBUG
+DEBUG=false
+SQLALCHEMY_ECHO=false
+
+# Notion import configuration, support public and internal
+NOTION_INTEGRATION_TYPE=public
+NOTION_CLIENT_SECRET=you-client-secret
+NOTION_CLIENT_ID=you-client-id
+NOTION_INTERNAL_SECRET=you-internal-secret
+
+ETL_TYPE=dify
+UNSTRUCTURED_API_URL=
+UNSTRUCTURED_API_KEY=
+SCARF_NO_ANALYTICS=true
+
+#ssrf
+SSRF_PROXY_HTTP_URL=
+SSRF_PROXY_HTTPS_URL=
+SSRF_DEFAULT_MAX_RETRIES=3
+SSRF_DEFAULT_TIME_OUT=5
+SSRF_DEFAULT_CONNECT_TIME_OUT=5
+SSRF_DEFAULT_READ_TIME_OUT=5
+SSRF_DEFAULT_WRITE_TIME_OUT=5
+
+BATCH_UPLOAD_LIMIT=10
+KEYWORD_DATA_SOURCE_TYPE=database
+
+# Workflow file upload limit
+WORKFLOW_FILE_UPLOAD_LIMIT=10
+
+# CODE EXECUTION CONFIGURATION
+CODE_EXECUTION_ENDPOINT=http://127.0.0.1:8194
+CODE_EXECUTION_API_KEY=dify-sandbox
+CODE_MAX_NUMBER=9223372036854775807
+CODE_MIN_NUMBER=-9223372036854775808
+CODE_MAX_STRING_LENGTH=80000
+TEMPLATE_TRANSFORM_MAX_LENGTH=80000
+CODE_MAX_STRING_ARRAY_LENGTH=30
+CODE_MAX_OBJECT_ARRAY_LENGTH=30
+CODE_MAX_NUMBER_ARRAY_LENGTH=1000
+
+# API Tool configuration
+API_TOOL_DEFAULT_CONNECT_TIMEOUT=10
+API_TOOL_DEFAULT_READ_TIMEOUT=60
+
+# HTTP Node configuration
+HTTP_REQUEST_MAX_CONNECT_TIMEOUT=300
+HTTP_REQUEST_MAX_READ_TIMEOUT=600
+HTTP_REQUEST_MAX_WRITE_TIMEOUT=600
+HTTP_REQUEST_NODE_MAX_BINARY_SIZE=10485760
+HTTP_REQUEST_NODE_MAX_TEXT_SIZE=1048576
+
+# Respect X-* headers to redirect clients
+RESPECT_XFORWARD_HEADERS_ENABLED=false
+
+# Log file path
+LOG_FILE=
+# Log file max size, the unit is MB
+LOG_FILE_MAX_SIZE=20
+# Log file max backup count
+LOG_FILE_BACKUP_COUNT=5
+# Log dateformat
+LOG_DATEFORMAT=%Y-%m-%d %H:%M:%S
+# Log Timezone
+LOG_TZ=UTC
+# Log format
+LOG_FORMAT=%(asctime)s,%(msecs)d %(levelname)-2s [%(filename)s:%(lineno)d] %(req_id)s %(message)s
+
+# Indexing configuration
+INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH=4000
+
+# Workflow runtime configuration
+WORKFLOW_MAX_EXECUTION_STEPS=500
+WORKFLOW_MAX_EXECUTION_TIME=1200
+WORKFLOW_CALL_MAX_DEPTH=5
+WORKFLOW_PARALLEL_DEPTH_LIMIT=3
+MAX_VARIABLE_SIZE=204800
+
+# App configuration
+APP_MAX_EXECUTION_TIME=1200
+APP_MAX_ACTIVE_REQUESTS=0
+
+
+# Celery beat configuration
+CELERY_BEAT_SCHEDULER_TIME=1
+
+# Position configuration
+POSITION_TOOL_PINS=
+POSITION_TOOL_INCLUDES=
+POSITION_TOOL_EXCLUDES=
+
+POSITION_PROVIDER_PINS=
+POSITION_PROVIDER_INCLUDES=
+POSITION_PROVIDER_EXCLUDES=
+
+# Reset password token expiry minutes
+RESET_PASSWORD_TOKEN_EXPIRY_MINUTES=5
+
+CREATE_TIDB_SERVICE_JOB_ENABLED=false
+
+# Maximum number of submitted thread count in a ThreadPool for parallel node execution
+MAX_SUBMIT_COUNT=100
+# Lockout duration in seconds
+LOGIN_LOCKOUT_DURATION=86400

api/.ruff.toml

@@ -0,0 +1,97 @@
+exclude = [
+    "migrations/*",
+]
+line-length = 120
+
+[format]
+quote-style = "double"
+
+[lint]
+preview = true
+select = [
+    "B", # flake8-bugbear rules
+    "C4", # flake8-comprehensions
+    "E", # pycodestyle E rules
+    "F", # pyflakes rules
+    "FURB", # refurb rules
+    "I", # isort rules
+    "N", # pep8-naming
+    "PT", # flake8-pytest-style rules
+    "PLC0208", # iteration-over-set
+    "PLC2801", # unnecessary-dunder-call
+    "PLC0414", # useless-import-alias
+    "PLE0604", # invalid-all-object
+    "PLE0605", # invalid-all-format
+    "PLR0402", # manual-from-import
+    "PLR1711", # useless-return
+    "PLR1714", # repeated-equality-comparison
+    "RUF013", # implicit-optional
+    "RUF019", # unnecessary-key-check
+    "RUF100", # unused-noqa
+    "RUF101", # redirected-noqa
+    "RUF200", # invalid-pyproject-toml
+    "RUF022", # unsorted-dunder-all
+    "S506", # unsafe-yaml-load
+    "SIM", # flake8-simplify rules
+    "TRY400", # error-instead-of-exception
+    "TRY401", # verbose-log-message
+    "UP", # pyupgrade rules
+    "W191", # tab-indentation
+    "W605", # invalid-escape-sequence
+]
+
+ignore = [
+    "E402", # module-import-not-at-top-of-file
+    "E711", # none-comparison
+    "E712", # true-false-comparison
+    "E721", # type-comparison
+    "E722", # bare-except
+    "E731", # lambda-assignment
+    "F821", # undefined-name
+    "F841", # unused-variable
+    "FURB113", # repeated-append
+    "FURB152", # math-constant
+    "UP007", # non-pep604-annotation
+    "UP032", # f-string
+    "UP045", # non-pep604-annotation-optional
+    "B005", # strip-with-multi-characters
+    "B006", # mutable-argument-default
+    "B007", # unused-loop-control-variable
+    "B026", # star-arg-unpacking-after-keyword-arg
+    "B903", # class-as-data-structure
+    "B904", # raise-without-from-inside-except
+    "B905", # zip-without-explicit-strict
+    "N806", # non-lowercase-variable-in-function
+    "N815", # mixed-case-variable-in-class-scope
+    "PT011", # pytest-raises-too-broad
+    "SIM102", # collapsible-if
+    "SIM103", # needless-bool
+    "SIM105", # suppressible-exception
+    "SIM107", # return-in-try-except-finally
+    "SIM108", # if-else-block-instead-of-if-exp
+    "SIM113", # enumerate-for-loop
+    "SIM117", # multiple-with-statements
+    "SIM210", # if-expr-with-true-false
+]
+
+[lint.per-file-ignores]
+"__init__.py" = [
+    "F401", # unused-import
+    "F811", # redefined-while-unused
+]
+"configs/*" = [
+    "N802", # invalid-function-name
+]
+"libs/gmpy2_pkcs10aep_cipher.py" = [
+    "N803", # invalid-argument-name
+]
+"tests/*" = [
+    "F811", # redefined-while-unused
+]
+
+[lint.pyflakes]
+allowed-unused-imports = [
+    "_pytest.monkeypatch",
+    "tests.integration_tests",
+    "tests.unit_tests",
+]

api/Dockerfile

@@ -0,0 +1,84 @@
+# base image
+FROM python:3.12-slim-bookworm AS base
+
+WORKDIR /app/api
+
+# Install Poetry
+ENV POETRY_VERSION=2.0.1
+
+# if you located in China, you can use aliyun mirror to speed up
+# RUN pip install --no-cache-dir poetry==${POETRY_VERSION} -i https://mirrors.aliyun.com/pypi/simple/
+
+RUN pip install --no-cache-dir poetry==${POETRY_VERSION}
+
+# Configure Poetry
+ENV POETRY_CACHE_DIR=/tmp/poetry_cache
+ENV POETRY_NO_INTERACTION=1
+ENV POETRY_VIRTUALENVS_IN_PROJECT=true
+ENV POETRY_VIRTUALENVS_CREATE=true
+ENV POETRY_REQUESTS_TIMEOUT=15
+
+FROM base AS packages
+
+# if you located in China, you can use aliyun mirror to speed up
+# RUN sed -i 's@deb.debian.org@mirrors.aliyun.com@g' /etc/apt/sources.list.d/debian.sources
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends gcc g++ libc-dev libffi-dev libgmp-dev libmpfr-dev libmpc-dev
+
+# Install Python dependencies
+COPY pyproject.toml poetry.lock ./
+RUN poetry install --sync --no-cache --no-root
+
+# production stage
+FROM base AS production
+
+ENV FLASK_APP=app.py
+ENV EDITION=SELF_HOSTED
+ENV DEPLOY_ENV=PRODUCTION
+ENV CONSOLE_API_URL=http://127.0.0.1:5001
+ENV CONSOLE_WEB_URL=http://127.0.0.1:3000
+ENV SERVICE_API_URL=http://127.0.0.1:5001
+ENV APP_WEB_URL=http://127.0.0.1:3000
+
+EXPOSE 5001
+
+# set timezone
+ENV TZ=UTC
+
+WORKDIR /app/api
+
+RUN \
+    apt-get update \
+    # Install dependencies
+    && apt-get install -y --no-install-recommends \
+        # basic environment
+        curl nodejs libgmp-dev libmpfr-dev libmpc-dev \
+        # For Security
+        expat libldap-2.5-0 perl libsqlite3-0 zlib1g \
+        # install a chinese font to support the use of tools like matplotlib
+        fonts-noto-cjk \
+        # install libmagic to support the use of python-magic guess MIMETYPE
+        libmagic1 \
+    && apt-get autoremove -y \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy Python environment and packages
+ENV VIRTUAL_ENV=/app/api/.venv
+COPY --from=packages ${VIRTUAL_ENV} ${VIRTUAL_ENV}
+ENV PATH="${VIRTUAL_ENV}/bin:${PATH}"
+
+# Download nltk data
+RUN python -c "import nltk; nltk.download('punkt'); nltk.download('averaged_perceptron_tagger')"
+
+# Copy source code
+COPY . /app/api/
+
+# Copy entrypoint
+COPY docker/entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+
+ARG COMMIT_SHA
+ENV COMMIT_SHA=${COMMIT_SHA}
+
+ENTRYPOINT ["/bin/bash", "/entrypoint.sh"]

api/README.md

@@ -0,0 +1,89 @@
+# Dify Backend API
+
+## Usage
+
+> [!IMPORTANT]
+> In the v0.6.12 release, we deprecated `pip` as the package management tool for Dify API Backend service and replaced it with `poetry`.
+
+1. Start the docker-compose stack
+
+   The backend require some middleware, including PostgreSQL, Redis, and Weaviate, which can be started together using `docker-compose`.
+
+   ```bash
+   cd ../docker
+   cp middleware.env.example middleware.env
+   # change the profile to other vector database if you are not using weaviate
+   docker compose -f docker-compose.middleware.yaml --profile weaviate -p dify up -d
+   cd ../api
+   ```
+
+2. Copy `.env.example` to `.env`
+
+   ```cli
+   cp .env.example .env 
+   ```
+3. Generate a `SECRET_KEY` in the `.env` file.
+
+   bash for Linux
+   ```bash for Linux
+   sed -i "/^SECRET_KEY=/c\SECRET_KEY=$(openssl rand -base64 42)" .env
+   ```
+   bash for Mac
+   ```bash for Mac
+   secret_key=$(openssl rand -base64 42)
+   sed -i '' "/^SECRET_KEY=/c\\
+   SECRET_KEY=${secret_key}" .env
+   ```
+
+4. Create environment.
+
+   Dify API service uses [Poetry](https://python-poetry.org/docs/) to manage dependencies. First, you need to add the poetry shell plugin, if you don't have it already, in order to run in a virtual environment. [Note: Poetry shell is no longer a native command so you need to install the poetry plugin beforehand]
+
+   ```bash
+   poetry self add poetry-plugin-shell
+   ```
+   
+   Then, You can execute `poetry shell` to activate the environment.
+
+5. Install dependencies
+
+   ```bash
+   poetry env use 3.12
+   poetry install
+   ```
+
+6. Run migrate
+
+   Before the first launch, migrate the database to the latest version.
+
+   ```bash
+   poetry run python -m flask db upgrade
+   ```
+
+7. Start backend
+
+   ```bash
+   poetry run python -m flask run --host 0.0.0.0 --port=5001 --debug
+   ```
+
+8. Start Dify [web](../web) service.
+9. Setup your application by visiting `http://localhost:3000`...
+10. If you need to handle and debug the async tasks (e.g. dataset importing and documents indexing), please start the worker service.
+
+   ```bash
+   poetry run python -m celery -A app.celery worker -P gevent -c 1 --loglevel INFO -Q dataset,generation,mail,ops_trace,app_deletion
+   ```
+
+## Testing
+
+1. Install dependencies for both the backend and the test environment
+
+   ```bash
+   poetry install -C api --with dev
+   ```
+
+2. Run the tests locally with mocked system environment variables in `tool.pytest_env` section in `pyproject.toml`
+
+   ```bash
+   poetry run -P api bash dev/pytest/pytest_all_tests.sh
+   ```

api/app.py

@@ -0,0 +1,41 @@
+import os
+import sys
+
+
+def is_db_command():
+    if len(sys.argv) > 1 and sys.argv[0].endswith("flask") and sys.argv[1] == "db":
+        return True
+    return False
+
+
+# create app
+if is_db_command():
+    from app_factory import create_migrations_app
+
+    app = create_migrations_app()
+else:
+    # It seems that JetBrains Python debugger does not work well with gevent,
+    # so we need to disable gevent in debug mode.
+    # If you are using debugpy and set GEVENT_SUPPORT=True, you can debug with gevent.
+    if (flask_debug := os.environ.get("FLASK_DEBUG", "0")) and flask_debug.lower() in {"false", "0", "no"}:
+        from gevent import monkey  # type: ignore
+
+        # gevent
+        monkey.patch_all()
+
+        from grpc.experimental import gevent as grpc_gevent  # type: ignore
+
+        # grpc gevent
+        grpc_gevent.init_gevent()
+
+        import psycogreen.gevent  # type: ignore
+
+        psycogreen.gevent.patch_psycopg()
+
+    from app_factory import create_app
+
+    app = create_app()
+    celery = app.extensions["celery"]
+
+if __name__ == "__main__":
+    app.run(host="0.0.0.0", port=5001)

api/app_factory.py

@@ -0,0 +1,101 @@
+import logging
+import time
+
+from configs import dify_config
+from dify_app import DifyApp
+
+
+# ----------------------------
+# Application Factory Function
+# ----------------------------
+def create_flask_app_with_configs() -> DifyApp:
+    """
+    create a raw flask app
+    with configs loaded from .env file
+    """
+    dify_app = DifyApp(__name__)
+    dify_app.config.from_mapping(dify_config.model_dump())
+
+    return dify_app
+
+
+def create_app() -> DifyApp:
+    start_time = time.perf_counter()
+    app = create_flask_app_with_configs()
+    initialize_extensions(app)
+    end_time = time.perf_counter()
+    if dify_config.DEBUG:
+        logging.info(f"Finished create_app ({round((end_time - start_time) * 1000, 2)} ms)")
+    return app
+
+
+def initialize_extensions(app: DifyApp):
+    from extensions import (
+        ext_app_metrics,
+        ext_blueprints,
+        ext_celery,
+        ext_code_based_extension,
+        ext_commands,
+        ext_compress,
+        ext_database,
+        ext_hosting_provider,
+        ext_import_modules,
+        ext_logging,
+        ext_login,
+        ext_mail,
+        ext_migrate,
+        ext_proxy_fix,
+        ext_redis,
+        ext_sentry,
+        ext_set_secretkey,
+        ext_storage,
+        ext_timezone,
+        ext_warnings,
+    )
+
+    extensions = [
+        ext_timezone,
+        ext_logging,
+        ext_warnings,
+        ext_import_modules,
+        ext_set_secretkey,
+        ext_compress,
+        ext_code_based_extension,
+        ext_database,
+        ext_app_metrics,
+        ext_migrate,
+        ext_redis,
+        ext_storage,
+        ext_celery,
+        ext_login,
+        ext_mail,
+        ext_hosting_provider,
+        ext_sentry,
+        ext_proxy_fix,
+        ext_blueprints,
+        ext_commands,
+    ]
+    for ext in extensions:
+        short_name = ext.__name__.split(".")[-1]
+        is_enabled = ext.is_enabled() if hasattr(ext, "is_enabled") else True
+        if not is_enabled:
+            if dify_config.DEBUG:
+                logging.info(f"Skipped {short_name}")
+            continue
+
+        start_time = time.perf_counter()
+        ext.init_app(app)
+        end_time = time.perf_counter()
+        if dify_config.DEBUG:
+            logging.info(f"Loaded {short_name} ({round((end_time - start_time) * 1000, 2)} ms)")
+
+
+def create_migrations_app():
+    app = create_flask_app_with_configs()
+    from extensions import ext_database, ext_migrate
+
+    # Initialize only required extensions
+    ext_database.init_app(app)
+    ext_migrate.init_app(app)
+
+    return app

api/commands.py

@@ -0,0 +1,651 @@
+import base64
+import json
+import logging
+import secrets
+from typing import Optional
+
+import click
+from flask import current_app
+from werkzeug.exceptions import NotFound
+
+from configs import dify_config
+from constants.languages import languages
+from core.rag.datasource.vdb.vector_factory import Vector
+from core.rag.datasource.vdb.vector_type import VectorType
+from core.rag.models.document import Document
+from events.app_event import app_was_created
+from extensions.ext_database import db
+from extensions.ext_redis import redis_client
+from libs.helper import email as email_validate
+from libs.password import hash_password, password_pattern, valid_password
+from libs.rsa import generate_key_pair
+from models import Tenant
+from models.dataset import Dataset, DatasetCollectionBinding, DocumentSegment
+from models.dataset import Document as DatasetDocument
+from models.model import Account, App, AppAnnotationSetting, AppMode, Conversation, MessageAnnotation
+from models.provider import Provider, ProviderModel
+from services.account_service import RegisterService, TenantService
+
+
+@click.command("reset-password", help="Reset the account password.")
+@click.option("--email", prompt=True, help="Account email to reset password for")
+@click.option("--new-password", prompt=True, help="New password")
+@click.option("--password-confirm", prompt=True, help="Confirm new password")
+def reset_password(email, new_password, password_confirm):
+    """
+    Reset password of owner account
+    Only available in SELF_HOSTED mode
+    """
+    if str(new_password).strip() != str(password_confirm).strip():
+        click.echo(click.style("Passwords do not match.", fg="red"))
+        return
+
+    account = db.session.query(Account).filter(Account.email == email).one_or_none()
+
+    if not account:
+        click.echo(click.style("Account not found for email: {}".format(email), fg="red"))
+        return
+
+    try:
+        valid_password(new_password)
+    except:
+        click.echo(click.style("Invalid password. Must match {}".format(password_pattern), fg="red"))
+        return
+
+    # generate password salt
+    salt = secrets.token_bytes(16)
+    base64_salt = base64.b64encode(salt).decode()
+
+    # encrypt password with salt
+    password_hashed = hash_password(new_password, salt)
+    base64_password_hashed = base64.b64encode(password_hashed).decode()
+    account.password = base64_password_hashed
+    account.password_salt = base64_salt
+    db.session.commit()
+    click.echo(click.style("Password reset successfully.", fg="green"))
+
+
+@click.command("reset-email", help="Reset the account email.")
+@click.option("--email", prompt=True, help="Current account email")
+@click.option("--new-email", prompt=True, help="New email")
+@click.option("--email-confirm", prompt=True, help="Confirm new email")
+def reset_email(email, new_email, email_confirm):
+    """
+    Replace account email
+    :return:
+    """
+    if str(new_email).strip() != str(email_confirm).strip():
+        click.echo(click.style("New emails do not match.", fg="red"))
+        return
+
+    account = db.session.query(Account).filter(Account.email == email).one_or_none()
+
+    if not account:
+        click.echo(click.style("Account not found for email: {}".format(email), fg="red"))
+        return
+
+    try:
+        email_validate(new_email)
+    except:
+        click.echo(click.style("Invalid email: {}".format(new_email), fg="red"))
+        return
+
+    account.email = new_email
+    db.session.commit()
+    click.echo(click.style("Email updated successfully.", fg="green"))
+
+
+@click.command(
+    "reset-encrypt-key-pair",
+    help="Reset the asymmetric key pair of workspace for encrypt LLM credentials. "
+    "After the reset, all LLM credentials will become invalid, "
+    "requiring re-entry."
+    "Only support SELF_HOSTED mode.",
+)
+@click.confirmation_option(
+    prompt=click.style(
+        "Are you sure you want to reset encrypt key pair? This operation cannot be rolled back!", fg="red"
+    )
+)
+def reset_encrypt_key_pair():
+    """
+    Reset the encrypted key pair of workspace for encrypt LLM credentials.
+    After the reset, all LLM credentials will become invalid, requiring re-entry.
+    Only support SELF_HOSTED mode.
+    """
+    if dify_config.EDITION != "SELF_HOSTED":
+        click.echo(click.style("This command is only for SELF_HOSTED installations.", fg="red"))
+        return
+
+    tenants = db.session.query(Tenant).all()
+    for tenant in tenants:
+        if not tenant:
+            click.echo(click.style("No workspaces found. Run /install first.", fg="red"))
+            return
+
+        tenant.encrypt_public_key = generate_key_pair(tenant.id)
+
+        db.session.query(Provider).filter(Provider.provider_type == "custom", Provider.tenant_id == tenant.id).delete()
+        db.session.query(ProviderModel).filter(ProviderModel.tenant_id == tenant.id).delete()
+        db.session.commit()
+
+        click.echo(
+            click.style(
+                "Congratulations! The asymmetric key pair of workspace {} has been reset.".format(tenant.id),
+                fg="green",
+            )
+        )
+
+
+@click.command("vdb-migrate", help="Migrate vector db.")
+@click.option("--scope", default="all", prompt=False, help="The scope of vector database to migrate, Default is All.")
+def vdb_migrate(scope: str):
+    if scope in {"knowledge", "all"}:
+        migrate_knowledge_vector_database()
+    if scope in {"annotation", "all"}:
+        migrate_annotation_vector_database()
+
+
+def migrate_annotation_vector_database():
+    """
+    Migrate annotation datas to target vector database .
+    """
+    click.echo(click.style("Starting annotation data migration.", fg="green"))
+    create_count = 0
+    skipped_count = 0
+    total_count = 0
+    page = 1
+    while True:
+        try:
+            # get apps info
+            apps = (
+                App.query.filter(App.status == "normal")
+                .order_by(App.created_at.desc())
+                .paginate(page=page, per_page=50)
+            )
+        except NotFound:
+            break
+
+        page += 1
+        for app in apps:
+            total_count = total_count + 1
+            click.echo(
+                f"Processing the {total_count} app {app.id}. " + f"{create_count} created, {skipped_count} skipped."
+            )
+            try:
+                click.echo("Creating app annotation index: {}".format(app.id))
+                app_annotation_setting = (
+                    db.session.query(AppAnnotationSetting).filter(AppAnnotationSetting.app_id == app.id).first()
+                )
+
+                if not app_annotation_setting:
+                    skipped_count = skipped_count + 1
+                    click.echo("App annotation setting disabled: {}".format(app.id))
+                    continue
+                # get dataset_collection_binding info
+                dataset_collection_binding = (
+                    db.session.query(DatasetCollectionBinding)
+                    .filter(DatasetCollectionBinding.id == app_annotation_setting.collection_binding_id)
+                    .first()
+                )
+                if not dataset_collection_binding:
+                    click.echo("App annotation collection binding not found: {}".format(app.id))
+                    continue
+                annotations = db.session.query(MessageAnnotation).filter(MessageAnnotation.app_id == app.id).all()
+                dataset = Dataset(
+                    id=app.id,
+                    tenant_id=app.tenant_id,
+                    indexing_technique="high_quality",
+                    embedding_model_provider=dataset_collection_binding.provider_name,
+                    embedding_model=dataset_collection_binding.model_name,
+                    collection_binding_id=dataset_collection_binding.id,
+                )
+                documents = []
+                if annotations:
+                    for annotation in annotations:
+                        document = Document(
+                            page_content=annotation.question,
+                            metadata={"annotation_id": annotation.id, "app_id": app.id, "doc_id": annotation.id},
+                        )
+                        documents.append(document)
+
+                vector = Vector(dataset, attributes=["doc_id", "annotation_id", "app_id"])
+                click.echo(f"Migrating annotations for app: {app.id}.")
+
+                try:
+                    vector.delete()
+                    click.echo(click.style(f"Deleted vector index for app {app.id}.", fg="green"))
+                except Exception as e:
+                    click.echo(click.style(f"Failed to delete vector index for app {app.id}.", fg="red"))
+                    raise e
+                if documents:
+                    try:
+                        click.echo(
+                            click.style(
+                                f"Creating vector index with {len(documents)} annotations for app {app.id}.",
+                                fg="green",
+                            )
+                        )
+                        vector.create(documents)
+                        click.echo(click.style(f"Created vector index for app {app.id}.", fg="green"))
+                    except Exception as e:
+                        click.echo(click.style(f"Failed to created vector index for app {app.id}.", fg="red"))
+                        raise e
+                click.echo(f"Successfully migrated app annotation {app.id}.")
+                create_count += 1
+            except Exception as e:
+                click.echo(
+                    click.style(
+                        "Error creating app annotation index: {} {}".format(e.__class__.__name__, str(e)), fg="red"
+                    )
+                )
+                continue
+
+    click.echo(
+        click.style(
+            f"Migration complete. Created {create_count} app annotation indexes. Skipped {skipped_count} apps.",
+            fg="green",
+        )
+    )
+
+
+def migrate_knowledge_vector_database():
+    """
+    Migrate vector database datas to target vector database .
+    """
+    click.echo(click.style("Starting vector database migration.", fg="green"))
+    create_count = 0
+    skipped_count = 0
+    total_count = 0
+    vector_type = dify_config.VECTOR_STORE
+    upper_collection_vector_types = {
+        VectorType.MILVUS,
+        VectorType.PGVECTOR,
+        VectorType.RELYT,
+        VectorType.WEAVIATE,
+        VectorType.ORACLE,
+        VectorType.ELASTICSEARCH,
+    }
+    lower_collection_vector_types = {
+        VectorType.ANALYTICDB,
+        VectorType.CHROMA,
+        VectorType.MYSCALE,
+        VectorType.PGVECTO_RS,
+        VectorType.TIDB_VECTOR,
+        VectorType.OPENSEARCH,
+        VectorType.TENCENT,
+        VectorType.BAIDU,
+        VectorType.VIKINGDB,
+        VectorType.UPSTASH,
+        VectorType.COUCHBASE,
+        VectorType.OCEANBASE,
+    }
+    page = 1
+    while True:
+        try:
+            datasets = (
+                Dataset.query.filter(Dataset.indexing_technique == "high_quality")
+                .order_by(Dataset.created_at.desc())
+                .paginate(page=page, per_page=50)
+            )
+        except NotFound:
+            break
+
+        page += 1
+        for dataset in datasets:
+            total_count = total_count + 1
+            click.echo(
+                f"Processing the {total_count} dataset {dataset.id}. {create_count} created, {skipped_count} skipped."
+            )
+            try:
+                click.echo("Creating dataset vector database index: {}".format(dataset.id))
+                if dataset.index_struct_dict:
+                    if dataset.index_struct_dict["type"] == vector_type:
+                        skipped_count = skipped_count + 1
+                        continue
+                collection_name = ""
+                dataset_id = dataset.id
+                if vector_type in upper_collection_vector_types:
+                    collection_name = Dataset.gen_collection_name_by_id(dataset_id)
+                elif vector_type == VectorType.QDRANT:
+                    if dataset.collection_binding_id:
+                        dataset_collection_binding = (
+                            db.session.query(DatasetCollectionBinding)
+                            .filter(DatasetCollectionBinding.id == dataset.collection_binding_id)
+                            .one_or_none()
+                        )
+                        if dataset_collection_binding:
+                            collection_name = dataset_collection_binding.collection_name
+                        else:
+                            raise ValueError("Dataset Collection Binding not found")
+                    else:
+                        collection_name = Dataset.gen_collection_name_by_id(dataset_id)
+
+                elif vector_type in lower_collection_vector_types:
+                    collection_name = Dataset.gen_collection_name_by_id(dataset_id).lower()
+                else:
+                    raise ValueError(f"Vector store {vector_type} is not supported.")
+
+                index_struct_dict = {"type": vector_type, "vector_store": {"class_prefix": collection_name}}
+                dataset.index_struct = json.dumps(index_struct_dict)
+                vector = Vector(dataset)
+                click.echo(f"Migrating dataset {dataset.id}.")
+
+                try:
+                    vector.delete()
+                    click.echo(
+                        click.style(f"Deleted vector index {collection_name} for dataset {dataset.id}.", fg="green")
+                    )
+                except Exception as e:
+                    click.echo(
+                        click.style(
+                            f"Failed to delete vector index {collection_name} for dataset {dataset.id}.", fg="red"
+                        )
+                    )
+                    raise e
+
+                dataset_documents = (
+                    db.session.query(DatasetDocument)
+                    .filter(
+                        DatasetDocument.dataset_id == dataset.id,
+                        DatasetDocument.indexing_status == "completed",
+                        DatasetDocument.enabled == True,
+                        DatasetDocument.archived == False,
+                    )
+                    .all()
+                )
+
+                documents = []
+                segments_count = 0
+                for dataset_document in dataset_documents:
+                    segments = (
+                        db.session.query(DocumentSegment)
+                        .filter(
+                            DocumentSegment.document_id == dataset_document.id,
+                            DocumentSegment.status == "completed",
+                            DocumentSegment.enabled == True,
+                        )
+                        .all()
+                    )
+
+                    for segment in segments:
+                        document = Document(
+                            page_content=segment.content,
+                            metadata={
+                                "doc_id": segment.index_node_id,
+                                "doc_hash": segment.index_node_hash,
+                                "document_id": segment.document_id,
+                                "dataset_id": segment.dataset_id,
+                            },
+                        )
+
+                        documents.append(document)
+                        segments_count = segments_count + 1
+
+                if documents:
+                    try:
+                        click.echo(
+                            click.style(
+                                f"Creating vector index with {len(documents)} documents of {segments_count}"
+                                f" segments for dataset {dataset.id}.",
+                                fg="green",
+                            )
+                        )
+                        vector.create(documents)
+                        click.echo(click.style(f"Created vector index for dataset {dataset.id}.", fg="green"))
+                    except Exception as e:
+                        click.echo(click.style(f"Failed to created vector index for dataset {dataset.id}.", fg="red"))
+                        raise e
+                db.session.add(dataset)
+                db.session.commit()
+                click.echo(f"Successfully migrated dataset {dataset.id}.")
+                create_count += 1
+            except Exception as e:
+                db.session.rollback()
+                click.echo(
+                    click.style("Error creating dataset index: {} {}".format(e.__class__.__name__, str(e)), fg="red")
+                )
+                continue
+
+    click.echo(
+        click.style(
+            f"Migration complete. Created {create_count} dataset indexes. Skipped {skipped_count} datasets.", fg="green"
+        )
+    )
+
+
+@click.command("convert-to-agent-apps", help="Convert Agent Assistant to Agent App.")
+def convert_to_agent_apps():
+    """
+    Convert Agent Assistant to Agent App.
+    """
+    click.echo(click.style("Starting convert to agent apps.", fg="green"))
+
+    proceeded_app_ids = []
+
+    while True:
+        # fetch first 1000 apps
+        sql_query = """SELECT a.id AS id FROM apps a
+            INNER JOIN app_model_configs am ON a.app_model_config_id=am.id
+            WHERE a.mode = 'chat'
+            AND am.agent_mode is not null
+            AND (
+				am.agent_mode like '%"strategy": "function_call"%'
+                OR am.agent_mode  like '%"strategy": "react"%'
+			)
+            AND (
+				am.agent_mode like '{"enabled": true%'
+                OR am.agent_mode like '{"max_iteration": %'
+			) ORDER BY a.created_at DESC LIMIT 1000
+        """
+
+        with db.engine.begin() as conn:
+            rs = conn.execute(db.text(sql_query))
+
+            apps = []
+            for i in rs:
+                app_id = str(i.id)
+                if app_id not in proceeded_app_ids:
+                    proceeded_app_ids.append(app_id)
+                    app = db.session.query(App).filter(App.id == app_id).first()
+                    if app is not None:
+                        apps.append(app)
+
+            if len(apps) == 0:
+                break
+
+        for app in apps:
+            click.echo("Converting app: {}".format(app.id))
+
+            try:
+                app.mode = AppMode.AGENT_CHAT.value
+                db.session.commit()
+
+                # update conversation mode to agent
+                db.session.query(Conversation).filter(Conversation.app_id == app.id).update(
+                    {Conversation.mode: AppMode.AGENT_CHAT.value}
+                )
+
+                db.session.commit()
+                click.echo(click.style("Converted app: {}".format(app.id), fg="green"))
+            except Exception as e:
+                click.echo(click.style("Convert app error: {} {}".format(e.__class__.__name__, str(e)), fg="red"))
+
+    click.echo(click.style("Conversion complete. Converted {} agent apps.".format(len(proceeded_app_ids)), fg="green"))
+
+
+@click.command("add-qdrant-doc-id-index", help="Add Qdrant doc_id index.")
+@click.option("--field", default="metadata.doc_id", prompt=False, help="Index field , default is metadata.doc_id.")
+def add_qdrant_doc_id_index(field: str):
+    click.echo(click.style("Starting Qdrant doc_id index creation.", fg="green"))
+    vector_type = dify_config.VECTOR_STORE
+    if vector_type != "qdrant":
+        click.echo(click.style("This command only supports Qdrant vector store.", fg="red"))
+        return
+    create_count = 0
+
+    try:
+        bindings = db.session.query(DatasetCollectionBinding).all()
+        if not bindings:
+            click.echo(click.style("No dataset collection bindings found.", fg="red"))
+            return
+        import qdrant_client
+        from qdrant_client.http.exceptions import UnexpectedResponse
+        from qdrant_client.http.models import PayloadSchemaType
+
+        from core.rag.datasource.vdb.qdrant.qdrant_vector import QdrantConfig
+
+        for binding in bindings:
+            if dify_config.QDRANT_URL is None:
+                raise ValueError("Qdrant URL is required.")
+            qdrant_config = QdrantConfig(
+                endpoint=dify_config.QDRANT_URL,
+                api_key=dify_config.QDRANT_API_KEY,
+                root_path=current_app.root_path,
+                timeout=dify_config.QDRANT_CLIENT_TIMEOUT,
+                grpc_port=dify_config.QDRANT_GRPC_PORT,
+                prefer_grpc=dify_config.QDRANT_GRPC_ENABLED,
+            )
+            try:
+                client = qdrant_client.QdrantClient(**qdrant_config.to_qdrant_params())
+                # create payload index
+                client.create_payload_index(binding.collection_name, field, field_schema=PayloadSchemaType.KEYWORD)
+                create_count += 1
+            except UnexpectedResponse as e:
+                # Collection does not exist, so return
+                if e.status_code == 404:
+                    click.echo(click.style(f"Collection not found: {binding.collection_name}.", fg="red"))
+                    continue
+                # Some other error occurred, so re-raise the exception
+                else:
+                    click.echo(
+                        click.style(
+                            f"Failed to create Qdrant index for collection: {binding.collection_name}.", fg="red"
+                        )
+                    )
+
+    except Exception as e:
+        click.echo(click.style("Failed to create Qdrant client.", fg="red"))
+
+    click.echo(click.style(f"Index creation complete. Created {create_count} collection indexes.", fg="green"))
+
+
+@click.command("create-tenant", help="Create account and tenant.")
+@click.option("--email", prompt=True, help="Tenant account email.")
+@click.option("--name", prompt=True, help="Workspace name.")
+@click.option("--language", prompt=True, help="Account language, default: en-US.")
+def create_tenant(email: str, language: Optional[str] = None, name: Optional[str] = None):
+    """
+    Create tenant account
+    """
+    if not email:
+        click.echo(click.style("Email is required.", fg="red"))
+        return
+
+    # Create account
+    email = email.strip()
+
+    if "@" not in email:
+        click.echo(click.style("Invalid email address.", fg="red"))
+        return
+
+    account_name = email.split("@")[0]
+
+    if language not in languages:
+        language = "en-US"
+
+    # Validates name encoding for non-Latin characters.
+    name = name.strip().encode("utf-8").decode("utf-8") if name else None
+
+    # generate random password
+    new_password = secrets.token_urlsafe(16)
+
+    # register account
+    account = RegisterService.register(
+        email=email,
+        name=account_name,
+        password=new_password,
+        language=language,
+        create_workspace_required=False,
+    )
+    TenantService.create_owner_tenant_if_not_exist(account, name)
+
+    click.echo(
+        click.style(
+            "Account and tenant created.\nAccount: {}\nPassword: {}".format(email, new_password),
+            fg="green",
+        )
+    )
+
+
+@click.command("upgrade-db", help="Upgrade the database")
+def upgrade_db():
+    click.echo("Preparing database migration...")
+    lock = redis_client.lock(name="db_upgrade_lock", timeout=60)
+    if lock.acquire(blocking=False):
+        try:
+            click.echo(click.style("Starting database migration.", fg="green"))
+
+            # run db migration
+            import flask_migrate  # type: ignore
+
+            flask_migrate.upgrade()
+
+            click.echo(click.style("Database migration successful!", fg="green"))
+
+        except Exception as e:
+            logging.exception("Failed to execute database migration")
+        finally:
+            lock.release()
+    else:
+        click.echo("Database migration skipped")
+
+
+@click.command("fix-app-site-missing", help="Fix app related site missing issue.")
+def fix_app_site_missing():
+    """
+    Fix app related site missing issue.
+    """
+    click.echo(click.style("Starting fix for missing app-related sites.", fg="green"))
+
+    failed_app_ids = []
+    while True:
+        sql = """select apps.id as id from apps left join sites on sites.app_id=apps.id
+where sites.id is null limit 1000"""
+        with db.engine.begin() as conn:
+            rs = conn.execute(db.text(sql))
+
+            processed_count = 0
+            for i in rs:
+                processed_count += 1
+                app_id = str(i.id)
+
+                if app_id in failed_app_ids:
+                    continue
+
+                try:
+                    app = db.session.query(App).filter(App.id == app_id).first()
+                    if not app:
+                        print(f"App {app_id} not found")
+                        continue
+
+                    tenant = app.tenant
+                    if tenant:
+                        accounts = tenant.get_accounts()
+                        if not accounts:
+                            print("Fix failed for app {}".format(app.id))
+                            continue
+
+                        account = accounts[0]
+                        print("Fixing missing site for app {}".format(app.id))
+                        app_was_created.send(app, account=account)
+                except Exception as e:
+                    failed_app_ids.append(app_id)
+                    click.echo(click.style("Failed to fix missing site for app {}".format(app_id), fg="red"))
+                    logging.exception(f"Failed to fix app related site missing issue, app_id: {app_id}")
+                    continue
+
+            if not processed_count:
+                break
+
+    click.echo(click.style("Fix for missing app-related sites completed successfully!", fg="green"))

api/dify_app.py

@@ -0,0 +1,5 @@
+from flask import Flask
+
+
+class DifyApp(Flask):
+    pass

api/mypy.ini

@@ -0,0 +1,10 @@
+[mypy]
+warn_return_any = True
+warn_unused_configs = True
+check_untyped_defs = True
+exclude = (?x)(
+    core/tools/provider/builtin/
+    | core/model_runtime/model_providers/
+    | tests/
+    | migrations/
+  )

api/poetry.toml

@@ -0,0 +1,4 @@
+[virtualenvs]
+in-project = true
+create = true
+prefer-active-python = true

api/pyproject.toml

@@ -0,0 +1,209 @@
+[project]
+name = "dify-api"
+requires-python = ">=3.11,<3.13"
+dynamic = [ "dependencies" ]
+
+[build-system]
+requires = ["poetry-core>=2.0.0"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.poetry]
+package-mode = false
+
+############################################################
+# [ Main ] Dependency group
+############################################################
+
+[tool.poetry.dependencies]
+anthropic = "~0.23.1"
+authlib = "1.3.1"
+azure-ai-inference = "~1.0.0b8"
+azure-ai-ml = "~1.20.0"
+azure-identity = "1.16.1"
+beautifulsoup4 = "4.12.2"
+boto3 = "1.36.12"
+bs4 = "~0.0.1"
+cachetools = "~5.3.0"
+celery = "~5.4.0"
+chardet = "~5.1.0"
+cohere = "~5.2.4"
+dashscope = { version = "~1.17.0", extras = ["tokenizer"] }
+fal-client = "0.5.6"
+flask = "~3.1.0"
+flask-compress = "~1.17"
+flask-cors = "~4.0.0"
+flask-login = "~0.6.3"
+flask-migrate = "~4.0.7"
+flask-restful = "~0.3.10"
+flask-sqlalchemy = "~3.1.1"
+gevent = "~24.11.1"
+gmpy2 = "~2.2.1"
+google-ai-generativelanguage = "0.6.9"
+google-api-core = "2.18.0"
+google-api-python-client = "2.90.0"
+google-auth = "2.29.0"
+google-auth-httplib2 = "0.2.0"
+google-cloud-aiplatform = "1.49.0"
+google-generativeai = "0.8.1"
+googleapis-common-protos = "1.63.0"
+gunicorn = "~23.0.0"
+httpx = { version = "~0.27.0", extras = ["socks"] }
+huggingface-hub = "~0.16.4"
+jieba = "0.42.1"
+langfuse = "~2.51.3"
+langsmith = "~0.1.77"
+mailchimp-transactional = "~1.0.50"
+markdown = "~3.5.1"
+nomic = "~3.1.2"
+novita-client = "~0.5.7"
+numpy = "~1.26.4"
+oci = "~2.135.1"
+openai = "~1.61.0"
+openpyxl = "~3.1.5"
+opik = "~1.3.4"
+pandas = { version = "~2.2.2", extras = ["performance", "excel"] }
+pandas-stubs = "~2.2.3.241009"
+psycogreen = "~1.0.2"
+psycopg2-binary = "~2.9.6"
+pycryptodome = "3.19.1"
+pydantic = "~2.9.2"
+pydantic-settings = "~2.6.0"
+pydantic_extra_types = "~2.9.0"
+pyjwt = "~2.8.0"
+pypdfium2 = "~4.30.0"
+python = ">=3.11,<3.13"
+python-docx = "~1.1.0"
+python-dotenv = "1.0.1"
+pyyaml = "~6.0.1"
+readabilipy = "0.2.0"
+redis = { version = "~5.0.3", extras = ["hiredis"] }
+replicate = "~0.22.0"
+resend = "~0.7.0"
+sagemaker = "~2.231.0"
+scikit-learn = "~1.5.1"
+sentry-sdk = { version = "~1.44.1", extras = ["flask"] }
+sqlalchemy = "~2.0.29"
+starlette = "0.41.0"
+tencentcloud-sdk-python-hunyuan = "~3.0.1294"
+tiktoken = "~0.8.0"
+tokenizers = "~0.15.0"
+transformers = "~4.35.0"
+unstructured = { version = "~0.16.1", extras = ["docx", "epub", "md", "msg", "ppt", "pptx"] }
+validators = "0.21.0"
+volcengine-python-sdk = {extras = ["ark"], version = "~1.0.98"}
+websocket-client = "~1.7.0"
+xinference-client = "0.15.2"
+yarl = "~1.18.3"
+youtube-transcript-api = "~0.6.2"
+zhipuai = "~2.1.5"
+# Before adding new dependency, consider place it in alphabet order (a-z) and suitable group.
+
+############################################################
+# [ Indirect ] dependency group
+# Related transparent dependencies with pinned version
+# required by main implementations
+############################################################
+[tool.poetry.group.indirect.dependencies]
+kaleido = "0.2.1"
+rank-bm25 = "~0.2.2"
+safetensors = "~0.4.3"
+
+############################################################
+# [ Tools ] dependency group
+############################################################
+[tool.poetry.group.tools.dependencies]
+arxiv = "2.1.0"
+cloudscraper = "1.2.71"
+duckduckgo-search = "~6.3.0"
+jsonpath-ng = "1.6.1"
+matplotlib = "~3.8.2"
+mplfonts = "~0.0.8"
+newspaper3k = "0.2.8"
+nltk = "3.9.1"
+numexpr = "~2.9.0"
+pydub = "~0.25.1"
+qrcode = "~7.4.2"
+twilio = "~9.0.4"
+vanna = { version = "0.7.5", extras = ["postgres", "mysql", "clickhouse", "duckdb", "oracle"] }
+wikipedia = "1.4.0"
+yfinance = "~0.2.40"
+
+############################################################
+# [ Storage ] dependency group
+# Required for storage clients
+############################################################
+[tool.poetry.group.storage.dependencies]
+azure-storage-blob = "12.13.0"
+bce-python-sdk = "~0.9.23"
+cos-python-sdk-v5 = "1.9.30"
+esdk-obs-python = "3.24.6.1"
+google-cloud-storage = "2.16.0"
+opendal = "~0.45.12"
+oss2 = "2.18.5"
+supabase = "~2.8.1"
+tos = "~2.7.1"
+
+############################################################
+# [ VDB ] dependency group
+# Required by vector store clients
+############################################################
+[tool.poetry.group.vdb.dependencies]
+alibabacloud_gpdb20160503 = "~3.8.0"
+alibabacloud_tea_openapi = "~0.3.9"
+chromadb = "0.5.20"
+clickhouse-connect = "~0.7.16"
+couchbase = "~4.3.0"
+elasticsearch = "8.14.0"
+opensearch-py = "2.4.0"
+oracledb = "~2.2.1"
+pgvecto-rs = { version = "~0.2.1", extras = ['sqlalchemy'] }
+pgvector = "0.2.5"
+pymilvus = "~2.5.0"
+pymochow = "1.3.1"
+pyobvector = "~0.1.6"
+qdrant-client = "1.7.3"
+tcvectordb = "1.3.2"
+tidb-vector = "0.0.9"
+upstash-vector = "0.6.0"
+volcengine-compat = "~1.0.156"
+weaviate-client = "~3.21.0"
+
+############################################################
+# [ Dev ] dependency group
+# Required for development and running tests
+############################################################
+[tool.poetry.group.dev]
+optional = true
+[tool.poetry.group.dev.dependencies]
+coverage = "~7.2.4"
+faker = "~32.1.0"
+mypy = "~1.13.0"
+pytest = "~8.3.2"
+pytest-benchmark = "~4.0.0"
+pytest-env = "~1.1.3"
+pytest-mock = "~3.14.0"
+types-beautifulsoup4 = "~4.12.0.20241020"
+types-flask-cors = "~5.0.0.20240902"
+types-flask-migrate = "~4.1.0.20250112"
+types-html5lib = "~1.1.11.20241018"
+types-openpyxl = "~3.1.5.20241225"
+types-protobuf = "~5.29.1.20241207"
+types-psutil = "~6.1.0.20241221"
+types-psycopg2 = "~2.9.21.20250121"
+types-python-dateutil = "~2.9.0.20241206"
+types-pytz = "~2024.2.0.20241221"
+types-pyyaml = "~6.0.12.20241230"
+types-regex = "~2024.11.6.20241221"
+types-requests = "~2.32.0.20241016"
+types-six = "~1.17.0.20241205"
+types-tqdm = "~4.67.0.20241221"
+
+############################################################
+# [ Lint ] dependency group
+# Required for code style linting
+############################################################
+[tool.poetry.group.lint]
+optional = true
+[tool.poetry.group.lint.dependencies]
+dotenv-linter = "~0.5.0"
+ruff = "~0.9.2"

api/pytest.ini

@@ -0,0 +1,31 @@
+[pytest]
+env =
+    ANTHROPIC_API_KEY = sk-ant-api11-IamNotARealKeyJustForMockTestKawaiiiiiiiiii-NotBaka-ASkksz
+    AZURE_OPENAI_API_BASE = https://difyai-openai.openai.azure.com
+    AZURE_OPENAI_API_KEY = xxxxb1707exxxxxxxxxxaaxxxxxf94
+    CHATGLM_API_BASE = http://a.abc.com:11451
+    CODE_EXECUTION_API_KEY = dify-sandbox
+    CODE_EXECUTION_ENDPOINT = http://127.0.0.1:8194
+    CODE_MAX_STRING_LENGTH = 80000
+    FIRECRAWL_API_KEY = fc-
+    FIREWORKS_API_KEY = fw_aaaaaaaaaaaaaaaaaaaa
+    GOOGLE_API_KEY = abcdefghijklmnopqrstuvwxyz
+    HUGGINGFACE_API_KEY = hf-awuwuwuwuwuwuwuwuwuwuwuwuwuwuwuwuwu
+    HUGGINGFACE_EMBEDDINGS_ENDPOINT_URL = c
+    HUGGINGFACE_TEXT2TEXT_GEN_ENDPOINT_URL = b
+    HUGGINGFACE_TEXT_GEN_ENDPOINT_URL = a
+    MIXEDBREAD_API_KEY = mk-aaaaaaaaaaaaaaaaaaaa
+    MOCK_SWITCH = true
+    NOMIC_API_KEY = nk-aaaaaaaaaaaaaaaaaaaa
+    OPENAI_API_KEY = sk-IamNotARealKeyJustForMockTestKawaiiiiiiiiii
+    TEI_EMBEDDING_SERVER_URL = http://a.abc.com:11451
+    TEI_RERANK_SERVER_URL = http://a.abc.com:11451
+    TEI_API_KEY = ttttttttttttttt
+    UPSTAGE_API_KEY = up-aaaaaaaaaaaaaaaaaaaa
+    VOYAGE_API_KEY = va-aaaaaaaaaaaaaaaaaaaa
+    XINFERENCE_CHAT_MODEL_UID = chat
+    XINFERENCE_EMBEDDINGS_MODEL_UID = embedding
+    XINFERENCE_GENERATION_MODEL_UID = generate
+    XINFERENCE_RERANK_MODEL_UID = rerank
+    XINFERENCE_SERVER_URL = http://a.abc.com:11451
+    GITEE_AI_API_KEY = aaaaaaaaaaaaaaaaaaaa