diff --git a/.dockerignore b/.dockerignore index b501ad1a9595541c90e0f9c78d98b20406594121..cd4afbe92f293bdc25787af1b7ba21c86cb2f8cb 100644 --- a/.dockerignore +++ b/.dockerignore @@ -8,4 +8,4 @@ node_modules/ .svelte-kit/ .env* !.env -!.env.local \ No newline at end of file +.env.local \ No newline at end of file diff --git a/.env b/.env index 232968a92e01a3f60222cfa6459b1c3d28204dfb..67993894d94d47fde852a675f860eed3ff092d9e 100644 --- a/.env +++ b/.env @@ -154,3 +154,5 @@ ALLOWED_USER_EMAILS=`[]` # if it's defined, only these emails will be allowed to USAGE_LIMITS=`{}` ALLOW_INSECURE_COOKIES=false # recommended to keep this to false but set to true if you need to run over http without tls +METRICS_PORT= +LOG_LEVEL=info \ No newline at end of file diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index b284c9f6ec3c4bf8e1e29f323756bdf7fa6ac908..0238051484f7bf7240aa69bb31fc1e828d2e3c6a 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -8,7 +8,7 @@ on: branches: - "*" paths: - - "Dockerfile.local" + - "Dockerfile" - "entrypoint.sh" workflow_dispatch: release: @@ -62,7 +62,7 @@ jobs: uses: docker/build-push-action@v5 with: context: . - file: Dockerfile.local + file: Dockerfile push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} @@ -116,7 +116,7 @@ jobs: uses: docker/build-push-action@v5 with: context: . - file: Dockerfile.local + file: Dockerfile push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} diff --git a/Dockerfile b/Dockerfile index f87b5a64935d2bc82cb98e31fcd7bba0c8a49561..ca205fbbfb4bad59f38d24611b40d98f30f63cf1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,9 @@ # syntax=docker/dockerfile:1 # read the doc: https://huggingface.co/docs/hub/spaces-sdks-docker # you will also find guides on how best to write your Dockerfile +ARG INCLUDE_DB=false + +# stage that install the dependencies FROM node:20 as builder-production WORKDIR /app @@ -12,31 +15,74 @@ RUN --mount=type=cache,target=/app/.npm \ FROM builder-production as builder +ARG APP_BASE= +ARG PUBLIC_APP_COLOR=blue + RUN --mount=type=cache,target=/app/.npm \ npm set cache /app/.npm && \ npm ci COPY --link --chown=1000 . . -RUN --mount=type=secret,id=DOTENV_LOCAL,dst=.env.local \ - npm run build +RUN npm run build -FROM node:20-slim -RUN npm install -g pm2 +# mongo image +FROM mongo:latest as mongo -RUN userdel -r node +# image to be used if INCLUDE_DB is false +FROM node:20-slim as local_db_false + +# image to be used if INCLUDE_DB is true +FROM node:20-slim as local_db_true + +RUN apt-get update +RUN apt-get install gnupg curl -y +# copy mongo from the other stage +COPY --from=mongo /usr/bin/mongo* /usr/bin/ + +ENV MONGODB_URL=mongodb://localhost:27017 +RUN mkdir -p /data/db +RUN chown -R 1000:1000 /data/db + +# final image +FROM local_db_${INCLUDE_DB} as final + +# build arg to determine if the database should be included +ARG INCLUDE_DB=false +ENV INCLUDE_DB=${INCLUDE_DB} + +# svelte requires APP_BASE at build time so it must be passed as a build arg +ARG APP_BASE= +# tailwind requires the primary theme to be known at build time so it must be passed as a build arg +ARG PUBLIC_APP_COLOR=blue -RUN useradd -m -u 1000 user +# install dotenv-cli +RUN npm install -g dotenv-cli + +# switch to a user that works for spaces +RUN userdel -r node +RUN useradd -m -u 1000 user USER user ENV HOME=/home/user \ PATH=/home/user/.local/bin:$PATH +WORKDIR /app -COPY --from=builder-production --chown=1000 /app/node_modules /app/node_modules -COPY --link --chown=1000 package.json /app/package.json -COPY --from=builder --chown=1000 /app/build /app/build +# add a .env.local if the user doesn't bind a volume to it +RUN touch /app/.env.local + +# get the default config, the entrypoint script and the server script +COPY --chown=1000 package.json /app/package.json +COPY --chown=1000 .env /app/.env +COPY --chown=1000 entrypoint.sh /app/entrypoint.sh COPY --chown=1000 gcp-*.json /app/ -CMD pm2 start /app/build/index.js -i $CPU_CORES --no-daemon +#import the build & dependencies +COPY --from=builder --chown=1000 /app/build /app/build +COPY --from=builder --chown=1000 /app/node_modules /app/node_modules + +RUN chmod +x /app/entrypoint.sh + +CMD ["/bin/bash", "-c", "/app/entrypoint.sh"] diff --git a/Dockerfile.local b/Dockerfile.local deleted file mode 100644 index c6046c3eb92038ba18f5f3d1d6a6fae51e4ae7bc..0000000000000000000000000000000000000000 --- a/Dockerfile.local +++ /dev/null @@ -1,28 +0,0 @@ -ARG INCLUDE_DB=false -FROM mongo:latest as mongo - -FROM node:20-slim as local_db_false - -FROM node:20-slim as local_db_true - -RUN apt-get update -RUN apt-get install gnupg curl -y - -COPY --from=mongo /usr/bin/mongo* /usr/bin/ - -FROM local_db_${INCLUDE_DB} as final -ARG INCLUDE_DB=false -ENV INCLUDE_DB=${INCLUDE_DB} - -WORKDIR /app - -COPY --link --chown=1000 package-lock.json package.json ./ -RUN --mount=type=cache,target=/app/.npm \ - npm set cache /app/.npm && \ - npm ci - -# copy the rest of the files, run regardless of -COPY --chown=1000 --link . . -RUN chmod +x /app/entrypoint.sh - -CMD ["/bin/bash", "-c", "/app/entrypoint.sh"] \ No newline at end of file diff --git a/entrypoint.sh b/entrypoint.sh index c9a9f41fe49cf23a956fb1ee606b0abf3e4c14e3..3fc46ee53aa072971739b5a251011b6327f53c32 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -2,7 +2,7 @@ ENV_LOCAL_PATH=/app/.env.local if test -z "${DOTENV_LOCAL}" ; then if ! test -f "${ENV_LOCAL_PATH}" ; then - echo "DOTENV_LOCAL was not found in the ENV variables and .env.local is not set using a bind volume. We are using the default .env config." + echo "DOTENV_LOCAL was not found in the ENV variables and .env.local is not set using a bind volume. Make sure to set environment variables properly. " fi; else echo "DOTENV_LOCAL was found in the ENV variables. Creating .env.local file." @@ -10,20 +10,10 @@ else fi; if [ "$INCLUDE_DB" = "true" ] ; then - echo "INCLUDE_DB is set to true." - - MONGODB_CONFIG="MONGODB_URL=mongodb://localhost:27017" - if ! grep -q "^${MONGODB_CONFIG}$" ${ENV_LOCAL_PATH}; then - echo "Appending MONGODB_URL" - touch /app/.env.local - echo -e "\n${MONGODB_CONFIG}" >> ${ENV_LOCAL_PATH} - fi - - mkdir -p /data/db - mongod & echo "Starting local MongoDB instance" - + nohup mongod & fi; -npm run build -npm run preview -- --host 0.0.0.0 --port 3000 \ No newline at end of file +export PUBLIC_VERSION=$(node -p "require('./package.json').version") + +dotenv -e /app/.env -c -- node /app/build/index.js -- --host 0.0.0.0 --port 3000 \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 0d637ce4db479a90249966dd82df40ccc91677c4..8a683fc33b3f54b9222ba04a5db987eb79f3f768 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,6 +17,7 @@ "browser-image-resizer": "^2.4.1", "date-fns": "^2.29.3", "dotenv": "^16.0.3", + "express": "^4.19.2", "handlebars": "^4.7.8", "highlight.js": "^11.7.0", "image-size": "^1.0.2", @@ -49,6 +50,7 @@ "@sveltejs/adapter-node": "^1.3.1", "@sveltejs/kit": "^1.30.4", "@tailwindcss/typography": "^0.5.9", + "@types/express": "^4.17.21", "@types/jsdom": "^21.1.1", "@types/minimist": "^1.2.5", "@types/parquetjs": "^0.10.3", @@ -62,6 +64,7 @@ "prettier": "^2.8.0", "prettier-plugin-svelte": "^2.10.1", "prettier-plugin-tailwindcss": "^0.2.7", + "prom-client": "^15.1.2", "svelte": "^4.2.8", "svelte-check": "^3.6.2", "ts-node": "^10.9.1", @@ -1263,6 +1266,15 @@ "node": ">= 8" } }, + "node_modules/@opentelemetry/api": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.8.0.tgz", + "integrity": "sha512-I/s6F7yKUDdtMsoBWXJe8Qz40Tui5vsuKCWJEWVL+5q9sSWRzzx6v2KeNsOBEwd94j0eWkpWCH4yB6rZg9Mf0w==", + "dev": true, + "engines": { + "node": ">=8.0.0" + } + }, "node_modules/@polka/url": { "version": "1.0.0-next.21", "resolved": "https://registry.npmjs.org/@polka/url/-/url-1.0.0-next.21.tgz", @@ -1987,6 +1999,16 @@ "integrity": "sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==", "devOptional": true }, + "node_modules/@types/body-parser": { + "version": "1.19.5", + "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.5.tgz", + "integrity": "sha512-fB3Zu92ucau0iQ0JMCFQE7b/dv8Ot07NI3KaZIkIUNXq82k4eBAqUaneXfleGY9JWskeS9y+u0nXMyspcuQrCg==", + "dev": true, + "dependencies": { + "@types/connect": "*", + "@types/node": "*" + } + }, "node_modules/@types/chai": { "version": "4.3.5", "resolved": "https://registry.npmjs.org/@types/chai/-/chai-4.3.5.tgz", @@ -2002,6 +2024,15 @@ "@types/chai": "*" } }, + "node_modules/@types/connect": { + "version": "3.4.38", + "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.38.tgz", + "integrity": "sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/cookie": { "version": "0.5.1", "resolved": "https://registry.npmjs.org/@types/cookie/-/cookie-0.5.1.tgz", @@ -2014,6 +2045,36 @@ "integrity": "sha512-/kYRxGDLWzHOB7q+wtSUQlFrtcdUccpfy+X+9iMBpHK8QLLhx2wIPYuS5DYtR9Wa/YlZAbIovy7qVdB1Aq6Lyw==", "dev": true }, + "node_modules/@types/express": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.21.tgz", + "integrity": "sha512-ejlPM315qwLpaQlQDTjPdsUFSc6ZsP4AN6AlWnogPjQ7CVi7PYF3YVz+CY3jE2pwYf7E/7HlDAN0rV2GxTG0HQ==", + "dev": true, + "dependencies": { + "@types/body-parser": "*", + "@types/express-serve-static-core": "^4.17.33", + "@types/qs": "*", + "@types/serve-static": "*" + } + }, + "node_modules/@types/express-serve-static-core": { + "version": "4.19.0", + "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.19.0.tgz", + "integrity": "sha512-bGyep3JqPCRry1wq+O5n7oiBgGWmeIJXPjXXCo8EK0u8duZGSYar7cGqd3ML2JUsLGeB7fmc06KYo9fLGWqPvQ==", + "dev": true, + "dependencies": { + "@types/node": "*", + "@types/qs": "*", + "@types/range-parser": "*", + "@types/send": "*" + } + }, + "node_modules/@types/http-errors": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@types/http-errors/-/http-errors-2.0.4.tgz", + "integrity": "sha512-D0CFMMtydbJAegzOyHjtiKPLlvnm3iTZyZRSZoLq2mRhDdmLfIWOCYPfQJ4cu2erKghU++QvjcUjp/5h7hESpA==", + "dev": true + }, "node_modules/@types/jsdom": { "version": "21.1.1", "resolved": "https://registry.npmjs.org/@types/jsdom/-/jsdom-21.1.1.tgz", @@ -2041,6 +2102,12 @@ "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.2.tgz", "integrity": "sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA==" }, + "node_modules/@types/mime": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.5.tgz", + "integrity": "sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==", + "dev": true + }, "node_modules/@types/minimist": { "version": "1.2.5", "resolved": "https://registry.npmjs.org/@types/minimist/-/minimist-1.2.5.tgz", @@ -2086,6 +2153,18 @@ "integrity": "sha512-Sk/uYFOBAB7mb74XcpizmH0KOR2Pv3D2Hmrh1Dmy5BmK3MpdSa5kqZcg6EKBdklU0bFXX9gCfzvpnyUehrPIuA==", "dev": true }, + "node_modules/@types/qs": { + "version": "6.9.15", + "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.15.tgz", + "integrity": "sha512-uXHQKES6DQKKCLh441Xv/dwxOq1TVS3JPUMlEqoEglvlhR6Mxnlew/Xq/LRVHpLyk7iK3zODe1qYHIMltO7XGg==", + "dev": true + }, + "node_modules/@types/range-parser": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.7.tgz", + "integrity": "sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==", + "dev": true + }, "node_modules/@types/resolve": { "version": "1.20.2", "resolved": "https://registry.npmjs.org/@types/resolve/-/resolve-1.20.2.tgz", @@ -2098,6 +2177,27 @@ "integrity": "sha512-OxepLK9EuNEIPxWNME+C6WwbRAOOI2o2BaQEGzz5Lu2e4Z5eDnEo+/aVEDMIXywoJitJ7xWd641wrGLZdtwRyw==", "dev": true }, + "node_modules/@types/send": { + "version": "0.17.4", + "resolved": "https://registry.npmjs.org/@types/send/-/send-0.17.4.tgz", + "integrity": "sha512-x2EM6TJOybec7c52BX0ZspPodMsQUd5L6PRwOunVyVUhXiBSKf3AezDL8Dgvgt5o0UfKNfuA0eMLr2wLT4AiBA==", + "dev": true, + "dependencies": { + "@types/mime": "^1", + "@types/node": "*" + } + }, + "node_modules/@types/serve-static": { + "version": "1.15.7", + "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.15.7.tgz", + "integrity": "sha512-W8Ym+h8nhuRwaKPaDw34QUkwsGi6Rc4yYqvKFo5rm2FUEhCFbzVWrxXUxuKK8TASjWsysJY0nsmNCGhCOIsrOw==", + "dev": true, + "dependencies": { + "@types/http-errors": "*", + "@types/node": "*", + "@types/send": "*" + } + }, "node_modules/@types/tough-cookie": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.2.tgz", @@ -2460,6 +2560,18 @@ "node": ">=6.5" } }, + "node_modules/accepts": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", + "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "dependencies": { + "mime-types": "~2.1.34", + "negotiator": "0.6.3" + }, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/acorn": { "version": "8.11.3", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.11.3.tgz", @@ -2590,6 +2702,11 @@ "dequal": "^2.0.3" } }, + "node_modules/array-flatten": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", + "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==" + }, "node_modules/array-union": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz", @@ -2726,6 +2843,12 @@ "integrity": "sha512-u4cBQNepWxYA55FunZSM7wMi55yQaN0otnhhilNoWHq0MfOfJeQx0v0mRRpolGOExPjZcl6FtB0BB8Xkb88F0g==", "optional": true }, + "node_modules/bintrees": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/bintrees/-/bintrees-1.0.2.tgz", + "integrity": "sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==", + "dev": true + }, "node_modules/bl": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", @@ -2742,6 +2865,67 @@ "integrity": "sha512-DRQrD6gJyy8FbiE4s+bDoXS9hiW3Vbx5uCdwvcCf3zLHL+Iv7LtGHLpr+GZV8rHG8tK766FGYBwRbu8pELTt+w==", "dev": true }, + "node_modules/body-parser": { + "version": "1.20.2", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.2.tgz", + "integrity": "sha512-ml9pReCu3M61kGlqoTm2umSXTlRTuGTx0bfYj+uIUKKYycG5NtSbeetV3faSU6R7ajOPw0g/J1PvK4qNy7s5bA==", + "dependencies": { + "bytes": "3.1.2", + "content-type": "~1.0.5", + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", + "on-finished": "2.4.1", + "qs": "6.11.0", + "raw-body": "2.5.2", + "type-is": "~1.6.18", + "unpipe": "1.0.0" + }, + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, + "node_modules/body-parser/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/body-parser/node_modules/iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/body-parser/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" + }, + "node_modules/body-parser/node_modules/qs": { + "version": "6.11.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz", + "integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==", + "dependencies": { + "side-channel": "^1.0.4" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -2874,6 +3058,14 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "engines": { + "node": ">= 0.8" + } + }, "node_modules/cac": { "version": "6.7.14", "resolved": "https://registry.npmjs.org/cac/-/cac-6.7.14.tgz", @@ -2887,7 +3079,6 @@ "version": "1.0.7", "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.7.tgz", "integrity": "sha512-GHTSNSYICQ7scH7sZ+M2rFopRoLh8t2bLSW6BbgrtLsahOIB5iyAVJf9GjWK3cYTDaMj4XdBpM1cA6pIS0Kv2w==", - "optional": true, "dependencies": { "es-define-property": "^1.0.0", "es-errors": "^1.3.0", @@ -3166,6 +3357,25 @@ "node": ">=10.18.0 <11 || >=12.14.0 <13 || >=14" } }, + "node_modules/content-disposition": { + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", + "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", + "dependencies": { + "safe-buffer": "5.2.1" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "engines": { + "node": ">= 0.6" + } + }, "node_modules/cookie": { "version": "0.5.0", "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.5.0.tgz", @@ -3175,6 +3385,11 @@ "node": ">= 0.6" } }, + "node_modules/cookie-signature": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", + "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==" + }, "node_modules/create-require": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", @@ -3409,7 +3624,6 @@ "version": "1.1.4", "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==", - "optional": true, "dependencies": { "es-define-property": "^1.0.0", "es-errors": "^1.3.0", @@ -3430,6 +3644,14 @@ "node": ">=0.4.0" } }, + "node_modules/depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "engines": { + "node": ">= 0.8" + } + }, "node_modules/dequal": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", @@ -3439,6 +3661,15 @@ "node": ">=6" } }, + "node_modules/destroy": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", + "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==", + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, "node_modules/detect-indent": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/detect-indent/-/detect-indent-6.1.0.tgz", @@ -3543,6 +3774,11 @@ "safe-buffer": "^5.0.1" } }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==" + }, "node_modules/electron-to-chromium": { "version": "1.4.359", "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.359.tgz", @@ -3553,6 +3789,14 @@ "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-10.3.0.tgz", "integrity": "sha512-QpLs9D9v9kArv4lfDEgg1X/gN5XLnf/A6l9cs8SPZLRZR3ZkY9+kwIQTxm+fsSej5UMYGE8fdoaZVIBlqG0XTw==" }, + "node_modules/encodeurl": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", + "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==", + "engines": { + "node": ">= 0.8" + } + }, "node_modules/end-of-stream": { "version": "1.4.4", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", @@ -3576,7 +3820,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.0.tgz", "integrity": "sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ==", - "optional": true, "dependencies": { "get-intrinsic": "^1.2.4" }, @@ -3588,7 +3831,6 @@ "version": "1.3.0", "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", - "optional": true, "engines": { "node": ">= 0.4" } @@ -3878,6 +4120,14 @@ "node": ">=0.10.0" } }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "engines": { + "node": ">= 0.6" + } + }, "node_modules/event-target-shim": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", @@ -3925,6 +4175,82 @@ "node": ">=6" } }, + "node_modules/express": { + "version": "4.19.2", + "resolved": "https://registry.npmjs.org/express/-/express-4.19.2.tgz", + "integrity": "sha512-5T6nhjsT+EOMzuck8JjBHARTHfMht0POzlA60WV2pMD3gyXw2LZnZ+ueGdNxG+0calOJcWKbpFcuzLZ91YWq9Q==", + "dependencies": { + "accepts": "~1.3.8", + "array-flatten": "1.1.1", + "body-parser": "1.20.2", + "content-disposition": "0.5.4", + "content-type": "~1.0.4", + "cookie": "0.6.0", + "cookie-signature": "1.0.6", + "debug": "2.6.9", + "depd": "2.0.0", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "finalhandler": "1.2.0", + "fresh": "0.5.2", + "http-errors": "2.0.0", + "merge-descriptors": "1.0.1", + "methods": "~1.1.2", + "on-finished": "2.4.1", + "parseurl": "~1.3.3", + "path-to-regexp": "0.1.7", + "proxy-addr": "~2.0.7", + "qs": "6.11.0", + "range-parser": "~1.2.1", + "safe-buffer": "5.2.1", + "send": "0.18.0", + "serve-static": "1.15.0", + "setprototypeof": "1.2.0", + "statuses": "2.0.1", + "type-is": "~1.6.18", + "utils-merge": "1.0.1", + "vary": "~1.1.2" + }, + "engines": { + "node": ">= 0.10.0" + } + }, + "node_modules/express/node_modules/cookie": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.6.0.tgz", + "integrity": "sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/express/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/express/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" + }, + "node_modules/express/node_modules/qs": { + "version": "6.11.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz", + "integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==", + "dependencies": { + "side-channel": "^1.0.4" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/extend": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", @@ -4040,6 +4366,36 @@ "node": ">=8" } }, + "node_modules/finalhandler": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.2.0.tgz", + "integrity": "sha512-5uXcUVftlQMFnWC9qu/svkWv3GTd2PfUhK/3PLkYNAe7FbqJMt3515HaxE6eRL74GdsriiwujiawdaB1BpEISg==", + "dependencies": { + "debug": "2.6.9", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "on-finished": "2.4.1", + "parseurl": "~1.3.3", + "statuses": "2.0.1", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/finalhandler/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/finalhandler/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" + }, "node_modules/find-up": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", @@ -4112,6 +4468,14 @@ "node": ">= 12.20" } }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "engines": { + "node": ">= 0.6" + } + }, "node_modules/fraction.js": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/fraction.js/-/fraction.js-4.2.0.tgz", @@ -4124,6 +4488,14 @@ "url": "https://www.patreon.com/infusion" } }, + "node_modules/fresh": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", + "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", + "engines": { + "node": ">= 0.6" + } + }, "node_modules/fs-constants": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", @@ -4221,7 +4593,6 @@ "version": "1.2.4", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.4.tgz", "integrity": "sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==", - "optional": true, "dependencies": { "es-errors": "^1.3.0", "function-bind": "^1.1.2", @@ -4352,7 +4723,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==", - "optional": true, "dependencies": { "get-intrinsic": "^1.1.3" }, @@ -4429,7 +4799,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==", - "optional": true, "dependencies": { "es-define-property": "^1.0.0" }, @@ -4441,7 +4810,6 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.0.3.tgz", "integrity": "sha512-SJ1amZAJUiZS+PhsVLf5tGydlaVB8EdFpaSO4gmiUKUOxk8qzn5AIy4ZeJUmh22znIdk/uMAUT2pl3FxzVUH+Q==", - "optional": true, "engines": { "node": ">= 0.4" }, @@ -4453,7 +4821,6 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", - "optional": true, "engines": { "node": ">= 0.4" }, @@ -4512,6 +4879,21 @@ "node": ">=12" } }, + "node_modules/http-errors": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", + "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==", + "dependencies": { + "depd": "2.0.0", + "inherits": "2.0.4", + "setprototypeof": "1.2.0", + "statuses": "2.0.1", + "toidentifier": "1.0.1" + }, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/http-proxy-agent": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz", @@ -4669,6 +5051,14 @@ "node": ">= 12" } }, + "node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "engines": { + "node": ">= 0.10" + } + }, "node_modules/is-arrayish": { "version": "0.3.2", "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz", @@ -5231,11 +5621,24 @@ "integrity": "sha512-GaqWWShW4kv/G9IEucWScBx9G1/vsFZZJUO+tD26M8J8z3Kw5RDQjaoZe03YAClgeS/SWPOcb4nkFBTEi5DUEA==", "dev": true }, + "node_modules/media-typer": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", + "engines": { + "node": ">= 0.6" + } + }, "node_modules/memory-pager": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/memory-pager/-/memory-pager-1.5.0.tgz", "integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==" }, + "node_modules/merge-descriptors": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", + "integrity": "sha512-cCi6g3/Zr1iqQi6ySbseM1Xvooa98N0w31jzUYrXPX2xqObmFGHJ0tQ5u74H3mVh7wLouTseZyYIq39g8cNp1w==" + }, "node_modules/merge-stream": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", @@ -5250,6 +5653,14 @@ "node": ">= 8" } }, + "node_modules/methods": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", + "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==", + "engines": { + "node": ">= 0.6" + } + }, "node_modules/micromatch": { "version": "4.0.5", "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz", @@ -5262,6 +5673,17 @@ "node": ">=8.6" } }, + "node_modules/mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/mime-db": { "version": "1.52.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", @@ -5468,6 +5890,14 @@ "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", "dev": true }, + "node_modules/negotiator": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", + "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", + "engines": { + "node": ">= 0.6" + } + }, "node_modules/neo-async": { "version": "2.6.2", "resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.2.tgz", @@ -5625,7 +6055,6 @@ "version": "1.13.1", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.1.tgz", "integrity": "sha512-5qoj1RUiKOMsCCNLV1CBiPYE10sziTsnmNxkAI/rZhiD63CF7IqdFGC/XzjWjpSgLf0LxXX3bDFIh0E18f6UhQ==", - "optional": true, "funding": { "url": "https://github.com/sponsors/ljharb" } @@ -5654,6 +6083,17 @@ "node": ">=14.0.0" } }, + "node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", @@ -5880,6 +6320,14 @@ "url": "https://github.com/inikulin/parse5?sponsor=1" } }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "engines": { + "node": ">= 0.8" + } + }, "node_modules/path-exists": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", @@ -5911,6 +6359,11 @@ "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==" }, + "node_modules/path-to-regexp": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", + "integrity": "sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ==" + }, "node_modules/path-type": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", @@ -6529,6 +6982,19 @@ "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-3.0.0.tgz", "integrity": "sha512-mqn0kFRl0EoqhnL0GQ0veqFHyIN1yig9RHh/InzORTUiZHFRAur+aMtRkELNwGs9aNwKS6tg/An4NYBPGwvtzQ==" }, + "node_modules/prom-client": { + "version": "15.1.2", + "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-15.1.2.tgz", + "integrity": "sha512-on3h1iXb04QFLLThrmVYg1SChBQ9N1c+nKAjebBjokBqipddH3uxmOUcEkTnzmJ8Jh/5TSUnUqS40i2QB2dJHQ==", + "dev": true, + "dependencies": { + "@opentelemetry/api": "^1.4.0", + "tdigest": "^0.1.1" + }, + "engines": { + "node": "^16 || ^18 || >=20" + } + }, "node_modules/protobufjs": { "version": "6.11.4", "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.11.4.tgz", @@ -6554,6 +7020,18 @@ "pbts": "bin/pbts" } }, + "node_modules/proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + } + }, "node_modules/psl": { "version": "1.9.0", "resolved": "https://registry.npmjs.org/psl/-/psl-1.9.0.tgz", @@ -6642,6 +7120,39 @@ "resolved": "https://registry.npmjs.org/quick-format-unescaped/-/quick-format-unescaped-4.0.4.tgz", "integrity": "sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==" }, + "node_modules/range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/raw-body": { + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz", + "integrity": "sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==", + "dependencies": { + "bytes": "3.1.2", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", + "unpipe": "1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/raw-body/node_modules/iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/rc": { "version": "1.2.8", "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", @@ -6944,6 +7455,47 @@ "node": ">=10" } }, + "node_modules/send": { + "version": "0.18.0", + "resolved": "https://registry.npmjs.org/send/-/send-0.18.0.tgz", + "integrity": "sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg==", + "dependencies": { + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "0.5.2", + "http-errors": "2.0.0", + "mime": "1.6.0", + "ms": "2.1.3", + "on-finished": "2.4.1", + "range-parser": "~1.2.1", + "statuses": "2.0.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/send/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/send/node_modules/debug/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" + }, + "node_modules/send/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + }, "node_modules/serpapi": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/serpapi/-/serpapi-1.1.1.tgz", @@ -6952,6 +7504,20 @@ "undici": "^5.12.0" } }, + "node_modules/serve-static": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.15.0.tgz", + "integrity": "sha512-XGuRDNjXUijsUL0vl6nSD7cwURuzEgglbOaFuZM9g3kwDXOWVTck0jLzjPzGD+TazWbboZYu52/9/XPdUgne9g==", + "dependencies": { + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "parseurl": "~1.3.3", + "send": "0.18.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, "node_modules/set-cookie-parser": { "version": "2.6.0", "resolved": "https://registry.npmjs.org/set-cookie-parser/-/set-cookie-parser-2.6.0.tgz", @@ -6962,7 +7528,6 @@ "version": "1.2.2", "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz", "integrity": "sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg==", - "optional": true, "dependencies": { "define-data-property": "^1.1.4", "es-errors": "^1.3.0", @@ -6975,6 +7540,11 @@ "node": ">= 0.4" } }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==" + }, "node_modules/sharp": { "version": "0.33.2", "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.33.2.tgz", @@ -7039,7 +7609,6 @@ "version": "1.0.6", "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.6.tgz", "integrity": "sha512-fDW/EZ6Q9RiO8eFG8Hj+7u/oW+XrPTIChwCOM2+th2A6OblDtYYIpve9m+KvI9Z4C9qSEXlaGR6bTEYHReuglA==", - "optional": true, "dependencies": { "call-bind": "^1.0.7", "es-errors": "^1.3.0", @@ -7232,6 +7801,14 @@ "integrity": "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==", "dev": true }, + "node_modules/statuses": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", + "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==", + "engines": { + "node": ">= 0.8" + } + }, "node_modules/std-env": { "version": "3.3.3", "resolved": "https://registry.npmjs.org/std-env/-/std-env-3.3.3.tgz", @@ -7703,6 +8280,15 @@ "streamx": "^2.15.0" } }, + "node_modules/tdigest": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.2.tgz", + "integrity": "sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==", + "dev": true, + "dependencies": { + "bintrees": "1.0.2" + } + }, "node_modules/text-table": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", @@ -7808,6 +8394,14 @@ "node": ">=8.0" } }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "engines": { + "node": ">=0.6" + } + }, "node_modules/totalist": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/totalist/-/totalist-3.0.0.tgz", @@ -7958,6 +8552,18 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/type-is": { + "version": "1.6.18", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "dependencies": { + "media-typer": "0.3.0", + "mime-types": "~2.1.24" + }, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/typescript": { "version": "5.2.2", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.2.2.tgz", @@ -8022,6 +8628,14 @@ "node": ">= 4.0.0" } }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", + "engines": { + "node": ">= 0.8" + } + }, "node_modules/unplugin": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/unplugin/-/unplugin-1.3.1.tgz", @@ -8126,6 +8740,14 @@ "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" }, + "node_modules/utils-merge": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", + "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==", + "engines": { + "node": ">= 0.4.0" + } + }, "node_modules/uuid": { "version": "9.0.1", "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", @@ -8149,6 +8771,14 @@ "resolved": "https://registry.npmjs.org/varint/-/varint-5.0.2.tgz", "integrity": "sha512-lKxKYG6H03yCZUpAGOPOsMcGxd1RHCu1iKvEHYDPmTyq2HueGhD73ssNBqqQWfvYs04G9iUFRvmAVLW20Jw6ow==" }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "engines": { + "node": ">= 0.8" + } + }, "node_modules/vite": { "version": "4.5.3", "resolved": "https://registry.npmjs.org/vite/-/vite-4.5.3.tgz", diff --git a/package.json b/package.json index fe0ad318bdfa86f51d32902165a52468179e9114..ef1d419c0c8766e0377cc25f14f27bf31fb85fc7 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,7 @@ "@sveltejs/adapter-node": "^1.3.1", "@sveltejs/kit": "^1.30.4", "@tailwindcss/typography": "^0.5.9", + "@types/express": "^4.17.21", "@types/jsdom": "^21.1.1", "@types/minimist": "^1.2.5", "@types/parquetjs": "^0.10.3", @@ -36,6 +37,7 @@ "prettier": "^2.8.0", "prettier-plugin-svelte": "^2.10.1", "prettier-plugin-tailwindcss": "^0.2.7", + "prom-client": "^15.1.2", "svelte": "^4.2.8", "svelte-check": "^3.6.2", "ts-node": "^10.9.1", @@ -57,6 +59,7 @@ "browser-image-resizer": "^2.4.1", "date-fns": "^2.29.3", "dotenv": "^16.0.3", + "express": "^4.19.2", "handlebars": "^4.7.8", "highlight.js": "^11.7.0", "image-size": "^1.0.2", diff --git a/scripts/populate.ts b/scripts/populate.ts index 2a3e12a541ec82585cd17335b21e0b0ec5924a16..8e06879a634d7f40010606ff49e3010967efb4b9 100644 --- a/scripts/populate.ts +++ b/scripts/populate.ts @@ -2,12 +2,13 @@ import readline from "readline"; import minimist from "minimist"; // @ts-expect-error: vite-node makes the var available but the typescript compiler doesn't see them -import { MONGODB_URL } from "$env/static/private"; +import { env } from "$env/dynamic/private"; import { faker } from "@faker-js/faker"; import { ObjectId } from "mongodb"; -import { collections } from "../src/lib/server/database.ts"; +// @ts-expect-error: vite-node makes the var available but the typescript compiler doesn't see them +import { collections } from "$lib/server/database"; import { models } from "../src/lib/server/models.ts"; import type { User } from "../src/lib/types/User"; import type { Assistant } from "../src/lib/types/Assistant"; @@ -17,6 +18,7 @@ import { defaultEmbeddingModel } from "../src/lib/server/embeddingModels.ts"; import { Message } from "../src/lib/types/Message.ts"; import { addChildren } from "../src/lib/utils/tree/addChildren.ts"; +import { generateSearchTokens } from "../src/lib/utils/searchTokens.ts"; const rl = readline.createInterface({ input: process.stdin, @@ -158,10 +160,11 @@ async function seed() { console.log("Creating assistants for all users"); await Promise.all( users.map(async (user) => { + const name = faker.animal.insect(); const assistants = faker.helpers.multiple( () => ({ _id: new ObjectId(), - name: faker.animal.insect(), + name, createdById: user._id, createdByName: user.username, createdAt: faker.date.recent({ days: 30 }), @@ -174,6 +177,8 @@ async function seed() { exampleInputs: faker.helpers.multiple(() => faker.lorem.sentence(), { count: faker.number.int({ min: 0, max: 4 }), }), + searchTokens: generateSearchTokens(name), + last24HoursCount: faker.number.int({ min: 0, max: 1000 }), }), { count: faker.number.int({ min: 3, max: 10 }) } ); @@ -241,7 +246,7 @@ async function seed() { try { rl.question( "You're about to run a seeding script on the following MONGODB_URL: \x1b[31m" + - MONGODB_URL + + env.MONGODB_URL + "\x1b[0m\n\n With the following flags: \x1b[31m" + flags.join("\x1b[0m , \x1b[31m") + "\x1b[0m\n \n\n Are you sure you want to continue? (yes/no): ", diff --git a/src/hooks.server.ts b/src/hooks.server.ts index 4272fe8c17e0dbc28589382f566b2d20ec910165..ff4f044eca4655e1bbeee64916464b437cadbdec 100644 --- a/src/hooks.server.ts +++ b/src/hooks.server.ts @@ -1,17 +1,6 @@ -import { - ADMIN_API_SECRET, - COOKIE_NAME, - ENABLE_ASSISTANTS, - EXPOSE_API, - MESSAGES_BEFORE_LOGIN, - PARQUET_EXPORT_SECRET, -} from "$env/static/private"; +import { env } from "$env/dynamic/private"; +import { env as envPublic } from "$env/dynamic/public"; import type { Handle, HandleServerError } from "@sveltejs/kit"; -import { - PUBLIC_GOOGLE_ANALYTICS_ID, - PUBLIC_ORIGIN, - PUBLIC_APP_DISCLAIMER, -} from "$env/static/public"; import { collections } from "$lib/server/database"; import { base } from "$app/paths"; import { findUser, refreshSessionCookie, requiresUser } from "$lib/server/auth"; @@ -22,16 +11,30 @@ import { checkAndRunMigrations } from "$lib/migrations/migrations"; import { building } from "$app/environment"; import { refreshAssistantsCounts } from "$lib/assistantStats/refresh-assistants-counts"; import { logger } from "$lib/server/logger"; +import { AbortedGenerations } from "$lib/server/abortedGenerations"; +import { MetricsServer } from "$lib/server/metrics"; +// TODO: move this code on a started server hook, instead of using a "building" flag if (!building) { await checkAndRunMigrations(); - if (ENABLE_ASSISTANTS) { + if (env.ENABLE_ASSISTANTS) { refreshAssistantsCounts(); } + + // Init metrics server + MetricsServer.getInstance(); + + // Init AbortedGenerations refresh process + AbortedGenerations.getInstance(); } export const handleError: HandleServerError = async ({ error, event }) => { // handle 404 + + if (building) { + throw error; + } + if (event.route.id === null) { return { message: `Page ${event.url.pathname} not found`, @@ -56,7 +59,14 @@ export const handleError: HandleServerError = async ({ error, event }) => { }; export const handle: Handle = async ({ event, resolve }) => { - if (event.url.pathname.startsWith(`${base}/api/`) && EXPOSE_API !== "true") { + logger.debug({ + locals: event.locals, + url: event.url.pathname, + params: event.params, + request: event.request, + }); + + if (event.url.pathname.startsWith(`${base}/api/`) && env.EXPOSE_API !== "true") { return new Response("API is disabled", { status: 403 }); } @@ -73,7 +83,7 @@ export const handle: Handle = async ({ event, resolve }) => { } if (event.url.pathname.startsWith(`${base}/admin/`) || event.url.pathname === `${base}/admin`) { - const ADMIN_SECRET = ADMIN_API_SECRET || PARQUET_EXPORT_SECRET; + const ADMIN_SECRET = env.ADMIN_API_SECRET || env.PARQUET_EXPORT_SECRET; if (!ADMIN_SECRET) { return errorResponse(500, "Admin API is not configured"); @@ -84,7 +94,7 @@ export const handle: Handle = async ({ event, resolve }) => { } } - const token = event.cookies.get(COOKIE_NAME); + const token = event.cookies.get(env.COOKIE_NAME); let secretSessionId: string; let sessionId: string; @@ -123,18 +133,18 @@ export const handle: Handle = async ({ event, resolve }) => { refreshSessionCookie(event.cookies, event.locals.sessionId); if (nativeFormContentTypes.includes(requestContentType)) { - const referer = event.request.headers.get("referer"); + const origin = event.request.headers.get("origin"); - if (!referer) { - return errorResponse(403, "Non-JSON form requests need to have a referer"); + if (!origin) { + return errorResponse(403, "Non-JSON form requests need to have an origin"); } const validOrigins = [ - new URL(event.request.url).origin, - ...(PUBLIC_ORIGIN ? [new URL(PUBLIC_ORIGIN).origin] : []), + new URL(event.request.url).host, + ...(envPublic.PUBLIC_ORIGIN ? [new URL(envPublic.PUBLIC_ORIGIN).host] : []), ]; - if (!validOrigins.includes(new URL(referer).origin)) { + if (!validOrigins.includes(new URL(origin).host)) { return errorResponse(403, "Invalid referer for POST request"); } } @@ -158,7 +168,7 @@ export const handle: Handle = async ({ event, resolve }) => { if ( !event.locals.user && requiresUser && - !((MESSAGES_BEFORE_LOGIN ? parseInt(MESSAGES_BEFORE_LOGIN) : 0) > 0) + !((env.MESSAGES_BEFORE_LOGIN ? parseInt(env.MESSAGES_BEFORE_LOGIN) : 0) > 0) ) { return errorResponse(401, ERROR_MESSAGES.authOnly); } @@ -169,7 +179,7 @@ export const handle: Handle = async ({ event, resolve }) => { if ( !requiresUser && !event.url.pathname.startsWith(`${base}/settings`) && - !!PUBLIC_APP_DISCLAIMER + !!envPublic.PUBLIC_APP_DISCLAIMER ) { const hasAcceptedEthicsModal = await collections.settings.countDocuments({ sessionId: event.locals.sessionId, @@ -192,7 +202,7 @@ export const handle: Handle = async ({ event, resolve }) => { } replaced = true; - return chunk.html.replace("%gaId%", PUBLIC_GOOGLE_ANALYTICS_ID); + return chunk.html.replace("%gaId%", envPublic.PUBLIC_GOOGLE_ANALYTICS_ID); }, }); diff --git a/src/lib/assistantStats/refresh-assistants-counts.ts b/src/lib/assistantStats/refresh-assistants-counts.ts index 07013e1989a3bf3e7e3c98120b2696003900a108..a5bb5f525964b5b0c0779e3f4ef3486adc529d9d 100644 --- a/src/lib/assistantStats/refresh-assistants-counts.ts +++ b/src/lib/assistantStats/refresh-assistants-counts.ts @@ -1,4 +1,4 @@ -import { client, collections } from "$lib/server/database"; +import { Database } from "$lib/server/database"; import { acquireLock, refreshLock } from "$lib/migrations/lock"; import type { ObjectId } from "mongodb"; import { subDays } from "date-fns"; @@ -15,44 +15,49 @@ async function refreshAssistantsCountsHelper() { } try { - await client.withSession((session) => - session.withTransaction(async () => { - await collections.assistants - .aggregate([ - { $project: { _id: 1 } }, - { $set: { last24HoursCount: 0 } }, - { - $unionWith: { - coll: "assistants.stats", - pipeline: [ - { $match: { "date.at": { $gte: subDays(new Date(), 1) }, "date.span": "hour" } }, - { - $group: { - _id: "$assistantId", - last24HoursCount: { $sum: "$count" }, + await Database.getInstance() + .getClient() + .withSession((session) => + session.withTransaction(async () => { + await Database.getInstance() + .getCollections() + .assistants.aggregate([ + { $project: { _id: 1 } }, + { $set: { last24HoursCount: 0 } }, + { + $unionWith: { + coll: "assistants.stats", + pipeline: [ + { + $match: { "date.at": { $gte: subDays(new Date(), 1) }, "date.span": "hour" }, }, - }, - ], + { + $group: { + _id: "$assistantId", + last24HoursCount: { $sum: "$count" }, + }, + }, + ], + }, }, - }, - { - $group: { - _id: "$_id", - last24HoursCount: { $sum: "$last24HoursCount" }, + { + $group: { + _id: "$_id", + last24HoursCount: { $sum: "$last24HoursCount" }, + }, }, - }, - { - $merge: { - into: "assistants", - on: "_id", - whenMatched: "merge", - whenNotMatched: "discard", + { + $merge: { + into: "assistants", + on: "_id", + whenMatched: "merge", + whenNotMatched: "discard", + }, }, - }, - ]) - .next(); - }) - ); + ]) + .next(); + }) + ); } catch (e) { logger.error("Refresh assistants counter failed!"); logger.error(e); diff --git a/src/lib/components/AnnouncementBanner.svelte b/src/lib/components/AnnouncementBanner.svelte index 7d6948a6b387ef45104b9b08294ac1e5920e607b..47ec7b02ef248faab47d38c37dd6260c625886b0 100644 --- a/src/lib/components/AnnouncementBanner.svelte +++ b/src/lib/components/AnnouncementBanner.svelte @@ -5,7 +5,7 @@
New {title} diff --git a/src/lib/components/DisclaimerModal.svelte b/src/lib/components/DisclaimerModal.svelte index 590bb088b3cb3e0f2bf40a18279cd7d3949319b8..f0a18abd1dd2fd536a05ddc9eafa9e8c5d3cd435 100644 --- a/src/lib/components/DisclaimerModal.svelte +++ b/src/lib/components/DisclaimerModal.svelte @@ -1,11 +1,7 @@
- + - {PUBLIC_APP_NAME} + {envPublic.PUBLIC_APP_NAME} Settings - {#if PUBLIC_APP_NAME === "HuggingChat"} + {#if envPublic.PUBLIC_APP_NAME === "HuggingChat"} 0 || assistant?.dynamicPrompt; - const prefix = PUBLIC_SHARE_PREFIX || `${PUBLIC_ORIGIN || $page.url.origin}${base}`; + const prefix = + envPublic.PUBLIC_SHARE_PREFIX || `${envPublic.PUBLIC_ORIGIN || $page.url.origin}${base}`; $: shareUrl = `${prefix}/assistant/${assistant?._id}`; diff --git a/src/lib/components/chat/ChatIntroduction.svelte b/src/lib/components/chat/ChatIntroduction.svelte index eb0a11b0537d947d411618437e64c900e99b981b..81fb51b09faf012417377ac77f379e6db72e701d 100644 --- a/src/lib/components/chat/ChatIntroduction.svelte +++ b/src/lib/components/chat/ChatIntroduction.svelte @@ -1,7 +1,5 @@ -{#if PUBLIC_APP_ASSETS === "chatui"} +{#if envPublic.PUBLIC_APP_ASSETS === "chatui"} {/if} diff --git a/src/lib/migrations/migrations.ts b/src/lib/migrations/migrations.ts index 6e7da313491c62d19fa8b75c5c76fb178d1c713f..2331644d8b8bd9cdd14c6f7c2827071274941374 100644 --- a/src/lib/migrations/migrations.ts +++ b/src/lib/migrations/migrations.ts @@ -1,4 +1,4 @@ -import { client, collections } from "$lib/server/database"; +import { Database } from "$lib/server/database"; import { migrations } from "./routines"; import { acquireLock, releaseLock, isDBLocked, refreshLock } from "./lock"; import { isHuggingChat } from "$lib/utils/isHuggingChat"; @@ -13,12 +13,15 @@ export async function checkAndRunMigrations() { } // check if all migrations have already been run - const migrationResults = await collections.migrationResults.find().toArray(); + const migrationResults = await Database.getInstance() + .getCollections() + .migrationResults.find() + .toArray(); logger.info("[MIGRATIONS] Begin check..."); // connect to the database - const connectedClient = await client.connect(); + const connectedClient = await Database.getInstance().getClient().connect(); const lockId = await acquireLock(LOCK_KEY); @@ -71,23 +74,25 @@ export async function checkAndRunMigrations() { }. Applying...` ); - await collections.migrationResults.updateOne( - { _id: migration._id }, - { - $set: { - name: migration.name, - status: "ongoing", + await Database.getInstance() + .getCollections() + .migrationResults.updateOne( + { _id: migration._id }, + { + $set: { + name: migration.name, + status: "ongoing", + }, }, - }, - { upsert: true } - ); + { upsert: true } + ); const session = connectedClient.startSession(); let result = false; try { await session.withTransaction(async () => { - result = await migration.up(connectedClient); + result = await migration.up(Database.getInstance()); }); } catch (e) { logger.info(`[MIGRATIONS] "${migration.name}" failed!`); @@ -96,16 +101,18 @@ export async function checkAndRunMigrations() { await session.endSession(); } - await collections.migrationResults.updateOne( - { _id: migration._id }, - { - $set: { - name: migration.name, - status: result ? "success" : "failure", + await Database.getInstance() + .getCollections() + .migrationResults.updateOne( + { _id: migration._id }, + { + $set: { + name: migration.name, + status: result ? "success" : "failure", + }, }, - }, - { upsert: true } - ); + { upsert: true } + ); } } diff --git a/src/lib/migrations/routines/01-update-search-assistants.ts b/src/lib/migrations/routines/01-update-search-assistants.ts index 9f12b27d3fbf7060afcf76b094211773fbde55c2..52c8b2f6c99a5e9d349690271c4e28761e351b53 100644 --- a/src/lib/migrations/routines/01-update-search-assistants.ts +++ b/src/lib/migrations/routines/01-update-search-assistants.ts @@ -1,5 +1,5 @@ import type { Migration } from "."; -import { getCollections } from "$lib/server/database"; +import { collections } from "$lib/server/database"; import { ObjectId, type AnyBulkWriteOperation } from "mongodb"; import type { Assistant } from "$lib/types/Assistant"; import { generateSearchTokens } from "$lib/utils/searchTokens"; @@ -7,8 +7,8 @@ import { generateSearchTokens } from "$lib/utils/searchTokens"; const migration: Migration = { _id: new ObjectId("5f9f3e3e3e3e3e3e3e3e3e3e"), name: "Update search assistants", - up: async (client) => { - const { assistants } = getCollections(client); + up: async () => { + const { assistants } = collections; let ops: AnyBulkWriteOperation[] = []; for await (const assistant of assistants @@ -40,8 +40,8 @@ const migration: Migration = { return true; }, - down: async (client) => { - const { assistants } = getCollections(client); + down: async () => { + const { assistants } = collections; await assistants.updateMany({}, { $unset: { searchTokens: "" } }); return true; }, diff --git a/src/lib/migrations/routines/02-update-assistants-models.ts b/src/lib/migrations/routines/02-update-assistants-models.ts index 73655a88f84c711175f07bd5f749d7a3e375375a..f7f0c9dd4543761e69efc55de923e2a284306c72 100644 --- a/src/lib/migrations/routines/02-update-assistants-models.ts +++ b/src/lib/migrations/routines/02-update-assistants-models.ts @@ -1,14 +1,14 @@ import type { Migration } from "."; -import { getCollections } from "$lib/server/database"; +import { collections } from "$lib/server/database"; import { ObjectId } from "mongodb"; const updateAssistantsModels: Migration = { _id: new ObjectId("5f9f3f3f3f3f3f3f3f3f3f3f"), name: "Update deprecated models in assistants with the default model", - up: async (client) => { + up: async () => { const models = (await import("$lib/server/models")).models; - const { assistants } = getCollections(client); + const { assistants } = collections; const modelIds = models.map((el) => el.id); // string[] const defaultModelId = models[0].id; diff --git a/src/lib/migrations/routines/index.ts b/src/lib/migrations/routines/index.ts index 96a6a07ab3c70a6d1211f70d501f5667a655fc4e..0d6eafa8f04e86e7bba3ba9462774d2c25589600 100644 --- a/src/lib/migrations/routines/index.ts +++ b/src/lib/migrations/routines/index.ts @@ -1,13 +1,14 @@ -import type { MongoClient, ObjectId } from "mongodb"; +import type { ObjectId } from "mongodb"; import updateSearchAssistant from "./01-update-search-assistants"; import updateAssistantsModels from "./02-update-assistants-models"; +import type { Database } from "$lib/server/database"; export interface Migration { _id: ObjectId; name: string; - up: (client: MongoClient) => Promise; - down?: (client: MongoClient) => Promise; + up: (client: Database) => Promise; + down?: (client: Database) => Promise; runForFreshInstall?: "only" | "never"; // leave unspecified to run for both runForHuggingChat?: "only" | "never"; // leave unspecified to run for both runEveryTime?: boolean; diff --git a/src/lib/server/abortedGenerations.ts b/src/lib/server/abortedGenerations.ts index 84ad9ec904163aaa2e32ad16af25e103a9518cee..548809d6b6c839524ff08e5fbcceb6d2adb2ae01 100644 --- a/src/lib/server/abortedGenerations.ts +++ b/src/lib/server/abortedGenerations.ts @@ -1,24 +1,38 @@ // Shouldn't be needed if we dove into sveltekit internals, see https://github.com/huggingface/chat-ui/pull/88#issuecomment-1523173850 -import { setTimeout } from "node:timers/promises"; -import { collections } from "./database"; import { logger } from "$lib/server/logger"; +import { collections } from "$lib/server/database"; -let closed = false; -process.on("SIGINT", () => { - closed = true; -}); +export class AbortedGenerations { + private static instance: AbortedGenerations; -export let abortedGenerations: Map = new Map(); + private abortedGenerations: Map = new Map(); -async function maintainAbortedGenerations() { - while (!closed) { - await setTimeout(1000); + private constructor() { + const interval = setInterval(this.updateList, 1000); + process.on("SIGINT", () => { + clearInterval(interval); + }); + } + + public static getInstance(): AbortedGenerations { + if (!AbortedGenerations.instance) { + AbortedGenerations.instance = new AbortedGenerations(); + } + + return AbortedGenerations.instance; + } + + public getList(): Map { + return this.abortedGenerations; + } + + private async updateList() { try { const aborts = await collections.abortedGenerations.find({}).sort({ createdAt: 1 }).toArray(); - abortedGenerations = new Map( + this.abortedGenerations = new Map( aborts.map(({ conversationId, createdAt }) => [conversationId.toString(), createdAt]) ); } catch (err) { @@ -26,5 +40,3 @@ async function maintainAbortedGenerations() { } } } - -maintainAbortedGenerations(); diff --git a/src/lib/server/auth.ts b/src/lib/server/auth.ts index 9a76d25fd667ed96c47ccdfc2ec113d51a874077..94eacdb476b5f2c21cd667d41f4ab92b9332e680 100644 --- a/src/lib/server/auth.ts +++ b/src/lib/server/auth.ts @@ -1,22 +1,11 @@ import { Issuer, BaseClient, type UserinfoResponse, TokenSet, custom } from "openid-client"; import { addHours, addWeeks } from "date-fns"; -import { - COOKIE_NAME, - OPENID_CLIENT_ID, - OPENID_CLIENT_SECRET, - OPENID_PROVIDER_URL, - OPENID_SCOPES, - OPENID_NAME_CLAIM, - OPENID_TOLERANCE, - OPENID_RESOURCE, - OPENID_CONFIG, - ALLOW_INSECURE_COOKIES, -} from "$env/static/private"; +import { env } from "$env/dynamic/private"; import { sha256 } from "$lib/utils/sha256"; import { z } from "zod"; import { dev } from "$app/environment"; import type { Cookies } from "@sveltejs/kit"; -import { collections } from "./database"; +import { collections } from "$lib/server/database"; import JSON5 from "json5"; import { logger } from "$lib/server/logger"; @@ -37,27 +26,27 @@ const stringWithDefault = (value: string) => export const OIDConfig = z .object({ - CLIENT_ID: stringWithDefault(OPENID_CLIENT_ID), - CLIENT_SECRET: stringWithDefault(OPENID_CLIENT_SECRET), - PROVIDER_URL: stringWithDefault(OPENID_PROVIDER_URL), - SCOPES: stringWithDefault(OPENID_SCOPES), - NAME_CLAIM: stringWithDefault(OPENID_NAME_CLAIM).refine( + CLIENT_ID: stringWithDefault(env.OPENID_CLIENT_ID), + CLIENT_SECRET: stringWithDefault(env.OPENID_CLIENT_SECRET), + PROVIDER_URL: stringWithDefault(env.OPENID_PROVIDER_URL), + SCOPES: stringWithDefault(env.OPENID_SCOPES), + NAME_CLAIM: stringWithDefault(env.OPENID_NAME_CLAIM).refine( (el) => !["preferred_username", "email", "picture", "sub"].includes(el), { message: "nameClaim cannot be one of the restricted keys." } ), - TOLERANCE: stringWithDefault(OPENID_TOLERANCE), - RESOURCE: stringWithDefault(OPENID_RESOURCE), + TOLERANCE: stringWithDefault(env.OPENID_TOLERANCE), + RESOURCE: stringWithDefault(env.OPENID_RESOURCE), }) - .parse(JSON5.parse(OPENID_CONFIG)); + .parse(JSON5.parse(env.OPENID_CONFIG)); export const requiresUser = !!OIDConfig.CLIENT_ID && !!OIDConfig.CLIENT_SECRET; export function refreshSessionCookie(cookies: Cookies, sessionId: string) { - cookies.set(COOKIE_NAME, sessionId, { + cookies.set(env.COOKIE_NAME, sessionId, { path: "/", // So that it works inside the space's iframe - sameSite: dev || ALLOW_INSECURE_COOKIES === "true" ? "lax" : "none", - secure: !dev && !(ALLOW_INSECURE_COOKIES === "true"), + sameSite: dev || env.ALLOW_INSECURE_COOKIES === "true" ? "lax" : "none", + secure: !dev && !(env.ALLOW_INSECURE_COOKIES === "true"), httpOnly: true, expires: addWeeks(new Date(), 2), }); diff --git a/src/lib/server/database.ts b/src/lib/server/database.ts index ba4759ee8a7a3677e8a54e77724a398f473c1f0c..4a8302ce9ca119fcb8217f798896286cf65079bc 100644 --- a/src/lib/server/database.ts +++ b/src/lib/server/database.ts @@ -1,4 +1,4 @@ -import { MONGODB_URL, MONGODB_DB_NAME, MONGODB_DIRECT_CONNECTION } from "$env/static/private"; +import { env } from "$env/dynamic/private"; import { GridFSBucket, MongoClient } from "mongodb"; import type { Conversation } from "$lib/types/Conversation"; import type { SharedConversation } from "$lib/types/SharedConversation"; @@ -14,149 +14,196 @@ import type { MigrationResult } from "$lib/types/MigrationResult"; import type { Semaphore } from "$lib/types/Semaphore"; import type { AssistantStats } from "$lib/types/AssistantStats"; import { logger } from "$lib/server/logger"; +import { building } from "$app/environment"; -if (!MONGODB_URL) { - throw new Error( - "Please specify the MONGODB_URL environment variable inside .env.local. Set it to mongodb://localhost:27017 if you are running MongoDB locally, or to a MongoDB Atlas free instance for example." - ); -} export const CONVERSATION_STATS_COLLECTION = "conversations.stats"; -const client = new MongoClient(MONGODB_URL, { - directConnection: MONGODB_DIRECT_CONNECTION === "true", -}); - -export const connectPromise = client.connect().catch(logger.error); - -export function getCollections(mongoClient: MongoClient) { - const db = mongoClient.db(MONGODB_DB_NAME + (import.meta.env.MODE === "test" ? "-test" : "")); - - const conversations = db.collection("conversations"); - const conversationStats = db.collection(CONVERSATION_STATS_COLLECTION); - const assistants = db.collection("assistants"); - const assistantStats = db.collection("assistants.stats"); - const reports = db.collection("reports"); - const sharedConversations = db.collection("sharedConversations"); - const abortedGenerations = db.collection("abortedGenerations"); - const settings = db.collection("settings"); - const users = db.collection("users"); - const sessions = db.collection("sessions"); - const messageEvents = db.collection("messageEvents"); - const bucket = new GridFSBucket(db, { bucketName: "files" }); - const migrationResults = db.collection("migrationResults"); - const semaphores = db.collection("semaphores"); - - return { - conversations, - conversationStats, - assistants, - assistantStats, - reports, - sharedConversations, - abortedGenerations, - settings, - users, - sessions, - messageEvents, - bucket, - migrationResults, - semaphores, - }; -} -const db = client.db(MONGODB_DB_NAME + (import.meta.env.MODE === "test" ? "-test" : "")); - -const collections = getCollections(client); - -const { - conversations, - conversationStats, - assistants, - assistantStats, - reports, - sharedConversations, - abortedGenerations, - settings, - users, - sessions, - messageEvents, - semaphores, -} = collections; - -export { client, db, collections }; - -client.on("open", () => { - conversations - .createIndex( - { sessionId: 1, updatedAt: -1 }, - { partialFilterExpression: { sessionId: { $exists: true } } } - ) - .catch(logger.error); - conversations - .createIndex( - { userId: 1, updatedAt: -1 }, - { partialFilterExpression: { userId: { $exists: true } } } - ) - .catch(logger.error); - conversations - .createIndex( - { "message.id": 1, "message.ancestors": 1 }, - { partialFilterExpression: { userId: { $exists: true } } } - ) - .catch(logger.error); - // To do stats on conversations - conversations.createIndex({ updatedAt: 1 }).catch(logger.error); - // Not strictly necessary, could use _id, but more convenient. Also for stats - conversations.createIndex({ createdAt: 1 }).catch(logger.error); - // To do stats on conversation messages - conversations.createIndex({ "messages.createdAt": 1 }, { sparse: true }).catch(logger.error); - // Unique index for stats - conversationStats - .createIndex( - { +export class Database { + private client: MongoClient; + + private static instance: Database; + + private constructor() { + if (!env.MONGODB_URL) { + throw new Error( + "Please specify the MONGODB_URL environment variable inside .env.local. Set it to mongodb://localhost:27017 if you are running MongoDB locally, or to a MongoDB Atlas free instance for example." + ); + } + + this.client = new MongoClient(env.MONGODB_URL, { + directConnection: env.MONGODB_DIRECT_CONNECTION === "true", + }); + + this.client.connect().catch((err) => { + logger.error("Connection error", err); + process.exit(1); + }); + this.client.db(env.MONGODB_DB_NAME + (import.meta.env.MODE === "test" ? "-test" : "")); + this.client.on("open", () => this.initDatabase()); + + // Disconnect DB on process kill + process.on("SIGINT", async () => { + await this.client.close(true); + + // https://github.com/sveltejs/kit/issues/9540 + setTimeout(() => { + process.exit(0); + }, 100); + }); + } + + public static getInstance(): Database { + if (!Database.instance) { + Database.instance = new Database(); + } + + return Database.instance; + } + + /** + * Return mongoClient + */ + public getClient(): MongoClient { + return this.client; + } + + /** + * Return map of database's collections + */ + public getCollections() { + const db = this.client.db( + env.MONGODB_DB_NAME + (import.meta.env.MODE === "test" ? "-test" : "") + ); + + const conversations = db.collection("conversations"); + const conversationStats = db.collection(CONVERSATION_STATS_COLLECTION); + const assistants = db.collection("assistants"); + const assistantStats = db.collection("assistants.stats"); + const reports = db.collection("reports"); + const sharedConversations = db.collection("sharedConversations"); + const abortedGenerations = db.collection("abortedGenerations"); + const settings = db.collection("settings"); + const users = db.collection("users"); + const sessions = db.collection("sessions"); + const messageEvents = db.collection("messageEvents"); + const bucket = new GridFSBucket(db, { bucketName: "files" }); + const migrationResults = db.collection("migrationResults"); + const semaphores = db.collection("semaphores"); + + return { + conversations, + conversationStats, + assistants, + assistantStats, + reports, + sharedConversations, + abortedGenerations, + settings, + users, + sessions, + messageEvents, + bucket, + migrationResults, + semaphores, + }; + } + + /** + * Init database once connected: Index creation + * @private + */ + private initDatabase() { + const { + conversations, + conversationStats, + assistants, + assistantStats, + reports, + sharedConversations, + abortedGenerations, + settings, + users, + sessions, + messageEvents, + semaphores, + } = this.getCollections(); + + conversations + .createIndex( + { sessionId: 1, updatedAt: -1 }, + { partialFilterExpression: { sessionId: { $exists: true } } } + ) + .catch(logger.error); + conversations + .createIndex( + { userId: 1, updatedAt: -1 }, + { partialFilterExpression: { userId: { $exists: true } } } + ) + .catch(logger.error); + conversations + .createIndex( + { "message.id": 1, "message.ancestors": 1 }, + { partialFilterExpression: { userId: { $exists: true } } } + ) + .catch(logger.error); + // Not strictly necessary, could use _id, but more convenient. Also for stats + // To do stats on conversation messages + conversations.createIndex({ "messages.createdAt": 1 }, { sparse: true }).catch(logger.error); + // Unique index for stats + conversationStats + .createIndex( + { + type: 1, + "date.field": 1, + "date.span": 1, + "date.at": 1, + distinct: 1, + }, + { unique: true } + ) + .catch(logger.error); + // Allow easy check of last computed stat for given type/dateField + conversationStats + .createIndex({ type: 1, "date.field": 1, - "date.span": 1, "date.at": 1, - distinct: 1, - }, - { unique: true } - ) - .catch(logger.error); - // Allow easy check of last computed stat for given type/dateField - conversationStats - .createIndex({ - type: 1, - "date.field": 1, - "date.at": 1, - }) - .catch(logger.error); - abortedGenerations.createIndex({ updatedAt: 1 }, { expireAfterSeconds: 30 }).catch(logger.error); - abortedGenerations.createIndex({ conversationId: 1 }, { unique: true }).catch(logger.error); - sharedConversations.createIndex({ hash: 1 }, { unique: true }).catch(logger.error); - settings.createIndex({ sessionId: 1 }, { unique: true, sparse: true }).catch(logger.error); - settings.createIndex({ userId: 1 }, { unique: true, sparse: true }).catch(logger.error); - settings.createIndex({ assistants: 1 }).catch(logger.error); - users.createIndex({ hfUserId: 1 }, { unique: true }).catch(logger.error); - users.createIndex({ sessionId: 1 }, { unique: true, sparse: true }).catch(logger.error); - // No unicity because due to renames & outdated info from oauth provider, there may be the same username on different users - users.createIndex({ username: 1 }).catch(logger.error); - messageEvents.createIndex({ createdAt: 1 }, { expireAfterSeconds: 60 }).catch(logger.error); - sessions.createIndex({ expiresAt: 1 }, { expireAfterSeconds: 0 }).catch(logger.error); - sessions.createIndex({ sessionId: 1 }, { unique: true }).catch(logger.error); - assistants.createIndex({ createdById: 1, userCount: -1 }).catch(logger.error); - assistants.createIndex({ userCount: 1 }).catch(logger.error); - assistants.createIndex({ featured: 1, userCount: -1 }).catch(logger.error); - assistants.createIndex({ modelId: 1, userCount: -1 }).catch(logger.error); - assistants.createIndex({ searchTokens: 1 }).catch(logger.error); - assistants.createIndex({ last24HoursCount: 1 }).catch(logger.error); - assistantStats - // Order of keys is important for the queries - .createIndex({ "date.span": 1, "date.at": 1, assistantId: 1 }, { unique: true }) - .catch(logger.error); - reports.createIndex({ assistantId: 1 }).catch(logger.error); - reports.createIndex({ createdBy: 1, assistantId: 1 }).catch(logger.error); - - // Unique index for semaphore and migration results - semaphores.createIndex({ key: 1 }, { unique: true }).catch(logger.error); - semaphores.createIndex({ createdAt: 1 }, { expireAfterSeconds: 60 }).catch(logger.error); -}); + }) + .catch(logger.error); + abortedGenerations + .createIndex({ updatedAt: 1 }, { expireAfterSeconds: 30 }) + .catch(logger.error); + abortedGenerations.createIndex({ conversationId: 1 }, { unique: true }).catch(logger.error); + sharedConversations.createIndex({ hash: 1 }, { unique: true }).catch(logger.error); + settings.createIndex({ sessionId: 1 }, { unique: true, sparse: true }).catch(logger.error); + settings.createIndex({ userId: 1 }, { unique: true, sparse: true }).catch(logger.error); + settings.createIndex({ assistants: 1 }).catch(logger.error); + users.createIndex({ hfUserId: 1 }, { unique: true }).catch(logger.error); + users.createIndex({ sessionId: 1 }, { unique: true, sparse: true }).catch(logger.error); + // No unicity because due to renames & outdated info from oauth provider, there may be the same username on different users + users.createIndex({ username: 1 }).catch(logger.error); + messageEvents.createIndex({ createdAt: 1 }, { expireAfterSeconds: 60 }).catch(logger.error); + sessions.createIndex({ expiresAt: 1 }, { expireAfterSeconds: 0 }).catch(logger.error); + sessions.createIndex({ sessionId: 1 }, { unique: true }).catch(logger.error); + assistants.createIndex({ createdById: 1, userCount: -1 }).catch(logger.error); + assistants.createIndex({ userCount: 1 }).catch(logger.error); + assistants.createIndex({ featured: 1, userCount: -1 }).catch(logger.error); + assistants.createIndex({ modelId: 1, userCount: -1 }).catch(logger.error); + assistants.createIndex({ searchTokens: 1 }).catch(logger.error); + assistants.createIndex({ last24HoursCount: 1 }).catch(logger.error); + assistantStats + // Order of keys is important for the queries + .createIndex({ "date.span": 1, "date.at": 1, assistantId: 1 }, { unique: true }) + .catch(logger.error); + reports.createIndex({ assistantId: 1 }).catch(logger.error); + reports.createIndex({ createdBy: 1, assistantId: 1 }).catch(logger.error); + + // Unique index for semaphore and migration results + semaphores.createIndex({ key: 1 }, { unique: true }).catch(logger.error); + semaphores.createIndex({ createdAt: 1 }, { expireAfterSeconds: 60 }).catch(logger.error); + } +} + +export const collections = building + ? ({} as unknown as ReturnType) + : Database.getInstance().getCollections(); diff --git a/src/lib/server/embeddingEndpoints/hfApi/embeddingHfApi.ts b/src/lib/server/embeddingEndpoints/hfApi/embeddingHfApi.ts index 140f223aa5246366fe66ca53f421c61f4447f258..86f84ac19b851333a9380fe08eda2e8d05653fe4 100644 --- a/src/lib/server/embeddingEndpoints/hfApi/embeddingHfApi.ts +++ b/src/lib/server/embeddingEndpoints/hfApi/embeddingHfApi.ts @@ -1,7 +1,7 @@ import { z } from "zod"; import type { EmbeddingEndpoint, Embedding } from "../embeddingEndpoints"; import { chunk } from "$lib/utils/chunk"; -import { HF_TOKEN } from "$env/static/private"; +import { env } from "$env/dynamic/private"; import { logger } from "$lib/server/logger"; export const embeddingEndpointHfApiSchema = z.object({ @@ -11,7 +11,7 @@ export const embeddingEndpointHfApiSchema = z.object({ authorization: z .string() .optional() - .transform((v) => (!v && HF_TOKEN ? "Bearer " + HF_TOKEN : v)), // if the header is not set but HF_TOKEN is, use it as the authorization header + .transform((v) => (!v && env.HF_TOKEN ? "Bearer " + env.HF_TOKEN : v)), // if the header is not set but HF_TOKEN is, use it as the authorization header }); export async function embeddingEndpointHfApi( diff --git a/src/lib/server/embeddingEndpoints/openai/embeddingEndpoints.ts b/src/lib/server/embeddingEndpoints/openai/embeddingEndpoints.ts index 89d7900bb28c09a8e87751b92159e3a499a4a4e7..527a97324984512fd836efde5e7e6e8aebb19345 100644 --- a/src/lib/server/embeddingEndpoints/openai/embeddingEndpoints.ts +++ b/src/lib/server/embeddingEndpoints/openai/embeddingEndpoints.ts @@ -1,14 +1,14 @@ import { z } from "zod"; import type { EmbeddingEndpoint, Embedding } from "../embeddingEndpoints"; import { chunk } from "$lib/utils/chunk"; -import { OPENAI_API_KEY } from "$env/static/private"; +import { env } from "$env/dynamic/private"; export const embeddingEndpointOpenAIParametersSchema = z.object({ weight: z.number().int().positive().default(1), model: z.any(), type: z.literal("openai"), url: z.string().url().default("https://api.openai.com/v1/embeddings"), - apiKey: z.string().default(OPENAI_API_KEY), + apiKey: z.string().default(env.OPENAI_API_KEY), }); export async function embeddingEndpointOpenAI( diff --git a/src/lib/server/embeddingEndpoints/tei/embeddingEndpoints.ts b/src/lib/server/embeddingEndpoints/tei/embeddingEndpoints.ts index 6c2f2f51bb649e33c78329f87dbfba7fc9d90bf9..c999ceba7da550ce5f7eb1ef6e263fb3142f4101 100644 --- a/src/lib/server/embeddingEndpoints/tei/embeddingEndpoints.ts +++ b/src/lib/server/embeddingEndpoints/tei/embeddingEndpoints.ts @@ -1,7 +1,7 @@ import { z } from "zod"; import type { EmbeddingEndpoint, Embedding } from "../embeddingEndpoints"; import { chunk } from "$lib/utils/chunk"; -import { HF_TOKEN } from "$env/static/private"; +import { env } from "$env/dynamic/private"; import { logger } from "$lib/server/logger"; export const embeddingEndpointTeiParametersSchema = z.object({ @@ -12,7 +12,7 @@ export const embeddingEndpointTeiParametersSchema = z.object({ authorization: z .string() .optional() - .transform((v) => (!v && HF_TOKEN ? "Bearer " + HF_TOKEN : v)), // if the header is not set but HF_TOKEN is, use it as the authorization header + .transform((v) => (!v && env.HF_TOKEN ? "Bearer " + env.HF_TOKEN : v)), // if the header is not set but HF_TOKEN is, use it as the authorization header }); const getModelInfoByUrl = async (url: string, authorization?: string) => { diff --git a/src/lib/server/embeddingModels.ts b/src/lib/server/embeddingModels.ts index 96f3795bd936e8a31e27e16d28c3c29dc53fba62..67ad8fe5b1edc61aa6cde738e9ee18ca477f304f 100644 --- a/src/lib/server/embeddingModels.ts +++ b/src/lib/server/embeddingModels.ts @@ -1,4 +1,4 @@ -import { TEXT_EMBEDDING_MODELS } from "$env/static/private"; +import { env } from "$env/dynamic/private"; import { z } from "zod"; import { sum } from "$lib/utils/sum"; @@ -29,7 +29,7 @@ const modelConfig = z.object({ // Default embedding model for backward compatibility const rawEmbeddingModelJSON = - TEXT_EMBEDDING_MODELS || + env.TEXT_EMBEDDING_MODELS || `[ { "name": "Xenova/gte-small", diff --git a/src/lib/server/endpoints/anthropic/endpointAnthropic.ts b/src/lib/server/endpoints/anthropic/endpointAnthropic.ts index e3ef2eb51fa445f6ddd0acf52356b19525e22665..4353c6b11a5fe0dc2b31608dc3ee0e821c09ed8d 100644 --- a/src/lib/server/endpoints/anthropic/endpointAnthropic.ts +++ b/src/lib/server/endpoints/anthropic/endpointAnthropic.ts @@ -1,5 +1,5 @@ import { z } from "zod"; -import { ANTHROPIC_API_KEY } from "$env/static/private"; +import { env } from "$env/dynamic/private"; import type { Endpoint } from "../endpoints"; import type { TextGenerationStreamOutput } from "@huggingface/inference"; @@ -8,7 +8,7 @@ export const endpointAnthropicParametersSchema = z.object({ model: z.any(), type: z.literal("anthropic"), baseURL: z.string().url().default("https://api.anthropic.com"), - apiKey: z.string().default(ANTHROPIC_API_KEY ?? "sk-"), + apiKey: z.string().default(env.ANTHROPIC_API_KEY ?? "sk-"), defaultHeaders: z.record(z.string()).optional(), defaultQuery: z.record(z.string()).optional(), }); diff --git a/src/lib/server/endpoints/cloudflare/endpointCloudflare.ts b/src/lib/server/endpoints/cloudflare/endpointCloudflare.ts index 50f9ac57efad9924a3305ab52924fed8d227f919..f09d2723adb6c2413b5ff8ad61491b63850b058a 100644 --- a/src/lib/server/endpoints/cloudflare/endpointCloudflare.ts +++ b/src/lib/server/endpoints/cloudflare/endpointCloudflare.ts @@ -1,15 +1,15 @@ import { z } from "zod"; import type { Endpoint } from "../endpoints"; import type { TextGenerationStreamOutput } from "@huggingface/inference"; -import { CLOUDFLARE_ACCOUNT_ID, CLOUDFLARE_API_TOKEN } from "$env/static/private"; +import { env } from "$env/dynamic/private"; import { logger } from "$lib/server/logger"; export const endpointCloudflareParametersSchema = z.object({ weight: z.number().int().positive().default(1), model: z.any(), type: z.literal("cloudflare"), - accountId: z.string().default(CLOUDFLARE_ACCOUNT_ID), - apiToken: z.string().default(CLOUDFLARE_API_TOKEN), + accountId: z.string().default(env.CLOUDFLARE_ACCOUNT_ID), + apiToken: z.string().default(env.CLOUDFLARE_API_TOKEN), }); export async function endpointCloudflare( diff --git a/src/lib/server/endpoints/cohere/endpointCohere.ts b/src/lib/server/endpoints/cohere/endpointCohere.ts index 524152fb9915fea2e1e48cf31bc4023ebe4b0c27..f1c5562fa022d3e5e40f7d8f240a32185f61e9c3 100644 --- a/src/lib/server/endpoints/cohere/endpointCohere.ts +++ b/src/lib/server/endpoints/cohere/endpointCohere.ts @@ -1,5 +1,5 @@ import { z } from "zod"; -import { COHERE_API_TOKEN } from "$env/static/private"; +import { env } from "$env/dynamic/private"; import type { Endpoint } from "../endpoints"; import type { TextGenerationStreamOutput } from "@huggingface/inference"; import type { Cohere, CohereClient } from "cohere-ai"; @@ -9,7 +9,7 @@ export const endpointCohereParametersSchema = z.object({ weight: z.number().int().positive().default(1), model: z.any(), type: z.literal("cohere"), - apiKey: z.string().default(COHERE_API_TOKEN), + apiKey: z.string().default(env.COHERE_API_TOKEN), raw: z.boolean().default(false), }); diff --git a/src/lib/server/endpoints/llamacpp/endpointLlamacpp.ts b/src/lib/server/endpoints/llamacpp/endpointLlamacpp.ts index 67b5ea954d0c5420517a03166669b21c2670aad9..b2b8d1478c2fb351869984f44df7feb04184fada 100644 --- a/src/lib/server/endpoints/llamacpp/endpointLlamacpp.ts +++ b/src/lib/server/endpoints/llamacpp/endpointLlamacpp.ts @@ -1,4 +1,4 @@ -import { HF_ACCESS_TOKEN, HF_TOKEN } from "$env/static/private"; +import { env } from "$env/dynamic/private"; import { buildPrompt } from "$lib/buildPrompt"; import type { TextGenerationStreamOutput } from "@huggingface/inference"; import type { Endpoint } from "../endpoints"; @@ -13,7 +13,7 @@ export const endpointLlamacppParametersSchema = z.object({ accessToken: z .string() .min(1) - .default(HF_TOKEN ?? HF_ACCESS_TOKEN), + .default(env.HF_TOKEN ?? env.HF_ACCESS_TOKEN), }); export function endpointLlamacpp( diff --git a/src/lib/server/endpoints/openai/endpointOai.ts b/src/lib/server/endpoints/openai/endpointOai.ts index 8bd28540dd4b82dd28ce7502fd95d25cd66320ce..945921b1b9a4bd86e8fcf3374c0e120007a3336c 100644 --- a/src/lib/server/endpoints/openai/endpointOai.ts +++ b/src/lib/server/endpoints/openai/endpointOai.ts @@ -2,7 +2,7 @@ import { z } from "zod"; import { openAICompletionToTextGenerationStream } from "./openAICompletionToTextGenerationStream"; import { openAIChatToTextGenerationStream } from "./openAIChatToTextGenerationStream"; import { buildPrompt } from "$lib/buildPrompt"; -import { OPENAI_API_KEY } from "$env/static/private"; +import { env } from "$env/dynamic/private"; import type { Endpoint } from "../endpoints"; export const endpointOAIParametersSchema = z.object({ @@ -10,7 +10,7 @@ export const endpointOAIParametersSchema = z.object({ model: z.any(), type: z.literal("openai"), baseURL: z.string().url().default("https://api.openai.com/v1"), - apiKey: z.string().default(OPENAI_API_KEY ?? "sk-"), + apiKey: z.string().default(env.OPENAI_API_KEY ?? "sk-"), completion: z .union([z.literal("completions"), z.literal("chat_completions")]) .default("chat_completions"), diff --git a/src/lib/server/endpoints/tgi/endpointTgi.ts b/src/lib/server/endpoints/tgi/endpointTgi.ts index 131d628ae2122e5ad3976a69458372574c3d299c..aed067397222cfab3633c59b37b5212860b9c054 100644 --- a/src/lib/server/endpoints/tgi/endpointTgi.ts +++ b/src/lib/server/endpoints/tgi/endpointTgi.ts @@ -1,4 +1,4 @@ -import { HF_ACCESS_TOKEN, HF_TOKEN } from "$env/static/private"; +import { env } from "$env/dynamic/private"; import { buildPrompt } from "$lib/buildPrompt"; import { textGenerationStream } from "@huggingface/inference"; import type { Endpoint } from "../endpoints"; @@ -9,7 +9,7 @@ export const endpointTgiParametersSchema = z.object({ model: z.any(), type: z.literal("tgi"), url: z.string().url(), - accessToken: z.string().default(HF_TOKEN ?? HF_ACCESS_TOKEN), + accessToken: z.string().default(env.HF_TOKEN ?? env.HF_ACCESS_TOKEN), authorization: z.string().optional(), }); diff --git a/src/lib/server/files/downloadFile.ts b/src/lib/server/files/downloadFile.ts index 4d2bddb1c30459c319a02200e4f33a9b38b957e2..91b430fc5d835a66e95e2010c07f79b0acc57d58 100644 --- a/src/lib/server/files/downloadFile.ts +++ b/src/lib/server/files/downloadFile.ts @@ -1,5 +1,5 @@ import { error } from "@sveltejs/kit"; -import { collections } from "../database"; +import { collections } from "$lib/server/database"; import type { Conversation } from "$lib/types/Conversation"; import type { SharedConversation } from "$lib/types/SharedConversation"; diff --git a/src/lib/server/files/uploadFile.ts b/src/lib/server/files/uploadFile.ts index 1c4a59b6f44f1914c9ae28b83a5c91c81e146af1..34452245741f05b7819489a109133e8c8e125f9c 100644 --- a/src/lib/server/files/uploadFile.ts +++ b/src/lib/server/files/uploadFile.ts @@ -1,6 +1,6 @@ import type { Conversation } from "$lib/types/Conversation"; import { sha256 } from "$lib/utils/sha256"; -import { collections } from "../database"; +import { collections } from "$lib/server/database"; export async function uploadFile(file: Blob, conv: Conversation): Promise { const sha = await sha256(await file.text()); diff --git a/src/lib/server/logger.ts b/src/lib/server/logger.ts index 76d9a2d77ab84a7dfbaef4bdf248b81d3674e8fd..b01b7692e3a33b20df5a86572cd8d748113ffda7 100644 --- a/src/lib/server/logger.ts +++ b/src/lib/server/logger.ts @@ -1,11 +1,11 @@ import pino from "pino"; import { dev } from "$app/environment"; +import { env } from "$env/dynamic/private"; let options: pino.LoggerOptions = {}; if (dev) { options = { - level: "debug", transport: { target: "pino-pretty", options: { @@ -15,4 +15,4 @@ if (dev) { }; } -export const logger = pino(options); +export const logger = pino({ ...options, level: env.LOG_LEVEL ?? "info" }); diff --git a/src/lib/server/metrics.ts b/src/lib/server/metrics.ts new file mode 100644 index 0000000000000000000000000000000000000000..a4728cc78ba71d4ee7e37546f576b7fc29963ee8 --- /dev/null +++ b/src/lib/server/metrics.ts @@ -0,0 +1,43 @@ +import { collectDefaultMetrics, Registry } from "prom-client"; +import express from "express"; +import { logger } from "$lib/server/logger"; +import { env } from "$env/dynamic/private"; + +export class MetricsServer { + private static instance: MetricsServer; + + private constructor() { + const app = express(); + const port = env.METRICS_PORT || "5565"; + + const server = app.listen(port, () => { + logger.info(`Metrics server listening on port ${port}`); + }); + + const register = new Registry(); + collectDefaultMetrics({ register }); + + app.get("/metrics", (req, res) => { + register.metrics().then((metrics) => { + res.set("Content-Type", "text/plain"); + res.send(metrics); + }); + }); + + process.on("SIGINT", async () => { + logger.info("Sigint received, disconnect metrics server ..."); + server.close(() => { + logger.info("Server stopped ..."); + }); + process.exit(); + }); + } + + public static getInstance(): MetricsServer { + if (!MetricsServer.instance) { + MetricsServer.instance = new MetricsServer(); + } + + return MetricsServer.instance; + } +} diff --git a/src/lib/server/models.ts b/src/lib/server/models.ts index 0ba2fa7c830a119ac82d891de5ec5eb9f920836d..7cdbaf5f5bb11e8a4d74bdc5a783e206ff19ed2b 100644 --- a/src/lib/server/models.ts +++ b/src/lib/server/models.ts @@ -1,11 +1,4 @@ -import { - HF_TOKEN, - HF_API_ROOT, - MODELS, - OLD_MODELS, - TASK_MODEL, - HF_ACCESS_TOKEN, -} from "$env/static/private"; +import { env } from "$env/dynamic/private"; import type { ChatTemplateInput } from "$lib/types/Template"; import { compileTemplate } from "$lib/utils/template"; import { z } from "zod"; @@ -72,7 +65,7 @@ const modelConfig = z.object({ embeddingModel: validateEmbeddingModelByName(embeddingModels).optional(), }); -const modelsRaw = z.array(modelConfig).parse(JSON5.parse(MODELS)); +const modelsRaw = z.array(modelConfig).parse(JSON5.parse(env.MODELS)); async function getChatPromptRender( m: z.infer @@ -147,8 +140,8 @@ const addEndpoint = (m: Awaited>) => ({ if (!m.endpoints) { return endpointTgi({ type: "tgi", - url: `${HF_API_ROOT}/${m.name}`, - accessToken: HF_TOKEN ?? HF_ACCESS_TOKEN, + url: `${env.HF_API_ROOT}/${m.name}`, + accessToken: env.HF_TOKEN ?? env.HF_ACCESS_TOKEN, weight: 1, model: m, }); @@ -199,7 +192,7 @@ export const models = await Promise.all(modelsRaw.map((e) => processModel(e).the export const defaultModel = models[0]; // Models that have been deprecated -export const oldModels = OLD_MODELS +export const oldModels = env.OLD_MODELS ? z .array( z.object({ @@ -208,7 +201,7 @@ export const oldModels = OLD_MODELS displayName: z.string().min(1).optional(), }) ) - .parse(JSON5.parse(OLD_MODELS)) + .parse(JSON5.parse(env.OLD_MODELS)) .map((m) => ({ ...m, id: m.id || m.name, displayName: m.displayName || m.name })) : []; @@ -219,9 +212,9 @@ export const validateModel = (_models: BackendModel[]) => { // if `TASK_MODEL` is string & name of a model in `MODELS`, then we use `MODELS[TASK_MODEL]`, else we try to parse `TASK_MODEL` as a model config itself -export const smallModel = TASK_MODEL - ? (models.find((m) => m.name === TASK_MODEL) || - (await processModel(modelConfig.parse(JSON5.parse(TASK_MODEL))).then((m) => +export const smallModel = env.TASK_MODEL + ? (models.find((m) => m.name === env.TASK_MODEL) || + (await processModel(modelConfig.parse(JSON5.parse(env.TASK_MODEL))).then((m) => addEndpoint(m) ))) ?? defaultModel diff --git a/src/lib/server/summarize.ts b/src/lib/server/summarize.ts index f18639637bcd9deb994ce8510c95affb07ab087a..4cef6174dc9d516ab20c99ef88ede10640d97078 100644 --- a/src/lib/server/summarize.ts +++ b/src/lib/server/summarize.ts @@ -1,10 +1,10 @@ -import { LLM_SUMMERIZATION } from "$env/static/private"; +import { env } from "$env/dynamic/private"; import { generateFromDefaultEndpoint } from "$lib/server/generateFromDefaultEndpoint"; import type { Message } from "$lib/types/Message"; import { logger } from "$lib/server/logger"; export async function summarize(prompt: string) { - if (!LLM_SUMMERIZATION) { + if (!env.LLM_SUMMERIZATION) { return prompt.split(/\s+/g).slice(0, 5).join(" "); } diff --git a/src/lib/server/usageLimits.ts b/src/lib/server/usageLimits.ts index 0323e83fb50a7c86c3662267b509d83f021be403..e1f2390388a60b9cb0f2045ee713ea82706f35a7 100644 --- a/src/lib/server/usageLimits.ts +++ b/src/lib/server/usageLimits.ts @@ -1,5 +1,5 @@ import { z } from "zod"; -import { USAGE_LIMITS, RATE_LIMIT } from "$env/static/private"; +import { env } from "$env/dynamic/private"; import JSON5 from "json5"; // RATE_LIMIT is the legacy way to define messages per minute limit @@ -12,7 +12,7 @@ export const usageLimitsSchema = z messagesPerMinute: z .preprocess((val) => { if (val === undefined) { - return RATE_LIMIT; + return env.RATE_LIMIT; } return val; }, z.coerce.number().optional()) @@ -20,4 +20,4 @@ export const usageLimitsSchema = z }) .optional(); -export const usageLimits = usageLimitsSchema.parse(JSON5.parse(USAGE_LIMITS)); +export const usageLimits = usageLimitsSchema.parse(JSON5.parse(env.USAGE_LIMITS)); diff --git a/src/lib/server/websearch/runWebSearch.ts b/src/lib/server/websearch/runWebSearch.ts index 8eba2ddd627f3a6b5c99b21e7558789ad0d85cfd..1d06a39c5a8dfc605d122944a2f15a4381009150 100644 --- a/src/lib/server/websearch/runWebSearch.ts +++ b/src/lib/server/websearch/runWebSearch.ts @@ -5,7 +5,7 @@ import { chunk } from "$lib/utils/chunk"; import { findSimilarSentences } from "$lib/server/sentenceSimilarity"; import { getWebSearchProvider } from "./searchWeb"; import { defaultEmbeddingModel, embeddingModels } from "$lib/server/embeddingModels"; -import { WEBSEARCH_ALLOWLIST, WEBSEARCH_BLOCKLIST, ENABLE_LOCAL_FETCH } from "$env/static/private"; +import { env } from "$env/dynamic/private"; import type { Conversation } from "$lib/types/Conversation"; import type { MessageUpdate } from "$lib/types/MessageUpdate"; @@ -22,8 +22,8 @@ const MAX_N_PAGES_EMBED = 5 as const; const listSchema = z.array(z.string()).default([]); -const allowList = listSchema.parse(JSON5.parse(WEBSEARCH_ALLOWLIST)); -const blockList = listSchema.parse(JSON5.parse(WEBSEARCH_BLOCKLIST)); +const allowList = listSchema.parse(JSON5.parse(env.WEBSEARCH_ALLOWLIST)); +const blockList = listSchema.parse(JSON5.parse(env.WEBSEARCH_BLOCKLIST)); export async function runWebSearch( conv: Conversation, @@ -52,7 +52,7 @@ export async function runWebSearch( let linksToUse = [...ragSettings.allowedLinks]; - if (ENABLE_LOCAL_FETCH !== "true") { + if (env.ENABLE_LOCAL_FETCH !== "true") { const localLinks = await Promise.all( linksToUse.map(async (link) => { try { diff --git a/src/lib/server/websearch/searchSearxng.ts b/src/lib/server/websearch/searchSearxng.ts index 5378f953b471e180121a195469660bb1e3ce1147..9507d5369690c7be7669beec8965150f88ca1424 100644 --- a/src/lib/server/websearch/searchSearxng.ts +++ b/src/lib/server/websearch/searchSearxng.ts @@ -1,4 +1,4 @@ -import { SEARXNG_QUERY_URL } from "$env/static/private"; +import { env } from "$env/dynamic/private"; import { logger } from "$lib/server/logger"; export async function searchSearxng(query: string) { @@ -6,7 +6,7 @@ export async function searchSearxng(query: string) { setTimeout(() => abortController.abort(), 10000); // Insert the query into the URL template - let url = SEARXNG_QUERY_URL.replace("", query); + let url = env.SEARXNG_QUERY_URL.replace("", query); // Check if "&format=json" already exists in the URL if (!url.includes("&format=json")) { diff --git a/src/lib/server/websearch/searchWeb.ts b/src/lib/server/websearch/searchWeb.ts index 94021e5c014230d91662644363cf4341f951f003..724be1227a7544c6944aaf69e8617d76f94e8e88 100644 --- a/src/lib/server/websearch/searchWeb.ts +++ b/src/lib/server/websearch/searchWeb.ts @@ -1,13 +1,6 @@ import type { YouWebSearch } from "../../types/WebSearch"; import { WebSearchProvider } from "../../types/WebSearch"; -import { - SERPAPI_KEY, - SERPER_API_KEY, - SERPSTACK_API_KEY, - USE_LOCAL_WEBSEARCH, - SEARXNG_QUERY_URL, - YDC_API_KEY, -} from "$env/static/private"; +import { env } from "$env/dynamic/private"; import { getJson } from "serpapi"; import type { GoogleParameters } from "serpapi"; import { searchWebLocal } from "./searchWebLocal"; @@ -15,9 +8,9 @@ import { searchSearxng } from "./searchSearxng"; // get which SERP api is providing web results export function getWebSearchProvider() { - if (YDC_API_KEY) { + if (env.YDC_API_KEY) { return WebSearchProvider.YOU; - } else if (SEARXNG_QUERY_URL) { + } else if (env.SEARXNG_QUERY_URL) { return WebSearchProvider.SEARXNG; } else { return WebSearchProvider.GOOGLE; @@ -26,22 +19,22 @@ export function getWebSearchProvider() { // Show result as JSON export async function searchWeb(query: string) { - if (USE_LOCAL_WEBSEARCH) { + if (env.USE_LOCAL_WEBSEARCH) { return await searchWebLocal(query); } - if (SEARXNG_QUERY_URL) { + if (env.SEARXNG_QUERY_URL) { return await searchSearxng(query); } - if (SERPER_API_KEY) { + if (env.SERPER_API_KEY) { return await searchWebSerper(query); } - if (YDC_API_KEY) { + if (env.YDC_API_KEY) { return await searchWebYouApi(query); } - if (SERPAPI_KEY) { + if (env.SERPAPI_KEY) { return await searchWebSerpApi(query); } - if (SERPSTACK_API_KEY) { + if (env.SERPSTACK_API_KEY) { return await searchSerpStack(query); } throw new Error("No You.com or Serper.dev or SerpAPI key found"); @@ -58,7 +51,7 @@ export async function searchWebSerper(query: string) { method: "POST", body: JSON.stringify(params), headers: { - "x-api-key": SERPER_API_KEY, + "x-api-key": env.SERPER_API_KEY, "Content-type": "application/json; charset=UTF-8", }, }); @@ -84,7 +77,7 @@ export async function searchWebSerpApi(query: string) { hl: "en", gl: "us", google_domain: "google.com", - api_key: SERPAPI_KEY, + api_key: env.SERPAPI_KEY, } satisfies GoogleParameters; // Show result as JSON @@ -97,7 +90,7 @@ export async function searchWebYouApi(query: string) { const response = await fetch(`https://api.ydc-index.io/search?query=${query}`, { method: "GET", headers: { - "X-API-Key": YDC_API_KEY, + "X-API-Key": env.YDC_API_KEY, "Content-type": "application/json; charset=UTF-8", }, }); @@ -123,7 +116,7 @@ export async function searchWebYouApi(query: string) { export async function searchSerpStack(query: string) { const response = await fetch( - `http://api.serpstack.com/search?access_key=${SERPSTACK_API_KEY}&query=${query}&hl=en&gl=us`, + `http://api.serpstack.com/search?access_key=${env.SERPSTACK_API_KEY}&query=${query}&hl=en&gl=us`, { method: "GET", headers: { diff --git a/src/lib/utils/getShareUrl.ts b/src/lib/utils/getShareUrl.ts index ef4259f6ad3a33c17cb29c676b3994a903663e9a..5278ab6fd6ef21b1c34468a0c9b81a164f78b52b 100644 --- a/src/lib/utils/getShareUrl.ts +++ b/src/lib/utils/getShareUrl.ts @@ -1,6 +1,8 @@ import { base } from "$app/paths"; -import { PUBLIC_ORIGIN, PUBLIC_SHARE_PREFIX } from "$env/static/public"; +import { env as envPublic } from "$env/dynamic/public"; export function getShareUrl(url: URL, shareId: string): string { - return `${PUBLIC_SHARE_PREFIX || `${PUBLIC_ORIGIN || url.origin}${base}`}/r/${shareId}`; + return `${ + envPublic.PUBLIC_SHARE_PREFIX || `${envPublic.PUBLIC_ORIGIN || url.origin}${base}` + }/r/${shareId}`; } diff --git a/src/lib/utils/isHuggingChat.ts b/src/lib/utils/isHuggingChat.ts index fbcbefbc546054af65fce8dc0d918b16bf05bc35..df1ad80039eb147a5427cd5ca1980e92b5c2c22a 100644 --- a/src/lib/utils/isHuggingChat.ts +++ b/src/lib/utils/isHuggingChat.ts @@ -1,3 +1,3 @@ -import { PUBLIC_APP_ASSETS } from "$env/static/public"; +import { env as envPublic } from "$env/dynamic/public"; -export const isHuggingChat = PUBLIC_APP_ASSETS === "huggingchat"; +export const isHuggingChat = envPublic.PUBLIC_APP_ASSETS === "huggingchat"; diff --git a/src/routes/+layout.server.ts b/src/routes/+layout.server.ts index c353bc4f47cb00bb8477345c2fc894fdbdba4786..a79c470f6885acbd324a4ed05b530f413f512b49 100644 --- a/src/routes/+layout.server.ts +++ b/src/routes/+layout.server.ts @@ -5,17 +5,7 @@ import { UrlDependency } from "$lib/types/UrlDependency"; import { defaultModel, models, oldModels, validateModel } from "$lib/server/models"; import { authCondition, requiresUser } from "$lib/server/auth"; import { DEFAULT_SETTINGS } from "$lib/types/Settings"; -import { - SERPAPI_KEY, - SERPER_API_KEY, - SERPSTACK_API_KEY, - MESSAGES_BEFORE_LOGIN, - YDC_API_KEY, - USE_LOCAL_WEBSEARCH, - SEARXNG_QUERY_URL, - ENABLE_ASSISTANTS, - ENABLE_ASSISTANTS_RAG, -} from "$env/static/private"; +import { env } from "$env/dynamic/private"; import { ObjectId } from "mongodb"; import type { ConvSidebar } from "$lib/types/ConvSidebar"; @@ -47,7 +37,7 @@ export const load: LayoutServerLoad = async ({ locals, depends }) => { }); } - const enableAssistants = ENABLE_ASSISTANTS === "true"; + const enableAssistants = env.ENABLE_ASSISTANTS === "true"; const assistantActive = !models.map(({ id }) => id).includes(settings?.activeModel ?? ""); @@ -87,7 +77,7 @@ export const load: LayoutServerLoad = async ({ locals, depends }) => { const assistants = await collections.assistants.find({ _id: { $in: assistantIds } }).toArray(); - const messagesBeforeLogin = MESSAGES_BEFORE_LOGIN ? parseInt(MESSAGES_BEFORE_LOGIN) : 0; + const messagesBeforeLogin = env.MESSAGES_BEFORE_LOGIN ? parseInt(env.MESSAGES_BEFORE_LOGIN) : 0; let loginRequired = false; @@ -136,12 +126,12 @@ export const load: LayoutServerLoad = async ({ locals, depends }) => { }) satisfies ConvSidebar[], settings: { searchEnabled: !!( - SERPAPI_KEY || - SERPER_API_KEY || - SERPSTACK_API_KEY || - YDC_API_KEY || - USE_LOCAL_WEBSEARCH || - SEARXNG_QUERY_URL + env.SERPAPI_KEY || + env.SERPER_API_KEY || + env.SERPSTACK_API_KEY || + env.YDC_API_KEY || + env.USE_LOCAL_WEBSEARCH || + env.SEARXNG_QUERY_URL ), ethicsModalAccepted: !!settings?.ethicsModalAcceptedAt, ethicsModalAcceptedAt: settings?.ethicsModalAcceptedAt ?? null, @@ -188,7 +178,7 @@ export const load: LayoutServerLoad = async ({ locals, depends }) => { }, assistant, enableAssistants, - enableAssistantsRAG: ENABLE_ASSISTANTS_RAG === "true", + enableAssistantsRAG: env.ENABLE_ASSISTANTS_RAG === "true", loginRequired, loginEnabled: requiresUser, guestMode: requiresUser && messagesBeforeLogin > 0, diff --git a/src/routes/+layout.svelte b/src/routes/+layout.svelte index bcfa7c05f37b0bbaa1fdfb2ad9139f122c5405fa..18f9a884b26608292974a4a053717d19c0122d09 100644 --- a/src/routes/+layout.svelte +++ b/src/routes/+layout.svelte @@ -7,13 +7,7 @@ import { page } from "$app/stores"; import { browser } from "$app/environment"; - import { - PUBLIC_APPLE_APP_ID, - PUBLIC_APP_DESCRIPTION, - PUBLIC_ORIGIN, - PUBLIC_PLAUSIBLE_SCRIPT_URL, - } from "$env/static/public"; - import { PUBLIC_APP_ASSETS, PUBLIC_APP_NAME } from "$env/static/public"; + import { env as envPublic } from "$env/dynamic/public"; import { error } from "$lib/stores/errors"; import { createSettingsStore } from "$lib/stores/settings"; @@ -134,7 +128,7 @@ - {PUBLIC_APP_NAME} + {envPublic.PUBLIC_APP_NAME} @@ -142,44 +136,49 @@ {#if !$page.url.pathname.includes("/assistant/") && $page.route.id !== "/assistants" && !$page.url.pathname.includes("/models/")} - + - + - + {/if} - {#if PUBLIC_PLAUSIBLE_SCRIPT_URL && PUBLIC_ORIGIN} + {#if envPublic.PUBLIC_PLAUSIBLE_SCRIPT_URL && envPublic.PUBLIC_ORIGIN} {/if} - {#if PUBLIC_APPLE_APP_ID} - + {#if envPublic.PUBLIC_APPLE_APP_ID} + {/if} diff --git a/src/routes/+page.svelte b/src/routes/+page.svelte index d0410802aa4a53a3ca7d1b9d8be6e40d838f0b59..abedc1c8862c1ac1a81770408cbdf3fe16ce8f5a 100644 --- a/src/routes/+page.svelte +++ b/src/routes/+page.svelte @@ -1,7 +1,7 @@ - {PUBLIC_APP_NAME} + {envPublic.PUBLIC_APP_NAME} " -H "Content-Type: application/json" -d '{"model": "OpenAssistant/oasst-sft-6-llama-30b-xor"}' export async function POST({ request }) { - if (!PARQUET_EXPORT_DATASET || !PARQUET_EXPORT_HF_TOKEN) { + if (!env.PARQUET_EXPORT_DATASET || !env.PARQUET_EXPORT_HF_TOKEN) { throw error(500, "Parquet export is not configured."); } @@ -144,10 +144,10 @@ export async function POST({ request }) { await uploadFile({ file: pathToFileURL(fileName) as URL, - credentials: { accessToken: PARQUET_EXPORT_HF_TOKEN }, + credentials: { accessToken: env.PARQUET_EXPORT_HF_TOKEN }, repo: { type: "dataset", - name: PARQUET_EXPORT_DATASET, + name: env.PARQUET_EXPORT_DATASET, }, }); diff --git a/src/routes/admin/stats/compute/+server.ts b/src/routes/admin/stats/compute/+server.ts index f814976f505e2ae4ff9df69cddbfb53a985624c4..d8d7f0ec605075c2acf490d1196dacd6fd701a10 100644 --- a/src/routes/admin/stats/compute/+server.ts +++ b/src/routes/admin/stats/compute/+server.ts @@ -1,6 +1,6 @@ import { json } from "@sveltejs/kit"; import type { ConversationStats } from "$lib/types/ConversationStats"; -import { CONVERSATION_STATS_COLLECTION, collections } from "$lib/server/database.js"; +import { CONVERSATION_STATS_COLLECTION, collections } from "$lib/server/database"; import { logger } from "$lib/server/logger"; // Triger like this: diff --git a/src/routes/api/assistants/+server.ts b/src/routes/api/assistants/+server.ts index 8b99a680bdf5be2327844e85dbda872259fa7ddf..ac588676cbc910ca1a45b2c0b21417801229b462 100644 --- a/src/routes/api/assistants/+server.ts +++ b/src/routes/api/assistants/+server.ts @@ -1,9 +1,9 @@ -import { collections } from "$lib/server/database.js"; +import { collections } from "$lib/server/database"; import type { Assistant } from "$lib/types/Assistant"; import type { User } from "$lib/types/User"; import { generateQueryTokens } from "$lib/utils/searchTokens.js"; import type { Filter } from "mongodb"; -import { REQUIRE_FEATURED_ASSISTANTS } from "$env/static/private"; +import { env } from "$env/dynamic/private"; const NUM_PER_PAGE = 24; @@ -27,11 +27,11 @@ export async function GET({ url, locals }) { // if there is no user, we show community assistants, so only show featured assistants const shouldBeFeatured = - REQUIRE_FEATURED_ASSISTANTS === "true" && !user ? { featured: true } : {}; + env.REQUIRE_FEATURED_ASSISTANTS === "true" && !user ? { featured: true } : {}; // if the user queried is not the current user, only show "public" assistants that have been shared before const shouldHaveBeenShared = - REQUIRE_FEATURED_ASSISTANTS === "true" && !createdByCurrentUser + env.REQUIRE_FEATURED_ASSISTANTS === "true" && !createdByCurrentUser ? { userCount: { $gt: 1 } } : {}; diff --git a/src/routes/assistant/[assistantId]/+page.server.ts b/src/routes/assistant/[assistantId]/+page.server.ts index ac14877dbc4367ef9498e6bdd4a6207b79039aab..fddb181b4b88dcc7d8830af23688676c991ca792 100644 --- a/src/routes/assistant/[assistantId]/+page.server.ts +++ b/src/routes/assistant/[assistantId]/+page.server.ts @@ -1,5 +1,5 @@ import { base } from "$app/paths"; -import { collections } from "$lib/server/database.js"; +import { collections } from "$lib/server/database"; import { redirect } from "@sveltejs/kit"; import { ObjectId } from "mongodb"; diff --git a/src/routes/assistant/[assistantId]/+page.svelte b/src/routes/assistant/[assistantId]/+page.svelte index 16608d07ef2b744869baf0802273de99f808ea00..f19b879023bad0ec39e5a45bcfd25e5b586987b2 100644 --- a/src/routes/assistant/[assistantId]/+page.svelte +++ b/src/routes/assistant/[assistantId]/+page.svelte @@ -6,7 +6,7 @@ import { useSettingsStore } from "$lib/stores/settings"; import type { PageData } from "./$types"; import { applyAction, enhance } from "$app/forms"; - import { PUBLIC_APP_NAME, PUBLIC_ORIGIN } from "$env/static/public"; + import { env as envPublic } from "$env/dynamic/public"; import { page } from "$app/stores"; import IconGear from "~icons/bi/gear-fill"; @@ -24,15 +24,16 @@ - + diff --git a/src/routes/assistants/+page.server.ts b/src/routes/assistants/+page.server.ts index 5be72ff54122f97ea954449a9957e906861d6843..0cf3663f704a4565a162e511f4c11ead8a6e9002 100644 --- a/src/routes/assistants/+page.server.ts +++ b/src/routes/assistants/+page.server.ts @@ -1,6 +1,6 @@ import { base } from "$app/paths"; -import { ENABLE_ASSISTANTS, REQUIRE_FEATURED_ASSISTANTS } from "$env/static/private"; -import { collections } from "$lib/server/database.js"; +import { env } from "$env/dynamic/private"; +import { Database, collections } from "$lib/server/database.js"; import { SortKey, type Assistant } from "$lib/types/Assistant"; import type { User } from "$lib/types/User"; import { generateQueryTokens } from "$lib/utils/searchTokens.js"; @@ -10,7 +10,7 @@ import type { Filter } from "mongodb"; const NUM_PER_PAGE = 24; export const load = async ({ url, locals }) => { - if (!ENABLE_ASSISTANTS) { + if (!env.ENABLE_ASSISTANTS) { throw redirect(302, `${base}/`); } @@ -34,11 +34,11 @@ export const load = async ({ url, locals }) => { // if there is no user, we show community assistants, so only show featured assistants const shouldBeFeatured = - REQUIRE_FEATURED_ASSISTANTS === "true" && !user ? { featured: true } : {}; + env.REQUIRE_FEATURED_ASSISTANTS === "true" && !user ? { featured: true } : {}; // if the user queried is not the current user, only show "public" assistants that have been shared before const shouldHaveBeenShared = - REQUIRE_FEATURED_ASSISTANTS === "true" && !createdByCurrentUser + env.REQUIRE_FEATURED_ASSISTANTS === "true" && !createdByCurrentUser ? { userCount: { $gt: 1 } } : {}; @@ -50,8 +50,9 @@ export const load = async ({ url, locals }) => { ...shouldBeFeatured, ...shouldHaveBeenShared, }; - const assistants = await collections.assistants - .find(filter) + const assistants = await Database.getInstance() + .getCollections() + .assistants.find(filter) .skip(NUM_PER_PAGE * pageIndex) .sort({ ...(sort === SortKey.TRENDING && { last24HoursCount: -1 }), @@ -60,7 +61,9 @@ export const load = async ({ url, locals }) => { .limit(NUM_PER_PAGE) .toArray(); - const numTotalItems = await collections.assistants.countDocuments(filter); + const numTotalItems = await Database.getInstance() + .getCollections() + .assistants.countDocuments(filter); return { assistants: JSON.parse(JSON.stringify(assistants)) as Array, diff --git a/src/routes/assistants/+page.svelte b/src/routes/assistants/+page.svelte index e8a84bbcfc7f3aa1c9112c2e301f834b454bfa0c..d2f90b6323cb8f455e3626701cdaff51698c6e08 100644 --- a/src/routes/assistants/+page.svelte +++ b/src/routes/assistants/+page.svelte @@ -1,7 +1,7 @@ - + - + diff --git a/src/routes/models/[...model]/thumbnail.png/ModelThumbnail.svelte b/src/routes/models/[...model]/thumbnail.png/ModelThumbnail.svelte index b76e362af73397948cfa1277562b4123cdae6463..86af4085e1f81f486278cebe2b9f9129421b2cc1 100644 --- a/src/routes/models/[...model]/thumbnail.png/ModelThumbnail.svelte +++ b/src/routes/models/[...model]/thumbnail.png/ModelThumbnail.svelte @@ -1,5 +1,5 @@