Repository: keephq/keep
Pull request: chore: docker-compose templatization
/close #531

- Templatization docker compose file
- Fixes Dockerfile.dev.api entry point
so that also calls config.py at the start of gunicorn server.
- Updates volume binding for backend in otel compose file

Problem statement:
Templatization of our docker-compose files 
Today we maintain 4 different docker-compose files:

- docker-compose.yml
- docker-compose.dev.yml
- docker-compose-with-otel.yaml
- docker-compose-with-auth.yml

The main difference between them is either extra services or different environment variables.
We need to maintain 1 "baseline" docker-compose file that other docker-compsoe files inherit and extend.

Hints from issue discussion:
@talboren will like to work on this please assign this to me

@talboren @shahargl apologizes for being late on this.
I hope no one working on this issue currently, will be starting work now.

@mohanish2504 happy to have you contribute that :)

Commit message: Merge branch 'main' into mohanish/docker_templatization

Patch to review:
diff --git a/docker-compose-with-auth.yml b/docker-compose-with-auth.yml
index d0bca7e2f8..30cc966e1f 100644
--- a/docker-compose-with-auth.yml
+++ b/docker-compose-with-auth.yml
@@ -1,52 +1,28 @@
 services:
   keep-frontend:
+    extends:
+      file: docker-compose.common.yml
+      service: keep-frontend-common
     image: us-central1-docker.pkg.dev/keephq/keep/keep-ui
-    ports:
-      - "3000:3000"
     environment:
       - AUTH_TYPE=SINGLE_TENANT
-      - NEXTAUTH_SECRET=secret
-      - NEXTAUTH_URL=http://localhost:3000
       - API_URL=http://keep-backend:8080
-      - NEXT_PUBLIC_API_URL=http://localhost:8080
-      - NEXT_PUBLIC_POSTHOG_KEY=phc_muk9qE3TfZsX3SZ9XxX52kCGJBclrjhkP9JxAQcm1PZ
-      - NEXT_PUBLIC_POSTHOG_HOST=https://app.posthog.com
-      - NEXT_PUBLIC_PUSHER_HOST=localhost
-      - NEXT_PUBLIC_PUSHER_PORT=6001
-      - NEXT_PUBLIC_PUSHER_APP_KEY=keepappkey
-      - NEXT_PUBLIC_KEEP_VERSION=0.2.9
     volumes:
       - ./state:/state
     depends_on:
       - keep-backend
+  
   keep-backend:
+    extends:
+      file: docker-compose.common.yml
+      service: keep-backend-common
     image: us-central1-docker.pkg.dev/keephq/keep/keep-api
-    ports:
-      - "8080:8080"
     environment:
       - AUTH_TYPE=SINGLE_TENANT
-      - PORT=8080
-      - SECRET_MANAGER_TYPE=FILE
-      - SECRET_MANAGER_DIRECTORY=/state
-      - DATABASE_CONNECTION_STRING=sqlite:////state/db.sqlite3?check_same_thread=False
-      - OPENAI_API_KEY=$OPENAI_API_KEY
-      - KEEP_JWT_SECRET=keepjwtsecret
-      - KEEP_DEFAULT_USERNAME=admin
-      - KEEP_DEFAULT_PASSWORD=keep
-      - PUSHER_APP_ID=1
-      - PUSHER_APP_KEY=keepappkey
-      - PUSHER_APP_SECRET=keepappsecret
-      - PUSHER_HOST=keep-websocket-server
-      - PUSHER_PORT=6001
     volumes:
       - ./state:/state
+  
   keep-websocket-server:
-    image: quay.io/soketi/soketi:1.4-16-debian
-    ports:
-      - "6001:6001"
-      - "9601:9601"
-    environment:
-      - SOKETI_USER_AUTHENTICATION_TIMEOUT=3000
-      - SOKETI_DEFAULT_APP_ID=1
-      - SOKETI_DEFAULT_APP_KEY=keepappkey
-      - SOKETI_DEFAULT_APP_SECRET=keepappsecret
+    extends:
+      file: docker-compose.common.yml
+      service: keep-websocket-server-common
diff --git a/docker-compose-with-otel.yaml b/docker-compose-with-otel.yaml
index d8889d4133..2d30d0996f 100644
--- a/docker-compose-with-otel.yaml
+++ b/docker-compose-with-otel.yaml
@@ -1,4 +1,3 @@
-version: "3"
 services:
   loki:
     image: grafana/loki:latest
@@ -85,63 +84,38 @@ services:
       - ./otel-shared/otel-collector-config.yaml:/etc/otelcol-contrib/config.yaml
 
   keep-frontend-dev:
-    build:
-      context: .
-      dockerfile: docker/Dockerfile.dev.ui
-    ports:
-      - "3000:3000"
+    extends:
+      file: docker-compose.common.yml
+      service: keep-frontend-common
     environment:
-      - NEXTAUTH_SECRET=secret
-      - NEXTAUTH_URL=http://localhost:3001
-      - NEXT_PUBLIC_API_URL=http://localhost:8080
       - API_URL=http://keep-backend-dev:8080
-      - NEXT_PUBLIC_POSTHOG_KEY=phc_muk9qE3TfZsX3SZ9XxX52kCGJBclrjhkP9JxAQcm1PZ
-      - NEXT_PUBLIC_POSTHOG_HOST=https://app.posthog.com
-      - NEXT_PUBLIC_PUSHER_HOST=keep-websocket-server
-      - NEXT_PUBLIC_PUSHER_PORT=6001
-      - NEXT_PUBLIC_PUSHER_APP_KEY=keepappkey
-      - NEXT_PUBLIC_KEEP_VERSION=0.2.9
+    build:
+      dockerfile: docker/Dockerfile.dev.ui
     volumes:
-      - ./keep/keep-ui:/app
+      - ./keep-ui:/app
       - /app/node_modules
       - /app/.next
     depends_on:
       - keep-backend-dev
+  
   keep-backend-dev:
+    extends:
+      file: docker-compose.common.yml
+      service: keep-backend-common
     build:
-      context: .
       dockerfile: docker/Dockerfile.dev.api
-    labels:
-      - vector_scrape=true
-      - service=keep-api
-    ports:
-      - "8080:8080"
     environment:
       - OTEL_SERVICE_NAME=keephq
       - OTLP_ENDPOINT=http://otel-collector:4317
       - METRIC_OTEL_ENABLED=true
-      - PORT=8080
-      - SECRET_MANAGER_TYPE=FILE
-      - SECRET_MANAGER_DIRECTORY=/state
-      - DATABASE_CONNECTION_STRING=sqlite:////state/db.sqlite3?check_same_thread=False
-      - PUSHER_APP_ID=1
-      - PUSHER_APP_KEY=keepappkey
-      - PUSHER_APP_SECRET=keepappsecret
-      - PUSHER_HOST=localhost
-      - PUSHER_PORT=6001
     volumes:
-      - ./keep/keep:/code/keep
+      - .:/app
       - ./state:/state
+  
   keep-websocket-server:
-    image: quay.io/soketi/soketi:1.4-16-debian
-    ports:
-      - "6001:6001"
-      - "9601:9601"
-    environment:
-      - SOKETI_USER_AUTHENTICATION_TIMEOUT=3000
-      - SOKETI_DEFAULT_APP_ID=1
-      - SOKETI_DEFAULT_APP_KEY=keepappkey
-      - SOKETI_DEFAULT_APP_SECRET=keepappsecret
+    extends:
+      file: docker-compose.common.yml
+      service: keep-websocket-server-common
 
   log_collector:
     image: timberio/vector:0.32.2-debian
diff --git a/docker-compose.common.yml b/docker-compose.common.yml
new file mode 100644
index 0000000000..ae0626d791
--- /dev/null
+++ b/docker-compose.common.yml
@@ -0,0 +1,42 @@
+services:
+
+  keep-frontend-common:
+    ports:
+      - '3000:3000'
+    environment:
+      - NEXTAUTH_SECRET=secret
+      - NEXTAUTH_URL=http://localhost:3000
+      - NEXT_PUBLIC_API_URL=http://localhost:8080
+      - NEXT_PUBLIC_POSTHOG_KEY=phc_muk9qE3TfZsX3SZ9XxX52kCGJBclrjhkP9JxAQcm1PZ
+      - NEXT_PUBLIC_POSTHOG_HOST=https://app.posthog.com
+      - NEXT_PUBLIC_PUSHER_HOST=localhost
+      - NEXT_PUBLIC_PUSHER_PORT=6001
+      - NEXT_PUBLIC_PUSHER_APP_KEY=keepappkey
+      - NEXT_PUBLIC_KEEP_VERSION=0.2.9
+
+  keep-backend-common:
+    ports:
+      - '8080:8080'
+    environment:
+      - PORT=8080
+      - SECRET_MANAGER_TYPE=FILE
+      - SECRET_MANAGER_DIRECTORY=/state
+      - DATABASE_CONNECTION_STRING=sqlite:///./db.sqlite3?check_same_thread=False
+      - OPENAI_API_KEY=$OPENAI_API_KEY
+      - PUSHER_APP_ID=1
+      - PUSHER_APP_KEY=keepappkey
+      - PUSHER_APP_SECRET=keepappsecret
+      - PUSHER_HOST=keep-websocket-server
+      - PUSHER_PORT=6001
+
+  keep-websocket-server-common:
+    image: quay.io/soketi/soketi:1.4-16-debian
+    ports:
+      - "6001:6001"
+      - "9601:9601"
+    environment:
+      - SOKETI_USER_AUTHENTICATION_TIMEOUT=3000
+      - SOKETI_DEBUG=1
+      - SOKETI_DEFAULT_APP_ID=1
+      - SOKETI_DEFAULT_APP_KEY=keepappkey
+      - SOKETI_DEFAULT_APP_SECRET=keepappsecret
\ No newline at end of file
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 4bacb24fb0..a1dc5c1c54 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -1,54 +1,32 @@
 services:
   keep-frontend-dev:
-    build:
-      dockerfile: docker/Dockerfile.dev.ui
-    ports:
-      - '3000:3000'
+    extends:
+      file: docker-compose.common.yml
+      service: keep-frontend-common
     environment:
-      - NEXTAUTH_SECRET=secret
-      - NEXTAUTH_URL=http://localhost:3000
-      - NEXT_PUBLIC_API_URL=http://localhost:8080
       - API_URL=http://keep-backend-dev:8080
-      - NEXT_PUBLIC_POSTHOG_KEY=phc_muk9qE3TfZsX3SZ9XxX52kCGJBclrjhkP9JxAQcm1PZ
-      - NEXT_PUBLIC_POSTHOG_HOST=https://app.posthog.com
-      - NEXT_PUBLIC_PUSHER_HOST=localhost
-      - NEXT_PUBLIC_PUSHER_PORT=6001
-      - NEXT_PUBLIC_PUSHER_APP_KEY=keepappkey
-      - NEXT_PUBLIC_KEEP_VERSION=0.2.9
+    build:
+      dockerfile: docker/Dockerfile.dev.ui
     volumes:
       - ./keep-ui:/app
       - /app/node_modules
       - /app/.next
     depends_on:
       - keep-backend-dev
+  
   keep-backend-dev:
+    extends:
+      file: docker-compose.common.yml
+      service: keep-backend-common
     build:
       dockerfile: docker/Dockerfile.dev.api
-    ports:
-      - '8080:8080'
     environment:
-      - PORT=8080
-      - SECRET_MANAGER_TYPE=FILE
-      - SECRET_MANAGER_DIRECTORY=/state
-      - DATABASE_CONNECTION_STRING=sqlite:////db.sqlite3?check_same_thread=False
-      - USE_NGROK=true
-      - OPENAI_API_KEY=$OPENAI_API_KEY
-      - PUSHER_APP_ID=1
-      - PUSHER_APP_KEY=keepappkey
-      - PUSHER_APP_SECRET=keepappsecret
-      - PUSHER_HOST=keep-websocket-server
-      - PUSHER_PORT=6001
+      - USE_NGROK=true 
     volumes:
       - .:/app
       - ./state:/state
+      
   keep-websocket-server:
-    image: quay.io/soketi/soketi:1.4-16-debian
-    ports:
-      - "6001:6001"
-      - "9601:9601"
-    environment:
-      - SOKETI_USER_AUTHENTICATION_TIMEOUT=3000
-      - SOKETI_DEBUG=1
-      - SOKETI_DEFAULT_APP_ID=1
-      - SOKETI_DEFAULT_APP_KEY=keepappkey
-      - SOKETI_DEFAULT_APP_SECRET=keepappsecret
+    extends:
+      file: docker-compose.common.yml
+      service: keep-websocket-server-common
diff --git a/docker-compose.yml b/docker-compose.yml
index 40a2f40140..6c88728ff6 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,49 +1,28 @@
 services:
   keep-frontend:
+    extends:
+      file: docker-compose.common.yml
+      service: keep-frontend-common
     image: us-central1-docker.pkg.dev/keephq/keep/keep-ui
-    ports:
-      - "3000:3000"
     environment:
       - AUTH_TYPE=NO_AUTH
-      - NEXTAUTH_SECRET=secret
-      - NEXTAUTH_URL=http://localhost:3000
       - API_URL=http://keep-backend:8080
-      - NEXT_PUBLIC_API_URL=http://localhost:8080
-      - NEXT_PUBLIC_POSTHOG_KEY=phc_muk9qE3TfZsX3SZ9XxX52kCGJBclrjhkP9JxAQcm1PZ
-      - NEXT_PUBLIC_POSTHOG_HOST=https://app.posthog.com
-      - NEXT_PUBLIC_PUSHER_HOST=localhost
-      - NEXT_PUBLIC_PUSHER_PORT=6001
-      - NEXT_PUBLIC_PUSHER_APP_KEY=keepappkey
-      - NEXT_PUBLIC_KEEP_VERSION=0.2.9
     volumes:
       - ./state:/state
     depends_on:
       - keep-backend
+
   keep-backend:
+    extends:
+      file: docker-compose.common.yml
+      service: keep-backend-common
     image: us-central1-docker.pkg.dev/keephq/keep/keep-api
-    ports:
-      - "8080:8080"
     environment:
-      - PORT=8080
       - AUTH_TYPE=NO_AUTH
-      - SECRET_MANAGER_TYPE=FILE
-      - SECRET_MANAGER_DIRECTORY=/state
-      - DATABASE_CONNECTION_STRING=sqlite:////state/db.sqlite3?check_same_thread=False
-      - OPENAI_API_KEY=$OPENAI_API_KEY
-      - PUSHER_APP_ID=1
-      - PUSHER_APP_KEY=keepappkey
-      - PUSHER_APP_SECRET=keepappsecret
-      - PUSHER_HOST=keep-websocket-server
-      - PUSHER_PORT=6001
     volumes:
       - ./state:/state
+  
   keep-websocket-server:
-    image: quay.io/soketi/soketi:1.4-16-debian
-    ports:
-      - "6001:6001"
-      - "9601:9601"
-    environment:
-      - SOKETI_USER_AUTHENTICATION_TIMEOUT=3000
-      - SOKETI_DEFAULT_APP_ID=1
-      - SOKETI_DEFAULT_APP_KEY=keepappkey
-      - SOKETI_DEFAULT_APP_SECRET=keepappsecret
+    extends:
+      file: docker-compose.common.yml
+      service: keep-websocket-server-common
diff --git a/docker/Dockerfile.dev.api b/docker/Dockerfile.dev.api
index ea66ecf0a0..344f112146 100644
--- a/docker/Dockerfile.dev.api
+++ b/docker/Dockerfile.dev.api
@@ -25,4 +25,4 @@ RUN echo "test"
 ENV PATH="/venv/bin:${PATH}"
 ENV VIRTUAL_ENV="/venv"
 
-CMD ["uvicorn", "keep.api.api:get_app", "--host", "0.0.0.0", "--port", "8080", "--reload"]
+ENTRYPOINT ["gunicorn", "keep.api.api:get_app", "--bind" , "0.0.0.0:8080" , "--workers", "1" , "-k" , "uvicorn.workers.UvicornWorker", "-c", "./keep/api/config.py"]


Code review comments:
diff_hunk: @@ -25,4 +25,4 @@ RUN echo "test"
 ENV PATH="/venv/bin:${PATH}"
 ENV VIRTUAL_ENV="/venv"
 
-CMD ["uvicorn", "keep.api.api:get_app", "--host", "0.0.0.0", "--port", "8080", "--reload"]
path: docker/Dockerfile.dev.api
text: @user1:
`--reload` is required