From ff52fe4d8791d625f9c0bf9e8ffdb068d776f6be Mon Sep 17 00:00:00 2001 From: Hari John Kuriakose Date: Sat, 18 Oct 2025 21:45:03 +0530 Subject: [PATCH 1/3] feat: change port to support rootless feat: (breaking) change localhost url and port to support rootless feat: support podman socket docs: add container runtime details --- backend/sample.env | 4 +- docker/CONTAINER_RUNTIME.md | 143 ++++++++++++++++++++++ docker/README.md | 8 ++ docker/docker-compose-dev-essentials.yaml | 14 ++- docker/docker-compose.yaml | 30 +++-- docker/dockerfiles/frontend.Dockerfile | 2 +- frontend/nginx.conf | 2 +- frontend/sample.env | 2 +- 8 files changed, 187 insertions(+), 18 deletions(-) create mode 100644 docker/CONTAINER_RUNTIME.md diff --git a/backend/sample.env b/backend/sample.env index 7553bc280d..772a8eeacb 100644 --- a/backend/sample.env +++ b/backend/sample.env @@ -11,7 +11,7 @@ DEFAULT_LOG_LEVEL="INFO" PATH_PREFIX="api/v1" # Django settings -DJANGO_APP_BACKEND_URL=http://frontend.unstract.localhost +DJANGO_APP_BACKEND_URL=http://frontend.unstract.localhost:8081 DJANGO_SECRET_KEY="1(xf&nc6!y7!l&!5xe&i_rx7e^m@fcut9fduv86ft=-b@2g6" # Postgres DB envs @@ -40,7 +40,7 @@ GOOGLE_OAUTH2_SECRET= SESSION_EXPIRATION_TIME_IN_SECOND=7200 # FE Web Application Dependencies -WEB_APP_ORIGIN_URL="http://frontend.unstract.localhost" +WEB_APP_ORIGIN_URL="http://frontend.unstract.localhost:8081" # API keys for trusted services INTERNAL_SERVICE_API_KEY= diff --git a/docker/CONTAINER_RUNTIME.md b/docker/CONTAINER_RUNTIME.md new file mode 100644 index 0000000000..d265f00722 --- /dev/null +++ b/docker/CONTAINER_RUNTIME.md @@ -0,0 +1,143 @@ +# Container Runtime Support (Docker & Podman) + +The Unstract docker-compose configuration supports both **Docker** and **Podman** automatically. + +## Automatic Detection + +By default, the configuration will automatically detect and use the appropriate socket: +- **Podman rootless**: `/run/user/$UID/podman/podman.sock` +- **Docker**: `/var/run/docker.sock` + +## Using Docker + +Just run docker-compose commands normally: + +```bash +VERSION=main docker-compose -f docker-compose.yaml up -d +``` + +No environment variables needed - Docker socket at `/var/run/docker.sock` will be used automatically. + +## Using Podman + +### Prerequisites + +1. **Enable Podman socket** (required for Traefik to discover containers): + ```bash + systemctl --user enable podman.socket + systemctl --user start podman.socket + ``` + +2. **Verify socket is running**: + ```bash + systemctl --user status podman.socket + # Should show: active (listening) + ``` + +### Run with Podman + +```bash +VERSION=main podman-compose -f docker-compose.yaml up -d +``` + +The Podman socket will be automatically detected via `$XDG_RUNTIME_DIR/podman/podman.sock`. + +## Manual Override + +If you need to specify a custom socket path, set the `DOCKER_SOCKET` environment variable: + +```bash +# Example: Custom Docker socket location +export DOCKER_SOCKET=/custom/path/docker.sock +VERSION=main docker-compose -f docker-compose.yaml up -d + +# Example: Custom Podman socket location +export DOCKER_SOCKET=/run/user/$(id -u)/podman/podman.sock +VERSION=main podman-compose -f docker-compose.yaml up -d +``` + +## Troubleshooting + +### Traefik shows "Cannot connect to Docker daemon" + +**For Podman users**: +1. Check if Podman socket is running: + ```bash + systemctl --user status podman.socket + ``` + +2. If inactive, start it: + ```bash + systemctl --user start podman.socket + ``` + +3. Verify socket file exists: + ```bash + ls -la $XDG_RUNTIME_DIR/podman/podman.sock + # Should show: srw-rw---- (socket file, not directory) + ``` + +4. If it's a directory (wrong), remove and restart: + ```bash + rmdir $XDG_RUNTIME_DIR/podman/podman.sock + systemctl --user restart podman.socket + ``` + +**For Docker users**: +1. Check if Docker daemon is running: + ```bash + systemctl status docker + ``` + +2. Verify socket permissions: + ```bash + ls -la /var/run/docker.sock + ``` + +### Port 8081 not accessible + +This is the Traefik HTTP port for Podman rootless compatibility. + +1. Check if Traefik container is running: + ```bash + podman ps | grep unstract-proxy + # or + docker ps | grep unstract-proxy + ``` + +2. Check Traefik logs: + ```bash + podman logs unstract-proxy + # or + docker logs unstract-proxy + ``` + +## Socket Path Priority + +The configuration uses this priority order: + +1. `$DOCKER_SOCKET` - if explicitly set +2. `$XDG_RUNTIME_DIR/podman/podman.sock` - for Podman rootless +3. `/run/user/1000/podman/podman.sock` - fallback for Podman +4. Falls back to default compose behavior (typically `/var/run/docker.sock`) + +## Technical Details + +The docker-compose files use this volume mount configuration: + +```yaml +volumes: + - ${DOCKER_SOCKET:-${XDG_RUNTIME_DIR:-/run/user/1000}/podman/podman.sock}:/var/run/docker.sock +``` + +This means: +- If `DOCKER_SOCKET` is set → use that +- Else if `XDG_RUNTIME_DIR` is set → use `$XDG_RUNTIME_DIR/podman/podman.sock` +- Else → use `/run/user/1000/podman/podman.sock` + +For Docker, you can set: +```bash +export DOCKER_SOCKET=/var/run/docker.sock +``` + +But it's usually not necessary since Docker Compose will use `/var/run/docker.sock` by default when the variable is unset. diff --git a/docker/README.md b/docker/README.md index 9461511d31..395fce7ddd 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,5 +1,13 @@ # Docker Commands +## Container Runtime Support + +This project supports both **Docker** and **Podman**. See [CONTAINER_RUNTIME.md](CONTAINER_RUNTIME.md) for detailed information. + +**Quick Start**: +- **Docker**: Use `docker compose` commands as shown below +- **Podman**: Use `podman-compose` commands AND enable the socket: `systemctl --user enable --now podman.socket` + ## Docker Build ```bash diff --git a/docker/docker-compose-dev-essentials.yaml b/docker/docker-compose-dev-essentials.yaml index cd5aa31fe8..4f71f7624c 100644 --- a/docker/docker-compose-dev-essentials.yaml +++ b/docker/docker-compose-dev-essentials.yaml @@ -77,13 +77,15 @@ services: --providers.docker=true --providers.docker.network=unstract-network --providers.file.filename=/proxy_overrides.yaml --providers.file.watch=true ports: - # The HTTP port - - "80:80" + # The HTTP port (changed to 8081 for rootless Podman compatibility) + - "8081:80" # The Web UI (enabled by --api.insecure=true) - "8080:8080" volumes: - # So that Traefik can listen to the Docker events - - /var/run/docker.sock:/var/run/docker.sock + # Universal socket mount - works with both Docker and Podman + # Podman rootless: /run/user/$UID/podman/podman.sock + # Docker: /var/run/docker.sock + - ${DOCKER_SOCKET:-${XDG_RUNTIME_DIR:-/run/user/1000}/podman/podman.sock}:/var/run/docker.sock # Proxy overrides for components run directly in Docker host - ./proxy_overrides.yaml:/proxy_overrides.yaml # Since any proxy overrides need to point to Docker host for relevant routes. @@ -142,8 +144,8 @@ services: env_file: - ./essentials.env ports: - - "5672:5672" # AMQP port - - "15672:15672" # Management UI port + - "5672:5672" # AMQP port + - "15672:15672" # Management UI port volumes: - rabbitmq_data:/var/lib/rabbitmq diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index f4226c0737..6466c87267 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -57,7 +57,6 @@ services: - ./workflow_data:/data - ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config - # Celery worker for managing logs and periodic tasks worker-logging: image: unstract/backend:${VERSION} @@ -153,8 +152,8 @@ services: - ../backend/.env - ./essentials.env depends_on: - - db - - rabbitmq + - db + - rabbitmq environment: - ENVIRONMENT=development - APPLICATION_NAME=unstract-celery-beat @@ -165,7 +164,7 @@ services: container_name: unstract-frontend restart: unless-stopped ports: - - "3000:80" + - "3000:8080" depends_on: - backend - reverse-proxy @@ -174,6 +173,7 @@ services: labels: - traefik.enable=true - traefik.http.routers.frontend.rule=Host(`frontend.unstract.localhost`) && !PathPrefix(`/api/v1`, `/deployment`) + - traefik.http.services.frontend.loadbalancer.server.port=8080 platform-service: image: unstract/platform-service:${VERSION} @@ -231,8 +231,10 @@ services: - ../runner/.env volumes: - ./workflow_data:/data - # Docker socket bind mount to spawn tool containers - - /var/run/docker.sock:/var/run/docker.sock + # Universal socket mount - works with both Docker and Podman + # Podman rootless: /run/user/$UID/podman/podman.sock + # Docker: /var/run/docker.sock + - ${DOCKER_SOCKET:-${XDG_RUNTIME_DIR:-/run/user/1000}/podman/podman.sock}:/var/run/docker.sock depends_on: - redis - rabbitmq @@ -310,7 +312,21 @@ services: container_name: unstract-worker-file-processing-v2 restart: unless-stopped # command: ["file-processing"] - command: [".venv/bin/celery", "-A", "worker", "worker", "--queues=file_processing,api_file_processing,file_processing_priority", "--loglevel=INFO", "--pool=prefork", "--concurrency=4", "--prefetch-multiplier=1", "--without-gossip", "--without-mingle", "--without-heartbeat"] + command: + [ + ".venv/bin/celery", + "-A", + "worker", + "worker", + "--queues=file_processing,api_file_processing,file_processing_priority", + "--loglevel=INFO", + "--pool=prefork", + "--concurrency=4", + "--prefetch-multiplier=1", + "--without-gossip", + "--without-mingle", + "--without-heartbeat", + ] ports: - "8087:8082" env_file: diff --git a/docker/dockerfiles/frontend.Dockerfile b/docker/dockerfiles/frontend.Dockerfile index 3c205ffcb6..4a8a2d5bc7 100644 --- a/docker/dockerfiles/frontend.Dockerfile +++ b/docker/dockerfiles/frontend.Dockerfile @@ -54,7 +54,7 @@ RUN mkdir -p /usr/share/nginx/html/config && \ COPY ../frontend/generate-runtime-config.sh /docker-entrypoint.d/40-env.sh RUN chmod +x /docker-entrypoint.d/40-env.sh -EXPOSE 80 +EXPOSE 8080 USER nginx diff --git a/frontend/nginx.conf b/frontend/nginx.conf index 8cb1b66d51..1eab901392 100644 --- a/frontend/nginx.conf +++ b/frontend/nginx.conf @@ -38,7 +38,7 @@ http { scgi_temp_path /tmp/scgi_temp 1 2; server { - listen 80; + listen 8080; root /usr/share/nginx/html; include /etc/nginx/mime.types; diff --git a/frontend/sample.env b/frontend/sample.env index d7d8b7d677..a3cbd160b0 100644 --- a/frontend/sample.env +++ b/frontend/sample.env @@ -1,4 +1,4 @@ -REACT_APP_BACKEND_URL=http://localhost:8000 +REACT_APP_BACKEND_URL=http://frontend.unstract.localhost:8081 # For development NODE_ENV=development From aac3bf341674f12864e57a3605bac007c52ee779 Mon Sep 17 00:00:00 2001 From: Hari John Kuriakose Date: Thu, 23 Oct 2025 17:13:07 +0530 Subject: [PATCH 2/3] chore: updated gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 8cb0de0377..01feee77fc 100644 --- a/.gitignore +++ b/.gitignore @@ -713,3 +713,6 @@ CONTRIBUTION_GUIDE.md .qodo .windsurfrules .windsurf/rules + +# MCP servers +.serena From 3a3f0722eecc9650ba3e232b07b991e4e1038da3 Mon Sep 17 00:00:00 2001 From: Hari John Kuriakose Date: Thu, 23 Oct 2025 21:51:11 +0530 Subject: [PATCH 3/3] refactor: simplify socket detection to default to Docker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Simplified container runtime socket configuration based on code review feedback. Docker is now the default with Podman as an explicit opt-in. Changes: - Simplified socket mount from complex nested fallback to simple default - Before: ${DOCKER_SOCKET:-${XDG_RUNTIME_DIR:-/run/user/1000}/podman/podman.sock} - After: ${DOCKER_SOCKET:-/var/run/docker.sock} - Docker works out of the box with no environment variables needed - Podman users set: export DOCKER_SOCKET=${XDG_RUNTIME_DIR}/podman/podman.sock - Updated CONTAINER_RUNTIME.md to reflect simplified approach - Fixed worker-file-processing-v2 command structure to match other workers (entrypoint + command pattern instead of array with binary) Benefits: - Cleaner default configuration for majority Docker users - Explicit opt-in for Podman users - Addresses CodeRabbit review concerns about socket fallback logic 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- docker/CONTAINER_RUNTIME.md | 47 ++++++++++++----------- docker/docker-compose-dev-essentials.yaml | 8 ++-- docker/docker-compose.yaml | 26 +++---------- 3 files changed, 35 insertions(+), 46 deletions(-) diff --git a/docker/CONTAINER_RUNTIME.md b/docker/CONTAINER_RUNTIME.md index d265f00722..68037d15bb 100644 --- a/docker/CONTAINER_RUNTIME.md +++ b/docker/CONTAINER_RUNTIME.md @@ -1,22 +1,22 @@ # Container Runtime Support (Docker & Podman) -The Unstract docker-compose configuration supports both **Docker** and **Podman** automatically. +The Unstract docker-compose configuration supports both **Docker** (default) and **Podman**. -## Automatic Detection +## Socket Detection -By default, the configuration will automatically detect and use the appropriate socket: -- **Podman rootless**: `/run/user/$UID/podman/podman.sock` -- **Docker**: `/var/run/docker.sock` +The configuration defaults to Docker and supports Podman via environment variable: +- **Docker**: Default (`/var/run/docker.sock`) - no configuration needed +- **Podman**: Set `DOCKER_SOCKET=${XDG_RUNTIME_DIR}/podman/podman.sock` ## Using Docker -Just run docker-compose commands normally: +Docker works out of the box with no additional configuration: ```bash VERSION=main docker-compose -f docker-compose.yaml up -d ``` -No environment variables needed - Docker socket at `/var/run/docker.sock` will be used automatically. +The Docker socket at `/var/run/docker.sock` is used automatically. ## Using Podman @@ -36,13 +36,16 @@ No environment variables needed - Docker socket at `/var/run/docker.sock` will b ### Run with Podman +Set the `DOCKER_SOCKET` environment variable to point to Podman socket, then run podman-compose: + ```bash +export DOCKER_SOCKET=${XDG_RUNTIME_DIR}/podman/podman.sock VERSION=main podman-compose -f docker-compose.yaml up -d ``` -The Podman socket will be automatically detected via `$XDG_RUNTIME_DIR/podman/podman.sock`. +**Note**: The `DOCKER_SOCKET` environment variable must be set to use Podman instead of the default Docker socket. -## Manual Override +## Custom Socket Path If you need to specify a custom socket path, set the `DOCKER_SOCKET` environment variable: @@ -51,7 +54,7 @@ If you need to specify a custom socket path, set the `DOCKER_SOCKET` environment export DOCKER_SOCKET=/custom/path/docker.sock VERSION=main docker-compose -f docker-compose.yaml up -d -# Example: Custom Podman socket location +# Example: Alternative Podman socket location export DOCKER_SOCKET=/run/user/$(id -u)/podman/podman.sock VERSION=main podman-compose -f docker-compose.yaml up -d ``` @@ -114,12 +117,13 @@ This is the Traefik HTTP port for Podman rootless compatibility. ## Socket Path Priority -The configuration uses this priority order: +The configuration uses this simple priority: + +1. `$DOCKER_SOCKET` - if explicitly set (use this for Podman or custom paths) +2. `/var/run/docker.sock` - default (Docker standard socket) -1. `$DOCKER_SOCKET` - if explicitly set -2. `$XDG_RUNTIME_DIR/podman/podman.sock` - for Podman rootless -3. `/run/user/1000/podman/podman.sock` - fallback for Podman -4. Falls back to default compose behavior (typically `/var/run/docker.sock`) +**Docker**: No configuration needed - uses default socket +**Podman**: Set `export DOCKER_SOCKET=${XDG_RUNTIME_DIR}/podman/podman.sock` ## Technical Details @@ -127,17 +131,16 @@ The docker-compose files use this volume mount configuration: ```yaml volumes: - - ${DOCKER_SOCKET:-${XDG_RUNTIME_DIR:-/run/user/1000}/podman/podman.sock}:/var/run/docker.sock + - ${DOCKER_SOCKET:-/var/run/docker.sock}:/var/run/docker.sock ``` This means: -- If `DOCKER_SOCKET` is set → use that -- Else if `XDG_RUNTIME_DIR` is set → use `$XDG_RUNTIME_DIR/podman/podman.sock` -- Else → use `/run/user/1000/podman/podman.sock` +- If `DOCKER_SOCKET` is set → use that path (for Podman or custom Docker socket) +- Else → use `/var/run/docker.sock` (Docker default) -For Docker, you can set: +**For Podman users:** ```bash -export DOCKER_SOCKET=/var/run/docker.sock +export DOCKER_SOCKET=${XDG_RUNTIME_DIR}/podman/podman.sock ``` -But it's usually not necessary since Docker Compose will use `/var/run/docker.sock` by default when the variable is unset. +This overrides the default Docker socket with the Podman socket path. diff --git a/docker/docker-compose-dev-essentials.yaml b/docker/docker-compose-dev-essentials.yaml index 4f71f7624c..9ae29072c1 100644 --- a/docker/docker-compose-dev-essentials.yaml +++ b/docker/docker-compose-dev-essentials.yaml @@ -82,10 +82,10 @@ services: # The Web UI (enabled by --api.insecure=true) - "8080:8080" volumes: - # Universal socket mount - works with both Docker and Podman - # Podman rootless: /run/user/$UID/podman/podman.sock - # Docker: /var/run/docker.sock - - ${DOCKER_SOCKET:-${XDG_RUNTIME_DIR:-/run/user/1000}/podman/podman.sock}:/var/run/docker.sock + # Socket mount for container runtime (defaults to Docker) + # Docker: /var/run/docker.sock (default) + # Podman: Set DOCKER_SOCKET=${XDG_RUNTIME_DIR}/podman/podman.sock + - ${DOCKER_SOCKET:-/var/run/docker.sock}:/var/run/docker.sock # Proxy overrides for components run directly in Docker host - ./proxy_overrides.yaml:/proxy_overrides.yaml # Since any proxy overrides need to point to Docker host for relevant routes. diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 6466c87267..1bdd43a164 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -231,10 +231,10 @@ services: - ../runner/.env volumes: - ./workflow_data:/data - # Universal socket mount - works with both Docker and Podman - # Podman rootless: /run/user/$UID/podman/podman.sock - # Docker: /var/run/docker.sock - - ${DOCKER_SOCKET:-${XDG_RUNTIME_DIR:-/run/user/1000}/podman/podman.sock}:/var/run/docker.sock + # Socket mount for container runtime (defaults to Docker) + # Docker: /var/run/docker.sock (default) + # Podman: Set DOCKER_SOCKET=${XDG_RUNTIME_DIR}/podman/podman.sock + - ${DOCKER_SOCKET:-/var/run/docker.sock}:/var/run/docker.sock depends_on: - redis - rabbitmq @@ -311,22 +311,8 @@ services: image: unstract/worker-unified:${VERSION} container_name: unstract-worker-file-processing-v2 restart: unless-stopped - # command: ["file-processing"] - command: - [ - ".venv/bin/celery", - "-A", - "worker", - "worker", - "--queues=file_processing,api_file_processing,file_processing_priority", - "--loglevel=INFO", - "--pool=prefork", - "--concurrency=4", - "--prefetch-multiplier=1", - "--without-gossip", - "--without-mingle", - "--without-heartbeat", - ] + entrypoint: .venv/bin/celery + command: "-A worker worker --queues=file_processing,api_file_processing,file_processing_priority --loglevel=INFO --pool=prefork --concurrency=4 --prefetch-multiplier=1 --without-gossip --without-mingle --without-heartbeat" ports: - "8087:8082" env_file: