From d68f3491099cfe3d20af700e352200cf6808a365 Mon Sep 17 00:00:00 2001 From: Jakob Stadlhuber Date: Wed, 24 Jul 2024 19:31:37 +0200 Subject: [PATCH] Update Kubernetes YAMLs and add worker service Refactored container configurations in worker, api, and playwright-service YAMLs to streamline syntax and add missing fields. Added a service definition for the worker component and included a new environment variable in the configmap for rate-limiting. These changes enhance configuration clarity and ensure proper resource definitions. --- examples/kubernetes/cluster-install/api.yaml | 60 ++++++++--------- .../kubernetes/cluster-install/configmap.yaml | 1 + .../cluster-install/playwright-service.yaml | 54 +++++++-------- .../kubernetes/cluster-install/worker.yaml | 67 +++++++++++-------- 4 files changed, 95 insertions(+), 87 deletions(-) diff --git a/examples/kubernetes/cluster-install/api.yaml b/examples/kubernetes/cluster-install/api.yaml index c709857c..54ecfbf6 100644 --- a/examples/kubernetes/cluster-install/api.yaml +++ b/examples/kubernetes/cluster-install/api.yaml @@ -15,37 +15,35 @@ spec: imagePullSecrets: - name: docker-registry-secret containers: - - name: api - image: ghcr.io/winkk-dev/firecrawl:latest - imagePullPolicy: Always - args: [ "pnpm", "run", "start:production" ] - ports: - - containerPort: 3002 - envFrom: - - configMapRef: - name: firecrawl-config - #- secretRef: - # name: firecrawl-secret - - livenessProbe: - httpGet: - path: /v0/health/liveness - port: 3002 - initialDelaySeconds: 30 - periodSeconds: 30 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 3 - name: api-container - - readinessProbe: - httpGet: - path: /v0/health/readiness - port: 3002 - initialDelaySeconds: 30 - periodSeconds: 30 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 3 - name: api-container + - name: api + image: ghcr.io/winkk-dev/firecrawl:latest + imagePullPolicy: Always + args: [ "pnpm", "run", "start:production" ] + ports: + - containerPort: 3002 + envFrom: + - configMapRef: + name: firecrawl-config + #- secretRef: + # name: firecrawl-secret + livenessProbe: + httpGet: + path: /v0/health/liveness + port: 3002 + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /v0/health/readiness + port: 3002 + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 --- apiVersion: v1 kind: Service diff --git a/examples/kubernetes/cluster-install/configmap.yaml b/examples/kubernetes/cluster-install/configmap.yaml index 32559de7..b56cfbcd 100644 --- a/examples/kubernetes/cluster-install/configmap.yaml +++ b/examples/kubernetes/cluster-install/configmap.yaml @@ -7,6 +7,7 @@ data: PORT: "3002" HOST: "0.0.0.0" REDIS_URL: "redis://redis:6379" + REDIS_RATE_LIMIT_URL: "redis://redis:6379" PLAYWRIGHT_MICROSERVICE_URL: "http://playwright-service:3000/html" USE_DB_AUTHENTICATION: "false" HDX_NODE_BETA_MODE: "1" diff --git a/examples/kubernetes/cluster-install/playwright-service.yaml b/examples/kubernetes/cluster-install/playwright-service.yaml index a9c24d42..43cf15f0 100644 --- a/examples/kubernetes/cluster-install/playwright-service.yaml +++ b/examples/kubernetes/cluster-install/playwright-service.yaml @@ -22,34 +22,32 @@ spec: imagePullSecrets: - name: docker-registry-secret containers: - - name: playwright-service - image: ghcr.io/winkk-dev/firecrawl-playwright:latest - imagePullPolicy: Always - ports: - - containerPort: 3000 - envFrom: - - configMapRef: - name: playwright-service-config - - livenessProbe: - httpGet: - path: /health/liveness - port: 3000 - initialDelaySeconds: 30 - periodSeconds: 30 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 3 - name: playwright-service-container - - readinessProbe: - httpGet: - path: /health/readiness - port: 3000 - initialDelaySeconds: 30 - periodSeconds: 30 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 3 - name: playwright-service-container + - name: playwright-service + image: ghcr.io/winkk-dev/firecrawl-playwright:latest + imagePullPolicy: Always + ports: + - containerPort: 3000 + envFrom: + - configMapRef: + name: playwright-service-config + livenessProbe: + httpGet: + path: /health/liveness + port: 3000 + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /health/readiness + port: 3000 + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 --- apiVersion: v1 kind: Service diff --git a/examples/kubernetes/cluster-install/worker.yaml b/examples/kubernetes/cluster-install/worker.yaml index b904bd78..6185f991 100644 --- a/examples/kubernetes/cluster-install/worker.yaml +++ b/examples/kubernetes/cluster-install/worker.yaml @@ -15,31 +15,42 @@ spec: imagePullSecrets: - name: docker-registry-secret containers: - - name: worker - image: ghcr.io/winkk-dev/firecrawl:latest - imagePullPolicy: Always - envFrom: - - configMapRef: - name: firecrawl-config - #- secretRef: - # name: firecrawl-secret - - livenessProbe: - httpGet: - path: /v0/health/liveness - port: 3002 - initialDelaySeconds: 30 - periodSeconds: 30 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 3 - name: worker-container - - readinessProbe: - httpGet: - path: /v0/health/readiness - port: 3002 - initialDelaySeconds: 30 - periodSeconds: 30 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 3 - name: worker-container \ No newline at end of file + - name: worker + image: ghcr.io/winkk-dev/firecrawl:latest + imagePullPolicy: Always + args: [ "pnpm", "run", "workers" ] + envFrom: + - configMapRef: + name: firecrawl-config + #- secretRef: + # name: firecrawl-secret + livenessProbe: + httpGet: + path: /v0/health/liveness + port: 3003 + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /v0/health/readiness + port: 3003 + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 +--- +apiVersion: v1 +kind: Service +metadata: + name: worker +spec: + selector: + app: worker + ports: + - protocol: TCP + port: 3003 + targetPort: 3003