chore: modernize CI/CD pipeline

- Migrate Gitea runner to Kubernetes backend (remove DIND) - Implement Kaniko for image builds - Use Git SHA for deterministic image tagging - Automate Kustomize manifest updates in CI - Update documentation with strict GitOps policies
2026-05-08 16:14:25 +02:00 · 2026-05-08 16:14:25 +02:00 · a092935834
parent 537e55e268
commit a092935834
8 changed files with 153 additions and 131 deletions
--- a/.gitea/workflows/build-push.yaml
+++ b/.gitea/workflows/build-push.yaml
@ -10,71 +10,61 @@ on:
      - 'apps/web/**'

 jobs:
-  build-worker:
+  build:
    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        component: [worker, api, web]
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Docker config
+        run: |
+          mkdir -p .docker
+          echo "{\"auths\":{\"https://index.docker.io/v1/\":{\"auth\":\"$(echo -n "${{ secrets.REGISTRY_USERNAME }}:${{ secrets.REGISTRY_PASSWORD }}" | base64 | tr -d '\n')\"}}}" > .docker/config.json
+
+      - name: Build and Push with Kaniko
+        uses: docker://gcr.io/kaniko-project/executor:debug
+        with:
+          args: >-
+            --dockerfile=apps/${{ matrix.component }}/Dockerfile
+            --context=dir://${{ github.workspace }}
+            --destination=frankchine/geocrop-${{ matrix.component }}:${{ github.sha }}
+            --destination=frankchine/geocrop-${{ matrix.component }}:latest
+            --cache=true
+        env:
+          DOCKER_CONFIG: ${{ github.workspace }}/.docker
+
+  deploy:
+    runs-on: ubuntu-latest
+    needs: build
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
        with:
-          repository: fchinembiri/geocrop-platform
-          url: https://git.techarvest.co.zw/fchinembiri/geocrop-platform..git
          token: ${{ secrets.GITEA_TOKEN }}

-      - name: Build and Push Worker
-        env:
-          REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }}
-          REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
+      - name: Update Manifests
        run: |
-          curl -L https://github.com/GoogleContainerTools/kaniko/releases/download/v1.19.2/executor-Linux-amd64.tar.gz | tar -xz -C /usr/local/bin
-          /kaniko/executor \
-            --dockerfile apps/worker/Dockerfile \
-            --context . \
-            --destination frankchine/geocrop-worker:latest \
-            --cache=true \
-            --registry-repository docker.io
-
-  build-api:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-        with:
-          repository: fchinembiri/geocrop-platform
-          url: https://git.techarvest.co.zw/fchinembiri/geocrop-platform..git
-          token: ${{ secrets.GITEA_TOKEN }}
-
-      - name: Build and Push API
-        env:
-          REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }}
-          REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
+          cd k8s/base
+          # Install kustomize if not present
+          if ! command -v kustomize &> /dev/null; then
+            curl -s "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" | bash
+            chmod +x kustomize
+            mv kustomize /usr/local/bin/
+          fi
+          
+          kustomize edit set image frankchine/geocrop-api=frankchine/geocrop-api:${{ github.sha }}
+          kustomize edit set image frankchine/geocrop-worker=frankchine/geocrop-worker:${{ github.sha }}
+          kustomize edit set image frankchine/geocrop-web=frankchine/geocrop-web:${{ github.sha }}
+          
+      - name: Commit and Push
        run: |
-          curl -L https://github.com/GoogleContainerTools/kaniko/releases/download/v1.19.2/executor-Linux-amd64.tar.gz | tar -xz -C /usr/local/bin
-          /kaniko/executor \
-            --dockerfile apps/api/Dockerfile \
-            --context . \
-            --destination frankchine/geocrop-api:latest \
-            --cache=true \
-            --registry-repository docker.io
-
-  build-web:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-        with:
-          repository: fchinembiri/geocrop-platform
-          url: https://git.techarvest.co.zw/fchinembiri/geocrop-platform..git
-          token: ${{ secrets.GITEA_TOKEN }}
-
-      - name: Build and Push Web
-        env:
-          REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }}
-          REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
-        run: |
-          curl -L https://github.com/GoogleContainerTools/kaniko/releases/download/v1.19.2/executor-Linux-amd64.tar.gz | tar -xz -C /usr/local/bin
-          /kaniko/executor \
-            --dockerfile apps/web/Dockerfile \
-            --context . \
-            --destination frankchine/geocrop-web:latest \
-            --cache=true \
-            --registry-repository docker.io
+          git config --global user.name "Gitea Action"
+          git config --global user.email "action@gitea.com"
+          # Ensure we push to the correct Gitea instance
+          git remote set-url origin http://x-access-token:${{ secrets.GITEA_TOKEN }}@gitea.geocrop.svc.cluster.local:3000/fchinembiri/geocrop-platform.git
+          git add k8s/base/kustomization.yaml
+          git commit -m "ci: update image tags to ${{ github.sha }} [skip ci]" || echo "No changes to commit"
+          git push origin main
--- a/AGENTS.md
+++ b/AGENTS.md
@ -13,19 +13,35 @@ This file provides foundational guidance for AI agents working within this repos
 ## 🚀 Build & Dev Commands

 ### Frontend
-`cd apps/web && npm install && npm run dev`
+```bash
+cd apps/web
+npm install
+npm run dev          # dev server on :5173
+npm run lint         # eslint
+npm run build        # runs tsc -b && vite build (typecheck before build)
+```

 ### API
-`cd apps/api && uvicorn main:app --host 0.0.0.0 --port 8000 --reload`
+```bash
+cd apps/api
+uvicorn main:app --host 0.0.0.0 --port 8000 --reload
+```

 ### Worker
-`cd apps/worker && python worker.py --worker`
+```bash
+cd apps/worker
+python worker.py --test   # syntax/import check (default when no --worker flag)
+python worker.py --worker  # start RQ worker listening on geocrop_tasks queue
+```

 ### Docker (Local Build)
 `docker build -t frankchine/geocrop-web:latest apps/web/`

 ## 🧠 Critical Patterns (Non-Obvious)

+### ⚠️ Build Order (Frontend)
+`npm run lint` → `npm run build` (build includes `tsc -b`). Run lint before build to catch issues early.
+
 ### 🚫 Scoping Mandate
 - **Kubernetes Only:** Focus exclusively on resources managed by Kubernetes. **NEVER** modify host-level Nginx, CloudPanel, or system services outside the cluster.

@ -46,32 +62,22 @@ This file provides foundational guidance for AI agents working within this repos
 - `geocrop-results/`: Output COGs (`results/<job_id>/...`).
 - `geocrop-datasets/`: Training CSVs.

-## 🚢 GitOps Workflow
- **CI**: Build and Push via `.gitea/workflows/build-push.yaml`.
- **CD**: ArgoCD tracks `k8s/base/` in the `geocrop-platform` application.
+### 🗂️ Repo Structure
+- `apps/web/`: React 19 + TypeScript + Vite + OpenLayers frontend.
+- `apps/api/`: FastAPI backend (auth/JWT, job queue via RQ).
+- `apps/worker/`: Python 3.11 worker. Entry: `worker.py` → `run_job()` → orchestrates STAC fetch → features → inference → COG export.
+- `training/`: Jupyter-based training scripts (`MinIOStorageClient` for data access).
+- `k8s/base/`: Kustomize manifests (ArgoCD target).
+
+### 🚢 GitOps Workflow & Policies (MANDATORY)
+- **CI**: Build images using **Kaniko** via `.gitea/workflows/build-push.yaml`.
+- **Tagging**: CI uses Git SHA for deterministic image tagging.
+- **CD**: CI updates `k8s/base/kustomization.yaml` with the new tag; ArgoCD auto-syncs.
+- **Strict GitOps**: 
+    - ALL changes MUST be pushed to Gitea.
+    - Deployments occur ONLY through the CI/CD pipeline via ArgoCD.
+    - Direct manual modifications to K8s resources or running containers are FORBIDDEN.
+    - No bypassing the GitOps flow.
 - **Secrets**: Managed via Kubernetes Secrets (e.g., `geocrop-secrets`, `geocrop-db-secret`).

-## 📊 Current Kubernetes State (geocrop namespace)

-| Deployment | Role | Status |
-|------------|------|--------|
-| `geocrop-web` | React Frontend | Running (1/1) |
-| `geocrop-api` | FastAPI Backend | Running (1/1) |
-| `geocrop-worker` | Inference Engine | Running (1/1) |
-| `gitea` | Source Control | Running (1/1) |
-| `gitea-runner` | CI Runner (Actions) | Running (1/1) |
-| `mlflow` | Experiment Tracking | Running (1/1) |
-| `jupyter-lab` | Data Science IDE | Running (1/1) |
-| `geocrop-db` | PostGIS Database | Running (1/1) |
-| `redis` | Job Broker | Running (1/1) |
-| `minio` | S3 Storage | Running (1/1) |
-| `geocrop-tiler` | Dynamic Tile Server | Running (2/2) |
-
-### 🌐 Endpoints
- **Portfolio**: `portfolio.techarvest.co.zw`
- **API Docs**: `api.portfolio.techarvest.co.zw/docs`
- **Gitea**: `git.techarvest.co.zw`
- **ArgoCD**: `cd.techarvest.co.zw`
- **MLflow**: `ml.techarvest.co.zw`
- **Jupyter**: `lab.techarvest.co.zw`
- **Tiler**: `tiles.portfolio.techarvest.co.zw`
--- a/CLAUDE.md
+++ b/CLAUDE.md
@ -19,12 +19,17 @@ cd apps/worker && python worker.py --worker
 # Root manifests in k8s/base/
 ```

-## 🚢 CI/CD & GitOps
+## 🚢 CI/CD & GitOps (STRICT)

 - **Source Control**: Gitea (`git.techarvest.co.zw`).
- **CI**: Gitea Actions (`.gitea/workflows/build-push.yaml`) builds and pushes images to Docker Hub.
- **CD**: ArgoCD (`cd.techarvest.co.zw`) tracks `k8s/base/` and auto-syncs to the `geocrop` namespace.
- **Git Repo**: `http://gitea.geocrop.svc.cluster.local:3000/fchinembiri/geocrop-platform..git`
+- **CI**: Gitea Actions builds images using **Kaniko** (DIND is deprecated).
+- **Tagging**: Deterministic SHA-based tagging (managed by CI).
+- **CD**: ArgoCD (`cd.techarvest.co.zw`) tracks `k8s/base/`.
+- **Policy**:
+    - All changes MUST be pushed to Gitea.
+    - Deployments occur ONLY via CI/CD + ArgoCD.
+    - Manual server/container modifications are forbidden.
+    - No bypassing GitOps for production state.

 ## 🌐 Endpoints

--- a/GEMINI.md
+++ b/GEMINI.md
@ -34,8 +34,14 @@ python worker.py --worker

 ### GitOps Workflow (CI/CD)
 1. **Push** code to Gitea (`git.techarvest.co.zw`).
-2. **CI**: Gitea Actions build and push Docker images to Docker Hub.
-3. **CD**: ArgoCD detects manifest changes or image updates and reconciles the cluster state.
+2. **CI**: Gitea Actions build images using **Kaniko** (no DIND). Images are tagged with the Git commit SHA.
+3. **CD**: CI pipeline updates `k8s/base/kustomization.yaml` with new SHA tags. ArgoCD detects these changes and reconciles the cluster state.
+
+## 🛑 Engineering Policy
+- **STRICT GitOps:** All approved changes MUST be committed and pushed to Gitea.
+- **NO Manual Deploys:** All deployments MUST occur ONLY through the CI/CD pipeline via ArgoCD.
+- **NO Hotfixes:** Direct manual modification of running containers or K8s resources is forbidden.
+- **Deterministic Tagging:** Never rely on `:latest` for production rollouts; always use SHA-based tags managed by CI.

 ### Kubernetes Deployment
 ```bash
--- a/k8s/base/gitea-runner-config.yaml
+++ b/k8s/base/gitea-runner-config.yaml
@ -18,14 +18,21 @@ data:
        - "ubuntu-24.04:docker://docker.gitea.com/runner-images:ubuntu-24.04"
        - "ubuntu-22.04:docker://docker.gitea.com/runner-images:ubuntu-22.04"
      envs:
-        DOCKER_HOST: "unix:///var/run/docker.sock"
+        DOCKER_HOST: "tcp://localhost:2376"
+        DOCKER_CERT_PATH: "/certs/client"
+        DOCKER_TLS_VERIFY: "1"
    cache:
      enabled: true
      dir: ""
      host: ""
      port: 0
    container:
-      privileged: true
-      network: host
-      docker_host: "unix:///var/run/docker.sock"
+      backend: kubernetes
+      kubernetes:
+        namespace: geocrop
+        service_account_name: gitea-runner-sa
+        privileged: false
+        pull_policy: IfNotPresent
      force_pull: true
+    valid_volumes:
+      - "/certs/client"
--- a/k8s/base/gitea-runner-rbac.yaml
+++ b/k8s/base/gitea-runner-rbac.yaml
@ -0,0 +1,38 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: gitea-runner-sa
+  namespace: geocrop
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: gitea-runner-role
+  namespace: geocrop
+rules:
+  - apiGroups: [""]
+    resources: ["pods", "pods/log", "pods/exec"]
+    verbs: ["get", "list", "watch", "create", "delete"]
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get", "list", "create", "delete"]
+  - apiGroups: [""]
+    resources: ["configmaps"]
+    verbs: ["get", "list", "create", "delete"]
+  - apiGroups: ["batch"]
+    resources: ["jobs"]
+    verbs: ["get", "list", "create", "delete"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: gitea-runner-rolebinding
+  namespace: geocrop
+subjects:
+  - kind: ServiceAccount
+    name: gitea-runner-sa
+    namespace: geocrop
+roleRef:
+  kind: Role
+  name: gitea-runner-role
+  apiGroup: rbac.authorization.k8s.io
--- a/k8s/base/gitea-runner.yaml
+++ b/k8s/base/gitea-runner.yaml
@ -15,11 +15,7 @@ spec:
      labels:
        app: gitea-runner
    spec:
-      securityContext:
-        supplementalGroups:
-          - 999
-      hostNetwork: true
-      dnsPolicy: ClusterFirstWithHostNet
+      serviceAccountName: gitea-runner-sa
      containers:
      - name: runner
        image: gitea/act_runner:latest
@ -32,44 +28,17 @@ spec:
          value: "k3s-runner"
        - name: CONFIG_FILE
          value: /config.yaml
-        - name: DOCKER_HOST
-          value: tcp://localhost:2376
-        - name: DOCKER_TLS_CERTDIR
-          value: /certs/client
        securityContext:
-          privileged: true
+          privileged: false
        volumeMounts:
        - name: runner-data
          mountPath: /data
-        - name: docker-certs
-          mountPath: /certs/client
-          readOnly: true
        - name: config
          mountPath: /config.yaml
          subPath: config.yaml
-      - name: dind
-        image: docker:dind
-        securityContext:
-          privileged: true
-        env:
-        - name: DOCKER_TLS_CERTDIR
-          value: /certs/client
-        - name: DOCKER_DRIVER
-          value: overlay2
-        volumeMounts:
-        - name: runner-data
-          mountPath: /data
-        - name: docker-graph-storage
-          mountPath: /var/lib/docker
-        - name: docker-certs
-          mountPath: /certs/client
      volumes:
      - name: runner-data
        emptyDir: {}
-      - name: docker-graph-storage
-        emptyDir: {}
-      - name: docker-certs
-        emptyDir: {}
      - name: config
        configMap:
          name: gitea-runner-config
--- a/k8s/base/kustomization.yaml
+++ b/k8s/base/kustomization.yaml
@ -8,6 +8,7 @@ resources:
  - postgres-postgis.yaml
  - gitea-runner.yaml
  - gitea-runner-config.yaml
+  - gitea-runner-rbac.yaml
  - 10-redis.yaml
  - 20-minio.yaml
  - 25-tiler.yaml