8.9 KiB
8.9 KiB
Manifest Suite: Sovereign MLOps Platform
1. Gitea Source Control (k8s/base/gitea.yaml)
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: gitea-data-pvc
namespace: geocrop
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 10Gi
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: gitea
namespace: geocrop
spec:
replicas: 1
selector:
matchLabels:
app: gitea
template:
metadata:
labels:
app: gitea
spec:
containers:
- name: gitea
image: gitea/gitea:1.21.6
env:
- name: USER_UID
value: "1000"
- name: USER_GID
value: "1000"
ports:
- containerPort: 3000
- containerPort: 2222
volumeMounts:
- name: gitea-data
mountPath: /data
volumes:
- name: gitea-data
persistentVolumeClaim:
claimName: gitea-data-pvc
---
apiVersion: v1
kind: Service
metadata:
name: gitea
namespace: geocrop
spec:
ports:
- port: 3000
targetPort: 3000
name: http
- port: 2222
targetPort: 2222
name: ssh
selector:
app: gitea
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: gitea-ingress
namespace: geocrop
annotations:
cert-manager.io/cluster-issuer: "letsencrypt-prod"
nginx.ingress.kubernetes.io/proxy-body-size: "500m"
spec:
ingressClassName: nginx
tls:
- hosts:
- git.techarvest.co.zw
secretName: gitea-tls
rules:
- host: git.techarvest.co.zw
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: gitea
port:
number: 3000
2. Terraform: Namespace (terraform/main.tf)
terraform {
required_providers {
kubernetes = {
source = "hashicorp/kubernetes"
version = "~> 2.0"
}
}
}
provider "kubernetes" {
config_path = "~/.kube/config"
}
resource "kubernetes_namespace" "geocrop" {
metadata {
name = "geocrop"
}
}
# Note: Resource quotas are intentionally omitted here and will be managed dynamically
# based on cluster telemetry to allow MLflow and Argo to consume available resources.
3. Standalone Postgres + PostGIS (k8s/base/postgres-postgis.yaml)
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: geocrop-db-pvc
namespace: geocrop
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 10Gi
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: geocrop-db
namespace: geocrop
spec:
replicas: 1
selector:
matchLabels:
app: geocrop-db
template:
metadata:
labels:
app: geocrop-db
spec:
containers:
- name: postgis
image: postgis/postgis:15-3.4
ports:
- containerPort: 5432
env:
- name: POSTGRES_DB
value: geocrop_gis
- name: POSTGRES_USER
value: postgres
- name: POSTGRES_PASSWORD
valueFrom:
secretKeyRef:
name: geocrop-db-secret
key: password
resources:
limits:
memory: "512Mi" # Lightweight DB limit
requests:
memory: "256Mi"
volumeMounts:
- name: db-data
mountPath: /var/lib/postgresql/data
volumes:
- name: db-data
persistentVolumeClaim:
claimName: geocrop-db-pvc
---
apiVersion: v1
kind: Service
metadata:
name: geocrop-db
namespace: geocrop
spec:
ports:
- port: 5433
targetPort: 5432
selector:
app: geocrop-db
3. MLflow Server (k8s/base/mlflow.yaml)
apiVersion: apps/v1
kind: Deployment
metadata:
name: mlflow
namespace: geocrop
spec:
replicas: 1
selector:
matchLabels:
app: mlflow
template:
metadata:
labels:
app: mlflow
spec:
containers:
- name: mlflow
image: ghcr.io/mlflow/mlflow:v2.10.2
command:
- mlflow
- server
- --host=0.0.0.0
- --port=5000
- --backend-store-uri=postgresql://postgres:$(DB_PASSWORD)@geocrop-db:5433/geocrop_gis
- --default-artifact-root=s3://geocrop-models/mlflow-artifacts
env:
- name: DB_PASSWORD
valueFrom:
secretKeyRef:
name: geocrop-db-secret
key: password
- name: AWS_ACCESS_KEY_ID
valueFrom:
secretKeyRef:
name: geocrop-secrets
key: minio-access-key
- name: AWS_SECRET_ACCESS_KEY
valueFrom:
secretKeyRef:
name: geocrop-secrets
key: minio-secret-key
- name: MLFLOW_S3_ENDPOINT_URL
value: http://minio.geocrop.svc.cluster.local:9000
ports:
- containerPort: 5000
resources:
limits:
memory: "512Mi"
---
apiVersion: v1
kind: Service
metadata:
name: mlflow
namespace: geocrop
spec:
ports:
- port: 5000
targetPort: 5000
selector:
app: mlflow
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: mlflow-ingress
namespace: geocrop
annotations:
cert-manager.io/cluster-issuer: "letsencrypt-prod"
spec:
ingressClassName: nginx
tls:
- hosts:
- ml.techarvest.co.zw
secretName: mlflow-tls
rules:
- host: ml.techarvest.co.zw
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: mlflow
port:
number: 5000
5. JupyterHub Data Science Workspace (k8s/base/jupyter.yaml)
apiVersion: apps/v1
kind: Deployment
metadata:
name: jupyter-lab
namespace: geocrop
spec:
replicas: 1
selector:
matchLabels:
app: jupyter-lab
template:
metadata:
labels:
app: jupyter-lab
spec:
containers:
- name: jupyter
image: jupyter/datascience-notebook:python-3.11
env:
- name: JUPYTER_ENABLE_LAB
value: "yes"
- name: AWS_ACCESS_KEY_ID
valueFrom:
secretKeyRef:
name: geocrop-secrets
key: minio-access-key
- name: AWS_SECRET_ACCESS_KEY
valueFrom:
secretKeyRef:
name: geocrop-secrets
key: minio-secret-key
- name: AWS_S3_ENDPOINT_URL
value: http://minio.geocrop.svc.cluster.local:9000
ports:
- containerPort: 8888
resources:
requests:
memory: "1Gi"
limits:
memory: "2Gi" # Explicitly higher limit for data science
---
apiVersion: v1
kind: Service
metadata:
name: jupyter-lab
namespace: geocrop
spec:
ports:
- port: 8888
targetPort: 8888
selector:
app: jupyter-lab
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: jupyter-ingress
namespace: geocrop
annotations:
cert-manager.io/cluster-issuer: "letsencrypt-prod"
spec:
ingressClassName: nginx
tls:
- hosts:
- lab.techarvest.co.zw
secretName: jupyter-tls
rules:
- host: lab.techarvest.co.zw
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: jupyter-lab
port:
number: 8888
5. Gitea Action: Build & Sync to Docker Hub (.gitea/workflows/build-push.yaml)
name: Build and Push Docker Images
on:
push:
branches:
- main
paths:
- 'apps/**'
jobs:
build-worker:
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v3
- name: Login to Docker Hub
uses: docker/login-action@v2
with:
username: frankchine
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Build and push Worker Image
uses: docker/build-push-action@v4
with:
context: ./apps/worker
push: true
tags: frankchine/geocrop-worker:latest, frankchine/geocrop-worker:${{ github.sha }}
build-api:
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v3
- name: Login to Docker Hub
uses: docker/login-action@v2
with:
username: frankchine
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Build and push API Image
uses: docker/build-push-action@v4
with:
context: ./apps/api
push: true
tags: frankchine/geocrop-api:latest, frankchine/geocrop-api:${{ github.sha }}
build-api: runs-on: ubuntu-latest steps: - name: Checkout repository uses: actions/checkout@v3
- name: Login to Docker Hub
uses: docker/login-action@v2
with:
username: frankchine
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Build and push API Image
uses: docker/build-push-action@v4
with:
context: ./apps/api
push: true
tags: frankchine/geocrop-api:latest, frankchine/geocrop-api:${{ github.sha }}