429 lines
8.9 KiB
Markdown
429 lines
8.9 KiB
Markdown
# Manifest Suite: Sovereign MLOps Platform
|
|
|
|
## 1. Gitea Source Control (`k8s/base/gitea.yaml`)
|
|
```yaml
|
|
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata:
|
|
name: gitea-data-pvc
|
|
namespace: geocrop
|
|
spec:
|
|
accessModes:
|
|
- ReadWriteOnce
|
|
resources:
|
|
requests:
|
|
storage: 10Gi
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: gitea
|
|
namespace: geocrop
|
|
spec:
|
|
replicas: 1
|
|
selector:
|
|
matchLabels:
|
|
app: gitea
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: gitea
|
|
spec:
|
|
containers:
|
|
- name: gitea
|
|
image: gitea/gitea:1.21.6
|
|
env:
|
|
- name: USER_UID
|
|
value: "1000"
|
|
- name: USER_GID
|
|
value: "1000"
|
|
ports:
|
|
- containerPort: 3000
|
|
- containerPort: 2222
|
|
volumeMounts:
|
|
- name: gitea-data
|
|
mountPath: /data
|
|
volumes:
|
|
- name: gitea-data
|
|
persistentVolumeClaim:
|
|
claimName: gitea-data-pvc
|
|
---
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: gitea
|
|
namespace: geocrop
|
|
spec:
|
|
ports:
|
|
- port: 3000
|
|
targetPort: 3000
|
|
name: http
|
|
- port: 2222
|
|
targetPort: 2222
|
|
name: ssh
|
|
selector:
|
|
app: gitea
|
|
---
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: Ingress
|
|
metadata:
|
|
name: gitea-ingress
|
|
namespace: geocrop
|
|
annotations:
|
|
cert-manager.io/cluster-issuer: "letsencrypt-prod"
|
|
nginx.ingress.kubernetes.io/proxy-body-size: "500m"
|
|
spec:
|
|
ingressClassName: nginx
|
|
tls:
|
|
- hosts:
|
|
- git.techarvest.co.zw
|
|
secretName: gitea-tls
|
|
rules:
|
|
- host: git.techarvest.co.zw
|
|
http:
|
|
paths:
|
|
- path: /
|
|
pathType: Prefix
|
|
backend:
|
|
service:
|
|
name: gitea
|
|
port:
|
|
number: 3000
|
|
```
|
|
|
|
## 2. Terraform: Namespace (`terraform/main.tf`)
|
|
```hcl
|
|
terraform {
|
|
required_providers {
|
|
kubernetes = {
|
|
source = "hashicorp/kubernetes"
|
|
version = "~> 2.0"
|
|
}
|
|
}
|
|
}
|
|
|
|
provider "kubernetes" {
|
|
config_path = "~/.kube/config"
|
|
}
|
|
|
|
resource "kubernetes_namespace" "geocrop" {
|
|
metadata {
|
|
name = "geocrop"
|
|
}
|
|
}
|
|
|
|
# Note: Resource quotas are intentionally omitted here and will be managed dynamically
|
|
# based on cluster telemetry to allow MLflow and Argo to consume available resources.
|
|
```
|
|
|
|
## 3. Standalone Postgres + PostGIS (`k8s/base/postgres-postgis.yaml`)
|
|
```yaml
|
|
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata:
|
|
name: geocrop-db-pvc
|
|
namespace: geocrop
|
|
spec:
|
|
accessModes:
|
|
- ReadWriteOnce
|
|
resources:
|
|
requests:
|
|
storage: 10Gi
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: geocrop-db
|
|
namespace: geocrop
|
|
spec:
|
|
replicas: 1
|
|
selector:
|
|
matchLabels:
|
|
app: geocrop-db
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: geocrop-db
|
|
spec:
|
|
containers:
|
|
- name: postgis
|
|
image: postgis/postgis:15-3.4
|
|
ports:
|
|
- containerPort: 5432
|
|
env:
|
|
- name: POSTGRES_DB
|
|
value: geocrop_gis
|
|
- name: POSTGRES_USER
|
|
value: postgres
|
|
- name: POSTGRES_PASSWORD
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: geocrop-db-secret
|
|
key: password
|
|
resources:
|
|
limits:
|
|
memory: "512Mi" # Lightweight DB limit
|
|
requests:
|
|
memory: "256Mi"
|
|
volumeMounts:
|
|
- name: db-data
|
|
mountPath: /var/lib/postgresql/data
|
|
volumes:
|
|
- name: db-data
|
|
persistentVolumeClaim:
|
|
claimName: geocrop-db-pvc
|
|
---
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: geocrop-db
|
|
namespace: geocrop
|
|
spec:
|
|
ports:
|
|
- port: 5433
|
|
targetPort: 5432
|
|
selector:
|
|
app: geocrop-db
|
|
```
|
|
|
|
## 3. MLflow Server (`k8s/base/mlflow.yaml`)
|
|
```yaml
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: mlflow
|
|
namespace: geocrop
|
|
spec:
|
|
replicas: 1
|
|
selector:
|
|
matchLabels:
|
|
app: mlflow
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: mlflow
|
|
spec:
|
|
containers:
|
|
- name: mlflow
|
|
image: ghcr.io/mlflow/mlflow:v2.10.2
|
|
command:
|
|
- mlflow
|
|
- server
|
|
- --host=0.0.0.0
|
|
- --port=5000
|
|
- --backend-store-uri=postgresql://postgres:$(DB_PASSWORD)@geocrop-db:5433/geocrop_gis
|
|
- --default-artifact-root=s3://geocrop-models/mlflow-artifacts
|
|
env:
|
|
- name: DB_PASSWORD
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: geocrop-db-secret
|
|
key: password
|
|
- name: AWS_ACCESS_KEY_ID
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: geocrop-secrets
|
|
key: minio-access-key
|
|
- name: AWS_SECRET_ACCESS_KEY
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: geocrop-secrets
|
|
key: minio-secret-key
|
|
- name: MLFLOW_S3_ENDPOINT_URL
|
|
value: http://minio.geocrop.svc.cluster.local:9000
|
|
ports:
|
|
- containerPort: 5000
|
|
resources:
|
|
limits:
|
|
memory: "512Mi"
|
|
---
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: mlflow
|
|
namespace: geocrop
|
|
spec:
|
|
ports:
|
|
- port: 5000
|
|
targetPort: 5000
|
|
selector:
|
|
app: mlflow
|
|
---
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: Ingress
|
|
metadata:
|
|
name: mlflow-ingress
|
|
namespace: geocrop
|
|
annotations:
|
|
cert-manager.io/cluster-issuer: "letsencrypt-prod"
|
|
spec:
|
|
ingressClassName: nginx
|
|
tls:
|
|
- hosts:
|
|
- ml.techarvest.co.zw
|
|
secretName: mlflow-tls
|
|
rules:
|
|
- host: ml.techarvest.co.zw
|
|
http:
|
|
paths:
|
|
- path: /
|
|
pathType: Prefix
|
|
backend:
|
|
service:
|
|
name: mlflow
|
|
port:
|
|
number: 5000
|
|
```
|
|
|
|
## 5. JupyterHub Data Science Workspace (`k8s/base/jupyter.yaml`)
|
|
```yaml
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: jupyter-lab
|
|
namespace: geocrop
|
|
spec:
|
|
replicas: 1
|
|
selector:
|
|
matchLabels:
|
|
app: jupyter-lab
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: jupyter-lab
|
|
spec:
|
|
containers:
|
|
- name: jupyter
|
|
image: jupyter/datascience-notebook:python-3.11
|
|
env:
|
|
- name: JUPYTER_ENABLE_LAB
|
|
value: "yes"
|
|
- name: AWS_ACCESS_KEY_ID
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: geocrop-secrets
|
|
key: minio-access-key
|
|
- name: AWS_SECRET_ACCESS_KEY
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: geocrop-secrets
|
|
key: minio-secret-key
|
|
- name: AWS_S3_ENDPOINT_URL
|
|
value: http://minio.geocrop.svc.cluster.local:9000
|
|
ports:
|
|
- containerPort: 8888
|
|
resources:
|
|
requests:
|
|
memory: "1Gi"
|
|
limits:
|
|
memory: "2Gi" # Explicitly higher limit for data science
|
|
---
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: jupyter-lab
|
|
namespace: geocrop
|
|
spec:
|
|
ports:
|
|
- port: 8888
|
|
targetPort: 8888
|
|
selector:
|
|
app: jupyter-lab
|
|
---
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: Ingress
|
|
metadata:
|
|
name: jupyter-ingress
|
|
namespace: geocrop
|
|
annotations:
|
|
cert-manager.io/cluster-issuer: "letsencrypt-prod"
|
|
spec:
|
|
ingressClassName: nginx
|
|
tls:
|
|
- hosts:
|
|
- lab.techarvest.co.zw
|
|
secretName: jupyter-tls
|
|
rules:
|
|
- host: lab.techarvest.co.zw
|
|
http:
|
|
paths:
|
|
- path: /
|
|
pathType: Prefix
|
|
backend:
|
|
service:
|
|
name: jupyter-lab
|
|
port:
|
|
number: 8888
|
|
```
|
|
|
|
## 5. Gitea Action: Build & Sync to Docker Hub (`.gitea/workflows/build-push.yaml`)
|
|
```yaml
|
|
name: Build and Push Docker Images
|
|
on:
|
|
push:
|
|
branches:
|
|
- main
|
|
paths:
|
|
- 'apps/**'
|
|
|
|
jobs:
|
|
build-worker:
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@v3
|
|
|
|
- name: Login to Docker Hub
|
|
uses: docker/login-action@v2
|
|
with:
|
|
username: frankchine
|
|
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
|
|
|
- name: Build and push Worker Image
|
|
uses: docker/build-push-action@v4
|
|
with:
|
|
context: ./apps/worker
|
|
push: true
|
|
tags: frankchine/geocrop-worker:latest, frankchine/geocrop-worker:${{ github.sha }}
|
|
|
|
build-api:
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@v3
|
|
|
|
- name: Login to Docker Hub
|
|
uses: docker/login-action@v2
|
|
with:
|
|
username: frankchine
|
|
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
|
|
|
- name: Build and push API Image
|
|
uses: docker/build-push-action@v4
|
|
with:
|
|
context: ./apps/api
|
|
push: true
|
|
tags: frankchine/geocrop-api:latest, frankchine/geocrop-api:${{ github.sha }}
|
|
```
|
|
|
|
|
|
build-api:
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@v3
|
|
|
|
- name: Login to Docker Hub
|
|
uses: docker/login-action@v2
|
|
with:
|
|
username: frankchine
|
|
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
|
|
|
- name: Build and push API Image
|
|
uses: docker/build-push-action@v4
|
|
with:
|
|
context: ./apps/api
|
|
push: true
|
|
tags: frankchine/geocrop-api:latest, frankchine/geocrop-api:${{ github.sha }}
|
|
```
|