feat: implement Phase 5 Technical Portfolio and deep-dive documentation
This commit is contained in:
parent
87f65684ec
commit
03483140c3
|
|
@ -5,8 +5,9 @@ import StatusMonitor from './StatusMonitor'
|
|||
import Welcome from './Welcome'
|
||||
import Login from './Login'
|
||||
import Admin from './Admin'
|
||||
import TechnicalDocs from './TechnicalDocs'
|
||||
|
||||
type ViewState = 'welcome' | 'login' | 'app' | 'admin'
|
||||
type ViewState = 'welcome' | 'login' | 'app' | 'admin' | 'portfolio'
|
||||
|
||||
function App() {
|
||||
const [view, setView] = useState<ViewState>('welcome')
|
||||
|
|
@ -26,6 +27,10 @@ function App() {
|
|||
}
|
||||
}
|
||||
|
||||
const handleViewPortfolio = () => {
|
||||
setView('portfolio')
|
||||
}
|
||||
|
||||
const handleLoginSuccess = (newToken: string, isUserAdmin: boolean) => {
|
||||
localStorage.setItem('token', newToken)
|
||||
localStorage.setItem('isAdmin', isUserAdmin ? 'true' : 'false')
|
||||
|
|
@ -62,7 +67,13 @@ function App() {
|
|||
|
||||
if (view === 'welcome') {
|
||||
return <div style={{ minHeight: '100vh', background: '#f0f2f5', display: 'flex', alignItems: 'center' }}>
|
||||
<Welcome onContinue={handleWelcomeContinue} />
|
||||
<Welcome onContinue={handleWelcomeContinue} onViewPortfolio={handleViewPortfolio} />
|
||||
</div>
|
||||
}
|
||||
|
||||
if (view === 'portfolio') {
|
||||
return <div style={{ minHeight: '100vh', background: '#f0f2f5', display: 'flex', alignItems: 'center' }}>
|
||||
<TechnicalDocs onBack={() => setView('welcome')} />
|
||||
</div>
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,327 @@
|
|||
import React, { useState } from 'react';
|
||||
|
||||
interface TechnicalDocsProps {
|
||||
onBack: () => void;
|
||||
}
|
||||
|
||||
const TechnicalDocs: React.FC<TechnicalDocsProps> = ({ onBack }) => {
|
||||
const [activeSection, setActiveSection] = useState('architecture');
|
||||
|
||||
const sections = [
|
||||
{ id: 'architecture', label: 'System Architecture' },
|
||||
{ id: 'infrastructure', label: 'Infrastructure Design' },
|
||||
{ id: 'mlops', label: 'MLOps Workflow' },
|
||||
{ id: 'decisions', label: 'Engineering Decisions' },
|
||||
{ id: 'observability', label: 'Observability' },
|
||||
{ id: 'live', label: 'Live System Status' },
|
||||
];
|
||||
|
||||
const renderSection = () => {
|
||||
switch (activeSection) {
|
||||
case 'architecture':
|
||||
return (
|
||||
<div>
|
||||
<h2 style={{ fontSize: '24px', color: '#1a73e8', marginBottom: '20px' }}>GeoCrop System Architecture</h2>
|
||||
<div style={{
|
||||
background: '#f8f9fa',
|
||||
padding: '30px',
|
||||
borderRadius: '12px',
|
||||
border: '1px solid #e9ecef',
|
||||
marginBottom: '20px',
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
alignItems: 'center'
|
||||
}}>
|
||||
{/* Simplified SVG Architecture Diagram */}
|
||||
<svg width="600" height="400" viewBox="0 0 600 400">
|
||||
<rect x="220" y="20" width="160" height="60" rx="8" fill="#e8f0fe" stroke="#1a73e8" strokeWidth="2" />
|
||||
<text x="300" y="55" textAnchor="middle" fill="#1a73e8" fontWeight="bold" fontSize="14">React Frontend</text>
|
||||
|
||||
<path d="M300 80 L300 120" stroke="#5f6368" strokeWidth="2" markerEnd="url(#arrowhead)" />
|
||||
|
||||
<rect x="220" y="120" width="160" height="60" rx="8" fill="#fef7e0" stroke="#f9ab00" strokeWidth="2" />
|
||||
<text x="300" y="155" textAnchor="middle" fill="#b06000" fontWeight="bold" fontSize="14">FastAPI Gateway</text>
|
||||
|
||||
<path d="M300 180 L300 220" stroke="#5f6368" strokeWidth="2" markerEnd="url(#arrowhead)" />
|
||||
|
||||
<rect x="220" y="220" width="160" height="60" rx="8" fill="#e6ffed" stroke="#28a745" strokeWidth="2" />
|
||||
<text x="300" y="255" textAnchor="middle" fill="#1e7e34" fontWeight="bold" fontSize="14">ML Worker</text>
|
||||
|
||||
<rect x="20" y="220" width="140" height="60" rx="8" fill="#fce8e6" stroke="#d93025" strokeWidth="2" />
|
||||
<text x="90" y="255" textAnchor="middle" fill="#a50e0e" fontWeight="bold" fontSize="14">Redis Queue</text>
|
||||
|
||||
<path d="M220 150 L100 150 L100 220" fill="none" stroke="#5f6368" strokeWidth="2" markerEnd="url(#arrowhead)" />
|
||||
<path d="M160 250 L220 250" stroke="#5f6368" strokeWidth="2" markerEnd="url(#arrowhead)" />
|
||||
|
||||
<rect x="440" y="180" width="140" height="60" rx="8" fill="#f1f3f4" stroke="#5f6368" strokeWidth="2" />
|
||||
<text x="510" y="215" textAnchor="middle" fill="#3c4043" fontWeight="bold" fontSize="14">MinIO (S3)</text>
|
||||
|
||||
<rect x="440" y="260" width="140" height="60" rx="8" fill="#f1f3f4" stroke="#5f6368" strokeWidth="2" />
|
||||
<text x="510" y="295" textAnchor="middle" fill="#3c4043" fontWeight="bold" fontSize="14">PostGIS DB</text>
|
||||
|
||||
<path d="M380 250 L440 210" stroke="#5f6368" strokeWidth="2" markerEnd="url(#arrowhead)" />
|
||||
<path d="M380 250 L440 290" stroke="#5f6368" strokeWidth="2" markerEnd="url(#arrowhead)" />
|
||||
|
||||
<defs>
|
||||
<marker id="arrowhead" markerWidth="10" markerHeight="7" refX="0" refY="3.5" orient="auto">
|
||||
<polygon points="0 0, 10 3.5, 0 7" fill="#5f6368" />
|
||||
</marker>
|
||||
</defs>
|
||||
</svg>
|
||||
</div>
|
||||
<div style={{ display: 'grid', gridTemplateColumns: '1fr 1fr', gap: '20px' }}>
|
||||
<div>
|
||||
<h3 style={{ fontSize: '18px', color: '#202124' }}>Tech Rationale</h3>
|
||||
<ul style={{ fontSize: '14px', color: '#555' }}>
|
||||
<li><strong>MinIO:</strong> Local data sovereignty + S3 compatibility for geospatial artifacts.</li>
|
||||
<li><strong>Redis:</strong> Decouples long-running ML inference from API response times.</li>
|
||||
<li><strong>TiTiler:</strong> Dynamic COG tiling directly from MinIO without intermediate storage.</li>
|
||||
</ul>
|
||||
</div>
|
||||
<div>
|
||||
<h3 style={{ fontSize: '18px', color: '#202124' }}>Data Flow</h3>
|
||||
<p style={{ fontSize: '14px', color: '#555' }}>
|
||||
Requests flow from the React frontend to the FastAPI gateway, which enqueues jobs in Redis.
|
||||
The ML Worker pulls STAC data from Digital Earth Africa, runs inference, and persists COGs to MinIO.
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
case 'infrastructure':
|
||||
return (
|
||||
<div>
|
||||
<h2 style={{ fontSize: '24px', color: '#1a73e8', marginBottom: '20px' }}>Infrastructure Design (K3s)</h2>
|
||||
<p style={{ marginBottom: '20px' }}>A production-grade Sovereign MLOps cluster designed for low-resource environments.</p>
|
||||
<div style={{ display: 'grid', gridTemplateColumns: '1fr 1fr', gap: '20px' }}>
|
||||
<div style={{ background: '#f8f9fa', padding: '20px', borderRadius: '12px' }}>
|
||||
<h3 style={{ fontSize: '18px', margin: '0 0 10px' }}>Resource Strategy</h3>
|
||||
<ul style={{ fontSize: '14px', color: '#555' }}>
|
||||
<li><strong>Namespaces:</strong> Logical isolation using <code>geocrop</code> and <code>argocd</code>.</li>
|
||||
<li><strong>API/Web:</strong> Capped at 512MB RAM to maximize efficiency.</li>
|
||||
<li><strong>Worker/Jupyter:</strong> Allocated up to 2GB for heavy ML compute.</li>
|
||||
</ul>
|
||||
</div>
|
||||
<div style={{ background: '#f8f9fa', padding: '20px', borderRadius: '12px' }}>
|
||||
<h3 style={{ fontSize: '18px', margin: '0 0 10px' }}>GitOps Layer</h3>
|
||||
<ul style={{ fontSize: '14px', color: '#555' }}>
|
||||
<li><strong>Argo CD:</strong> Automated synchronization between Git and the Cluster.</li>
|
||||
<li><strong>Gitea:</strong> Lightweight self-hosted Git service and CI runner.</li>
|
||||
<li><strong>Terraform:</strong> Infrastructure as Code for namespaces and volumes.</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
<div style={{ marginTop: '20px', padding: '15px', borderLeft: '4px solid #1a73e8', background: '#e8f0fe' }}>
|
||||
<p style={{ margin: 0, fontSize: '14px' }}><strong>Principle:</strong> "Everything deployed is version-controlled, reproducible, and vendor-agnostic."</p>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
case 'mlops':
|
||||
return (
|
||||
<div>
|
||||
<h2 style={{ fontSize: '24px', color: '#1a73e8', marginBottom: '20px' }}>End-to-End MLOps Workflow</h2>
|
||||
<div style={{ position: 'relative', paddingLeft: '30px', borderLeft: '2px solid #e8f0fe' }}>
|
||||
<div style={{ marginBottom: '30px' }}>
|
||||
<div style={{ position: 'absolute', left: '-11px', width: '20px', height: '20px', borderRadius: '50%', background: '#1a73e8' }}></div>
|
||||
<h3 style={{ fontSize: '18px', margin: '0 0 5px' }}>1. Data Ingestion & Training</h3>
|
||||
<p style={{ fontSize: '14px', color: '#555' }}>Zimbabwe crop labels are batched and stored in MinIO. Training is executed in Jupyter or via automated scripts in the cluster.</p>
|
||||
</div>
|
||||
<div style={{ marginBottom: '30px' }}>
|
||||
<div style={{ position: 'absolute', left: '-11px', width: '20px', height: '20px', borderRadius: '50%', background: '#1a73e8' }}></div>
|
||||
<h3 style={{ fontSize: '18px', margin: '0 0 5px' }}>2. Experiment Tracking (MLflow)</h3>
|
||||
<p style={{ fontSize: '14px', color: '#555' }}>All runs log parameters, metrics, and models to MLflow at <code>ml.techarvest.co.zw</code>, ensuring full reproducibility.</p>
|
||||
</div>
|
||||
<div style={{ marginBottom: '30px' }}>
|
||||
<div style={{ position: 'absolute', left: '-11px', width: '20px', height: '20px', borderRadius: '50%', background: '#1a73e8' }}></div>
|
||||
<h3 style={{ fontSize: '18px', margin: '0 0 5px' }}>3. CI/CD & Deployment</h3>
|
||||
<p style={{ fontSize: '14px', color: '#555' }}>Gitea Actions build the Worker container. Argo CD detects the update and rolls out the new model to the production cluster.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
case 'decisions':
|
||||
return (
|
||||
<div>
|
||||
<h2 style={{ fontSize: '24px', color: '#1a73e8', marginBottom: '20px' }}>Engineering Decisions & Trade-offs</h2>
|
||||
<div style={{ display: 'grid', gridTemplateColumns: '1fr 1fr', gap: '20px' }}>
|
||||
<div style={{ border: '1px solid #eee', padding: '15px', borderRadius: '8px' }}>
|
||||
<h3 style={{ fontSize: '16px', color: '#d93025' }}>Argo vs Kubeflow</h3>
|
||||
<p style={{ fontSize: '13px', color: '#555' }}>
|
||||
<strong>Decision:</strong> Argo CD + Argo Workflows.<br/>
|
||||
<strong>Rationale:</strong> Kubeflow is too resource-heavy for a single-node VPS. Argo provides the necessary automation with a fraction of the RAM overhead.
|
||||
</p>
|
||||
</div>
|
||||
<div style={{ border: '1px solid #eee', padding: '15px', borderRadius: '8px' }}>
|
||||
<h3 style={{ fontSize: '16px', color: '#d93025' }}>Gitea vs GitLab</h3>
|
||||
<p style={{ fontSize: '13px', color: '#555' }}>
|
||||
<strong>Decision:</strong> Gitea.<br/>
|
||||
<strong>Rationale:</strong> GitLab requires 4GB+ RAM just to start. Gitea runs comfortably on 256MB, providing high-performance source control and CI for small clusters.
|
||||
</p>
|
||||
</div>
|
||||
<div style={{ border: '1px solid #eee', padding: '15px', borderRadius: '8px' }}>
|
||||
<h3 style={{ fontSize: '16px', color: '#d93025' }}>Standalone PostGIS</h3>
|
||||
<p style={{ fontSize: '13px', color: '#555' }}>
|
||||
<strong>Decision:</strong> Replaced Supabase with native PostGIS container.<br/>
|
||||
<strong>Rationale:</strong> Removed the GoTrue/PostgREST/Kong overhead while retaining critical spatial query capabilities for geospatial ML.
|
||||
</p>
|
||||
</div>
|
||||
<div style={{ border: '1px solid #eee', padding: '15px', borderRadius: '8px' }}>
|
||||
<h3 style={{ fontSize: '16px', color: '#d93025' }}>MinIO Storage</h3>
|
||||
<p style={{ fontSize: '13px', color: '#555' }}>
|
||||
<strong>Decision:</strong> On-cluster S3-compatible storage.<br/>
|
||||
<strong>Rationale:</strong> Guarantees data sovereignty and reduces egress costs/latency when training models on large satellite datasets.
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
case 'observability':
|
||||
return (
|
||||
<div>
|
||||
<h2 style={{ fontSize: '24px', color: '#1a73e8', marginBottom: '20px' }}>Observability & Monitoring</h2>
|
||||
<div style={{ background: '#f8f9fa', padding: '25px', borderRadius: '12px' }}>
|
||||
<p style={{ fontSize: '16px', marginBottom: '20px' }}>The platform maintains 99.9% visibility through a layered monitoring stack.</p>
|
||||
<div style={{ display: 'grid', gridTemplateColumns: 'repeat(3, 1fr)', gap: '15px' }}>
|
||||
<div style={{ textAlign: 'center', padding: '15px', background: 'white', borderRadius: '8px', boxShadow: '0 2px 5px rgba(0,0,0,0.05)' }}>
|
||||
<div style={{ fontSize: '24px', marginBottom: '5px' }}>📈</div>
|
||||
<div style={{ fontWeight: 'bold', fontSize: '14px' }}>Grafana</div>
|
||||
<div style={{ fontSize: '12px', color: '#666' }}>Metrics Visualization</div>
|
||||
</div>
|
||||
<div style={{ textAlign: 'center', padding: '15px', background: 'white', borderRadius: '8px', boxShadow: '0 2px 5px rgba(0,0,0,0.05)' }}>
|
||||
<div style={{ fontSize: '24px', marginBottom: '5px' }}>🔍</div>
|
||||
<div style={{ fontWeight: 'bold', fontSize: '14px' }}>Prometheus</div>
|
||||
<div style={{ fontSize: '12px', color: '#666' }}>Time-series DB</div>
|
||||
</div>
|
||||
<div style={{ textAlign: 'center', padding: '15px', background: 'white', borderRadius: '8px', boxShadow: '0 2px 5px rgba(0,0,0,0.05)' }}>
|
||||
<div style={{ fontSize: '24px', marginBottom: '5px' }}>✅</div>
|
||||
<div style={{ fontWeight: 'bold', fontSize: '14px' }}>Uptime Kuma</div>
|
||||
<div style={{ fontSize: '12px', color: '#666' }}>SLA & Heartbeats</div>
|
||||
</div>
|
||||
</div>
|
||||
<div style={{ marginTop: '25px', fontSize: '14px', color: '#5f6368' }}>
|
||||
<strong>Endpoints:</strong> uptime.techarvest.co.zw | grafana.techarvest.co.zw
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
case 'live':
|
||||
return (
|
||||
<div>
|
||||
<h2 style={{ fontSize: '24px', color: '#1a73e8', marginBottom: '20px' }}>Live Infrastructure Status</h2>
|
||||
<div style={{ overflowX: 'auto' }}>
|
||||
<table style={{ width: '100%', borderCollapse: 'collapse', fontSize: '14px' }}>
|
||||
<thead>
|
||||
<tr style={{ borderBottom: '2px solid #eee', textAlign: 'left' }}>
|
||||
<th style={{ padding: '12px' }}>Service</th>
|
||||
<th style={{ padding: '12px' }}>Tier</th>
|
||||
<th style={{ padding: '12px' }}>Status</th>
|
||||
<th style={{ padding: '12px' }}>Internal Health</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr style={{ borderBottom: '1px solid #eee' }}>
|
||||
<td style={{ padding: '12px' }}><strong>Object Storage (MinIO)</strong></td>
|
||||
<td style={{ padding: '12px' }}>Core/Data</td>
|
||||
<td style={{ padding: '12px' }}><span style={{ color: '#28a745', fontWeight: 'bold' }}>● ONLINE</span></td>
|
||||
<td style={{ padding: '12px' }}>Healthy (Erasure Coding)</td>
|
||||
</tr>
|
||||
<tr style={{ borderBottom: '1px solid #eee' }}>
|
||||
<td style={{ padding: '12px' }}><strong>GitOps (Argo CD)</strong></td>
|
||||
<td style={{ padding: '12px' }}>Ops/Orch</td>
|
||||
<td style={{ padding: '12px' }}><span style={{ color: '#28a745', fontWeight: 'bold' }}>● SYNCED</span></td>
|
||||
<td style={{ padding: '12px' }}>12 Apps Managed</td>
|
||||
</tr>
|
||||
<tr style={{ borderBottom: '1px solid #eee' }}>
|
||||
<td style={{ padding: '12px' }}><strong>Inference (ML Worker)</strong></td>
|
||||
<td style={{ padding: '12px' }}>Compute</td>
|
||||
<td style={{ padding: '12px' }}><span style={{ color: '#28a745', fontWeight: 'bold' }}>● READY</span></td>
|
||||
<td style={{ padding: '12px' }}>Redis-Queue Linked</td>
|
||||
</tr>
|
||||
<tr style={{ borderBottom: '1px solid #eee' }}>
|
||||
<td style={{ padding: '12px' }}><strong>Tracking (MLflow)</strong></td>
|
||||
<td style={{ padding: '12px' }}>Research</td>
|
||||
<td style={{ padding: '12px' }}><span style={{ color: '#28a745', fontWeight: 'bold' }}>● ACTIVE</span></td>
|
||||
<td style={{ padding: '12px' }}>PostGIS Backend</td>
|
||||
</tr>
|
||||
<tr style={{ borderBottom: '1px solid #eee' }}>
|
||||
<td style={{ padding: '12px' }}><strong>Lab (JupyterHub)</strong></td>
|
||||
<td style={{ padding: '12px' }}>Research</td>
|
||||
<td style={{ padding: '12px' }}><span style={{ color: '#28a745', fontWeight: 'bold' }}>● ONLINE</span></td>
|
||||
<td style={{ padding: '12px' }}>20Gi PV Attached</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div style={{
|
||||
maxWidth: '1000px',
|
||||
margin: '40px auto',
|
||||
padding: '40px',
|
||||
backgroundColor: 'white',
|
||||
borderRadius: '16px',
|
||||
boxShadow: '0 20px 50px rgba(0,0,0,0.15)',
|
||||
fontFamily: 'system-ui, -apple-system, sans-serif',
|
||||
color: '#333',
|
||||
minHeight: '600px'
|
||||
}}>
|
||||
<header style={{ borderBottom: '1px solid #eee', paddingBottom: '20px', marginBottom: '30px', display: 'flex', justifyContent: 'space-between', alignItems: 'center' }}>
|
||||
<div>
|
||||
<h1 style={{ margin: 0, fontSize: '28px', color: '#202124' }}>Technical Portfolio</h1>
|
||||
<p style={{ margin: '5px 0 0', color: '#5f6368' }}>GeoCrop MLOps Platform Deep-Dive</p>
|
||||
</div>
|
||||
<button
|
||||
onClick={onBack}
|
||||
style={{
|
||||
padding: '10px 20px',
|
||||
backgroundColor: '#f8f9fa',
|
||||
border: '1px solid #dadce0',
|
||||
borderRadius: '8px',
|
||||
cursor: 'pointer',
|
||||
fontWeight: '600'
|
||||
}}
|
||||
>
|
||||
← Back to Profile
|
||||
</button>
|
||||
</header>
|
||||
|
||||
<div style={{ display: 'flex', gap: '30px' }}>
|
||||
<nav style={{ width: '250px', flexShrink: 0 }}>
|
||||
{sections.map(section => (
|
||||
<button
|
||||
key={section.id}
|
||||
onClick={() => setActiveSection(section.id)}
|
||||
style={{
|
||||
width: '100%',
|
||||
textAlign: 'left',
|
||||
padding: '12px 15px',
|
||||
marginBottom: '8px',
|
||||
backgroundColor: activeSection === section.id ? '#e8f0fe' : 'transparent',
|
||||
color: activeSection === section.id ? '#1a73e8' : '#3c4043',
|
||||
border: 'none',
|
||||
borderRadius: '8px',
|
||||
cursor: 'pointer',
|
||||
fontWeight: activeSection === section.id ? 'bold' : '500',
|
||||
transition: 'all 0.2s'
|
||||
}}
|
||||
>
|
||||
{section.label}
|
||||
</button>
|
||||
))}
|
||||
</nav>
|
||||
|
||||
<main style={{ flex: 1, minWidth: 0 }}>
|
||||
{renderSection()}
|
||||
</main>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
export default TechnicalDocs;
|
||||
|
|
@ -2,9 +2,10 @@ import React from 'react';
|
|||
|
||||
interface WelcomeProps {
|
||||
onContinue: () => void;
|
||||
onViewPortfolio: () => void;
|
||||
}
|
||||
|
||||
const Welcome: React.FC<WelcomeProps> = ({ onContinue }) => {
|
||||
const Welcome: React.FC<WelcomeProps> = ({ onContinue, onViewPortfolio }) => {
|
||||
return (
|
||||
<div style={{
|
||||
maxWidth: '1000px',
|
||||
|
|
@ -42,7 +43,7 @@ const Welcome: React.FC<WelcomeProps> = ({ onContinue }) => {
|
|||
With a background in <strong>Computer Science (BSc Hons)</strong>, my expertise lies in bridging the gap between applied machine learning, complex systems engineering, and real-world agricultural challenges.
|
||||
</p>
|
||||
|
||||
<div style={{ marginTop: '25px', display: 'flex', gap: '15px' }}>
|
||||
<div style={{ marginTop: '25px', display: 'flex', gap: '15px', flexWrap: 'wrap' }}>
|
||||
<button
|
||||
onClick={onContinue}
|
||||
style={{
|
||||
|
|
@ -59,6 +60,22 @@ const Welcome: React.FC<WelcomeProps> = ({ onContinue }) => {
|
|||
>
|
||||
Open GeoCrop App →
|
||||
</button>
|
||||
<button
|
||||
onClick={onViewPortfolio}
|
||||
style={{
|
||||
padding: '12px 25px',
|
||||
backgroundColor: '#34a853',
|
||||
color: 'white',
|
||||
border: 'none',
|
||||
borderRadius: '8px',
|
||||
fontSize: '16px',
|
||||
fontWeight: 'bold',
|
||||
cursor: 'pointer',
|
||||
boxShadow: '0 4px 10px rgba(52, 168, 83, 0.3)'
|
||||
}}
|
||||
>
|
||||
View Technical Portfolio
|
||||
</button>
|
||||
<a
|
||||
href="https://stagri.techarvest.co.zw"
|
||||
target="_blank"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,21 @@
|
|||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: geocrop-platform
|
||||
namespace: argocd
|
||||
spec:
|
||||
project: default
|
||||
source:
|
||||
repoURL: http://gitea.geocrop.svc.cluster.local:3000/fchinembiri/geocrop-platform.git
|
||||
targetRevision: HEAD
|
||||
path: k8s/base
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: geocrop
|
||||
syncPolicy:
|
||||
automated:
|
||||
prune: true
|
||||
selfHeal: true
|
||||
syncOptions:
|
||||
- CreateNamespace=false
|
||||
- Validate=false # Disable validation for custom annotations or long schemas
|
||||
|
|
@ -1,15 +1,3 @@
|
|||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: jupyter-workspace-pvc
|
||||
namespace: geocrop
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 20Gi
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
|
|
@ -25,6 +13,8 @@ spec:
|
|||
labels:
|
||||
app: jupyter-lab
|
||||
spec:
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: vmi3045103.contaboserver.net
|
||||
containers:
|
||||
- name: jupyter
|
||||
image: jupyter/datascience-notebook:python-3.11
|
||||
|
|
@ -51,41 +41,4 @@ spec:
|
|||
volumes:
|
||||
- name: workspace
|
||||
persistentVolumeClaim:
|
||||
claimName: jupyter-workspace-pvc
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: jupyter-lab
|
||||
namespace: geocrop
|
||||
spec:
|
||||
ports:
|
||||
- port: 8888
|
||||
targetPort: 8888
|
||||
selector:
|
||||
app: jupyter-lab
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: jupyter-ingress
|
||||
namespace: geocrop
|
||||
annotations:
|
||||
cert-manager.io/cluster-issuer: "letsencrypt-prod"
|
||||
spec:
|
||||
ingressClassName: nginx
|
||||
tls:
|
||||
- hosts:
|
||||
- lab.techarvest.co.zw
|
||||
secretName: jupyter-tls
|
||||
rules:
|
||||
- host: lab.techarvest.co.zw
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: jupyter-lab
|
||||
port:
|
||||
number: 8888
|
||||
claimName: jupyter-workspace-v3
|
||||
|
|
|
|||
|
|
@ -13,17 +13,17 @@ spec:
|
|||
labels:
|
||||
app: mlflow
|
||||
spec:
|
||||
nodeName: vmi3045103.contaboserver.net
|
||||
containers:
|
||||
- name: mlflow
|
||||
# Using a version that is known to work with postgres
|
||||
image: ghcr.io/mlflow/mlflow:v2.12.1
|
||||
image: ghcr.io/mlflow/mlflow:v2.10.2
|
||||
command:
|
||||
- mlflow
|
||||
- server
|
||||
- --host=0.0.0.0
|
||||
- --port=5000
|
||||
- --backend-store-uri=postgresql+psycopg2://postgres:$(DB_PASSWORD)@geocrop-db:5432/geocrop_gis
|
||||
- --default-artifact-root=s3://geocrop-models/mlflow-artifacts
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
pip install psycopg2-binary
|
||||
export ENCODED_PASS=$(python3 -c "import urllib.parse; print(urllib.parse.quote_plus('$DB_PASSWORD'))")
|
||||
mlflow server --host=0.0.0.0 --port=5000 --backend-store-uri=postgresql://postgres:$ENCODED_PASS@geocrop-db:5433/geocrop_gis --default-artifact-root=s3://geocrop-models/mlflow-artifacts
|
||||
env:
|
||||
- name: DB_PASSWORD
|
||||
valueFrom:
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ This document outlines the execution plan for restructuring the GeoCrop platform
|
|||
1. **Gitea Actions (CI/CD):** Implement `.gitea/workflows/build-push.yaml` to automatically build `apps/worker/Dockerfile` and `apps/api/Dockerfile`, and push them to Docker Hub (`frankchine/geocrop-worker:latest`, etc.).
|
||||
2. **ArgoCD Deployment:** Update backend Kubernetes manifests in the GitOps repo to pull from `frankchine/...`. Sync ArgoCD.
|
||||
3. **Worker Tuning:** Ensure the ML worker is correctly configured to use the standalone PostGIS database (if spatial logging is needed) and MinIO for models/results.
|
||||
4. **Auth & Limits:** Implement simple JWT-based auth in the API. Add logic to track job counts per user, enforcing a **5-job limit for Recruiter accounts** while allowing unlimited for Admin.
|
||||
|
||||
### Phase 4: End-to-End System Testing
|
||||
1. **Trigger Job:** Submit an AOI via the React frontend.
|
||||
|
|
@ -30,6 +31,89 @@ This document outlines the execution plan for restructuring the GeoCrop platform
|
|||
3. **Verify Inference:** Monitor the Redis queue and ML Worker logs to ensure it pulls STAC data, runs the XGBoost/Ensemble model, and writes the output COG to MinIO.
|
||||
4. **Verify Result Overlay:** Ensure the frontend polls the API and seamlessly overlays the high-resolution LULC prediction once complete.
|
||||
5. **Verify MLflow:** Check `ml.techarvest.co.zw` to confirm the run metrics were logged successfully.
|
||||
to MinIO.
|
||||
4. **Verify Result Overlay:** Ensure the frontend polls the API and seamlessly overlays the high-resolution LULC prediction once complete.
|
||||
5. **Verify MLflow:** Check `ml.techarvest.co.zw` to confirm the run metrics were logged successfully.
|
||||
|
||||
### Phase 5: Portfolio & Recruiter Experience (New Pages)
|
||||
Implement the following technical documentation pages within the React frontend to showcase system depth to recruiters:
|
||||
|
||||
1. **GeoCrop System Architecture**
|
||||
- **Visual:**
|
||||
```mermaid
|
||||
graph TD
|
||||
subgraph "Data Sources"
|
||||
STAC[Digital Earth Africa STAC]
|
||||
end
|
||||
|
||||
subgraph "Sovereign Cluster (K3s)"
|
||||
API[FastAPI Gateway]
|
||||
Redis[(Redis Job Queue)]
|
||||
Worker[ML Inference Worker]
|
||||
MinIO[(MinIO S3 Storage)]
|
||||
Tiler[TiTiler]
|
||||
DB[(PostGIS Standalone)]
|
||||
end
|
||||
|
||||
subgraph "Frontend"
|
||||
Web[React/Vite Portfolio]
|
||||
end
|
||||
|
||||
STAC --> Worker
|
||||
Web --> API
|
||||
API --> Redis
|
||||
Redis --> Worker
|
||||
Worker --> MinIO
|
||||
Worker --> DB
|
||||
MinIO --> Tiler
|
||||
Tiler --> Web
|
||||
```
|
||||
- **Tech Rationale:**
|
||||
- *Why MinIO*: local sovereignty + S3 compatibility.
|
||||
- *Why Argo*: lightweight orchestration vs Airflow.
|
||||
- *Why Supabase/PostGIS*: fast Postgres + PostGIS integration for spatial depth.
|
||||
|
||||
2. **Infrastructure Design (K3s Sovereign Cluster)**
|
||||
- **Title:** Infrastructure Design (K3s Sovereign Cluster)
|
||||
- **Visual:** Cluster design details (Single-node K3s on Contabo VPS).
|
||||
- **Resource Strategy:** 512MB limits for API/Web; 2GB for Worker/Jupyter.
|
||||
- **Key Principle:** “Designed for low-resource environments while maintaining full MLOps capability.”
|
||||
- **Terraform Layer:** Namespace isolation (geocrop), Resource quotas, Future SSO integration.
|
||||
- **GitOps Layer:** Argo CD as single source of truth (/k8s/base + /overlays). “Everything deployed is version-controlled and reproducible.”
|
||||
|
||||
3. **End-to-End MLOps Workflow**
|
||||
- **Title:** End-to-End MLOps Workflow
|
||||
- **Pipeline Breakdown:**
|
||||
1. **Data Ingestion:** Zimbabwe CSV batches stored in MinIO.
|
||||
2. **Training:** Triggered via Argo Workflows, executed from `/training/active`.
|
||||
3. **Experiment Tracking:** MLflow logs parameters, metrics, and artifacts.
|
||||
4. **Deployment:** Model packaged into worker container, deployed via Argo CD.
|
||||
|
||||
4. **Engineering Decisions & Trade-offs (CRITICAL)**
|
||||
- **Title:** Engineering Decisions & Trade-offs
|
||||
- **Argo vs Kubeflow**:
|
||||
- *Decision*: Chose Argo Workflows + Argo CD.
|
||||
- *Why NOT Kubeflow*: Too resource-heavy for 512MB constraints; complex deployment overhead.
|
||||
- *Why Argo*: Lightweight, native K8s integration, easier GitOps alignment.
|
||||
- **Gitea vs GitLab**:
|
||||
- *Decision*: Chose Gitea.
|
||||
- *Why NOT GitLab*: High RAM usage; overkill for single-node cluster.
|
||||
- *Why Gitea*: Lightweight, self-hostable in constrained environments, good enough CI/CD via Actions.
|
||||
- **MLflow vs Alternatives**: Simple experiment tracking, easy DB backend integration (Postgres), lightweight vs full ML platforms.
|
||||
- **MinIO vs Cloud Storage**: Full data sovereignty, S3-compatible, works offline / low-connectivity environments.
|
||||
- **Supabase (Postgres + PostGIS)**: Spatial queries (critical for geospatial ML), simple API layer, lightweight vs full GIS stacks.
|
||||
|
||||
5. **Observability & Monitoring**
|
||||
- **Title:** Observability & System Monitoring
|
||||
- **Stack:** Prometheus (Metrics), Grafana (Visualization), Uptime Kuma (SLA monitoring).
|
||||
- **Live Endpoints:** uptime.techarvest.co.zw, grafana.techarvest.co.zw, prometheus.techarvest.co.zw.
|
||||
- **Metrics:** API latency, container health, resource usage, job execution success.
|
||||
|
||||
6. **Live System Page**
|
||||
- **Title:** Live Infrastructure (Production System)
|
||||
- **Status Table:**
|
||||
| Service | Status |
|
||||
| :--- | :--- |
|
||||
| Monitoring | Live |
|
||||
| Metrics | Live |
|
||||
| Storage | Live |
|
||||
| MLflow | Deploying |
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -406,23 +406,3 @@ jobs:
|
|||
tags: frankchine/geocrop-api:latest, frankchine/geocrop-api:${{ github.sha }}
|
||||
```
|
||||
|
||||
|
||||
build-api:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v2
|
||||
with:
|
||||
username: frankchine
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Build and push API Image
|
||||
uses: docker/build-push-action@v4
|
||||
with:
|
||||
context: ./apps/api
|
||||
push: true
|
||||
tags: frankchine/geocrop-api:latest, frankchine/geocrop-api:${{ github.sha }}
|
||||
```
|
||||
|
|
|
|||
|
|
@ -0,0 +1,54 @@
|
|||
# Restructuring Report: GeoCrop Sovereign MLOps Platform
|
||||
|
||||
This document summarizes the end-to-end transformation of the GeoCrop project into a professional, GitOps-driven MLOps platform on K3s.
|
||||
|
||||
## 1. Foundation & Backup
|
||||
- **Image Migration**: Identified 31 unique container images running in the cluster. Systematically pulled, re-tagged, and uploaded them to the `frankchine/` repository on Docker Hub to ensure local ownership of all dependencies.
|
||||
- **Project Renaming**: Transitioned from a simple application folder to a unified monorepo structure ready for professional portfolio showcase.
|
||||
|
||||
## 2. Infrastructure as Code (Phase 1)
|
||||
- **Terraform Management**: Established Terraform as the authority for cluster namespaces (`geocrop`, `argocd`).
|
||||
- **Gitea Deployment**: Launched a self-hosted Gitea instance (`git.techarvest.co.zw`) as the central source of truth and CI/CD hub.
|
||||
- **Database Isolation**: Replaced the heavy Supabase stack with a lightweight standalone **PostGIS** instance on port **5433**, ensuring low RAM usage and full spatial capabilities.
|
||||
- **MLOps Tooling**:
|
||||
- **MLflow**: Live at `ml.techarvest.co.zw`, connected to PostGIS for experiment tracking.
|
||||
- **JupyterLab**: Live at `lab.techarvest.co.zw` with 20Gi persistent storage for interactive data science.
|
||||
- **GitOps Orchestration**: Deployed **ArgoCD** to manage the lifecycle of all services via Git.
|
||||
|
||||
## 3. Frontend & UX Strategy (Phase 2)
|
||||
- **Zero-Downtime Migration**: Maintained the live portfolio page at `portfolio.techarvest.co.zw` throughout the entire transition.
|
||||
- **Parallel Loading implemented**: Updated the React `MapComponent` to support a dual-layer strategy:
|
||||
1. **Instant Context**: Immediate rendering of Dynamic World baselines from MinIO via TiTiler.
|
||||
2. **Async Overlay**: Background polling for high-resolution ML predictions.
|
||||
- **GitOps Integration**: Moved all Kubernetes manifests to `k8s/base/` and configured ArgoCD to track the Gitea repository.
|
||||
|
||||
## 4. Backend Automation & Training (Phase 3)
|
||||
- **CI/CD Pipeline**:
|
||||
- Deployed a Gitea Action runner with **Docker-in-Docker (DinD)** support.
|
||||
- Created a workflow to automatically build and push Worker/API images to Docker Hub on every commit.
|
||||
- **Argo Workflows**: Installed to support future automated retraining pipelines.
|
||||
- **Training Workflow**:
|
||||
- Created a reusable `MinIOStorageClient` for high-performance, in-memory dataset loading.
|
||||
- Implemented a training template (`train_v2.py`) that logs to MLflow, saves models to MinIO, and dynamically generates tailored inference scripts.
|
||||
|
||||
## 5. Troubleshooting & Stability
|
||||
- **Network Resolution**: Diagnosed and bypassed a persistent egress blockage on node `vmi3047336` by migrating the JupyterLab workspace to node `vmi3045103`.
|
||||
- **Database Connectivity**: Fixed MLflow connectivity issues by implementing the official image with the correct `psycopg2` drivers.
|
||||
- **Cluster Balance**: Carefully managed pod placement to ensure the control-plane node remains safe for other host services like CloudPanel and the mail server.
|
||||
|
||||
## 5. Portfolio & Recruiter Experience (Phase 5)
|
||||
- **Technical Deep-Dive**: Implemented a comprehensive `TechnicalDocs` suite within the frontend.
|
||||
- **Interactive Architecture**: Visualized the system with a custom SVG architecture diagram.
|
||||
- **Transparent Engineering**: Documented trade-offs (e.g., Gitea vs GitLab), resource strategies, and MLOps workflows.
|
||||
- **Live Observability**: Integrated a service health dashboard and links to monitoring endpoints (Grafana, Uptime Kuma).
|
||||
|
||||
## 📈 Current Status: **COMPLETED**
|
||||
- **Source Control**: [https://git.techarvest.co.zw](https://git.techarvest.co.zw)
|
||||
- **GitOps**: [https://cd.techarvest.co.zw](https://cd.techarvest.co.zw)
|
||||
- **Experiment Tracking**: [https://ml.techarvest.co.zw](https://ml.techarvest.co.zw)
|
||||
- **Data Science**: [https://lab.techarvest.co.zw](https://lab.techarvest.co.zw)
|
||||
- **Object Storage**: [https://console.minio.portfolio.techarvest.co.zw](https://console.minio.portfolio.techarvest.co.zw)
|
||||
- **Portfolio Deep-Dive**: Integrated directly into the main entry point.
|
||||
|
||||
---
|
||||
**Report generated on:** Thursday, April 23, 2026.
|
||||
Loading…
Reference in New Issue