#!/bin/bash #=============================================================================== # DW COG Migration Script # # Purpose: Upload Dynamic World COGs from local storage to MinIO # Source: ~/geocrop/data/dw_cogs/ # Target: s3://geocrop-baselines/dw/zim/summer/ # # Usage: ./ops/01_upload_dw_cogs.sh [--dry-run] #=============================================================================== set -euo pipefail # Configuration SOURCE_DIR="${SOURCE_DIR:-$HOME/geocrop/data/dw_cogs}" TARGET_BUCKET="geocrop-minio/geocrop-baselines" TARGET_PREFIX="dw/zim/summer" MINIO_ALIAS="geocrop-minio" # Colors for output RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' NC='\033[0m' # No Color log_info() { echo -e "${GREEN}[INFO]${NC} $1"; } log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; } log_error() { echo -e "${RED}[ERROR]${NC} $1"; } # Check if mc is installed if ! command -v mc &> /dev/null; then log_error "MinIO client (mc) not found. Please install it first." exit 1 fi # Check if source directory exists if [ ! -d "$SOURCE_DIR" ]; then log_error "Source directory not found: $SOURCE_DIR" exit 1 fi # Check if MinIO alias exists if ! mc alias list "$MINIO_ALIAS" &> /dev/null; then log_error "MinIO alias '$MINIO_ALIAS' not configured. Run:" echo " mc alias set $MINIO_ALIAS http://localhost:9000 minioadmin minioadmin123" exit 1 fi # Count local files log_info "Counting local TIF files..." LOCAL_COUNT=$(find "$SOURCE_DIR" -maxdepth 1 -type f -name '*.tif' | wc -l) LOCAL_SIZE=$(du -sh "$SOURCE_DIR" | cut -f1) log_info "Found $LOCAL_COUNT TIF files ($LOCAL_SIZE)" log_info "Target: $TARGET_BUCKET/$TARGET_PREFIX/" # Dry run mode DRY_RUN="" if [ "${1:-}" = "--dry-run" ]; then DRY_RUN="--dry-run" log_warn "DRY RUN MODE - No files will be uploaded" fi # List first 10 files for verification log_info "First 10 files in source directory:" find "$SOURCE_DIR" -maxdepth 1 -type f -name '*.tif' | sort | head -10 | while read -r f; do echo " - $(basename "$f")" done # Confirm before proceeding (unless dry-run) if [ -z "$DRY_RUN" ]; then echo "" read -p "Proceed with upload? (y/n) " -n 1 -r echo "" if [[ ! $REPLY =~ ^[Yy]$ ]]; then log_info "Upload cancelled by user" exit 0 fi fi # Perform the upload using mirror # --overwrite ensures files are updated if they exist # --preserve preserves file attributes if [ -z "$DRY_RUN" ]; then log_info "Starting upload..." mc mirror $DRY_RUN --overwrite --preserve \ "$SOURCE_DIR" \ "$TARGET_BUCKET/$TARGET_PREFIX/" if [ $? -eq 0 ]; then log_info "Upload completed successfully!" else log_error "Upload failed!" exit 1 fi fi # Verify upload log_info "Verifying upload..." UPLOADED_COUNT=$(mc ls "$TARGET_BUCKET/$TARGET_PREFIX/" 2>/dev/null | grep -c '\.tif$' || echo "0") log_info "Uploaded $UPLOADED_COUNT files to MinIO" # List first 10 objects in bucket log_info "First 10 objects in bucket:" mc ls "$TARGET_BUCKET/$TARGET_PREFIX/" | head -10 | while read -r line; do echo " $line" done echo "" log_info "Migration complete!" log_info "Local files: $LOCAL_COUNT" log_info "Uploaded files: $UPLOADED_COUNT"