114 lines
3.2 KiB
Bash
Executable File
114 lines
3.2 KiB
Bash
Executable File
#!/bin/bash
|
|
#===============================================================================
|
|
# DW COG Migration Script
|
|
#
|
|
# Purpose: Upload Dynamic World COGs from local storage to MinIO
|
|
# Source: ~/geocrop/data/dw_cogs/
|
|
# Target: s3://geocrop-baselines/dw/zim/summer/
|
|
#
|
|
# Usage: ./ops/01_upload_dw_cogs.sh [--dry-run]
|
|
#===============================================================================
|
|
|
|
set -euo pipefail
|
|
|
|
# Configuration
|
|
SOURCE_DIR="${SOURCE_DIR:-$HOME/geocrop/data/dw_cogs}"
|
|
TARGET_BUCKET="geocrop-minio/geocrop-baselines"
|
|
TARGET_PREFIX="dw/zim/summer"
|
|
MINIO_ALIAS="geocrop-minio"
|
|
|
|
# Colors for output
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
NC='\033[0m' # No Color
|
|
|
|
log_info() { echo -e "${GREEN}[INFO]${NC} $1"; }
|
|
log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
|
|
log_error() { echo -e "${RED}[ERROR]${NC} $1"; }
|
|
|
|
# Check if mc is installed
|
|
if ! command -v mc &> /dev/null; then
|
|
log_error "MinIO client (mc) not found. Please install it first."
|
|
exit 1
|
|
fi
|
|
|
|
# Check if source directory exists
|
|
if [ ! -d "$SOURCE_DIR" ]; then
|
|
log_error "Source directory not found: $SOURCE_DIR"
|
|
exit 1
|
|
fi
|
|
|
|
# Check if MinIO alias exists
|
|
if ! mc alias list "$MINIO_ALIAS" &> /dev/null; then
|
|
log_error "MinIO alias '$MINIO_ALIAS' not configured. Run:"
|
|
echo " mc alias set $MINIO_ALIAS http://localhost:9000 minioadmin minioadmin123"
|
|
exit 1
|
|
fi
|
|
|
|
# Count local files
|
|
log_info "Counting local TIF files..."
|
|
LOCAL_COUNT=$(find "$SOURCE_DIR" -maxdepth 1 -type f -name '*.tif' | wc -l)
|
|
LOCAL_SIZE=$(du -sh "$SOURCE_DIR" | cut -f1)
|
|
|
|
log_info "Found $LOCAL_COUNT TIF files ($LOCAL_SIZE)"
|
|
log_info "Target: $TARGET_BUCKET/$TARGET_PREFIX/"
|
|
|
|
# Dry run mode
|
|
DRY_RUN=""
|
|
if [ "${1:-}" = "--dry-run" ]; then
|
|
DRY_RUN="--dry-run"
|
|
log_warn "DRY RUN MODE - No files will be uploaded"
|
|
fi
|
|
|
|
# List first 10 files for verification
|
|
log_info "First 10 files in source directory:"
|
|
find "$SOURCE_DIR" -maxdepth 1 -type f -name '*.tif' | sort | head -10 | while read -r f; do
|
|
echo " - $(basename "$f")"
|
|
done
|
|
|
|
# Confirm before proceeding (unless dry-run)
|
|
if [ -z "$DRY_RUN" ]; then
|
|
echo ""
|
|
read -p "Proceed with upload? (y/n) " -n 1 -r
|
|
echo ""
|
|
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
|
|
log_info "Upload cancelled by user"
|
|
exit 0
|
|
fi
|
|
fi
|
|
|
|
# Perform the upload using mirror
|
|
# --overwrite ensures files are updated if they exist
|
|
# --preserve preserves file attributes
|
|
if [ -z "$DRY_RUN" ]; then
|
|
log_info "Starting upload..."
|
|
|
|
mc mirror $DRY_RUN --overwrite --preserve \
|
|
"$SOURCE_DIR" \
|
|
"$TARGET_BUCKET/$TARGET_PREFIX/"
|
|
|
|
if [ $? -eq 0 ]; then
|
|
log_info "Upload completed successfully!"
|
|
else
|
|
log_error "Upload failed!"
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
# Verify upload
|
|
log_info "Verifying upload..."
|
|
UPLOADED_COUNT=$(mc ls "$TARGET_BUCKET/$TARGET_PREFIX/" 2>/dev/null | grep -c '\.tif$' || echo "0")
|
|
log_info "Uploaded $UPLOADED_COUNT files to MinIO"
|
|
|
|
# List first 10 objects in bucket
|
|
log_info "First 10 objects in bucket:"
|
|
mc ls "$TARGET_BUCKET/$TARGET_PREFIX/" | head -10 | while read -r line; do
|
|
echo " $line"
|
|
done
|
|
|
|
echo ""
|
|
log_info "Migration complete!"
|
|
log_info "Local files: $LOCAL_COUNT"
|
|
log_info "Uploaded files: $UPLOADED_COUNT"
|