68 lines
2.3 KiB
Bash
68 lines
2.3 KiB
Bash
#!/bin/bash
|
|
#===============================================================================
|
|
# Upload DW COGs to MinIO
|
|
#
|
|
# This script uploads all 132 files from data/dw_cogs/ to MinIO
|
|
# with the correct structure per the storage contract.
|
|
#
|
|
# Run from geocrop root directory:
|
|
# bash ops/upload_dw_cogs.sh
|
|
#===============================================================================
|
|
|
|
set -euo pipefail
|
|
|
|
# Configuration
|
|
SOURCE_DIR="data/dw_cogs"
|
|
MINIO_ALIAS="local"
|
|
BUCKET="geocrop-baselines"
|
|
|
|
# Setup mc alias
|
|
mc alias set ${MINIO_ALIAS} http://localhost:9000 minioadmin minioadmin123 2>/dev/null || true
|
|
mc alias set ${MINIO_ALIAS} http://minio:9000 minioadmin minioadmin123 2>/dev/null || true
|
|
|
|
echo "Starting upload of DW COGs..."
|
|
|
|
# Upload Agreement files
|
|
echo "Uploading Agreement files..."
|
|
for f in ${SOURCE_DIR}/DW_Zim_Agreement_*.tif; do
|
|
if [ -f "$f" ]; then
|
|
season=$(basename "$f" | sed 's/DW_Zim_Agreement_\(...._....\)-.*/\1/')
|
|
mc cp "$f" "${MINIO_ALIAS}/${BUCKET}/dw/zim/summer/${season}/agreement/"
|
|
echo " Uploaded: $(basename $f)"
|
|
fi
|
|
done
|
|
|
|
# Upload HighestConf files
|
|
echo "Uploading HighestConf files..."
|
|
for f in ${SOURCE_DIR}/DW_Zim_HighestConf_*.tif; do
|
|
if [ -f "$f" ]; then
|
|
season=$(basename "$f" | sed 's/DW_Zim_HighestConf_\(...._....\)-.*/\1/')
|
|
mc cp "$f" "${MINIO_ALIAS}/${BUCKET}/dw/zim/summer/${season}/highest_conf/"
|
|
echo " Uploaded: $(basename $f)"
|
|
fi
|
|
done
|
|
|
|
# Upload Mode files
|
|
echo "Uploading Mode files..."
|
|
for f in ${SOURCE_DIR}/DW_Zim_Mode_*.tif; do
|
|
if [ -f "$f" ]; then
|
|
season=$(basename "$f" | sed 's/DW_Zim_Mode_\(...._....\)-.*/\1/')
|
|
mc cp "$f" "${MINIO_ALIAS}/${BUCKET}/dw/zim/summer/${season}/mode/"
|
|
echo " Uploaded: $(basename $f)"
|
|
fi
|
|
done
|
|
|
|
echo ""
|
|
echo "=== Upload Complete ==="
|
|
echo "Verifying files in MinIO..."
|
|
|
|
# Count files
|
|
AGREEMENT_COUNT=$(mc ls ${MINIO_ALIAS}/${BUCKET}/ --recursive 2>/dev/null | grep -c "Agreement" || echo "0")
|
|
HIGHESTCONF_COUNT=$(mc ls ${MINIO_ALIAS}/${BUCKET}/ --recursive 2>/dev/null | grep -c "HighestConf" || echo "0")
|
|
MODE_COUNT=$(mc ls ${MINIO_ALIAS}/${BUCKET}/ --recursive 2>/dev/null | grep -c "Mode" || echo "0")
|
|
|
|
echo "Agreement: $AGREEMENT_COUNT files"
|
|
echo "HighestConf: $HIGHESTCONF_COUNT files"
|
|
echo "Mode: $MODE_COUNT files"
|
|
echo "Total: $((AGREEMENT_COUNT + HIGHESTCONF_COUNT + MODE_COUNT)) files"
|