WIP - [AAP-38819] - Setting up pytest + S3 (Minio) + example test

tools/docker/docker-compose.yaml

@@ -0,0 +1,111 @@
+version: '3.8'
+
+services:
+  minio:
+    image: minio/minio:latest
+    container_name: minio
+    ports:
+      - "9000:9000"
+      - "9001:9001"
+    environment:
+      MINIO_ROOT_USER: minioaccess
+      MINIO_ROOT_PASSWORD: miniosecret
+    volumes:
+      - minio_data:/data
+    command: server /data --console-address ":9001"
+
+  minio-setup:
+    image: minio/mc:latest
+    depends_on:
+      - minio
+    volumes:
+      - minio_config:/tmp
+    entrypoint: |
+      sh -c '
+      echo "Waiting for MinIO to be ready...";
+      sleep 5;
+      until /usr/bin/mc alias set local http://minio:9000 minioaccess miniosecret; do
+        echo "MinIO is not ready yet... waiting...";
+        sleep 2;
+      done;
+
+      echo "MinIO is ready! Creating bucket and user...";
+      /usr/bin/mc mb --ignore-existing local/metricsutilitys3 || exit 1;
+      /usr/bin/mc admin user add local mynewuser mysecretpassword || exit 1;
+      /usr/bin/mc admin policy attach local readwrite --user=mynewuser || exit 1;
+      /usr/bin/mc admin user svcacct add local mynewuser > /tmp/minio_keys.txt;
+
+      echo "Extracting credentials...";
+      ACCESS_KEY=""
+      SECRET_KEY=""
+
+      while IFS= read -r line; do
+        case "$$line" in
+          "Access Key:"*) ACCESS_KEY="$${line#Access Key: }" ;;
+          "Secret Key:"*) SECRET_KEY="$${line#Secret Key: }" ;;
+        esac
+      done < /tmp/minio_keys.txt
+
+      if [ -z "$${ACCESS_KEY}" ] || [ -z "$${SECRET_KEY}" ]; then
+        echo "ERROR: Failed to extract credentials!"
+        exit 1
+      fi
+
+      echo "ACCESS_KEY=$${ACCESS_KEY}" > /tmp/minio_env.sh
+      echo "SECRET_KEY=$${SECRET_KEY}" >> /tmp/minio_env.sh
+      chmod +x /tmp/minio_env.sh
+      echo "Setup complete.";
+      '
+
+  metrics-utility:
+    image: python:3.11-slim
+    container_name: metrics-utility
+    depends_on:
+      - minio-setup
+    volumes:
+      - .:/app
+      - minio_config:/tmp
+    working_dir: /app
+    environment:
+      UV_VENV: /app/.venv
+      METRICS_UTILITY_SHIP_TARGET: s3
+      METRICS_UTILITY_SHIP_PATH: metrics-utility/shipped_data
+      METRICS_UTILITY_BUCKET_NAME: metricsutilitys3
+      METRICS_UTILITY_BUCKET_ENDPOINT: "http://minio:9000"
+      METRICS_UTILITY_BUCKET_REGION: "us-east-1"
+      METRICS_UTILITY_REPORT_TYPE: CCSPv2
+      METRICS_UTILITY_OPTIONAL_CCSP_REPORT_SHEETS: "jobs,managed_nodes,usage_by_organizations,managed_nodes_by_organizations"
+    entrypoint: |
+      bash -c '
+      echo "Waiting for MinIO credentials...";
+      while [ ! -s /tmp/minio_env.sh ]; do
+        sleep 2;
+      done;
+
+      source /tmp/minio_env.sh;
+
+      if [ -z "$$ACCESS_KEY" ] || [ -z "$$SECRET_KEY" ]; then
+        echo "ERROR: Loaded credentials are empty!"
+        exit 1
+      fi
+
+      export METRICS_UTILITY_BUCKET_ACCESS_KEY=$$ACCESS_KEY;
+      export METRICS_UTILITY_BUCKET_SECRET_KEY=$$SECRET_KEY;
+
+      echo "Listing all environment variables:";
+      printenv | sort;
+
+      echo "Installing dependencies...";
+      pip install uv || exit 1;
+      uv venv || exit 1;
+      uv pip install . || exit 1;
+      . .venv/bin/activate || exit 1;
+
+      echo "Running automation billing data collection...";
+      python manage.py gather_automation_controller_billing_data --ship --since=20d || exit 1;
+      echo "Execution complete.";
+      '
+
+volumes:
+  minio_data:
+  minio_config: