data ingest test: Use more replicas

Might make it easier to reproduce MaterializeInc/database-issues#8990
MaterializeInc · Feb 27, 2025 · 657ac8d · 657ac8d
1 parent f3f74c5
commit 657ac8d
Show file tree

Hide file tree

Showing 2 changed files with 48 additions and 12 deletions.
diff --git a/ci/nightly/pipeline.template.yml b/ci/nightly/pipeline.template.yml
@@ -1492,18 +1492,49 @@ steps:
       - ./ci/plugins/mzcompose:
           composition: retain-history
 
-  - id: data-ingest
-    label: "Data Ingest"
-    depends_on: build-aarch64
-    timeout_in_minutes: 90
-    parallelism: 2
-    agents:
-      queue: hetzner-aarch64-8cpu-16gb
-    plugins:
-      - ./ci/plugins/mzcompose:
-          composition: data-ingest
-          # Don't run with Azurite since it's pretty slow, see https://github.com/MaterializeInc/database-issues/issues/8892 for details
-          #args: [--azurite]
+  - group: "Data Ingest"
+    key: data-ingest
+    steps:
+    - id: data-ingest-1-replica
+      label: "Data Ingest (1 replica)"
+      depends_on: build-aarch64
+      timeout_in_minutes: 90
+      parallelism: 2
+      agents:
+        queue: hetzner-aarch64-8cpu-16gb
+      plugins:
+        - ./ci/plugins/mzcompose:
+            composition: data-ingest
+            # Don't run with Azurite since it's pretty slow, see https://github.com/MaterializeInc/database-issues/issues/8892 for details
+            args: [--replicas=1]
+
+    - id: data-ingest-2-replicas
+      label: "Data Ingest (2 replicas)"
+      depends_on: build-aarch64
+      timeout_in_minutes: 90
+      parallelism: 2
+      agents:
+        queue: hetzner-aarch64-8cpu-16gb
+      plugins:
+        - ./ci/plugins/mzcompose:
+            composition: data-ingest
+            # Don't run with Azurite since it's pretty slow, see https://github.com/MaterializeInc/database-issues/issues/8892 for details
+            args: [--replicas=2]
+      skip: "Reenable when database-issues#8990 is fixed"
+
+    - id: data-ingest-8-replicas
+      label: "Data Ingest (8 replicas)"
+      depends_on: build-aarch64
+      timeout_in_minutes: 90
+      parallelism: 2
+      agents:
+        queue: hetzner-aarch64-16cpu-32gb
+      plugins:
+        - ./ci/plugins/mzcompose:
+            composition: data-ingest
+            # Don't run with Azurite since it's pretty slow, see https://github.com/MaterializeInc/database-issues/issues/8892 for details
+            args: [--replicas=8]
+      skip: "Reenable when database-issues#8990 is fixed"
 
   - group: "Parallel Workload"
     key: parallel-workload

diff --git a/test/data-ingest/mzcompose.py b/test/data-ingest/mzcompose.py
@@ -134,6 +134,11 @@ def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None:
         c.up(*services)
 
         if args.replicas > 1:
+            c.sql(
+                "ALTER SYSTEM SET max_replicas_per_cluster = 32",
+                user="mz_system",
+                port=6877,
+            )
             c.sql("DROP CLUSTER quickstart CASCADE", user="mz_system", port=6877)
             replica_names = [f"r{replica_id}" for replica_id in range(0, args.replicas)]
             replica_string = ",".join(