ci/cli: add full backup/restore test

This test simulates migration of a Rancher Manager server.

Signed-off-by: Loic Devulder <[email protected]>

.github/workflows/cli-k3s-full-backup-matrix.yaml

@@ -0,0 +1,52 @@
+# This workflow calls the master E2E workflow with custom variables
+name: CLI-K3s
+
+on:
+  workflow_dispatch:
+    inputs:
+      destroy_runner:
+        description: Destroy the auto-generated self-hosted runner
+        default: true
+        type: boolean
+      k8s_downstream_version:
+        description: Rancher cluster downstream version to use
+        default: '"v1.27.13+k3s1"'
+        type: string
+      k8s_upstream_version:
+        description: Rancher cluster upstream version to use
+        default: '"v1.27.13+k3s1"'
+        type: string
+      qase_run_id:
+        description: Qase run ID where the results will be reported
+        type: string
+      rancher_version:
+        description: Rancher Manager channel/version/head_version to use
+        default: '"stable/latest"'
+        type: string
+# schedule:
+#   # From Monday to Saturday at 3am UTC (10pm in us-central1)
+#   - cron: '0 3 * * 1-6'
+
+jobs:
+  cli:
+    strategy:
+      fail-fast: false
+      max-parallel: 4
+      matrix:
+        k8s_downstream_version: ${{ fromJSON(format('[{0}]', inputs.k8s_downstream_version || '"v1.27.13+k3s1"')) }}
+        k8s_upstream_version: ${{ fromJSON(format('[{0}]', inputs.k8s_upstream_version || '"v1.27.13+k3s1"')) }}
+        rancher_version: ${{ fromJSON(format('[{0}]', inputs.rancher_version || '"stable/latest","latest/devel/2.8","latest/devel/2.9"')) }}
+    uses: ./.github/workflows/master_e2e.yaml
+    secrets:
+      credentials: ${{ secrets.GCP_CREDENTIALS }}
+      pat_token: ${{ secrets.SELF_HOSTED_RUNNER_PAT_TOKEN }}
+      qase_api_token: ${{ secrets.QASE_API_TOKEN }}
+    with:
+      destroy_runner: ${{ github.event_name == 'schedule' && true || inputs.destroy_runner }}
+      full_backup_restore: true
+      k8s_downstream_version: ${{ matrix.k8s_downstream_version }}
+      k8s_upstream_version: ${{ matrix.k8s_upstream_version }}
+      qase_run_id: ${{ github.event_name == 'schedule' && 'auto' || inputs.qase_run_id }}
+      rancher_version: ${{ matrix.rancher_version }}
+      test_type: cli
+      zone: us-central1-c

.github/workflows/master_e2e.yaml

@@ -45,6 +45,10 @@ on:
         description: Force OS downgrade
         default: false
         type: boolean
+      full_backup_restore:
+         description: Test migration-like backup/restore functionality
+         default: false
+        type: boolean
       k8s_downstream_version:
         description: Rancher cluster downstream version to use
         type: string
@@ -224,6 +228,7 @@ jobs:
       destroy_runner: ${{ inputs.destroy_runner }}
       elemental_ui_version: ${{ inputs.elemental_ui_version }}
       force_downgrade: ${{ inputs.force_downgrade }}
+      full_backup_restore: ${{ inputs.full_backup_restore }}
       k8s_downstream_version: ${{ inputs.k8s_downstream_version }}
       node_number: ${{ inputs.node_number }}
       operator_repo: ${{ inputs.operator_repo }}

.github/workflows/sub_cli.yaml

@@ -32,6 +32,9 @@ on:
       force_downgrade:
         required: true
         type: boolean
+      full_backup_restore:
+        required: true
+        type: boolean
       k8s_downstream_version:
         required: true
         type: string
@@ -334,10 +337,20 @@ jobs:
         run: |
           cd tests && make e2e-upgrade-node && make e2e-check-app
 
-      - name: Test Backup/Restore Elemental resources with Rancher Manager
+      - name: Test full Backup/Restore Rancher Manager/Elemental resources
         id: test_backup_restore
         run: |
-          cd tests && make e2e-backup-restore && make e2e-check-app
+          cd tests
+
+          # Run simple or full backup/restore test
+          if ${{ inputs.full_backup_restore == true }}; then
+            make e2e-full-backup-restore
+          else
+            make e2e-simple-backup-restore
+          fi
+
+          # Check the installed application
+          make e2e-check-app
 
       - name: Extract ISO version
         id: iso_version
@@ -411,7 +424,7 @@ jobs:
           OPERATOR_REPO: ${{ inputs.operator_repo }}
         # Don't test Operator uninstall if we want to keep the runner for debugging purposes
         if: ${{ inputs.destroy_runner == true }}
-        run: cd tests && make e2e-uninstall-operator
+        run: cd tests && make e2e-uninstall-operator && make e2e-check-app
 
       # This step must be called in each worklow that wants a summary!
       - name: Get logs and add summary

.github/workflows/sub_test_choice.yaml

@@ -41,6 +41,9 @@ on:
       force_downgrade:
         required: true
         type: boolean
+      full_backup_restore:
+        required: true
+        type: boolean
       k8s_downstream_version:
         required: true
         type: string
@@ -168,6 +171,7 @@ jobs:
       cluster_type: ${{ inputs.cluster_type }}
       destroy_runner: ${{ inputs.destroy_runner }}
       force_downgrade: ${{ inputs.force_downgrade }}
+      full_backup_restore: ${{ inputs.full_backup_restore }}
       k8s_downstream_version: ${{ inputs.k8s_downstream_version }}
       node_number: ${{ inputs.node_number }}
       operator_repo: ${{ inputs.operator_repo }}

tests/Makefile

@@ -61,15 +61,15 @@ e2e-airgap-rancher: deps
 e2e-bootstrap-node: deps
 	ginkgo --timeout $(GINKGO_TIMEOUT)s --label-filter bootstrap -r -v ./e2e
 
-e2e-backup-restore: deps
-	ginkgo --label-filter test-backup-restore -r -v ./e2e
-
 e2e-check-app: deps
 	ginkgo --label-filter check-app -r -v ./e2e
 
 e2e-configure-rancher: deps
 	ginkgo --label-filter configure -r -v ./e2e
 
+e2e-full-backup-restore: deps
+	ginkgo --label-filter test-full-backup-restore -r -v ./e2e
+
 e2e-get-logs: deps
 	ginkgo --label-filter logs -r -v ./e2e
 
@@ -91,11 +91,14 @@ e2e-iso-image: deps
 e2e-multi-cluster: deps
 	ginkgo --timeout $(GINKGO_TIMEOUT)s --label-filter multi-cluster -r -v ./e2e
 
+e2e-prepare-archive: deps
+	ginkgo --label-filter prepare-archive -r -v ./e2e
+
 e2e-reset: deps
 	ginkgo --label-filter reset -r -v ./e2e
 
-e2e-prepare-archive: deps
-	ginkgo --label-filter prepare-archive -r -v ./e2e
+e2e-simple-backup-restore: deps
+	ginkgo --label-filter test-simple-backup-restore -r -v ./e2e
 
 e2e-ui-rancher: deps
 	ginkgo --label-filter ui -r -v ./e2e

tests/assets/restore.yaml

@@ -7,4 +7,4 @@ metadata:
 spec:
   backupFilename: %BACKUP_FILE%
   deleteTimeoutSeconds: 10
-  prune: true
+  prune: %PRUNE%

tests/e2e/backup-restore_test.go

@@ -15,6 +15,7 @@ limitations under the License.
 package e2e_test
 
 import (
+	"os/exec"
 	"strings"
 	"time"
 
@@ -25,6 +26,11 @@ import (
 	"github.com/rancher-sandbox/ele-testhelpers/tools"
 )
 
+const (
+	backupResourceName  = "elemental-backup"
+	restoreResourceName = "elemental-restore"
+)
+
 var _ = Describe("E2E - Install Backup/Restore Operator", Label("install-backup-restore"), func() {
 	// Create kubectl context
 	// Default timeout is too small, so New() cannot be used
@@ -92,10 +98,159 @@ var _ = Describe("E2E - Install Backup/Restore Operator", Label("install-backup-
 	})
 })
 
-var _ = Describe("E2E - Test Backup/Restore", Label("test-backup-restore"), func() {
-	backupResourceName := "elemental-backup"
-	restoreResourceName := "elemental-restore"
+var _ = Describe("E2E - Test full Backup/Restore", Label("test-full-backup-restore"), func() {
+	// Create kubectl context
+	// Default timeout is too small, so New() cannot be used
+	k := &kubectl.Kubectl{
+		Namespace:    "",
+		PollTimeout:  tools.SetTimeout(300 * time.Second),
+		PollInterval: 500 * time.Millisecond,
+	}
+
+	var backupFile string
 
+	It("Do a backup", func() {
+		// TODO: use another case id for full backup/restore test
+		// Report to Qase
+		// testCaseID = 65
+
+		By("Adding a backup resource", func() {
+			err := kubectl.Apply(clusterNS, backupYaml)
+			Expect(err).To(Not(HaveOccurred()))
+		})
+
+		By("Checking that the backup has been done", func() {
+			out, err := kubectl.RunWithoutErr("get", "backup", backupResourceName,
+				"-o", "jsonpath={.metadata.name}")
+			Expect(err).To(Not(HaveOccurred()))
+			Expect(out).To(ContainSubstring(backupResourceName))
+
+			// Check operator logs
+			Eventually(func() string {
+				out, _ := kubectl.RunWithoutErr("logs", "-l", "app.kubernetes.io/name=rancher-backup",
+					"--tail=-1", "--since=5m",
+					"--namespace", "cattle-resources-system")
+				return out
+			}, tools.SetTimeout(5*time.Minute), 10*time.Second).Should(ContainSubstring("Done with backup"))
+		})
+
+		By("Copying the backup file", func() {
+			// Get local storage path
+			claimName, err := kubectl.RunWithoutErr("get", "pod", "-l", "app.kubernetes.io/name=rancher-backup",
+				"--namespace", "cattle-resources-system",
+				"-o", "jsonpath={.items[*].spec.volumes[?(@.name==\"pv-storage\")].persistentVolumeClaim.claimName}")
+			Expect(err).To(Not(HaveOccurred()))
+
+			localPath, err := kubectl.RunWithoutErr("get", "pv",
+				"--namespace", "cattle-resources-system",
+				"-o", "jsonpath={.items[?(@.spec.claimRef.name==\""+claimName+"\")].spec.local.path}")
+			Expect(err).To(Not(HaveOccurred()))
+
+			// Get the backup file from the previous backup
+			backupFile, err = kubectl.RunWithoutErr("get", "backup", backupResourceName, "-o", "jsonpath={.status.filename}")
+			Expect(err).To(Not(HaveOccurred()))
+
+			// Copy the file
+			err = exec.Command("sudo", "cp", localPath+"/"+backupFile, ".").Run()
+			Expect(err).To(Not(HaveOccurred()))
+		})
+	})
+
+	It("Wipe the whole upstream cluster", func() {
+		By("Uninstalling K8s", func() {
+			if strings.Contains(k8sUpstreamVersion, "rke2") {
+				err, out := exec.Command("sudo", "rke2-uninstall.sh").CombinedOutput()
+				Expect(err).To(Not(HaveOccurred()), out)
+			} else {
+				err, out := exec.Command("k3s-uninstall.sh").CombinedOutput()
+				Expect(err).To(Not(HaveOccurred()), out)
+			}
+		})
+	})
+
+	It("Do a restore", func() {
+		// TODO: use another case id for full backup/restore test
+		// Report to Qase
+		// testCaseID = 66
+
+		if strings.Contains(k8sUpstreamVersion, "rke2") {
+			By("Installing RKE2", func() {
+				InstallRKE2()
+			})
+
+			By("Starting RKE2", func() {
+				StartRKE2()
+			})
+
+			By("Waiting for RKE2 to be started", func() {
+				WaitForRKE2(k)
+			})
+
+			By("Installing local-path-provisionner", func() {
+				InstallLocalStorage(k)
+			})
+		} else {
+			By("Installing K3s", func() {
+				By("Installing K3s", func() {
+					InstallK3s()
+				})
+
+				By("Starting K3s", func() {
+					StartK3s()
+				})
+
+				By("Waiting for K3s to be started", func() {
+					WaitForK3s(k)
+				})
+			})
+		}
+
+		By("Adding a restore resource", func() {
+			// Set the backup file in the restore resource
+			err := tools.Sed("%BACKUP_FILE%", backupFile, restoreYaml)
+			Expect(err).To(Not(HaveOccurred()))
+
+			// "prune" option should be set to true here
+			err = tools.Sed("%PRUNE%", "false", restoreYaml)
+			Expect(err).To(Not(HaveOccurred()))
+
+			// And apply
+			err = kubectl.Apply(clusterNS, restoreYaml)
+			Expect(err).To(Not(HaveOccurred()))
+		})
+
+		By("Checking that the restore has been done", func() {
+			// Wait until resources are available again
+			Eventually(func() string {
+				out, _ := kubectl.RunWithoutErr("get", "restore", restoreResourceName,
+					"-o", "jsonpath={.metadata.name}")
+				return out
+			}, tools.SetTimeout(5*time.Minute), 10*time.Second).Should(ContainSubstring(restoreResourceName))
+
+			// Check operator logs
+			Eventually(func() string {
+				out, _ := kubectl.RunWithoutErr("logs", "-l app.kubernetes.io/name=rancher-backup",
+					"--tail=-1", "--since=5m",
+					"--namespace", "cattle-resources-system")
+				return out
+			}, tools.SetTimeout(5*time.Minute), 10*time.Second).Should(ContainSubstring("Done restoring"))
+		})
+
+		By("Installing CertManager", func() {
+			InstallCertManager(k)
+		})
+
+		By("Installing Rancher Manager", func() {
+			InstallRancher(k)
+		})
+
+		By("Checking cluster state after restore", func() {
+			WaitCluster(clusterNS, clusterName)
+		})
+	})
+})
+
+var _ = Describe("E2E - Test simple Backup/Restore", Label("test-simple-backup-restore"), func() {
 	It("Do a backup", func() {
 		// Report to Qase
 		testCaseID = 65
@@ -150,6 +305,10 @@ var _ = Describe("E2E - Test Backup/Restore", Label("test-backup-restore"), func
 			err = tools.Sed("%BACKUP_FILE%", backupFile, restoreYaml)
 			Expect(err).To(Not(HaveOccurred()))
 
+			// "prune" option should be set to true here
+			err = tools.Sed("%PRUNE%", "true", restoreYaml)
+			Expect(err).To(Not(HaveOccurred()))
+
 			// And apply
 			err = kubectl.Apply(clusterNS, restoreYaml)
 			Expect(err).To(Not(HaveOccurred()))