Implement skunkworks cluster upgrade test

Author: alexbrand

integration/framework.go

@@ -115,3 +115,60 @@ func WithMiniInfrastructure(distro linuxDistro, provisioner infrastructureProvis
 
 	f(nodes.worker[0], sshKey)
 }
+
+// SubDescribe allows you to define specifications inside another spec.
+// We have found the need for this because Gingko does not support
+// serializing a subset of tests when running in parallel. This means
+// that we must define multiple specs inside a parent It() block.
+// Use this when it is truly needed.
+//
+// Example:
+// Describe("the foo service", func() {
+//	It("should be deployed successfully", func() {
+//		// some assertions here...
+//		sub := SubDescribe("should return 200", func() error {
+//			// call service and return error if not 200
+//		})
+//	})
+// })
+func SubDescribe(name string) *subTest {
+	return &subTest{name: name}
+}
+
+type subTest struct {
+	name   string
+	specs  []string
+	errors []error
+}
+
+func (sub *subTest) It(name string, f func() error) {
+	By(fmt.Sprintf("Running spec: %s - %s", sub.name, name))
+	sub.specs = append(sub.specs, name)
+	sub.errors = append(sub.errors, f())
+}
+
+func (sub *subTest) Check() {
+	var failed bool
+	for _, e := range sub.errors {
+		if e != nil {
+			failed = true
+			break
+		}
+	}
+	if failed {
+		// Print report and fail test
+		sub.printReport()
+		Fail("Failed: " + sub.name)
+	}
+}
+
+func (sub *subTest) printReport() {
+	fmt.Println(sub.name)
+	for i, spec := range sub.specs {
+		if sub.errors[i] != nil {
+			fmt.Printf("%s: FAILED: %v\n", spec, sub.errors[i])
+		} else {
+			fmt.Printf("%s: PASSED\n", spec)
+		}
+	}
+}

integration/install.go

@@ -3,14 +3,14 @@ package integration
 import (
 	"bufio"
 	"html/template"
-	"log"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"time"
 
 	homedir "github.com/mitchellh/go-homedir"
 	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
 )
 
 func leaveIt() bool {
@@ -169,11 +169,8 @@ func completesInTime(dothis func(), howLong time.Duration) bool {
 	}
 }
 
-func FailIfError(err error, message ...string) {
-	if err != nil {
-		log.Printf(message[0]+": %v\n%v", err, message[1:])
-		Fail(message[0])
-	}
+func FailIfError(err error, message ...interface{}) {
+	Expect(err).ToNot(HaveOccurred(), message...)
 }
 
 func FailIfSuccess(err error) {

integration/storage.go

@@ -95,7 +95,7 @@ func verifyGlusterVolume(storageNode NodeDeets, sshKey string, name string, repl
 	FailIfError(err, "Gluster volume verification failed")
 }
 
-func createVolume(planFile *os.File, name string, replicationCount int, distributionCount int, allowAddress string) {
+func createVolume(planFile *os.File, name string, replicationCount int, distributionCount int, allowAddress string) error {
 	cmd := exec.Command("./kismatic", "volume", "add",
 		"-f", planFile.Name(),
 		"--replica-count", strconv.Itoa(replicationCount),
@@ -107,8 +107,7 @@ func createVolume(planFile *os.File, name string, replicationCount int, distribu
 	}
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
-	err := cmd.Run()
-	FailIfError(err, "Error running volume add command")
+	return cmd.Run()
 }
 
 func standupGlusterCluster(planFile *os.File, nodes []NodeDeets, sshKey string, distro linuxDistro) {

integration/upgrade.go

@@ -0,0 +1,48 @@
+package integration
+
+import (
+	"encoding/json"
+	"os/exec"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+type infoOutput struct {
+	EarliestVersion string
+	LatestVersion   string
+	Nodes           []struct {
+		Version string
+	}
+}
+
+type versionOutput struct {
+	Version string
+}
+
+func assertClusterVersionIsCurrent() {
+	By("Calling ./kismatic version to get Kismatic version")
+	cmd := exec.Command("./kismatic", "version", "-o", "json")
+	out, err := cmd.Output()
+	FailIfError(err)
+	ver := versionOutput{}
+	err = json.Unmarshal(out, &ver)
+	FailIfError(err)
+	assertClusterVersion(ver.Version)
+}
+
+func assertClusterVersion(version string) {
+	By("Calling ./kismatic info to get the cluster's version")
+	cmd := exec.Command("./kismatic", "info", "-f", "kismatic-testing.yaml", "-o", "json")
+	out, err := cmd.Output()
+	FailIfError(err)
+	info := infoOutput{}
+	err = json.Unmarshal(out, &info)
+	FailIfError(err)
+
+	Expect(info.EarliestVersion).To(Equal(version))
+	Expect(info.LatestVersion).To(Equal(version))
+	for _, n := range info.Nodes {
+		Expect(n.Version).To(Equal(version))
+	}
+}

integration/upgrade_test.go

@@ -8,16 +8,15 @@ import (
 	. "github.com/onsi/gomega"
 )
 
-var _ = Describe("kismatic upgrade tests", func() {
+var _ = Describe("Upgrade", func() {
 	BeforeEach(func() {
 		dir := setupTestWorkingDirWithVersion("v1.2.2")
 		os.Chdir(dir)
 	})
-
-	Describe("Doing an offline upgrade of a kubernetes cluster", func() {
+	Describe("Upgrading a cluster using offline mode", func() {
 		Context("Using a minikube layout", func() {
 			Context("Using Ubuntu 16.04", func() {
-				ItOnAWS("should result in an upgraded cluster [slow] [upgrade]", func(aws infrastructureProvisioner) {
+				ItOnAWS("should be upgraded [slow] [upgrade]", func(aws infrastructureProvisioner) {
 					WithMiniInfrastructure(Ubuntu1604LTS, aws, func(node NodeDeets, sshKey string) {
 						// Install previous version cluster
 						err := installKismaticMini(node, sshKey)
@@ -40,7 +39,7 @@ var _ = Describe("kismatic upgrade tests", func() {
 			})
 
 			Context("Using CentOS 7", func() {
-				ItOnAWS("should result in an upgraded cluster [slow] [upgrade]", func(aws infrastructureProvisioner) {
+				ItOnAWS("should be upgraded [slow] [upgrade]", func(aws infrastructureProvisioner) {
 					WithMiniInfrastructure(CentOS7, aws, func(node NodeDeets, sshKey string) {
 						// Install previous version cluster
 						err := installKismaticMini(node, sshKey)
@@ -63,5 +62,57 @@ var _ = Describe("kismatic upgrade tests", func() {
 			})
 		})
 
+		// This spec will be used for testing non-destructive kismatic features on
+		// an upgraded cluster.
+		// This spec is open to modification when new assertions have to be made.
+		Context("Using a skunkworks cluster", func() {
+			ItOnAWS("should result in an upgraded cluster [slow] [upgrade]", func(aws infrastructureProvisioner) {
+				WithInfrastructureAndDNS(NodeCount{Etcd: 3, Master: 2, Worker: 3, Ingress: 2, Storage: 2}, CentOS7, aws, func(nodes provisionedNodes, sshKey string) {
+					opts := installOptions{allowPackageInstallation: true}
+					err := installKismatic(nodes, opts, sshKey)
+					FailIfError(err)
+
+					pwd, err := os.Getwd()
+					FailIfError(err)
+					err = extractCurrentKismatic(pwd)
+					FailIfError(err)
+
+					cmd := exec.Command("./kismatic", "upgrade", "offline", "-f", "kismatic-testing.yaml")
+					cmd.Stdout = os.Stdout
+					cmd.Stderr = os.Stderr
+					err = cmd.Run()
+					FailIfError(err)
+
+					assertClusterVersionIsCurrent()
+
+					sub := SubDescribe("Using an upgraded cluster")
+					defer sub.Check()
+
+					sub.It("should allow adding a new storage volume", func() error {
+						planFile, err := os.Open("kismatic-testing.yaml")
+						if err != nil {
+							return err
+						}
+						return createVolume(planFile, "test-vol", 1, 1, "")
+					})
+
+					sub.It("should allow adding a worker node", func() error {
+						return nil
+					})
+
+					sub.It("should have an accessible dashboard", func() error {
+						return nil
+					})
+
+					sub.It("should be able to deploy a workload with ingress", func() error {
+						return nil
+					})
+
+					sub.It("should not have kube-apiserver systemd service", func() error {
+						return nil
+					})
+				})
+			})
+		})
 	})
 })