@@ -12,6 +12,230 @@ import (
1212var _ = Describe ("Upgrade" , func () {
1313
1414 Describe ("Upgrading a cluster using online mode" , func () {
15+ Context ("From KET version v1.6.2" , func () {
16+ BeforeEach (func () {
17+ dir := setupTestWorkingDirWithVersion ("v1.6.2" )
18+ os .Chdir (dir )
19+ })
20+
21+ Context ("Using a minikube layout" , func () {
22+ Context ("Using CentOS 7" , func () {
23+ ItOnAWS ("should be upgraded [slow] [upgrade]" , func (aws infrastructureProvisioner ) {
24+ WithMiniInfrastructure (CentOS7 , aws , func (node NodeDeets , sshKey string ) {
25+ installAndUpgradeMinikube (node , sshKey , true )
26+ })
27+ })
28+ })
29+
30+ Context ("Using RedHat 7" , func () {
31+ ItOnAWS ("should be upgraded [slow] [upgrade]" , func (aws infrastructureProvisioner ) {
32+ WithMiniInfrastructure (RedHat7 , aws , func (node NodeDeets , sshKey string ) {
33+ installAndUpgradeMinikube (node , sshKey , true )
34+ })
35+ })
36+ })
37+ })
38+
39+ // This spec will be used for testing non-destructive kismatic features on
40+ // an upgraded cluster.
41+ // This spec is open to modification when new assertions have to be made.
42+ Context ("Using a skunkworks cluster" , func () {
43+ ItOnAWS ("should result in an upgraded cluster [slow] [upgrade]" , func (aws infrastructureProvisioner ) {
44+ WithInfrastructureAndDNS (NodeCount {Etcd : 3 , Master : 2 , Worker : 3 , Ingress : 2 , Storage : 2 }, Ubuntu1604LTS , aws , func (nodes provisionedNodes , sshKey string ) {
45+ // reserve one of the workers for the add-worker test
46+ allWorkers := nodes .worker
47+ nodes .worker = allWorkers [0 : len (nodes .worker )- 1 ]
48+
49+ // Standup cluster with previous version
50+ opts := installOptions {}
51+ err := installKismatic (nodes , opts , sshKey )
52+ FailIfError (err )
53+
54+ // Extract current version of kismatic
55+ extractCurrentKismaticInstaller ()
56+
57+ // Perform upgrade
58+ upgradeCluster (true )
59+
60+ sub := SubDescribe ("Using an upgraded cluster" )
61+ defer sub .Check ()
62+
63+ sub .It ("should have working storage volumes" , func () error {
64+ return testStatefulWorkload (nodes , sshKey )
65+ })
66+
67+ sub .It ("should allow adding a worker node" , func () error {
68+ newWorker := allWorkers [len (allWorkers )- 1 ]
69+ return addWorkerToCluster (newWorker , sshKey , []string {})
70+ })
71+
72+ sub .It ("should be able to deploy a workload with ingress" , func () error {
73+ return verifyIngressNodes (nodes .master [0 ], nodes .ingress , sshKey )
74+ })
75+
76+ // Use master[0] public IP
77+ sub .It ("should have an accessible dashboard" , func () error {
78+ return canAccessDashboard (fmt .Sprintf ("https://admin:abbazabba@%s:6443/ui" , nodes .master [0 ].PublicIP ))
79+ })
80+
81+ sub .It ("should respect network policies" , func () error {
82+ return verifyNetworkPolicy (nodes .master [0 ], sshKey , false )
83+ })
84+
85+ // This test should always be last
86+ sub .It ("should still be a highly available cluster after upgrade" , func () error {
87+ By ("Removing a Kubernetes master node" )
88+ if err = aws .TerminateNode (nodes .master [0 ]); err != nil {
89+ return fmt .Errorf ("could not remove node: %v" , err )
90+ }
91+ By ("Re-running Kuberang" )
92+ if err = runViaSSH ([]string {"sudo kuberang" }, []NodeDeets {nodes .master [1 ]}, sshKey , 5 * time .Minute ); err != nil {
93+ return err
94+ }
95+ return nil
96+ })
97+ })
98+ })
99+ })
100+
101+ Context ("Using a cluster that has no internet access [slow] [upgrade]" , func () {
102+ Context ("With nodes running CentOS 7" , func () {
103+ ItOnAWS ("should result in an upgraded cluster" , func (aws infrastructureProvisioner ) {
104+ distro := CentOS7
105+ WithInfrastructure (NodeCount {Etcd : 1 , Master : 1 , Worker : 2 , Ingress : 1 , Storage : 1 }, distro , aws , func (nodes provisionedNodes , sshKey string ) {
106+ // One of the nodes will function as a repo mirror and image registry
107+ repoNode := nodes .worker [1 ]
108+ nodes .worker = nodes .worker [0 :1 ]
109+ // Standup cluster with previous version
110+ opts := installOptions {
111+ disconnectedInstallation : false , // we want KET to install the packages, so let it use the package repo
112+ modifyHostsFiles : true ,
113+ }
114+ err := installKismatic (nodes , opts , sshKey )
115+ FailIfError (err )
116+
117+ // Extract current version of kismatic
118+ extractCurrentKismaticInstaller ()
119+
120+ By ("Creating a package repository" )
121+ err = createPackageRepositoryMirror (repoNode , distro , sshKey )
122+ FailIfError (err , "Error creating local package repo" )
123+
124+ By ("Deploying a docker registry" )
125+ caFile , err := deployAuthenticatedDockerRegistry (repoNode , dockerRegistryPort , sshKey )
126+ FailIfError (err , "Failed to deploy docker registry" )
127+
128+ By ("Seeding the local registry" )
129+ err = seedRegistry (repoNode , caFile , dockerRegistryPort , sshKey )
130+ FailIfError (err , "Error seeding local registry" )
131+
132+ err = disableInternetAccess (nodes .allNodes (), sshKey )
133+ FailIfError (err )
134+
135+ By ("Configuring repository on nodes" )
136+ for _ , n := range nodes .allNodes () {
137+ err = copyFileToRemote ("test-resources/disconnected-installation/configure-rpm-mirrors.sh" , "/tmp/configure-rpm-mirrors.sh" , n , sshKey , 15 * time .Second )
138+ FailIfError (err , "Failed to copy script to nodes" )
139+ }
140+ cmds := []string {
141+ "chmod +x /tmp/configure-rpm-mirrors.sh" ,
142+ fmt .Sprintf ("sudo /tmp/configure-rpm-mirrors.sh http://%s" , repoNode .PrivateIP ),
143+ }
144+ err = runViaSSH (cmds , nodes .allNodes (), sshKey , 5 * time .Minute )
145+ FailIfError (err , "Failed to run mirror configuration script" )
146+
147+ if err := verifyNoInternetAccess (nodes .allNodes (), sshKey ); err == nil {
148+ Fail ("was able to ping google with outgoing connections blocked" )
149+ }
150+
151+ // Cleanup old cluster file and create a new one
152+ By ("Recreating kismatic-testing.yaml file" )
153+ err = os .Remove ("kismatic-testing.yaml" )
154+ FailIfError (err )
155+ opts = installOptions {
156+ disconnectedInstallation : true ,
157+ modifyHostsFiles : true ,
158+ dockerRegistryCAPath : caFile ,
159+ dockerRegistryServer : fmt .Sprintf ("%s:%d" , repoNode .PrivateIP , dockerRegistryPort ),
160+ dockerRegistryUsername : "kismaticuser" ,
161+ dockerRegistryPassword : "kismaticpassword" ,
162+ }
163+ writePlanFile (buildPlan (nodes , opts , sshKey ))
164+
165+ upgradeCluster (true )
166+ })
167+ })
168+ })
169+
170+ Context ("With nodes running Ubuntu 16.04" , func () {
171+ ItOnAWS ("should result in an upgraded cluster" , func (aws infrastructureProvisioner ) {
172+ distro := Ubuntu1604LTS
173+ WithInfrastructure (NodeCount {Etcd : 1 , Master : 1 , Worker : 2 , Ingress : 1 , Storage : 1 }, distro , aws , func (nodes provisionedNodes , sshKey string ) {
174+ // One of the nodes will function as a repo mirror and image registry
175+ repoNode := nodes .worker [1 ]
176+ nodes .worker = nodes .worker [0 :1 ]
177+ // Standup cluster with previous version
178+ opts := installOptions {
179+ disconnectedInstallation : false , // we want KET to install the packages, so let it use the package repo
180+ modifyHostsFiles : true ,
181+ }
182+ err := installKismatic (nodes , opts , sshKey )
183+ FailIfError (err )
184+
185+ extractCurrentKismaticInstaller ()
186+
187+ By ("Creating a package repository" )
188+ err = createPackageRepositoryMirror (repoNode , distro , sshKey )
189+ FailIfError (err , "Error creating local package repo" )
190+
191+ By ("Deploying a docker registry" )
192+ caFile , err := deployAuthenticatedDockerRegistry (repoNode , dockerRegistryPort , sshKey )
193+ FailIfError (err , "Failed to deploy docker registry" )
194+
195+ By ("Seeding the local registry" )
196+ err = seedRegistry (repoNode , caFile , dockerRegistryPort , sshKey )
197+ FailIfError (err , "Error seeding local registry" )
198+
199+ err = disableInternetAccess (nodes .allNodes (), sshKey )
200+ FailIfError (err )
201+
202+ By ("Configuring repository on nodes" )
203+ for _ , n := range nodes .allNodes () {
204+ err = copyFileToRemote ("test-resources/disconnected-installation/configure-deb-mirrors.sh" , "/tmp/configure-deb-mirrors.sh" , n , sshKey , 15 * time .Second )
205+ FailIfError (err , "Failed to copy script to nodes" )
206+ }
207+ cmds := []string {
208+ "chmod +x /tmp/configure-deb-mirrors.sh" ,
209+ fmt .Sprintf ("sudo /tmp/configure-deb-mirrors.sh http://%s" , repoNode .PrivateIP ),
210+ }
211+ err = runViaSSH (cmds , nodes .allNodes (), sshKey , 5 * time .Minute )
212+ FailIfError (err , "Failed to run mirror configuration script" )
213+
214+ if err := verifyNoInternetAccess (nodes .allNodes (), sshKey ); err == nil {
215+ Fail ("was able to ping google with outgoing connections blocked" )
216+ }
217+
218+ // Cleanup old cluster file and create a new one
219+ By ("Recreating kismatic-testing.yaml file" )
220+ err = os .Remove ("kismatic-testing.yaml" )
221+ FailIfError (err )
222+ opts = installOptions {
223+ disconnectedInstallation : true ,
224+ modifyHostsFiles : true ,
225+ dockerRegistryCAPath : caFile ,
226+ dockerRegistryServer : fmt .Sprintf ("%s:%d" , repoNode .PrivateIP , dockerRegistryPort ),
227+ dockerRegistryUsername : "kismaticuser" ,
228+ dockerRegistryPassword : "kismaticpassword" ,
229+ }
230+ writePlanFile (buildPlan (nodes , opts , sshKey ))
231+
232+ upgradeCluster (true )
233+ })
234+ })
235+ })
236+ })
237+ })
238+
15239 Context ("From KET version v1.6.1" , func () {
16240 BeforeEach (func () {
17241 dir := setupTestWorkingDirWithVersion ("v1.6.1" )
0 commit comments