Enable RBAC support in Kubernetes API server (#499)

* Enable RBAC and improve cert gen code

* Add integration test for RBAC

* Set RBAC proposal state to implemented

Author: alexbrand

ansible/roles/authorization-policy/templates/basicauth.csv

@@ -1 +1 @@
-{{ kubernetes_admin_password }},admin,1
+{{ kubernetes_admin_password }},admin,1,"system:masters"

ansible/roles/kube-apiserver/templates/kube-apiserver.service.debug

@@ -9,7 +9,7 @@ ExecStart={{ bin_dir }}/kube-apiserver \
   --allow-privileged=true \
   --anonymous-auth=false \
   --apiserver-count={{ kubernetes_master_apiserver_count }} \
-  --authorization-mode=ABAC \
+  --authorization-mode=RBAC,ABAC \
   --authorization-policy-file={{ kubernetes_authorization_policy_path }} \
   --basic-auth-file={{ kubernetes_basic_auth_path }} \
   --bind-address=0.0.0.0 \

ansible/roles/kube-apiserver/templates/kube-apiserver.yaml

@@ -22,7 +22,7 @@ spec:
       - --allow-privileged=true
       - --anonymous-auth=false
       - --apiserver-count={{ kubernetes_master_apiserver_count }}
-      - --authorization-mode=ABAC
+      - --authorization-mode=RBAC,ABAC
       - --authorization-policy-file={{ kubernetes_authorization_policy_path }}
       - --basic-auth-file={{ kubernetes_basic_auth_path }}
       - --bind-address=0.0.0.0

integration/install_test.go

@@ -193,6 +193,11 @@ var _ = Describe("kismatic", func() {
 						return verifyNetworkPolicy(nodes.master[0], sshKey)
 					})
 
+					sub.It("should allow creating RBAC policy", func() error {
+						// Run on worker because master uses unauth API endpoint (i.e. localhost:8080)
+						return verifyRBAC(nodes.worker[0], sshKey)
+					})
+
 					// This test should always be last
 					sub.It("should still be a highly available cluster after removing a master node", func() error {
 						By("Removing a Kubernetes master node")

integration/rbac.go

@@ -0,0 +1,49 @@
+package integration
+
+import (
+	"fmt"
+	"strings"
+	"time"
+)
+
+func verifyRBAC(master NodeDeets, sshKey string) error {
+	// copy rbac policy over to master node
+	err := copyFileToRemote("test-resources/rbac/pod-reader.yaml", "/tmp/pod-reader.yaml", master, sshKey, 10*time.Second)
+	if err != nil {
+		return err
+	}
+	// we need the CA key to generate a new user cert
+	err = copyFileToRemote("generated/keys/ca-key.pem", "/tmp/ca-key.pem", master, sshKey, 10*time.Second)
+	if err != nil {
+		return err
+	}
+	// create using kubectl
+	commands := []string{
+		// create the RBAC policy
+		"sudo kubectl create -f /tmp/pod-reader.yaml",
+		// generate a private key for jane
+		"sudo openssl genrsa -out /tmp/jane-key.pem 2048",
+		// generate a CSR for jane
+		"sudo openssl req -new -key /tmp/jane-key.pem -out /tmp/jane-csr.pem -subj \"/CN=jane/O=some-group\"",
+		// generate certificate for jane
+		"sudo openssl x509 -req -in /tmp/jane-csr.pem -CA /etc/kubernetes/ca.pem -CAkey /tmp/ca-key.pem -CAcreateserial -out /tmp/jane.pem -days 10",
+		// configure new user in kubeconfig
+		"sudo kubectl config set-credentials jane --client-certificate=/tmp/jane.pem --client-key=/tmp/jane-key.pem",
+	}
+	err = runViaSSH(commands, []NodeDeets{master}, sshKey, 30*time.Second)
+	if err != nil {
+		return err
+	}
+	// Using kubectl to get pods should succeed
+	err = runViaSSH([]string{"sudo kubectl get pods --user=jane"}, []NodeDeets{master}, sshKey, 30*time.Second)
+	if err != nil {
+		return fmt.Errorf("failed to get pods as jane: %v", err)
+	}
+	// This command is expected to fail, so we ignore the error.
+	// We expect the output to contain the string "Forbidden"
+	out, _ := executeCmd("sudo kubectl get nodes --user=jane", master.PublicIP, master.SSHUser, sshKey)
+	if !strings.Contains(out, "Forbidden") {
+		return fmt.Errorf("expected a forbidden response from the server, but output did not indicate this. Output was: %s", out)
+	}
+	return nil
+}

integration/test-resources/rbac/pod-reader.yaml

@@ -0,0 +1,24 @@
+---
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  namespace: default
+  name: pod-reader
+rules:
+- apiGroups: [""] # "" indicates the core API group
+  resources: ["pods"]
+  verbs: ["get", "watch", "list"]
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: read-pods
+  namespace: default
+subjects:
+- kind: User
+  name: jane
+  apiGroup: rbac.authorization.k8s.io
+roleRef:
+  kind: Role
+  name: pod-reader
+  apiGroup: rbac.authorization.k8s.io

pkg/install/add_worker_test.go

@@ -301,7 +301,7 @@ func (f *fakePKI) GenerateClusterCA(p *Plan) (*tls.CA, error) {
 	f.generateCACalled = true
 	return nil, f.err
 }
-func (f *fakePKI) GenerateClusterCertificates(p *Plan, ca *tls.CA, users []string) error { return f.err }
+func (f *fakePKI) GenerateClusterCertificates(p *Plan, ca *tls.CA) error { return f.err }
 
 type fakeRunner struct {
 	eventChan         chan ansible.Event

pkg/install/execute.go

@@ -747,7 +747,7 @@ func (ae *ansibleExecutor) generateTLSAssets(p *Plan) error {
 	}
 
 	// Generate node and user certificates
-	err = ae.pki.GenerateClusterCertificates(p, ca, []string{"admin"})
+	err = ae.pki.GenerateClusterCertificates(p, ca)
 	if err != nil {
 		return fmt.Errorf("error generating certificates for the cluster: %v", err)
 	}

pkg/install/pki.go

@@ -11,11 +11,16 @@ import (
 )
 
 const (
-	certOrganization = "Apprenda"
-	certOrgUnit      = "Kismatic"
-	certCountry      = "US"
-	certState        = "NY"
-	certLocality     = "Troy"
+	certOrganization             = "Apprenda"
+	certOrgUnit                  = "Kismatic"
+	certCountry                  = "US"
+	certState                    = "NY"
+	certLocality                 = "Troy"
+	adminUser                    = "admin"
+	adminGroup                   = "system:masters"
+	dockerRegistryCertFilename   = "docker"
+	serviceAccountCertFilename   = "service-account"
+	serviceAccountCertCommonName = "kube-service-account"
 )
 
 // The PKI provides a way for generating certificates for the cluster described by the Plan
@@ -25,7 +30,7 @@ type PKI interface {
 	GenerateNodeCertificate(plan *Plan, node Node, ca *tls.CA) error
 	GetClusterCA() (*tls.CA, error)
 	GenerateClusterCA(p *Plan) (*tls.CA, error)
-	GenerateClusterCertificates(p *Plan, ca *tls.CA, users []string) error
+	GenerateClusterCertificates(p *Plan, ca *tls.CA) error
 }
 
 // LocalPKI is a file-based PKI
@@ -98,7 +103,7 @@ func (lp *LocalPKI) GenerateClusterCA(p *Plan) (*tls.CA, error) {
 }
 
 // GenerateClusterCertificates creates a Certificates for all nodes on the cluster
-func (lp *LocalPKI) GenerateClusterCertificates(p *Plan, ca *tls.CA, users []string) error {
+func (lp *LocalPKI) GenerateClusterCertificates(p *Plan, ca *tls.CA) error {
 	if lp.Log == nil {
 		lp.Log = ioutil.Discard
 	}
@@ -134,17 +139,15 @@ func (lp *LocalPKI) GenerateClusterCertificates(p *Plan, ca *tls.CA, users []str
 	if err := lp.generateServiceAccountCert(p, ca); err != nil {
 		return err
 	}
-	// Finally, create certs for user if they are missing
-	for _, user := range users {
-		if err := lp.generateUserCert(p, user, ca); err != nil {
-			return err
-		}
+	// Create the admin user's certificate
+	if err := lp.generateUserCert(p, ca, adminUser, []string{adminGroup}); err != nil {
+		return err
 	}
 	return nil
 }
 
 // ValidateClusterCertificates validates all certificates in the cluster
-func (lp *LocalPKI) ValidateClusterCertificates(p *Plan, users []string) (warn []error, err []error) {
+func (lp *LocalPKI) ValidateClusterCertificates(p *Plan) (warn []error, err []error) {
 	if lp.Log == nil {
 		lp.Log = ioutil.Discard
 	}
@@ -183,20 +186,18 @@ func (lp *LocalPKI) ValidateClusterCertificates(p *Plan, users []string) (warn [
 	if err != nil {
 		err = append(err, saErr)
 	}
-	// Finally, create certs for user if they are missing
-	for _, user := range users {
-		_, userWarn, userErr := lp.validateUserCert(user)
-		warn = append(warn, userWarn...)
-		if err != nil {
-			err = append(err, userErr)
-		}
+	// Validate admin certificate
+	_, userWarn, userErr := lp.validateUserCert(adminUser, []string{adminGroup})
+	warn = append(warn, userWarn...)
+	if err != nil {
+		err = append(err, userErr)
 	}
 	return warn, err
 }
 
 // GenerateNodeCertificate creates a private key and certificate for the given node
 func (lp *LocalPKI) GenerateNodeCertificate(plan *Plan, node Node, ca *tls.CA) error {
-	CN := node.Host
+	commonName := node.Host
 	// Build list of SANs
 	clusterSANs, err := clusterCertsSubjectAlternateNames(plan)
 	if err != nil {
@@ -216,7 +217,7 @@ func (lp *LocalPKI) GenerateNodeCertificate(plan *Plan, node Node, ca *tls.CA) e
 	}
 
 	// Don't generate if the key pair exists and valid
-	valid, warn, err := tls.CertExistsAndValid(CN, nodeSANs, node.Host, lp.GeneratedCertsDirectory)
+	valid, warn, err := tls.CertExistsAndValid(commonName, nodeSANs, []string{}, node.Host, lp.GeneratedCertsDirectory)
 	if err != nil {
 		return err
 	}
@@ -232,7 +233,7 @@ func (lp *LocalPKI) GenerateNodeCertificate(plan *Plan, node Node, ca *tls.CA) e
 
 	util.PrettyPrintOk(lp.Log, "Generating certificates for host %q", node.Host)
 
-	key, cert, err := generateCert(CN, plan, nodeSANs, ca)
+	key, cert, err := generateCert(ca, commonName, nodeSANs)
 	if err != nil {
 		return fmt.Errorf("error during cluster cert generation: %v", err)
 	}
@@ -263,17 +264,17 @@ func (lp *LocalPKI) validateNodeCertificate(p *Plan, node Node) (valid bool, war
 		}
 	}
 
-	return tls.CertExistsAndValid(CN, nodeSANs, node.Host, lp.GeneratedCertsDirectory)
+	return tls.CertExistsAndValid(CN, nodeSANs, []string{}, node.Host, lp.GeneratedCertsDirectory)
 }
 
 func (lp *LocalPKI) generateDockerRegistryCert(p *Plan, ca *tls.CA) error {
 	// Default registry will be deployed on the first master
 	n := p.Master.Nodes[0]
-	CN := n.Host
+	commonName := n.Host
 	SANs := []string{n.Host, n.IP, n.InternalIP}
 
 	// Don't generate if the key pair exists and valid
-	valid, warn, err := tls.CertExistsAndValid(CN, SANs, "docker", lp.GeneratedCertsDirectory)
+	valid, warn, err := tls.CertExistsAndValid(commonName, SANs, []string{}, dockerRegistryCertFilename, lp.GeneratedCertsDirectory)
 	if err != nil {
 		return err
 	}
@@ -289,11 +290,11 @@ func (lp *LocalPKI) generateDockerRegistryCert(p *Plan, ca *tls.CA) error {
 
 	util.PrettyPrintOk(lp.Log, "Generating certificates for docker registry")
 
-	dockerKey, dockerCert, err := generateCert(CN, p, SANs, ca)
+	dockerKey, dockerCert, err := generateCert(ca, commonName, SANs)
 	if err != nil {
 		return fmt.Errorf("error during user cert generation: %v", err)
 	}
-	err = tls.WriteCert(dockerKey, dockerCert, "docker", lp.GeneratedCertsDirectory)
+	err = tls.WriteCert(dockerKey, dockerCert, dockerRegistryCertFilename, lp.GeneratedCertsDirectory)
 	if err != nil {
 		return fmt.Errorf("error writing cert files for docker registry")
 	}
@@ -306,16 +307,13 @@ func (lp *LocalPKI) validateDockerRegistryCert(p *Plan) (valid bool, warn []erro
 	CN := n.Host
 	SANs := []string{n.Host, n.IP, n.InternalIP}
 
-	return tls.CertExistsAndValid(CN, SANs, "docker", lp.GeneratedCertsDirectory)
+	return tls.CertExistsAndValid(CN, SANs, []string{}, dockerRegistryCertFilename, lp.GeneratedCertsDirectory)
 }
 
 func (lp *LocalPKI) generateServiceAccountCert(p *Plan, ca *tls.CA) error {
-	CN := "kube-service-account"
 	SANs := []string{}
-	certName := "service-account"
-
 	// Don't generate if the key pair exists and valid
-	valid, warn, err := tls.CertExistsAndValid(CN, SANs, certName, lp.GeneratedCertsDirectory)
+	valid, warn, err := tls.CertExistsAndValid(serviceAccountCertCommonName, SANs, []string{}, serviceAccountCertFilename, lp.GeneratedCertsDirectory)
 	if err != nil {
 		return err
 	}
@@ -328,32 +326,28 @@ func (lp *LocalPKI) generateServiceAccountCert(p *Plan, ca *tls.CA) error {
 		util.PrettyPrintOk(lp.Log, "Found key and certificate for service accounts")
 		return nil
 	}
-
 	util.PrettyPrintOk(lp.Log, "Generating certificates for service accounts")
 
-	key, cert, err := generateCert(CN, p, SANs, ca)
+	key, cert, err := generateCert(ca, serviceAccountCertCommonName, SANs)
 	if err != nil {
 		return fmt.Errorf("error generating service account certs: %v", err)
 	}
-	if err = tls.WriteCert(key, cert, certName, lp.GeneratedCertsDirectory); err != nil {
+	if err = tls.WriteCert(key, cert, serviceAccountCertFilename, lp.GeneratedCertsDirectory); err != nil {
 		return fmt.Errorf("error writing generated service account cert: %v", err)
 	}
 	return nil
 }
 
 func (lp *LocalPKI) validateServiceAccountCert() (valid bool, warn []error, err error) {
-	CN := "kube-service-account"
 	SANs := []string{}
-	certName := "service-account"
-
-	return tls.CertExistsAndValid(CN, SANs, certName, lp.GeneratedCertsDirectory)
+	return tls.CertExistsAndValid(serviceAccountCertCommonName, SANs, []string{}, serviceAccountCertFilename, lp.GeneratedCertsDirectory)
 }
 
-func (lp *LocalPKI) generateUserCert(p *Plan, user string, ca *tls.CA) error {
+func (lp *LocalPKI) generateUserCert(p *Plan, ca *tls.CA, user string, groups []string) error {
 	SANs := []string{user}
 
 	// Don't generate if the key pair exists and valid
-	valid, warn, err := tls.CertExistsAndValid(user, SANs, user, lp.GeneratedCertsDirectory)
+	valid, warn, err := tls.CertExistsAndValid(user, SANs, groups, user, lp.GeneratedCertsDirectory)
 	if err != nil {
 		return err
 	}
@@ -369,7 +363,7 @@ func (lp *LocalPKI) generateUserCert(p *Plan, user string, ca *tls.CA) error {
 
 	util.PrettyPrintOk(lp.Log, "Generating certificates for user %q", user)
 
-	adminKey, adminCert, err := generateCert(user, p, SANs, ca)
+	adminKey, adminCert, err := generateCert(ca, user, SANs, groups...)
 	if err != nil {
 		return fmt.Errorf("error during user cert generation: %v", err)
 	}
@@ -380,15 +374,14 @@ func (lp *LocalPKI) generateUserCert(p *Plan, user string, ca *tls.CA) error {
 	return nil
 }
 
-func (lp *LocalPKI) validateUserCert(user string) (valid bool, warn []error, err error) {
+func (lp *LocalPKI) validateUserCert(user string, groups []string) (valid bool, warn []error, err error) {
 	SANs := []string{user}
-
-	return tls.CertExistsAndValid(user, SANs, user, lp.GeneratedCertsDirectory)
+	return tls.CertExistsAndValid(user, SANs, groups, user, lp.GeneratedCertsDirectory)
 }
 
-func generateCert(cnName string, p *Plan, hostList []string, ca *tls.CA) (key, cert []byte, err error) {
+func generateCert(ca *tls.CA, commonName string, hostList []string, organizations ...string) (key, cert []byte, err error) {
 	req := csr.CertificateRequest{
-		CN: cnName,
+		CN: commonName,
 		KeyRequest: &csr.BasicKeyRequest{
 			A: "rsa",
 			S: 2048,
@@ -406,7 +399,7 @@ func generateCert(cnName string, p *Plan, hostList []string, ca *tls.CA) (key, c
 	}
 	key, cert, err = tls.NewCert(ca, req)
 	if err != nil {
-		return nil, nil, fmt.Errorf("error generating certs for %q: %v", cnName, err)
+		return nil, nil, fmt.Errorf("error generating certs for %q: %v", commonName, err)
 	}
 	return key, cert, err
 }