Skip to content

Commit eafdfe2

Browse files
alexbranddkoshkin
alexbrand
authored and
dkoshkin
committed
[WIP] Support for direct-lvm Docker storage driver (#480)
* Support for direct-lvm Docker storage driver * Add lvm2 package to inspector checks * Use filepath.Abs * Run block volume checks during preflight * Remove lvm2 package to inspector checks * Add lvm integration tests * Minor lvm play changes
1 parent 1080dfa commit eafdfe2

20 files changed

+286
-12
lines changed

‎ansible/_docker.yaml

+4-2
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
---
22
- hosts: master:worker:ingress:storage
33
any_errors_fatal: true
4-
name: "{{ play_name | default('Start Docker') }}"
4+
name: "{{ play_name | default('Install Docker') }}"
55
serial: "{{ serial_count | default('100%') }}"
66
remote_user: root
77
become_method: sudo
@@ -11,4 +11,6 @@
1111
roles:
1212
- role: packages-docker
1313
when: allow_package_installation|bool == true
14-
- docker
14+
- role: docker-storage
15+
when: docker_direct_lvm_enabled|bool == true
16+
- role: docker

‎ansible/group_vars/all.yaml

+7
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,13 @@ docker_certificates_key_file_name: docker-key.pem
7474
docker_certificates_cert_path: "{{ docker_install_dir }}/{{ docker_certificates_cert_file_name }}"
7575
docker_certificates_key_path: "{{ docker_install_dir }}/{{ docker_certificates_key_file_name }}"
7676
#===============================================================================
77+
# docker configuration
78+
# docker_direct_lvm_enabled: no default
79+
# docker_direct_lvm_block_device_path: no default
80+
# docker_direct_lvm_deferred_deletion_enabled: no default
81+
docker_device_mapper_thin_pool_autoextend_threshold: 80
82+
docker_device_mapper_thin_pool_autoextend_percent: 20
83+
#===============================================================================
7784
# calico
7885
# directories
7986
calico_dir: /etc/calico

‎ansible/roles/docker-storage/tasks/main.yaml

+33
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
---
2+
- name: install lvm2 package
3+
package:
4+
name: lvm2
5+
state: present
6+
when: "ansible_os_family == 'RedHat'"
7+
# Not using lvol and lvg ansible modules here as they are in preview
8+
- name: create a physical volume on the block device
9+
command: pvcreate {{ docker_direct_lvm_block_device_path }}
10+
when: "ansible_os_family == 'RedHat'"
11+
- name: create docker volume group
12+
command: vgcreate docker {{ docker_direct_lvm_block_device_path }}
13+
when: "ansible_os_family == 'RedHat'"
14+
- name: create thinpool logical volume
15+
command: lvcreate --wipesignatures y -n thinpool docker -l 95%VG
16+
when: "ansible_os_family == 'RedHat'"
17+
- name: create thinpoolmeta logical volume
18+
command: lvcreate --wipesignatures y -n thinpoolmeta docker -l 1%VG
19+
when: "ansible_os_family == 'RedHat'"
20+
- name: convert the pool to a thin pool
21+
command: lvconvert -y --zero n -c 512K --thinpool docker/thinpool --poolmetadata docker/thinpoolmeta
22+
when: "ansible_os_family == 'RedHat'"
23+
- name: create auto extension profile
24+
template:
25+
src: docker-thinpool.profile
26+
dest: /etc/lvm/profile/docker-thinpool.profile
27+
when: "ansible_os_family == 'RedHat'"
28+
- name: apply lvm profile
29+
command: lvchange --metadataprofile docker-thinpool docker/thinpool
30+
when: "ansible_os_family == 'RedHat'"
31+
- name: enable monitoring to ensure autoextend executes
32+
command: lvchange --metadataprofile docker-thinpool docker/thinpool
33+
when: "ansible_os_family == 'RedHat'"

‎ansible/roles/docker-storage/templates/docker-thinpool.profile

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
activation {
2+
thin_pool_autoextend_threshold={{docker_device_mapper_thin_pool_autoextend_threshold}}
3+
thin_pool_autoextend_percent={{docker_device_mapper_thin_pool_autoextend_percent}}
4+
}

‎ansible/roles/docker/tasks/main.yaml

+9
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,13 @@
11
---
2+
- name: create /etc/docker directory
3+
file:
4+
path: /etc/docker
5+
state: directory
6+
- name: write docker config file
7+
template:
8+
src: daemon.json
9+
dest: /etc/docker/daemon.json
10+
when: docker_direct_lvm_enabled|bool == true
211
# start and verify that Docker installed succesfully and is running
312
- name: start docker service
413
service:

‎ansible/roles/docker/templates/daemon.json

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
{
2+
"storage-driver": "devicemapper",
3+
"storage-opts": [
4+
"dm.thinpooldev=/dev/mapper/docker-thinpool",
5+
"dm.use_deferred_removal=true",
6+
"dm.use_deferred_deletion={{docker_direct_lvm_deferred_deletion_enabled}}"
7+
]
8+
}

‎ansible/roles/preflight/tasks/direct_lvm_preflight.yaml

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
---
2+
- name: stat the block device path
3+
stat:
4+
path: "{{ docker_direct_lvm_block_device_path }}"
5+
register: block_device_stat
6+
- name: fail if the block device does not exists
7+
fail:
8+
msg: "Block device specified for docker storage does not exist."
9+
when: block_device_stat.stat.exists == False
10+
- name: fail if the provided path is not a block device
11+
fail:
12+
msg: "{{ docker_direct_lvm_block_device_path }} is not a block device."
13+
when: block_device_stat.stat.isblk == False
14+
- name: fail if the block device is already mounted
15+
fail:
16+
msg: "Block deviced specified for docker storage is currently mounted. This should be an unmounted, unused device"
17+
with_items: ansible_mounts
18+
when: item.device == "{{ docker_direct_lvm_block_device_path }}"

‎ansible/roles/preflight/tasks/main.yaml

+4
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,10 @@
7373
loop_var: outer_item # Define this (even thought we don't use it) so that ansible doesn't complain.
7474
when: "'worker' in group_names"
7575

76+
- name: Validate devicemapper direct-lvm block device
77+
include: direct_lvm_preflight.yaml
78+
when: "ansible_os_family == 'RedHat' and docker_direct_lvm_enabled|bool == true and ('master' in group_names or 'worker' in group_names or 'ingress' in group_names or 'storage' in group_names)"
79+
7680
# setup Kismatic Inspector
7781
- name: copy Kismatic Inspector to node
7882
copy:

‎integration/aws/client.go

+11-1
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ func (c *Client) prepareSession() error {
106106

107107
// CreateNode is for creating a machine on AWS using the given AMI and InstanceType.
108108
// Returns the ID of the newly created machine.
109-
func (c Client) CreateNode(ami AMI, instanceType InstanceType) (string, error) {
109+
func (c Client) CreateNode(ami AMI, instanceType InstanceType, addBlockDevice bool) (string, error) {
110110
api, err := c.getEC2APIClient()
111111
if err != nil {
112112
return "", err
@@ -129,6 +129,16 @@ func (c Client) CreateNode(ami AMI, instanceType InstanceType) (string, error) {
129129
KeyName: aws.String(c.Config.Keyname),
130130
SecurityGroupIds: []*string{aws.String(c.Config.SecurityGroupID)},
131131
}
132+
if addBlockDevice {
133+
ebs := ec2.BlockDeviceMapping{
134+
DeviceName: aws.String("/dev/sdb"),
135+
Ebs: &ec2.EbsBlockDevice{
136+
DeleteOnTermination: aws.Bool(true),
137+
VolumeSize: aws.Int64(10),
138+
},
139+
}
140+
req.BlockDeviceMappings = append(req.BlockDeviceMappings, &ebs)
141+
}
132142
res, err := api.RunInstances(req)
133143
if err != nil {
134144
return "", err

‎integration/framework.go

+22
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,28 @@ func WithMiniInfrastructure(distro linuxDistro, provisioner infrastructureProvis
116116
f(nodes.worker[0], sshKey)
117117
}
118118

119+
// WithMiniInfrastructureAndBlockDevice runs the spec with the requested infrastructure and an additiona block device
120+
// The block will be under /dev/xvdb on AWS
121+
func WithMiniInfrastructureAndBlockDevice(distro linuxDistro, provisioner infrastructureProvisioner, f miniInfraDependentTest) {
122+
By("Provisioning minikube node")
123+
start := time.Now()
124+
nodes, err := provisioner.ProvisionNodes(NodeCount{Worker: 1}, distro, []string{"block_device"}...)
125+
if !leaveIt() {
126+
defer provisioner.TerminateNodes(nodes)
127+
}
128+
Expect(err).ToNot(HaveOccurred())
129+
fmt.Println("Provisioning node took", time.Since(start))
130+
131+
By("Waiting until nodes are SSH-accessible")
132+
start = time.Now()
133+
sshKey := provisioner.SSHKey()
134+
err = waitForSSH(nodes, sshKey)
135+
Expect(err).ToNot(HaveOccurred())
136+
fmt.Println("Waiting for SSH took", time.Since(start))
137+
138+
f(nodes.worker[0], sshKey)
139+
}
140+
119141
// SubDescribe allows you to define specifications inside another spec.
120142
// We have found the need for this because Gingko does not support
121143
// serializing a subset of tests when running in parallel. This means

‎integration/install.go

+2
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ type installOptions struct {
3636
dockerRegistryPort int
3737
dockerRegistryCAPath string
3838
modifyHostsFiles bool
39+
useDirectLVM bool
3940
}
4041

4142
func installKismaticMini(node NodeDeets, sshKey string) error {
@@ -82,6 +83,7 @@ func buildPlan(nodes provisionedNodes, installOpts installOptions, sshKey string
8283
DockerRegistryIP: installOpts.dockerRegistryIP,
8384
DockerRegistryPort: installOpts.dockerRegistryPort,
8485
ModifyHostsFiles: installOpts.modifyHostsFiles,
86+
UseDirectLVM: installOpts.useDirectLVM,
8587
}
8688
return plan
8789
}

‎integration/install_test.go

+38
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,44 @@ var _ = Describe("kismatic", func() {
119119
})
120120
})
121121

122+
Context("when using direct-lvm docker storage", func() {
123+
installOpts := installOptions{
124+
allowPackageInstallation: true,
125+
useDirectLVM: true,
126+
}
127+
Context("when targetting CentOS", func() {
128+
ItOnAWS("should install successfully", func(aws infrastructureProvisioner) {
129+
WithMiniInfrastructureAndBlockDevice(CentOS7, aws, func(node NodeDeets, sshKey string) {
130+
theNode := []NodeDeets{node}
131+
nodes := provisionedNodes{
132+
etcd: theNode,
133+
master: theNode,
134+
worker: theNode,
135+
ingress: theNode,
136+
}
137+
err := installKismatic(nodes, installOpts, sshKey)
138+
Expect(err).ToNot(HaveOccurred())
139+
})
140+
})
141+
})
142+
143+
Context("when targetting RHEL", func() {
144+
ItOnAWS("should install successfully", func(aws infrastructureProvisioner) {
145+
WithMiniInfrastructureAndBlockDevice(RedHat7, aws, func(node NodeDeets, sshKey string) {
146+
theNode := []NodeDeets{node}
147+
nodes := provisionedNodes{
148+
etcd: theNode,
149+
master: theNode,
150+
worker: theNode,
151+
ingress: theNode,
152+
}
153+
err := installKismatic(nodes, installOpts, sshKey)
154+
Expect(err).ToNot(HaveOccurred())
155+
})
156+
})
157+
})
158+
})
159+
122160
// This spec will be used for testing non-destructive kismatic features on
123161
// a new cluster.
124162
// This spec is open to modification when new assertions have to be made

‎integration/plan_patterns.go

+8-1
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ type PlanAWS struct {
2323
DockerRegistryPort int
2424
DockerRegistryCAPath string
2525
ModifyHostsFiles bool
26+
UseDirectLVM bool
2627
}
2728

2829
const planAWSOverlay = `cluster:
@@ -44,7 +45,13 @@ const planAWSOverlay = `cluster:
4445
ssh:
4546
user: {{.SSHUser}}
4647
ssh_key: {{.SSHKeyFile}}
47-
ssh_port: 22
48+
ssh_port: 22{{if .UseDirectLVM}}
49+
docker:
50+
storage:
51+
direct_lvm:
52+
enabled: true
53+
block_device: "/dev/xvdb"
54+
enable_deferred_deletion: false{{end}}
4855
docker_registry:
4956
setup_internal: {{.AutoConfiguredDockerRegistry}}
5057
address: {{.DockerRegistryIP}}

‎integration/provision.go

+15-8
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ const (
2727
)
2828

2929
type infrastructureProvisioner interface {
30-
ProvisionNodes(NodeCount, linuxDistro) (provisionedNodes, error)
30+
ProvisionNodes(NodeCount, linuxDistro, ...string) (provisionedNodes, error)
3131
TerminateNodes(provisionedNodes) error
3232
TerminateNode(NodeDeets) error
3333
SSHKey() string
@@ -147,7 +147,7 @@ func AWSClientFromEnvironment() (infrastructureProvisioner, bool) {
147147
return p, true
148148
}
149149

150-
func (p awsProvisioner) ProvisionNodes(nodeCount NodeCount, distro linuxDistro) (provisionedNodes, error) {
150+
func (p awsProvisioner) ProvisionNodes(nodeCount NodeCount, distro linuxDistro, opts ...string) (provisionedNodes, error) {
151151
var ami aws.AMI
152152
switch distro {
153153
case Ubuntu1604LTS:
@@ -159,38 +159,45 @@ func (p awsProvisioner) ProvisionNodes(nodeCount NodeCount, distro linuxDistro)
159159
default:
160160
panic(fmt.Sprintf("Used an unsupported distribution: %s", distro))
161161
}
162+
var addBlockDevice bool
163+
for _, o := range opts {
164+
if o == "block_device" {
165+
addBlockDevice = true
166+
}
167+
}
168+
162169
provisioned := provisionedNodes{}
163170
var i uint16
164171
for i = 0; i < nodeCount.Etcd; i++ {
165-
nodeID, err := p.client.CreateNode(ami, aws.T2Medium)
172+
nodeID, err := p.client.CreateNode(ami, aws.T2Medium, addBlockDevice)
166173
if err != nil {
167174
return provisioned, err
168175
}
169176
provisioned.etcd = append(provisioned.etcd, NodeDeets{id: nodeID})
170177
}
171178
for i = 0; i < nodeCount.Master; i++ {
172-
nodeID, err := p.client.CreateNode(ami, aws.T2Medium)
179+
nodeID, err := p.client.CreateNode(ami, aws.T2Medium, addBlockDevice)
173180
if err != nil {
174181
return provisioned, err
175182
}
176183
provisioned.master = append(provisioned.master, NodeDeets{id: nodeID})
177184
}
178185
for i = 0; i < nodeCount.Worker; i++ {
179-
nodeID, err := p.client.CreateNode(ami, aws.T2Medium)
186+
nodeID, err := p.client.CreateNode(ami, aws.T2Medium, addBlockDevice)
180187
if err != nil {
181188
return provisioned, err
182189
}
183190
provisioned.worker = append(provisioned.worker, NodeDeets{id: nodeID})
184191
}
185192
for i = 0; i < nodeCount.Ingress; i++ {
186-
nodeID, err := p.client.CreateNode(ami, aws.T2Medium)
193+
nodeID, err := p.client.CreateNode(ami, aws.T2Medium, addBlockDevice)
187194
if err != nil {
188195
return provisioned, err
189196
}
190197
provisioned.ingress = append(provisioned.ingress, NodeDeets{id: nodeID})
191198
}
192199
for i = 0; i < nodeCount.Storage; i++ {
193-
nodeID, err := p.client.CreateNode(ami, aws.T2Medium)
200+
nodeID, err := p.client.CreateNode(ami, aws.T2Medium, addBlockDevice)
194201
if err != nil {
195202
return provisioned, err
196203
}
@@ -329,7 +336,7 @@ func packetClientFromEnv() (infrastructureProvisioner, bool) {
329336
return p, true
330337
}
331338

332-
func (p packetProvisioner) ProvisionNodes(nodeCount NodeCount, distro linuxDistro) (provisionedNodes, error) {
339+
func (p packetProvisioner) ProvisionNodes(nodeCount NodeCount, distro linuxDistro, _ ...string) (provisionedNodes, error) {
333340
var packetDistro packet.OS
334341
switch distro {
335342
case Ubuntu1604LTS:

‎pkg/ansible/clustercatalog.go

+4
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,10 @@ type ClusterCatalog struct {
6363

6464
DiagnosticsDirectory string `yaml:"diagnostics_dir"`
6565
DiagnosticsDateTime string `yaml:"diagnostics_date_time"`
66+
67+
DockerDirectLVMEnabled bool `yaml:"docker_direct_lvm_enabled"`
68+
DockerDirectLVMBlockDevicePath string `yaml:"docker_direct_lvm_block_device_path"`
69+
DockerDirectLVMDeferredDeletionEnabled bool `yaml:"docker_direct_lvm_deferred_deletion_enabled"`
6670
}
6771

6872
type NFSVolume struct {

‎pkg/install/execute.go

+6
Original file line numberDiff line numberDiff line change
@@ -697,6 +697,12 @@ func (ae *ansibleExecutor) buildClusterCatalog(p *Plan) (*ansible.ClusterCatalog
697697
cc.DockerRegistryPort = "8443"
698698
} // Else just use DockerHub
699699

700+
// Setup docker options
701+
cc.DockerDirectLVMEnabled = p.Docker.Storage.DirectLVM.Enabled
702+
if cc.DockerDirectLVMEnabled {
703+
cc.DockerDirectLVMBlockDevicePath = p.Docker.Storage.DirectLVM.BlockDevice
704+
cc.DockerDirectLVMDeferredDeletionEnabled = p.Docker.Storage.DirectLVM.EnableDeferredDeletion
705+
}
700706
if ae.options.RestartServices {
701707
cc.EnableRestart()
702708
}

‎pkg/install/plan.go

+3
Original file line numberDiff line numberDiff line change
@@ -218,4 +218,7 @@ var commentMap = map[string]string{
218218
"nfs": "A set of NFS volumes for use by on-cluster persistent workloads, managed by Kismatic.",
219219
"nfs_host": "The host name or ip address of an NFS server.",
220220
"mount_path": "The mount path of an NFS share. Must start with /",
221+
"direct_lvm": "Configure devicemapper in direct-lvm mode (RHEL/CentOS only).",
222+
"block_device": "Path to the block device that will be used for direct-lvm mode. This device will be wiped and used exclusively by docker.",
223+
"enable_deferred_deletion": "Set to true if you want to enable deferred deletion when using direct-lvm mode.",
221224
}

0 commit comments

Comments
 (0)