Skip to content

Commit feb6352

Browse files
udimudim
committed
Run website precommits in a Docker container.
Build artifacts have been moved under build/website.
1 parent 15e2595 commit feb6352

8 files changed

+98
-87
lines changed

‎.test-infra/jenkins/CommonJobProperties.groovy

+3-13
Original file line numberDiff line numberDiff line change
@@ -63,16 +63,14 @@ class CommonJobProperties {
6363
String branch = 'master',
6464
int timeout = 100,
6565
boolean allowRemotePoll = true,
66-
boolean localPerfTest = false,
67-
String dockerImage = '') {
66+
boolean localPerfTest = false) {
6867
setTopLevelJobProperties(
6968
context,
7069
'beam',
7170
branch,
7271
timeout,
7372
allowRemotePoll,
74-
localPerfTest,
75-
dockerImage)
73+
localPerfTest)
7674
}
7775

7876
// Sets common top-level job properties. Accessed through one of the above
@@ -82,8 +80,7 @@ class CommonJobProperties {
8280
String defaultBranch,
8381
int defaultTimeout,
8482
boolean allowRemotePoll = true,
85-
boolean localPerfTest = false,
86-
String dockerImage='') {
83+
boolean localPerfTest = false) {
8784
def jenkinsExecutorLabel = 'beam'
8885
if (localPerfTest) {
8986
jenkinsExecutorLabel = 'beam-perf'
@@ -118,13 +115,6 @@ class CommonJobProperties {
118115
}
119116

120117
context.wrappers {
121-
if (dockerImage) {
122-
throw new UnsupportedOperationException(
123-
'[INFRA-16524] Running jobs within Docker not yet supported.')
124-
// buildInDocker {
125-
// image(dockerImage)
126-
// }
127-
}
128118
// Abort the build if it's stuck for more minutes than specified.
129119
timeout {
130120
absolute(defaultTimeout)

‎.test-infra/jenkins/PrecommitJobBuilder.groovy

+1-5
Original file line numberDiff line numberDiff line change
@@ -26,9 +26,6 @@ class PrecommitJobBuilder {
2626
/** Base name for each post-commit suite job, i.e. 'Go'. */
2727
String nameBase
2828

29-
/** The Docker image to run within, or blank to not use Docker. */
30-
String dockerImage
31-
3229
/** The Gradle task to execute. */
3330
String gradleTask
3431

@@ -99,8 +96,7 @@ class PrecommitJobBuilder {
9996
commonJobProperties.setTopLevelMainJobProperties(delegate,
10097
'master',
10198
timeoutMins,
102-
allowRemotePoll, // needed for included regions PR triggering; see [JENKINS-23606]
103-
dockerImage)
99+
allowRemotePoll) // needed for included regions PR triggering; see [JENKINS-23606]
104100
steps {
105101
gradle {
106102
rootBuildScriptDir(commonJobProperties.checkoutDir)

‎.test-infra/jenkins/WebsiteShared.groovy

+1
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
* limitations under the License.
1717
*/
1818

19+
// TODO(BEAM-4505): Remove this file once apache/beam-site jobs stop using it.
1920

2021
@groovy.transform.Field static final String install_ruby_and_gems_bash = '''
2122
maxKeyFetchAttempts=5

‎.test-infra/jenkins/job_PreCommit_Website.groovy

+1-3
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,7 @@ import PrecommitJobBuilder
2121
PrecommitJobBuilder builder = new PrecommitJobBuilder(
2222
scope: this,
2323
nameBase: 'Website',
24-
// TODO(BEAM-4696): Run these within docker so we don't need install step
25-
// dockerImage: 'ruby:2.5',
26-
gradleTask: ':setupJenkins :websitePreCommit',
24+
gradleTask: ':websitePreCommit',
2725
triggerPathPatterns: ['^website/.*$']
2826
)
2927
builder.build()

‎build.gradle

-5
Original file line numberDiff line numberDiff line change
@@ -141,12 +141,7 @@ rat {
141141
// Ruby build files
142142
"**/Gemfile.lock",
143143
"**/Rakefile",
144-
"**/_config.yml",
145-
"**/_config_test.yml",
146144
"**/.htaccess",
147-
"website/.bundle/**/*",
148-
"website/.sass-cache/**/*",
149-
"website/.testcontent/**/*",
150145
"website/src/_sass/_bootstrap.scss",
151146
"website/src/_sass/bootstrap/**/*",
152147
"website/src/js/bootstrap*.js",

‎website/Dockerfile

+33
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
###############################################################################
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing, software
13+
# distributed under the License is distributed on an "AS IS" BASIS,
14+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
# See the License for the specific language governing permissions and
16+
# limitations under the License.
17+
###############################################################################
18+
19+
# This image contains Ruby and dependencies required to build and test the Beam
20+
# website. It is used by tasks in build.gradle.
21+
22+
FROM ruby:2.5
23+
24+
WORKDIR /ruby
25+
RUN gem install bundler
26+
# Update buildDockerImage's inputs.files if you change this list.
27+
ADD Gemfile Gemfile.lock /ruby/
28+
RUN bundle install --deployment --path $GEM_HOME
29+
30+
# Required for website testing using HTMLProofer.
31+
ENV LC_ALL C.UTF-8
32+
33+
CMD sleep 3600

‎website/build.gradle

+59-25
Original file line numberDiff line numberDiff line change
@@ -19,47 +19,81 @@
1919
// Define common lifecycle tasks and artifact types
2020
apply plugin: "base"
2121

22-
task cleanWebsite(type: Delete) {
23-
delete '.sass-cache', 'vendor', '.testcontent', 'content'
22+
def dockerImageTag = 'beam-website'
23+
def dockerWorkDir = "/repo"
24+
def buildDir = "$project.rootDir/build/website"
25+
26+
task buildDockerImage(type: Exec) {
27+
inputs.files 'Gemfile', 'Gemfile.lock'
28+
commandLine 'docker', 'build', '-t', dockerImageTag, '.'
2429
}
25-
clean.dependsOn cleanWebsite
2630

27-
task installBundler(type: Exec) {
28-
commandLine './install_bundler.sh'
31+
task createDockerContainer(type: Exec) {
32+
dependsOn buildDockerImage
33+
standardOutput = new ByteArrayOutputStream()
34+
ext.containerId = {
35+
return standardOutput.toString().trim()
36+
}
37+
commandLine '/bin/bash', '-c',
38+
"docker create -v $project.rootDir:$dockerWorkDir -u \$(id -u):\$(id -g) $dockerImageTag"
2939
}
3040

31-
task installGems(type: Exec) {
32-
mustRunAfter cleanWebsite, installBundler
33-
inputs.file 'Gemfile.lock'
34-
outputs.dir 'vendor'
35-
commandLine 'bundle', 'install', '--deployment'
41+
task startDockerContainer(type: Exec) {
42+
dependsOn createDockerContainer
43+
ext.containerId = {
44+
return createDockerContainer.containerId()
45+
}
46+
commandLine 'docker', 'start',
47+
"${->createDockerContainer.containerId()}" // Lazily evaluate containerId.
3648
}
3749

38-
// TODO(BEAM-4696): Remove this once we can run Jenkins jobs within Docker
39-
task setupJenkins {
40-
dependsOn installBundler, installGems
50+
task stopAndRemoveDockerContainer(type: Exec) {
51+
commandLine 'docker', 'rm', '-f', "${->createDockerContainer.containerId()}"
4152
}
4253

54+
task setupBuildDir(type: Copy) {
55+
from('.') {
56+
include 'Gemfile*'
57+
include 'Rakefile'
58+
}
59+
into buildDir
60+
}
61+
62+
task cleanWebsite(type: Delete) {
63+
delete buildDir
64+
}
65+
clean.dependsOn cleanWebsite
66+
4367
task buildWebsite(type: Exec) {
44-
dependsOn installGems
45-
mustRunAfter cleanWebsite, setupJenkins
68+
dependsOn startDockerContainer, setupBuildDir
69+
finalizedBy stopAndRemoveDockerContainer
4670
inputs.files 'Gemfile.lock', '_config.yml'
4771
inputs.dir 'src'
48-
outputs.dir '.sass-cache'
49-
outputs.dir 'content'
50-
outputs.dir '.testcontent'
51-
commandLine 'bundle', 'exec', 'jekyll', 'build',
52-
'--config', '_config.yml',
53-
'--incremental'
72+
outputs.dir "$buildDir/.sass-cache"
73+
outputs.dir "$buildDir/content"
74+
commandLine 'docker', 'exec',
75+
"${->startDockerContainer.containerId()}", '/bin/bash', '-c',
76+
"""cd $dockerWorkDir/build/website && \
77+
bundle exec jekyll build \
78+
--config $dockerWorkDir/website/_config.yml \
79+
--incremental \
80+
--source $dockerWorkDir/website/src
81+
"""
5482
}
5583
build.dependsOn buildWebsite
5684

5785
task testWebsite(type: Exec) {
58-
dependsOn buildWebsite
59-
inputs.files 'Gemfile.lock', 'Rakefile'
60-
inputs.dir '.testcontent'
61-
commandLine 'bundle', 'exec', 'rake', 'test'
86+
dependsOn startDockerContainer, buildWebsite
87+
finalizedBy stopAndRemoveDockerContainer
88+
89+
inputs.files "$buildDir/Rakefile"
90+
inputs.dir "$buildDir/content"
91+
commandLine 'docker', 'exec',
92+
"${->startDockerContainer.containerId()}", '/bin/bash', '-c',
93+
"""cd $dockerWorkDir/build/website && \
94+
bundle exec rake test"""
6295
}
96+
6397
check.dependsOn testWebsite
6498

6599
task preCommit {

‎website/install_bundler.sh

-36
This file was deleted.

0 commit comments

Comments
 (0)