Skip to content

Commit dab5faa

Browse files
davidspekdavidspek
committed
Init
0 parents  commit dab5faa

40 files changed

+1192
-0
lines changed

‎.github/workflows/build-all.yml

+154
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,154 @@
1+
name: Build Notebook Images
2+
3+
on:
4+
push:
5+
branches: [ master ]
6+
tags:
7+
- 'v*.*.*'
8+
pull_request:
9+
branches:
10+
- 'master'
11+
12+
jobs:
13+
build-matrix:
14+
name: Create Build Matrix
15+
runs-on: ubuntu-latest
16+
steps:
17+
-
18+
name: Checkout
19+
id: set-matrix
20+
uses: actions/checkout@v2
21+
with:
22+
fetch-depth: 0 # No shallow clone, we need all history
23+
-
24+
name: Bump version and push tag
25+
if: github.event_name != 'pull_request'
26+
id: tag_version
27+
uses: mathieudutour/[email protected]
28+
with:
29+
github_token: ${{ secrets.GITHUB_TOKEN }}
30+
-
31+
name: Create a GitHub release
32+
if: github.event_name != 'pull_request'
33+
uses: actions/create-release@v1
34+
env:
35+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
36+
with:
37+
tag_name: ${{ steps.tag_version.outputs.new_tag }}
38+
release_name: Release ${{ steps.tag_version.outputs.new_tag }}
39+
body: ${{ steps.tag_version.outputs.changelog }}
40+
- name: generate matrix
41+
id: generate-matrix
42+
env:
43+
RELEASE: ${{ steps.tag_version.outputs.new_tag }}
44+
# run: echo "::set-output name=matrix::{\"include\":[$(for changed_folder in $(dirname $(git diff --name-only ${{ github.event.before }}..${{ github.event.after }}) | sort -u); do find $changed_folder -name "*Dockerfile"; done | sed 's/^\|$/"/g'|paste -sd, -)]}"
45+
run: |
46+
if [ -z "${RELEASE}" ];
47+
then
48+
RELEASE="pr"
49+
CHANGED_DIRS=$(git diff-tree --no-commit-id --name-only -r ${{ github.event.pull_request.head.sha }} | xargs -I {} dirname {})
50+
else
51+
CHANGED_DIRS=$(git diff-tree --no-commit-id --name-only -r ${{ github.sha }} | xargs -I {} dirname {})
52+
fi
53+
if [ -z "${CHANGED_DIRS}" ];
54+
then
55+
CHANGED_DIRS=$(dirname $(git diff --name-only ${{ github.event.before }}..${{ github.event.after }}))
56+
if [ "${CHANGED_DIRS}" == "." ];
57+
then
58+
CHANGED_DIRS=
59+
fi
60+
fi
61+
echo "${CHANGED_DIRS}"
62+
DOCKERFILES=$(for CHANGED_DIR in ${CHANGED_DIRS}; do find ${CHANGED_DIR} -name "*Dockerfile"; done | sort -u)
63+
echo "${DOCKERFILES}"
64+
MATRIX_PROJECTS_JSON="["
65+
MATRIX_INCLUDE_JSON="["
66+
for DOCKERFILE in ${DOCKERFILES}; do
67+
DIR=$(dirname ${DOCKERFILE})
68+
if [[ "$(basename ${DOCKERFILE})" == *"cuda"* ]]
69+
then
70+
MATRIX_PROJECTS_JSON+=$(sed 's/^/"/;s/$/"/' <<< "${DIR}"-cuda)
71+
PROJECT="${DIR}"-cuda
72+
else
73+
MATRIX_PROJECTS_JSON+=$(sed 's/^/"/;s/$/"/' <<< "${DIR}")
74+
PROJECT="${DIR}"
75+
fi
76+
echo "{$MATRIX_PROJECTS_JSON}"
77+
MATRIX_INCLUDE_JSON+="{\"path\": \"${DIR}\", \"project\": \"${PROJECT}\", \"dockerfile\": \"${DOCKERFILE}\", \"version\": \"${RELEASE}\"}"
78+
echo "${MATRIC_INCLUDE_JSON}"
79+
done
80+
echo "{$MATRIX_PROJECTS_JSON}"
81+
echo "${MATRIC_INCLUDE_JSON}"
82+
MATRIX_INCLUDE_JSON="${MATRIX_INCLUDE_JSON//\}\{/\}, \{}"
83+
MATRIX_INCLUDE_JSON+="]"
84+
MATRIX_PROJECTS_JSON="${MATRIX_PROJECTS_JSON//\"\"/\", \"}"
85+
MATRIX_PROJECTS_JSON+="]"
86+
MATRIX_JSON="{\"include\": ${MATRIX_INCLUDE_JSON}}"
87+
echo "${MATRIX_JSON}"
88+
89+
CONTINUE_DOCKER_JOB="no"
90+
if [[ "${MATRIX_PROJECTS_JSON}" != "[]" ]]; then
91+
CONTINUE_DOCKER_JOB="yes"
92+
fi
93+
94+
echo "${CONTINUE_DOCKER_JOB}"
95+
96+
echo "::set-output name=continue::${CONTINUE_DOCKER_JOB}"
97+
echo "::set-output name=matrix::${MATRIX_JSON}"
98+
outputs:
99+
matrix: ${{ steps.generate-matrix.outputs.matrix }}
100+
continue: ${{ steps.generate-matrix.outputs.continue }}
101+
build-images:
102+
if: needs.build-matrix.outputs.continue == 'yes'
103+
name: Build and push notebook images
104+
runs-on: ubuntu-latest
105+
needs: build-matrix
106+
strategy:
107+
matrix: ${{ fromJson(needs.build-matrix.outputs.matrix) }}
108+
steps:
109+
-
110+
name: Checkout
111+
uses: actions/checkout@v2
112+
-
113+
name: Get latest tag
114+
id: latest_tag
115+
uses: DavidSpek/[email protected]
116+
with:
117+
img: 'ghcr.io/pluralsh/kubeflow-notebooks-${{ matrix.project }}'
118+
-
119+
name: Docker meta
120+
id: meta
121+
uses: crazy-max/ghaction-docker-meta@v3
122+
with:
123+
# list of Docker images to use as base name for tags
124+
images: |
125+
ghcr.io/pluralsh/kubeflow-notebooks-${{ matrix.project }}
126+
# generate Docker tags based on the following events/attributes
127+
tags: |
128+
type=ref,event=branch
129+
type=ref,event=pr
130+
type=semver,pattern={{raw}},value=${{ matrix.project }}
131+
type=sha
132+
-
133+
name: Set up QEMU
134+
uses: docker/setup-qemu-action@v1
135+
-
136+
name: Set up Docker Buildx
137+
uses: docker/setup-buildx-action@v1
138+
-
139+
name: Login to GHCR
140+
if: github.event_name != 'pull_request'
141+
uses: docker/login-action@v1
142+
with:
143+
registry: ghcr.io
144+
username: ${{ github.repository_owner }}
145+
password: ${{ secrets.MY_GITHUB_TOKEN }}
146+
-
147+
name: Build and push
148+
uses: docker/build-push-action@v2
149+
with:
150+
context: ${{ matrix.path }}
151+
file: ${{ matrix.dockerfile }}
152+
push: ${{ github.event_name != 'pull_request' }}
153+
tags: ${{ steps.meta.outputs.tags }}
154+
labels: ${{ steps.meta.outputs.labels }}

‎README.md

+83
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
# Example Notebook Servers
2+
3+
> 🛑️️ These server images are provided as __examples only__ and are supported on a best-effort basis.
4+
> Contributions are greatly appreciated.
5+
6+
## Overview
7+
8+
In this folder, we have tried to make an extendable image structure which you can easily augment with additional tools and packages.
9+
10+
![flow-chart of kubeflow notebook server images](image-flow-chart.png)
11+
12+
__The following images are considered 'base' images, which you can extend:__
13+
14+
Name | Description
15+
--- | ---
16+
[./base](./base) | the common base for all other images
17+
[./jupyter](./jupyter) | the base [JupyterLab](https://github.com/jupyterlab/jupyterlab) image
18+
[./codeserver](./codeserver) | the base [code-server](https://github.com/cdr/code-server) (Visual Studio Code) image
19+
[./rstudio](./rstudio) | the base [RStudio](https://github.com/rstudio/rstudio) image
20+
21+
__Important points about the images:__
22+
23+
- they make use of the [s6-overlay](https://github.com/just-containers/s6-overlay) init system
24+
- they all run as the non-root `jovyan` user
25+
26+
## How do I extend these images?
27+
28+
> ⚠️ any changes made by users __after spawning__ a Kubeflow notebook will only last the lifetime of the pod, unless they are installed into a PVC-backed directory
29+
30+
### Adding conda/pip packages
31+
32+
Extend one of the base images and install any `pip` or `conda` packages your Kubeflow Notebook users are likely to need.
33+
34+
As a guide, look at [jupyter-pytorch-full.cpu](./jupyter-pytorch-full/cpu.Dockerfile) for a `pip install ...` example, and the [rstudio-tidyverse](./rstudio-tidyverse/Dockerfile) for `conda install ...`.
35+
36+
__WARNING:__ a common cause of errors is users running `pip install --user ...`, causing the home-directory (which is backed by a PVC) to contain a different or incompatible version of a package contained in `/opt/conda/...`
37+
38+
### Adding apt-get packages
39+
40+
Extend one of the base images and install any `apt-get` packages your Kubeflow Notebook users are likely to need.
41+
42+
__WARNING:__ ensure you swap to `root` in the Dockerfile before running `apt-get`, and swap back to `jovyan` after.
43+
44+
### Adding container startup scripts
45+
46+
Some use-cases might require custom scripts to run during the startup of the Notebook Server container, or advanced users might want to add additional services that run inside the container (for example, an Apache or NGINX web server).
47+
To make this easy, we use the [s6-overlay](https://github.com/just-containers/s6-overlay).
48+
49+
The [s6-overlay](https://github.com/just-containers/s6-overlay) differs from other init systems, such as the popular [tini](https://github.com/krallin/tini).
50+
While `tini` was created to handle a single process running in a container as PID 1, the `s6-overlay` is built to manage multiple processes and allows the creator of the image to determine which process failures should silently restart, and which should cause the container to exit.
51+
52+
__Custom startup scripts:__
53+
54+
Scripts that need to run during the startup of the container can be placed in `/etc/cont-init.d/`, and are executed in ascending alphanumeric order.
55+
56+
An example of a startup script can be found in [./rstudio/s6/cont-init.d/02-rstudio-env-fix](./rstudio/s6/cont-init.d/02-rstudio-env-fix).
57+
This script uses the [with-contenv](https://github.com/just-containers/s6-overlay#container-environment) helper so that environment variables (passed to container) are available in the script.
58+
The purpose of this script is to snapshot any `KUBERNETES_*` environment variables into the `Renviron.site` at pod startup, as without these variables `kubectl` does not work.
59+
60+
__Custom service scripts:__
61+
62+
Extra services to be monitored by `s6-overlay` should be placed in their own folder under `/etc/services.d/` containing a script called `run` and optionally a finishing script `finish`.
63+
64+
An example of a service can be found in [jupyter/s6/services.d/jupyterlab](jupyter/s6/services.d/jupyterlab) which is used to start JupyterLab itself.
65+
For more information about the `run` and `finish` scripts, please see the [s6-overlay documentation](https://github.com/just-containers/s6-overlay#writing-a-service-script).
66+
67+
__WARNING:__ our example images run `s6-overlay` as `$NB_USER` not `root`, meaning any files or scripts related to `s6-overlay` should be owned by the `$NB_USER` user
68+
69+
There may be cases when you need to run a service as root.
70+
To do this, you can change the Dockerfile to have `USER root` at the end, and then use `s6-setuidgid` to run the user-facing services as `$NB_USER`.
71+
72+
For example, here is a `run` script for `code-server`:
73+
74+
```bash
75+
#!/usr/bin/with-contenv bash
76+
77+
export SHELL='/bin/bash'
78+
exec s6-setuidgid $NB_USER \
79+
/usr/local/bin/code-server \
80+
--bind-addr 0.0.0.0:8888 \
81+
--disable-telemetry \
82+
--auth none
83+
```

‎base/Dockerfile

+76
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
FROM ubuntu:20.04
2+
3+
# common environemnt variables
4+
ENV NB_USER jovyan
5+
ENV NB_UID 1000
6+
ENV NB_PREFIX /
7+
ENV HOME /home/$NB_USER
8+
ENV SHELL /bin/bash
9+
10+
# args - software versions
11+
ARG KUBECTL_ARCH="amd64"
12+
ARG KUBECTL_VERSION=v1.21.0
13+
ARG S6_ARCH="amd64"
14+
# renovate: datasource=github-tags depName=just-containers/s6-overlay versioning=loose
15+
ARG S6_VERSION=v2.2.0.3
16+
17+
# set shell to bash
18+
SHELL ["/bin/bash", "-c"]
19+
20+
# install - usefull linux packages
21+
RUN export DEBIAN_FRONTEND=noninteractive \
22+
&& apt-get -yq update \
23+
&& apt-get -yq upgrade \
24+
&& apt-get -yq install --no-install-recommends \
25+
apt-transport-https \
26+
bash \
27+
bzip2 \
28+
ca-certificates \
29+
curl \
30+
git \
31+
gnupg \
32+
gnupg2 \
33+
locales \
34+
lsb-release \
35+
nano \
36+
software-properties-common \
37+
tzdata \
38+
unzip \
39+
vim \
40+
wget \
41+
zip \
42+
&& apt-get clean \
43+
&& rm -rf /var/lib/apt/lists/*
44+
45+
# install - s6 overlay
46+
RUN export GNUPGHOME=/tmp/ \
47+
&& curl -sL "https://github.com/just-containers/s6-overlay/releases/download/${S6_VERSION}/s6-overlay-${S6_ARCH}-installer" -o /tmp/s6-overlay-${S6_VERSION}-installer \
48+
&& curl -sL "https://github.com/just-containers/s6-overlay/releases/download/${S6_VERSION}/s6-overlay-${S6_ARCH}-installer.sig" -o /tmp/s6-overlay-${S6_VERSION}-installer.sig \
49+
&& gpg --keyserver keys.gnupg.net --keyserver pgp.surfnet.nl --recv-keys 6101B2783B2FD161 \
50+
&& gpg -q --verify /tmp/s6-overlay-${S6_VERSION}-installer.sig /tmp/s6-overlay-${S6_VERSION}-installer \
51+
&& chmod +x /tmp/s6-overlay-${S6_VERSION}-installer \
52+
&& /tmp/s6-overlay-${S6_VERSION}-installer / \
53+
&& rm /tmp/s6-overlay-${S6_VERSION}-installer.sig /tmp/s6-overlay-${S6_VERSION}-installer
54+
55+
# install - kubectl
56+
RUN curl -sL "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/${KUBECTL_ARCH}/kubectl" -o /usr/local/bin/kubectl \
57+
&& curl -sL "https://dl.k8s.io/${KUBECTL_VERSION}/bin/linux/${KUBECTL_ARCH}/kubectl.sha256" -o /tmp/kubectl.sha256 \
58+
&& echo "$(cat /tmp/kubectl.sha256) /usr/local/bin/kubectl" | sha256sum --check \
59+
&& rm /tmp/kubectl.sha256 \
60+
&& chmod +x /usr/local/bin/kubectl
61+
62+
# create user and set required ownership
63+
RUN useradd -M -s /bin/bash -N -u ${NB_UID} ${NB_USER} \
64+
&& mkdir -p ${HOME} \
65+
&& chown -R ${NB_USER}:users ${HOME} \
66+
&& chown -R ${NB_USER}:users /usr/local/bin \
67+
&& chown -R ${NB_USER}:users /etc/s6
68+
69+
# set locale configs
70+
RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen \
71+
&& locale-gen
72+
ENV LANG en_US.UTF-8
73+
ENV LANGUAGE en_US.UTF-8
74+
ENV LC_ALL en_US.UTF-8
75+
76+
USER $NB_UID

‎codeserver-python/Dockerfile

+70
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
FROM kubeflownotebooks/codeserver:v0.3.53
2+
3+
USER root
4+
5+
# args - software versions
6+
ARG CODESERVER_PYTHON_VERSION=2021.5.842923320
7+
ARG CODESERVER_GITLENS_VERSION=11.4.1
8+
ARG CODESERVER_GIT_GRAPH_VERSION=1.30.0
9+
ARG MINIFORGE_ARCH="x86_64"
10+
# renovate: datasource=github-tags depName=conda-forge/miniforge versioning=loose
11+
ARG MINIFORGE_VERSION=4.10.3-5
12+
ARG PIP_VERSION=21.1.2
13+
ARG PYTHON_VERSION=3.8.10
14+
15+
# update - ensure apt packages are always updated
16+
RUN export DEBIAN_FRONTEND=noninteractive \
17+
&& apt-get -yq update \
18+
&& apt-get -yq upgrade \
19+
&& apt-get clean \
20+
&& rm -rf /var/lib/apt/lists/*
21+
22+
# setup environment for conda
23+
ENV CONDA_DIR /opt/conda
24+
ENV PATH "${CONDA_DIR}/bin:${PATH}"
25+
RUN mkdir -p ${CONDA_DIR} \
26+
&& chown -R ${NB_USER}:users ${CONDA_DIR}
27+
28+
USER $NB_UID
29+
30+
# install - conda, pip, python
31+
RUN curl -sL "https://github.com/conda-forge/miniforge/releases/download/${MINIFORGE_VERSION}/Miniforge3-${MINIFORGE_VERSION}-Linux-${MINIFORGE_ARCH}.sh" -o /tmp/Miniforge3.sh \
32+
&& curl -sL "https://github.com/conda-forge/miniforge/releases/download/${MINIFORGE_VERSION}/Miniforge3-${MINIFORGE_VERSION}-Linux-${MINIFORGE_ARCH}.sh.sha256" -o /tmp/Miniforge3.sh.sha256 \
33+
&& echo "$(cat /tmp/Miniforge3.sh.sha256 | awk '{ print $1; }') /tmp/Miniforge3.sh" | sha256sum --check \
34+
&& rm /tmp/Miniforge3.sh.sha256 \
35+
&& /bin/bash /tmp/Miniforge3.sh -b -f -p ${CONDA_DIR} \
36+
&& rm /tmp/Miniforge3.sh \
37+
&& conda config --system --set auto_update_conda false \
38+
&& conda config --system --set show_channel_urls true \
39+
&& echo "conda ${MINIFORGE_VERSION:0:-2}" >> ${CONDA_DIR}/conda-meta/pinned \
40+
&& echo "python ${PYTHON_VERSION}" >> ${CONDA_DIR}/conda-meta/pinned \
41+
&& conda install -y -q \
42+
python=${PYTHON_VERSION} \
43+
conda=${MINIFORGE_VERSION:0:-2} \
44+
pip=${PIP_VERSION} \
45+
&& conda update -y -q --all \
46+
&& conda clean -a -f -y \
47+
&& chown -R ${NB_USER}:users ${CONDA_DIR} \
48+
&& chown -R ${NB_USER}:users ${HOME}
49+
50+
# install - requirements.txt
51+
COPY --chown=jovyan:users requirements.txt /tmp
52+
RUN python3 -m pip install -r /tmp/requirements.txt --quiet --no-cache-dir \
53+
&& rm -f /tmp/requirements.txt \
54+
&& chown -R ${NB_USER}:users ${CONDA_DIR} \
55+
&& chown -R ${NB_USER}:users ${HOME}
56+
57+
# install - codeserver extensions
58+
RUN code-server --install-extension "ms-python.python@${CODESERVER_PYTHON_VERSION}" \
59+
&& code-server --install-extension "eamodio.gitlens@${CODESERVER_GITLENS_VERSION}" \
60+
&& code-server --install-extension "mhutchie.git-graph@${CODESERVER_GIT_GRAPH_VERSION}"
61+
62+
# s6 - copy scripts
63+
COPY --chown=jovyan:users s6/ /etc
64+
65+
# s6 - 01-copy-tmp-home
66+
USER root
67+
RUN mkdir -p /tmp_home \
68+
&& cp -r ${HOME} /tmp_home \
69+
&& chown -R ${NB_USER}:users /tmp_home
70+
USER ${NB_UID}

‎codeserver-python/requirements.txt

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
kfp==1.7.2
2+
kfp-server-api==1.7.0
3+
kfserving==0.6.0

‎codeserver-python/s6/cont-init.d/02-conda-init

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
#!/usr/bin/with-contenv bash
2+
conda init bash
3+
conda activate base

0 commit comments

Comments
 (0)