Clean up ansible playbooks

- remove duplication
- correctly tag for more convenient usage
- standardize nodejs install and upgrade both web/shell to 14
- update wetty to maintainted verstion

ansible/README.md

@@ -12,3 +12,18 @@ run the picoCTF platform. By using the same playbooks across the board we
 achieve a robust, repeatable, and consistent experience across development and
 production.  Additionally this allows the picoCTF platform to be deployed in
 a wide variety of configurations with minor configuration changes.
+
+## Tags
+
+Tags are a convenient way of only running some tasks from the overall playbook.
+Some common tags:
+
+
+- `dependency`: runs dependencies
+- `sync`: syncs source code
+- `web`
+  - `web-api`
+  - `web-static`
+- `shell`
+  - `shell-api`
+  - `deploy-all`

ansible/common/tasks/apt_update.yml

@@ -0,0 +1,8 @@
+---
+
+- name: Run apt-get update
+  apt:
+    update_cache: yes
+    cache_valid_time: 3600
+  environment:
+    DEBIAN_FRONTEND: noninteractive

ansible/common/tasks/upgrade.yml → ansible/common/tasks/apt_upgrade.yml

@@ -1,11 +1,7 @@
 ---
 # Playbook that does a general system wide upgrade
-- name: Run apt-get update
-  apt:
-    update_cache: yes
-    cache_valid_time: 3600
-  environment:
-    DEBIAN_FRONTEND: noninteractive
+
+- include: apt_update.yml
 
 - name: Run apt-get dist-upgrade
   apt:

ansible/common/tasks/main.yml

@@ -1,40 +1,20 @@
 ---
 # Playbook that runs tasks common across all servers
 
-- include: upgrade.yml
+- include: apt_upgrade.yml
   tags:
     - dependency
-    - network
 
 - include: utils.yml
   tags:
     - dependency
-    - network
 
 - include: set_hostname.yml
 
-# Local development environments are the only place the platform should
-# be loaded in directly via filesystem sync without being cloned from a
-# specific source and branch
-- include: clone_repo.yml
-  when: "'git' in sync_mode" # coule be git or git_private
+- include: sync_source.yml
   tags:
-    - network
-
-# Source should be synchronized over from a local directory on the control machine
-- name: Synchronize picoCTF source code
-  synchronize:
-    src: "{{pico_src_dir}}"
-    dest: "{{pico_base_dir}}"
-    archive: no
-    recursive: yes
-  when: "'source' in sync_mode"
-
-- name: Ensure admin user owns picoCTF directory
-  file:
-    path: "{{ pico_base_dir}}"
-    owner: "{{ ansible_user }}"
-    group: "{{ ansible_user }}"
-    state: directory
-    recurse: yes
-  when: ('git' in sync_mode) or ('source' in sync_mode)
+    - sync
+    - web-api
+    - web-static
+    - shell-api
+    - deploy-all

ansible/common/tasks/sync_source.yml

@@ -0,0 +1,25 @@
+
+# Local development environments are the only place the platform should
+# be loaded in directly via filesystem sync without being cloned from a
+# specific source and branch
+- include: clone_repo.yml
+  when: "'git' in sync_mode" # coule be git or git_private
+
+# Source should be synchronized over from a local directory on the control machine
+- name: Synchronize picoCTF source code
+  synchronize:
+    src: "{{pico_src_dir}}"
+    dest: "{{pico_base_dir}}"
+    archive: no
+    recursive: yes
+    delete: yes
+  when: "'source' in sync_mode"
+
+- name: Ensure admin user owns picoCTF directory
+  file:
+    path: "{{ pico_base_dir}}"
+    owner: "{{ ansible_user }}"
+    group: "{{ ansible_user }}"
+    state: directory
+    recurse: yes
+  when: ('git' in sync_mode) or ('source' in sync_mode)

ansible/common/tasks/utils.yml

@@ -1,13 +1,6 @@
 ---
 # Playbook that installs base utilities we would want on any server
 
-- name: Update apt
-  apt:
-    update_cache: yes
-    cache_valid_time: 3600
-  environment:
-    DEBIAN_FRONTEND: noninteractive
-
 - name: Install base packages
   apt:
     state: present

ansible/mongodb/tasks/install.yml

@@ -12,18 +12,13 @@
     repo: "{{ mongodb_repo }}"
     state: present
 
-- name: Update apt
-  apt:
-    update_cache: yes
-  environment:
-    DEBIAN_FRONTEND: noninteractive
-
 - name: Install mongo
   apt:
     name: [
       'mongodb-org', # from PPA
     ]
     state: present
+    update_cache: yes
   environment:
     DEBIAN_FRONTEND: noninteractive
 
@@ -46,6 +41,6 @@
 - name: Ensure redis is running and started on boot
   service:
     name: redis-server
-    state: restarted
+    state: started
     enabled: yes
   when: not aws_elasticache

ansible/mongodb/tasks/main.yml

@@ -8,7 +8,6 @@
 - include: install.yml
   tags:
     - dependency
-    - network
 
 - include: configure_storage.yml
 

ansible/pico-web/tasks/nodejs.yml → ansible/nodejs/tasks/main.yml

@@ -8,43 +8,30 @@
     path: /etc/apt/sources.list.d/nodesource.list
   register: nodejs
 
-- name: Download nodejs 10.x setup script (LTS)
+# LTS until 2023-04-30
+# https://nodejs.org/en/about/releases/
+- name: Download nodejs 14.x setup script (LTS)
   get_url:
-    url: "https://deb.nodesource.com/setup_10.x"
+    url: "https://deb.nodesource.com/setup_14.x"
     dest: "/tmp/node_setup.sh"
     mode: 0700
+  when: nodejs.stat.exists == False
 
 # Since shell commands are not idempotent we manually
 # check above if already have the nodejs repo (implying
-# all nodejs steps where previously successful)
+# all nodejs steps were previously successful)
 - name: Setup nodejs repositories
   shell: "/tmp/node_setup.sh"
   become: yes
   when: nodejs.stat.exists == False
 
-- name: Update apt cache
-  apt:
-    update_cache: yes
-  environment:
-    DEBIAN_FRONTEND: noninteractive
-
 - name: Install nodejs and dependencies
   apt:
     name: [
       'build-essential',
       'nodejs',
       ]
     state: latest
+    update_cache: yes
   environment:
     DEBIAN_FRONTEND: noninteractive
-
-# Extracted from picoCTF-platform/scripts/web_setup.sh
-- name: Install nodejs packages (globally)
-  npm:
-    name: "{{ item }}"
-    global: yes
-  with_items:
-    - "@babel/core@^7.0.0"
-    - "@babel/cli@^7.0.0"
-    - "@babel/preset-env@^7.0.0"
-    - "@babel/preset-react@^7.0.0"

ansible/pico-shell/defaults/main.yml

@@ -9,7 +9,7 @@ virtualenv_dir    : "/picoCTF-env"
 ###
 # wetty configuration
 ###
-wetty_js_path     : "/opt/wetty/node_modules/wetty.js"
+wetty_js_path     : "/opt/wetty/node_modules/wetty"
 
 # configure nginx
 site_config_name  : "shell"
@@ -19,6 +19,7 @@ wetty_server_path : "http://127.0.0.1:{{ wetty_port }}/"
 # problem settings
 deb_dir           : "/picoCTF-debs"
 bundle_dir        : "/picoCTF-bundles"
+deploy_number     : 2
 
 # pam module settings
 web_address_internal       : "http://{{ hostvars['web']['ansible_host'] }}"

ansible/pico-shell/tasks/dependencies.yml

@@ -1,26 +1,26 @@
 ---
 # Playbook that installs the dependencies for the picoCTF-shell server
 
-- name: Update apt
-  apt:
-    update_cache: yes
-    cache_valid_time: 3600
-  environment:
-    DEBIAN_FRONTEND: noninteractive
+# static website frontend build dependencies
+- include_role:
+    name: nodejs
 
-# Add repositories for NodeJS and yarn
-- name: Add nodeJS and yarn repository/keys
-  shell: |
-    curl -sL https://deb.nodesource.com/setup_11.x | sudo -E bash -
-    curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add -
-    echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list
-  args:
-    warn: False
+- name: Add an apt signing key for yarn
+  apt_key:
+    url: https://dl.yarnpkg.com/debian/pubkey.gpg
+    state: present
+
+- name: Add yarn ppa
+  lineinfile:
+    path: /etc/apt/sources.list.d/yarn.list
+    line: "deb https://dl.yarnpkg.com/debian/ stable main"
+    create: yes
 
 # Extracted from picoCTF-platform/scripts/shell_setup.sh
 - name: Install picoCTF-shell system dependencies
   apt:
     state: present
+    update_cache: yes
     name: [
       'dpkg-dev',
       'dpkg',
@@ -180,16 +180,14 @@
         'ptpython',
         'pwntools',
         ]
+# pwntools is not ready for python3; skipping to avoid confusion
 
-- name: Install common pip3 packages for CTF shell servers
+# Fix "Invalid environment marker: platform_python_implementation != 'PyPy'"
+- name: Upgrade setuptools
   pip:
-    name: [
-        'ipython',
-        'ptpython',
-        'pwntools',
-        ]
-    executable: pip3
-  when: False # pwntools is not ready for python3; skipping to avoid confusion
+    name: "setuptools"
+    virtualenv: "{{ virtualenv_dir }}"
+    extra_args: "--upgrade"
 
 - name: Ensure wetty directory exists
   file:
@@ -198,12 +196,12 @@
     owner: root
     group: root
 
-- name: Install wetty.js
+- name: Install wetty
   npm:
+    name: wetty
     executable: /usr/bin/npm --unsafe-perm=true
-    name: wetty.js
-    global: no
     production: yes
+    global: no
     path: /opt/wetty
 
 - name: Install quota

ansible/pico-shell/tasks/deploy_problems.yml

@@ -14,7 +14,7 @@
   with_items: "{{ problem_source_dirs }}"
 
 - name: Deploy problems
-  command: "{{ virtualenv_dir }}/bin/shell_manager deploy -n 2 all"
+  command: "{{ virtualenv_dir }}/bin/shell_manager deploy -n {{deploy_number}} all"
 
 - name: Install bundles
   command: "{{ virtualenv_dir }}/bin/shell_manager install-bundle {{ item }}"

ansible/pico-shell/tasks/main.yml

@@ -2,43 +2,22 @@
 # Playbook that installs and configures picoCTF-shell servers
 
 - include: dependencies.yml
+  tags:
+    - dependency
+
+- include: shell_manager-config.yml
 
 - include: shell_manager.yml
+  tags:
+    - shell-api
 
 - include_role:
     name: nginx
 
 - include: pam_and_services.yml
 
 - include: deploy_problems.yml
+  tags:
+    deploy-all
 
 - include: permissions.yml
-
-- name: Hidepid
-  command: mount -o remount,hidepid=2 /proc
-
-- name: Hidepid persistence
-  command: echo "proc   /proc   proc  defaults,hidepid=2  0   0"  >> /etc/fstab
-
-- name: Copy over 99-motd
-  copy:
-    src: 99-motd
-    dest: /etc/update-motd.d/99-motd
-    owner: root
-    group: root
-    mode: 0755
-
-- name: Disable inter-user messages by default
-  lineinfile:
-    path: /etc/profile
-    line: mesg n
-
-- name: Remove /var/log/wtmp
-  file:
-    path: /var/log/wtmp
-    state: absent
-
-- name: Restrict /var/run/utmp
-  file:
-    path: /var/run/utmp
-    mode: go-r