Improve env_prod automation

- minor fixes and clarifications in terrafrom files. thanks @archang31 - restrict character set in gen_vault to prevent issues with escaping - remove run_py_in_venv.sh and update playbooks, prevents hacky arg passing.
hi-liang · Jun 5, 2020 · 18e01de · 18e01de
1 parent d5a1915
commit 18e01de
Show file tree

Hide file tree

Showing 12 changed files with 69 additions and 69 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,7 +1,7 @@
 **/*-console.log
 **/.terraform
 /.vagrant
-**/terraform.tfstate*
+*terraform.tfstate*
 /dist
 **/*.egg-info
 **/.pytest_cache

diff --git a/ansible/nginx/tasks/main.yml b/ansible/nginx/tasks/main.yml
@@ -39,7 +39,8 @@
 # referenced keys exist.
 - name: Request certificate (certbot)
   vars:
-    certbot_domain: "{{nginx_server_name}}"
+    certbot_domain : "{{ nginx_server_name }}"
+    certbot_email  : "{{ admin_email }}"
   include_role:
     name: certbot
     tasks_from: request_certificates

diff --git a/ansible/pico-web/tasks/auto_add_web_user.yml b/ansible/pico-web/tasks/auto_add_web_user.yml
@@ -2,8 +2,8 @@
 # Task list to add a user to the web site
 
 - name: "Run add-web-user.py to create user ({{ web_user }})"
-  command: >
-    {{scripts_dir}}/run_py_in_venv.sh {{virtualenv_dir}}
-    {{scripts_dir}}/add-web-user.py
-    -u {{ web_user }} -p {{ web_pw }} -e {{ web_email }}
-
+  shell:
+    cmd: >
+      . {{virtualenv_dir}}/bin/activate;
+      {{scripts_dir}}/add-web-user.py
+      -u '{{ web_user }}' -p '{{ web_pw }}' -e '{{ web_email }}'
diff --git a/ansible/pico-web/tasks/auto_configure.yml b/ansible/pico-web/tasks/auto_configure.yml
@@ -8,6 +8,7 @@
     web_pw    : "{{ web_admin_pw }}"
     web_email : "{{ admin_email }}"
   when: auto_add_web_admin
+  tags: ["XXX"]
 
 - name: Autoload Shell Server into web interface
   script: >

diff --git a/ansible/tasks/util_ssh_keyscan.yml b/ansible/tasks/util_ssh_keyscan.yml
@@ -2,6 +2,11 @@
 # Runs ssh-keyscan to add a host to SSH known host if if does not exist.
 # This is nessecary the first time after creating a new machine.
 
+- name: Remove any prior ssh host key
+  become: false
+  local_action:
+    module: command ssh-keygen -R '{{ ansible_host }}'
+
 # reference: https://gist.github.com/shirou/6928012
 - name: Ensure ssh host key known
   become: false
@@ -10,4 +15,4 @@
     dest: ~/.ssh/known_hosts
     create: yes
     state: present
-    line: "{{ lookup('pipe', 'ssh-keyscan -trsa ' + ansible_host ) }}"
+    line: "{{ lookup('pipe', 'ssh-keyscan -trsa -H ' + ansible_host ) }}"
diff --git a/env_prod/gen_vault.py b/env_prod/gen_vault.py
@@ -58,7 +58,8 @@
 
 def gen_random_string(n=24):
     """Returns a random n-length string, suitable for a password or random secret"""
-    char_set = string.ascii_letters +  string.digits + "!#$%&()*+,-./:;<=>?@[\]^_{|}~."
+    # RFC 3986 section 2.3. unreserved characters (no special escapes required)
+    char_set = string.ascii_letters +  string.digits + "-._~"
     return "".join(random.choices(char_set, k=n))
 
 def gen_random_config():

diff --git a/env_prod/inventory.yml b/env_prod/inventory.yml
@@ -10,20 +10,20 @@ all:
 
   hosts:
     shell:
-      hostname                    : "picoCTF-shell-PROD"
-      ansible_host                : "shell.example.com"   # UPDATE: with fully qualified domain name
+      hostname     : "picoCTF-shell-PROD"
+      ansible_host : "shell.example.com"   # UPDATE: with fully qualified domain name
     web:
-      hostname                    : "picoCTF-web-PROD"
-      ansible_host                : "ctf.example.com"     # UPDATE: with fully qualified domain name
+      hostname     : "picoCTF-web-PROD"
+      ansible_host : "ctf.example.com"     # UPDATE: with fully qualified domain name
 
   vars:
       # UPDATE: this is the default key specified in terraform/variables.tf
       # that was automatically added to the machines when launched
-      ansible_ssh_private_key_file: "~/.ssh/picoCTF_production_rsa"
+      ansible_ssh_private_key_file : "~/.ssh/picoCTF_production_rsa"
 
       # UPDATE: with an administrator's email
       # Used in auto_add_web_admin and to get SSL/TLS certificates
-      admin_email               : "[email protected]"
+      admin_email : "[email protected]"
 
       ###
       # *** Reasonable defaults ***
@@ -35,50 +35,50 @@ all:
       ###
 
       # ubuntu is the default user for the AWS Ubuntu AMI
-      ansible_user                : "ubuntu"
+      ansible_user : "ubuntu"
 
       ## Source Settings:
       # This is how the picoCTF platform source code will be sent to the remote
       # hosts. Since you already have a copy of the source code checked out
       # this simply pushes your current copy to the remote host (preventing any
       # complex interactions with private git repositories). See ansible/common/tasks
-      sync_mode                   : "source"
-      pico_src_dir                : "{{ playbook_dir }}/../"
+      sync_mode    : "source"
+      pico_src_dir : "{{ playbook_dir }}/../"
 
       # SSH admin keys (Optional)
-      admin_keys                  : []
+      admin_keys : []
 
       ## Web automation and Features (Optional):
       # These are currently setup to fully configure and start an event. This
       # is appropriate for testing purposes but if you are planning on running
       # a live event you will likely want to disable some of these.
-      auto_add_web_admin          : True
-      auto_add_shell              : True
-      auto_load_problems          : True
-      auto_start_competition      : True
-      enable_docker               : True
+      auto_add_web_admin     : True
+      auto_add_shell         : True
+      auto_load_problems     : True
+      auto_start_competition : True
+      enable_docker          : True
 
       ## HTTPS:
       # You can automatically obtain free SSL/TLS certificates from Let's
       # Encrypt by using certbot. In order for this functionality to work
       # you must have a domain registered and pointing at your hosts. The
       # `ansible_host` for both `web` and `shell` should be domain names.
-      enable_certbot              : True
+      enable_certbot : True
 
       # Set whether `ansible_host` is an IP address or a domain name (DNS) In a
       # production environment we recommend DNS so that you can easily use SSL/TLS
-      host_type                   : "DNS"  # valid options 'IP' or 'DNS'
+      host_type : "DNS"  # valid options 'IP' or 'DNS'
 
       ## Database settings:
       # Should not need to be changed.
-      on_aws                      : True
-      mongodb_web_user            : "picoWeb"
-      env_db_users:
+      on_aws           : True
+      mongodb_web_user : "picoWeb"
+      env_db_users     :
         - {
-          name      : "{{ mongodb_web_user }}",
-          password  : "{{ picoWeb_db_password }}",
-          database  : "{{ db_name }}",
-          roles     : readWrite
+          name     : "{{ mongodb_web_user }}",
+          password : "{{ picoWeb_db_password }}",
+          database : "{{ db_name }}",
+          roles    : readWrite
           }
 
       ###

diff --git a/env_prod/terraform/Makefile b/env_prod/terraform/Makefile
@@ -16,3 +16,6 @@ taint-shell:
 
 taint-machines: taint-web taint-shell
 	terraform apply
+
+taint-db-ebs:
+	terraform taint aws_ebs_volume.db_data_journal
diff --git a/env_prod/terraform/README.md b/env_prod/terraform/README.md
@@ -50,10 +50,9 @@ applicable. At a high level it is as simple as:
 
 ### 1. Configuration
 
-Add your AWS credentials for a `picoCTF` profile (or whatever profile you set in
-`variables.tf`)
+Add your AWS credentials for a `picoCTF` profile (or update in `variables.tf`)
 
-`~/.aws.credentials`
+`~/.aws/credentials`
 ```
 [picoCTF]
 aws_access_key_id=XXX_KEY_ID_______XXX

diff --git a/env_prod/terraform/main.tf b/env_prod/terraform/main.tf
@@ -14,11 +14,12 @@ terraform {
 }
 
 # AWS Specific config (single region)
-# Configured to get access_key and secret_key from  environment variables
+# Configured to get access_key and secret_key from a shared credentials file
 # For additional methods: https://www.terraform.io/docs/providers/aws/
 provider "aws" {
   region  = local.region
   profile = local.profile
+  shared_credentials_file = local.shared_credentials_file
 }
 
 
@@ -38,7 +39,7 @@ resource "aws_key_pair" "auth" {
 ###
 # Network configuration:
 # This is a simple network configuration where all machines are on a virtual network
-# that is attached via an gateway to the internet. All machines placed in this  subnet
+# that is attached via an gateway to the internet. All machines placed in this subnet
 # receive a public IP address
 ###
 
@@ -62,7 +63,7 @@ resource "aws_route" "internet_access" {
 }
 
 # Create a public facing subnet to launch our instances into
-# Maps public ip automatically so every instance gets a public ip
+# Maps public IP automatically so every instance gets a public IP
 # Security Groups are then used to restrict access
 resource "aws_subnet" "public" {
     vpc_id                  = aws_vpc.network.id
@@ -93,7 +94,7 @@ resource "aws_instance" "web" {
 
     # metadata
     key_name = aws_key_pair.auth.id
-    tags     = merge(local.default_tags, map("Name", "picoCTF-web"))
+    tags     = merge(local.default_tags, map("Name", local.web_name))
 }
 
 resource "aws_instance" "shell" {
@@ -110,28 +111,28 @@ resource "aws_instance" "shell" {
 
     # metadata
     key_name = aws_key_pair.auth.id
-    tags     = merge(local.default_tags, map("Name", "picoCTF-shell"))
+    tags     = merge(local.default_tags, map("Name", local.shell_name))
 }
 
 
 ###
 # Elastic IP:
 # This simplifies configuration and administration by allowing us to rebuild
-# and recreate the servers while maintaining the same public ip.
+# and recreate the servers while maintaining the same public IP.
 ###
 
 # Create Elastic IP for web server
 resource "aws_eip" "web" {
     instance = aws_instance.web.id
     vpc      = true
-    tags     = merge(local.default_tags, map("Name", "picoCTF-web"))
+    tags     = merge(local.default_tags, map("Name", local.web_name))
 }
 
 # Create Elastic IP for shell server
 resource "aws_eip" "shell" {
     instance = aws_instance.shell.id
     vpc      = true
-    tags     = merge(local.default_tags, map("Name", "picoCTF-shell"))
+    tags     = merge(local.default_tags, map("Name", local.shell_name))
 }
 
 
@@ -147,7 +148,7 @@ resource "aws_eip" "shell" {
 resource "aws_ebs_volume" "db_data_journal" {
     availability_zone = local.az
     size              = local.db_ebs_data_size
-    tags              = merge(local.default_tags, map("Name", "picoCTF-db-ebs"))
+    tags              = merge(local.default_tags, map("Name", local.db_name))
 }
 
 # Attach data and journal volume to the instance running the database

diff --git a/env_prod/terraform/variables.tf b/env_prod/terraform/variables.tf
@@ -1,4 +1,4 @@
-# Varaibles used to configure a production deployment
+# Variables used to configure a production deployment
 
 ###
 # Input Variables:
@@ -12,8 +12,9 @@ locals {
   ctf = "picoCTF"
 
   # AWS Settings: choose best for where your CTF is
-  region  = "us-east-1"
-  profile = "picoCTF" 
+  region                  = "us-east-1"
+  profile                 = "picoCTF"
+  shared_credentials_file = "~/.aws/credentials"
 
   # SSH Settings: local path to key that will be authorized on the machines
   public_key_path = "~/.ssh/picoCTF_production_rsa.pub"
@@ -30,21 +31,21 @@ locals {
 
 locals {
   # Name for AWS to track SSH key by.
-  key_name        = local.ctf
+  key_name = local.ctf
 
-  # selected automatically from availible zones
+  # selected automatically from available zones
   az = "${data.aws_availability_zones.available.names[0]}"
 
   # Network settings for Virtual Private Cloud
-  vpc_cidr            = "10.0.0.0/16"
+  vpc_cidr           = "10.0.0.0/16"
   public_subnet_cidr = "10.0.1.0/24"
 
   # Internal IP address for the servers
   web_private_ip   = "10.0.1.10"
   shell_private_ip = "10.0.1.20"
 
   # EBS Volumes: NOTE changing the instance type may require changing the device_name
-  db_ebs_data_size = "10"
+  db_ebs_data_size        = "10"
   db_ebs_data_device_name = "/dev/xvdf"
 
   # Tags that allow visibility via AWS Consolse
@@ -54,12 +55,16 @@ locals {
     Event       = local.ctf
   }
 
+  # Additional name tags which will be added to relevant resources
+  web_name   = "picoCTF-web"
+  shell_name = "picoCTF-shell"
+  db_name    = "picoCTF-db"
 
 }
 
 ###
 # Data Sources:
-# These are automatic data soruces which will pull valid values directly from
+# These are automatic data sources which will pull valid values directly from
 # AWS with no additional configuration required.
 ###
 

diff --git a/scripts/run_py_in_venv.sh b/scripts/run_py_in_venv.sh