Skip to content

Commit c8a9352

Browse files
kammerdienernutrinaGerald Iakobinyi-Pich
authored
Infra as code production (#10778)
* Infrastructure for staging environment. * adding env * Fixing static url location for profile page * Fixing yet another static file issue * fixing bucket name * bumping memory and replicas * removing DD_ENV * removing dd agent * adding new infra * updates * fixing CI * update rds * updating db * updating read replica * setting replicas * adding database * swapping rds instance * remove db, update connections, and add redirect * adding ci update * updating GITHUB envs * fixing github values * adjust database * swapping to test env * Swapping to private subnets * final few changes * updating database * updating static urls * fixing variables * changing static build * update prod dockerfile * moving to original dockerfile * updating ci * updating celery * fixing branch issue * fixing indent * fixing branch Co-authored-by: Gerald Iakobinyi-Pich <[email protected]> Co-authored-by: Gerald Iakobinyi-Pich <[email protected]>
1 parent cc3256a commit c8a9352

File tree

9 files changed

+3307
-4
lines changed

9 files changed

+3307
-4
lines changed

‎.github/workflows/ci-prod.yml

+232
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,232 @@
1+
name: Build & deploy to production
2+
3+
on:
4+
# run it during push to branch
5+
push:
6+
branches: [ "stable" ]
7+
8+
jobs:
9+
build-and-test:
10+
name: Build and Test
11+
12+
# run only when code is compiling and tests are passing
13+
runs-on: ubuntu-latest
14+
15+
outputs:
16+
dockerTag: ${{ steps.compute.outputs.docker_tag }}
17+
18+
services:
19+
# Label used to access the service container
20+
postgres:
21+
# Docker Hub image
22+
image: postgres:11.5
23+
# Provide the password for postgres
24+
env:
25+
POSTGRES_DB: testdb
26+
# Set health checks to wait until postgres has started
27+
options: >-
28+
--health-cmd pg_isready
29+
--health-interval 10s
30+
--health-timeout 5s
31+
--health-retries 5
32+
ports:
33+
# Maps tcp port 5432 on service container to the host
34+
- 5432:5432
35+
36+
redis:
37+
image: redis
38+
# Set health checks to wait until redis has started
39+
options: >-
40+
--health-cmd "redis-cli ping"
41+
--health-interval 10s
42+
--health-timeout 5s
43+
--health-retries 5
44+
ports:
45+
- 6379:6379
46+
47+
env:
48+
DJANGO_SETTINGS_MODULE: app.settings
49+
SUPRESS_DEBUG_TOOLBAR: 1
50+
GITCOIN_API_USER: ${{ secrets.GITCOIN_API_USER }}
51+
GITHUB_API_TOKEN: ${{ secrets.GITCOIN_API_TOKEN }}
52+
POLYGONSCAN_API_KEY: ${{ secrets.POLYGONSCAN_API_KEY }}
53+
54+
# steps to perform in job
55+
steps:
56+
- name: Checkout code
57+
uses: actions/checkout@v2
58+
59+
- name: Use Node.js 14
60+
uses: actions/setup-node@v2
61+
with:
62+
node-version: 14
63+
cache: "yarn"
64+
65+
- name: Use Python 3.7
66+
uses: "actions/setup-python@v2"
67+
with:
68+
python-version: 3.7
69+
cache: "pip"
70+
71+
- name: Setup Env
72+
run: |
73+
echo "PYTHONPATH=/home/runner/work/web/web/app" >> $GITHUB_ENV
74+
cp app/app/ci.env app/app/.env
75+
pip install pip==20.0.2 setuptools wheel --upgrade
76+
77+
- name: Fetch and Install GeoIP database files
78+
run: |
79+
sudo apt-get update && sudo apt-get install -y libmaxminddb-dev libsodium-dev libsecp256k1-dev
80+
cp dist/*.gz ./
81+
gunzip GeoLite2-City.mmdb.tar.gz && gunzip GeoLite2-Country.mmdb.tar.gz
82+
tar -xvf GeoLite2-City.mmdb.tar && tar -xvf GeoLite2-Country.mmdb.tar
83+
sudo mkdir -p /opt/GeoIP/
84+
sudo mv GeoLite2-City_20200128/*.mmdb /opt/GeoIP/
85+
sudo mv GeoLite2-Country_20200128/*.mmdb /opt/GeoIP/
86+
87+
- name: Install libvips, Node, and Python dependencies
88+
run: |
89+
sudo apt-get install -y libvips libvips-dev
90+
node --version
91+
yarn install
92+
pip install -r requirements/test.txt
93+
yarn run eslint
94+
yarn run stylelint
95+
(cd app; python ./manage.py collectstatic --noinput --disable-collectfast)
96+
97+
# - name: Run management commands
98+
# run: |
99+
# python app/manage.py migrate
100+
# python app/manage.py fetch_gas_prices
101+
102+
# - name: Run Python and UI tests
103+
# run: |
104+
# pytest -p no:ethereum -p no:warnings
105+
# bin/ci/cypress-run
106+
107+
# - name: Generate Markdown documentation and static docs page
108+
# run: pydocmd build
109+
110+
# - name: Deploy to Github Pages 🚀
111+
# uses: peaceiris/actions-gh-pages@v3
112+
# if: github.ref == 'refs/heads/master'
113+
# with:
114+
# github_token: ${{ secrets.GITHUB_TOKEN }}
115+
# publish_dir: _build/site
116+
# cname: docs.gitcoin.coind
117+
118+
- name: Compute some values
119+
id: compute
120+
run: |
121+
echo "::set-output name=docker_tag::gitcoin/web:${GITHUB_SHA: -10}"
122+
123+
- name: Login to Docker Hub
124+
uses: docker/login-action@v1
125+
with:
126+
username: ${{ secrets.DOCKER_USERNAME }}
127+
password: ${{ secrets.DOCKER_PASSWORD }}
128+
129+
- name: Set up Docker Buildx
130+
id: buildx
131+
uses: docker/setup-buildx-action@v1
132+
133+
- name: Deploy to Docker Hub 🚀
134+
uses: docker/build-push-action@v2
135+
with:
136+
context: ./
137+
file: ./Dockerfile-prod
138+
builder: ${{ steps.buildx.outputs.name }}
139+
push: true
140+
tags: |
141+
${{ steps.compute.outputs.docker_tag }}
142+
gitcoin/web:production-gha
143+
cache-from: type=registry,ref=${{ secrets.DOCKER_USERNAME }}/web:buildcache-production
144+
cache-to: type=registry,ref=${{ secrets.DOCKER_USERNAME }}/web:buildcache-production,mode=max
145+
146+
deploy:
147+
name: Deploy
148+
needs: build-and-test
149+
environment: production
150+
runs-on: ubuntu-latest
151+
152+
steps:
153+
154+
- name: Checkout code
155+
uses: actions/checkout@v2
156+
157+
- name: Use Node.js
158+
uses: actions/setup-node@v2
159+
with:
160+
# node-version: ${{ matrix.node-version }}
161+
cache: "npm"
162+
cache-dependency-path: infra/production/package-lock.json
163+
164+
# Install pulumi dependencies
165+
# Select the new pulumi stack
166+
- run: |
167+
npm install
168+
pulumi stack select -c gitcoin/production/dev
169+
pulumi config -s gitcoin/production/dev set aws:region us-west-2 --non-interactive
170+
working-directory: infra/production
171+
env:
172+
PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_ACCESS_TOKEN }}
173+
174+
# Run pulumi actions
175+
- uses: pulumi/actions@v3
176+
id: pulumi
177+
with:
178+
command: up
179+
stack-name: gitcoin/production/dev
180+
upsert: false
181+
work-dir: infra/production
182+
env:
183+
PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_ACCESS_TOKEN }}
184+
PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }}
185+
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
186+
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
187+
AWS_REGION: ${{ secrets.AWS_REGION }}
188+
DB_NAME: ${{ secrets.DB_NAME }}
189+
DB_PASSWORD: ${{ secrets.DB_PASSWORD }}
190+
DB_USER: ${{ secrets.DB_USER }}
191+
DOCKER_GTC_WEB_IMAGE: ${{ needs.build-and-test.outputs.dockerTag }}
192+
DATADOG_KEY: ${{ secrets.DATADOG_KEY }}
193+
ROUTE_53_ZONE: ${{ secrets.ROUTE_53_ZONE }}
194+
DOMAIN: ${{ secrets.DOMAIN }}
195+
SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
196+
GITHUB_CLIENT_ID: ${{ secrets.GTC_GITHUB_CLIENT_ID }}
197+
GITHUB_CLIENT_SECRET: ${{ secrets.GTC_GITHUB_CLIENT_SECRET }}
198+
TEMP_DATABASE: ${{ secrets.TEMP_DATABASE }}
199+
DATABASE_URL: ${{ secrets.DATABASE_URL }}
200+
READ_REPLICA_1_DATABASE_URL: ${{secrets.READ_REPLICA_1_DATABASE_URL}}
201+
READ_REPLICA_2_DATABASE_URL: ${{secrets.READ_REPLICA_2_DATABASE_URL}}
202+
READ_REPLICA_3_DATABASE_URL: ${{secrets.READ_REPLICA_3_DATABASE_URL}}
203+
READ_REPLICA_4_DATABASE_URL: ${{secrets.READ_REPLICA_4_DATABASE_URL}}
204+
GITHUB_API_TOKEN: ${{ secrets.GTC_GITHUB_API_TOKEN }}
205+
GITHUB_API_USER: ${{ secrets.GTC_GITHUB_API_USER }}
206+
GITHUB_APP_NAME: ${{ secrets.GTC_GITHUB_APP_NAME }}
207+
208+
# The static files are already bundled and located in the folder /code/app/static in the container
209+
- name: Copy static files to bucket
210+
run: |
211+
mkdir static_files_to_deploy
212+
mkdir docker_bin
213+
214+
cat <<EOT >> docker_bin/static_files.sh
215+
#!/bin/bash
216+
cp -Rf /code/app/static/* /static_files_to_deploy/
217+
EOT
218+
219+
docker run -v $(pwd)/static_files_to_deploy:/static_files_to_deploy -v $(pwd)/docker_bin:/code/app/bin -e DATABASE_URL=${{ steps.pulumi.outputs.rdsConnectionUrl }} ${{ needs.build-and-test.outputs.dockerTag }} sh /code/app/bin/static_files.sh
220+
221+
echo "Syncing to bucket: ${{ steps.pulumi.outputs.bucketName }}"
222+
echo "Source folder: $(pwd)/static_files_to_deploy"
223+
224+
aws s3 sync $(pwd)/static_files_to_deploy s3://${{ steps.pulumi.outputs.bucketName }}/static --acl public-read --delete
225+
env:
226+
# We need AWS_EC2_METADATA_DISABLED, because: https://github.com/actions/checkout/issues/440
227+
AWS_EC2_METADATA_DISABLED: true
228+
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
229+
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
230+
BUNDLE_USE_CHECKSUM: 'false'
231+
232+

‎.github/workflows/ci-stage.yml

-2
Original file line numberDiff line numberDiff line change
@@ -202,14 +202,12 @@ jobs:
202202
run: |
203203
mkdir static_files_to_deploy
204204
mkdir docker_bin
205-
206205
cat <<EOT >> docker_bin/static_files.sh
207206
#!/bin/bash
208207
cp -Rf /code/app/static/* /static_files_to_deploy/
209208
EOT
210209
211210
docker run -v $(pwd)/static_files_to_deploy:/static_files_to_deploy -v $(pwd)/docker_bin:/code/app/bin -e DATABASE_URL=${{ steps.pulumi.outputs.rdsConnectionUrl }} ${{ needs.build-and-test.outputs.dockerTag }} sh /code/app/bin/static_files.sh
212-
213211
echo "Syncing to bucket: ${{ steps.pulumi.outputs.bucketName }}"
214212
echo "Source folder: $(pwd)/static_files_to_deploy"
215213

‎infra/production/Pulumi.yaml

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
name: production
2+
runtime: nodejs
3+
description: Gitcoin production environment

0 commit comments

Comments
 (0)