Skip to content

Commit 1cc096a

Browse files
kammerdienernutrinaGerald Iakobinyi-Pich
authored
Infra as code staging (#10777)
* Infrastructure for staging environment. * adding env * Fixing static url location for profile page * Fixing yet another static file issue * fixing bucket name * bumping memory and replicas * removing DD_ENV * removing dd agent * infra updates * updating correct files * moving celery to a separate service * updating media url * updating values * updating more values * adding Rule * fixing arn * adding in flag * fix path * small webpack change * fixing webpack back * changing build * fixing path * fixing RDS * fixing based on comments Co-authored-by: Gerald Iakobinyi-Pich <[email protected]> Co-authored-by: Gerald Iakobinyi-Pich <[email protected]>
1 parent cebc3b2 commit 1cc096a

File tree

10 files changed

+3421
-4
lines changed

10 files changed

+3421
-4
lines changed

‎.github/workflows/ci-stage.yml

+224
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,224 @@
1+
name: Build & deploy to staging
2+
3+
on:
4+
# run it during push to branch
5+
push:
6+
branches: [ master ]
7+
8+
jobs:
9+
build-and-test:
10+
name: Build and Test
11+
12+
# run only when code is compiling and tests are passing
13+
runs-on: ubuntu-latest
14+
15+
outputs:
16+
dockerTag: ${{ steps.compute.outputs.docker_tag }}
17+
18+
services:
19+
# Label used to access the service container
20+
postgres:
21+
# Docker Hub image
22+
image: postgres:11.5
23+
# Provide the password for postgres
24+
env:
25+
POSTGRES_DB: testdb
26+
# Set health checks to wait until postgres has started
27+
options: >-
28+
--health-cmd pg_isready
29+
--health-interval 10s
30+
--health-timeout 5s
31+
--health-retries 5
32+
ports:
33+
# Maps tcp port 5432 on service container to the host
34+
- 5432:5432
35+
36+
redis:
37+
image: redis
38+
# Set health checks to wait until redis has started
39+
options: >-
40+
--health-cmd "redis-cli ping"
41+
--health-interval 10s
42+
--health-timeout 5s
43+
--health-retries 5
44+
ports:
45+
- 6379:6379
46+
47+
env:
48+
DJANGO_SETTINGS_MODULE: app.settings
49+
SUPRESS_DEBUG_TOOLBAR: 1
50+
GITCOIN_API_USER: ${{ secrets.GITCOIN_API_USER }}
51+
GITHUB_API_TOKEN: ${{ secrets.GITCOIN_API_TOKEN }}
52+
POLYGONSCAN_API_KEY: ${{ secrets.POLYGONSCAN_API_KEY }}
53+
54+
# steps to perform in job
55+
steps:
56+
- name: Checkout code
57+
uses: actions/checkout@v2
58+
59+
- name: Use Node.js 14
60+
uses: actions/setup-node@v2
61+
with:
62+
node-version: 14
63+
cache: "yarn"
64+
65+
- name: Use Python 3.7
66+
uses: "actions/setup-python@v2"
67+
with:
68+
python-version: 3.7
69+
cache: "pip"
70+
71+
- name: Setup Env
72+
run: |
73+
echo "PYTHONPATH=/home/runner/work/web/web/app" >> $GITHUB_ENV
74+
cp app/app/ci.env app/app/.env
75+
pip install pip==20.0.2 setuptools wheel --upgrade
76+
77+
- name: Fetch and Install GeoIP database files
78+
run: |
79+
sudo apt-get update && sudo apt-get install -y libmaxminddb-dev libsodium-dev libsecp256k1-dev
80+
cp dist/*.gz ./
81+
gunzip GeoLite2-City.mmdb.tar.gz && gunzip GeoLite2-Country.mmdb.tar.gz
82+
tar -xvf GeoLite2-City.mmdb.tar && tar -xvf GeoLite2-Country.mmdb.tar
83+
sudo mkdir -p /opt/GeoIP/
84+
sudo mv GeoLite2-City_20200128/*.mmdb /opt/GeoIP/
85+
sudo mv GeoLite2-Country_20200128/*.mmdb /opt/GeoIP/
86+
87+
- name: Install libvips, Node, and Python dependencies
88+
run: |
89+
sudo apt-get install -y libvips libvips-dev
90+
node --version
91+
yarn install
92+
pip install -r requirements/test.txt
93+
yarn run eslint
94+
yarn run stylelint
95+
(cd app; python ./manage.py collectstatic --noinput --disable-collectfast)
96+
97+
# - name: Run management commands
98+
# run: |
99+
# python app/manage.py migrate
100+
# python app/manage.py fetch_gas_prices
101+
102+
# - name: Run Python and UI tests
103+
# run: |
104+
# pytest -p no:ethereum -p no:warnings
105+
# bin/ci/cypress-run
106+
107+
# - name: Generate Markdown documentation and static docs page
108+
# run: pydocmd build
109+
110+
# - name: Deploy to Github Pages 🚀
111+
# uses: peaceiris/actions-gh-pages@v3
112+
# if: github.ref == 'refs/heads/master'
113+
# with:
114+
# github_token: ${{ secrets.GITHUB_TOKEN }}
115+
# publish_dir: _build/site
116+
# cname: docs.gitcoin.coind
117+
118+
- name: Compute some values
119+
id: compute
120+
run: |
121+
echo "::set-output name=docker_tag::gitcoin/web:${GITHUB_SHA: -10}"
122+
123+
- name: Login to Docker Hub
124+
uses: docker/login-action@v1
125+
with:
126+
username: ${{ secrets.DOCKER_USERNAME }}
127+
password: ${{ secrets.DOCKER_PASSWORD }}
128+
129+
- name: Set up Docker Buildx
130+
id: buildx
131+
uses: docker/setup-buildx-action@v1
132+
133+
- name: Deploy to Docker Hub 🚀
134+
uses: docker/build-push-action@v2
135+
with:
136+
context: ./
137+
file: ./Dockerfile-prod
138+
builder: ${{ steps.buildx.outputs.name }}
139+
push: true
140+
tags: |
141+
${{ steps.compute.outputs.docker_tag }}
142+
gitcoin/web:staging-gha
143+
cache-from: type=registry,ref=${{ secrets.DOCKER_USERNAME }}/web:buildcache-staging
144+
cache-to: type=registry,ref=${{ secrets.DOCKER_USERNAME }}/web:buildcache-staging,mode=max
145+
146+
deploy:
147+
name: Deploy
148+
needs: build-and-test
149+
environment: staging
150+
runs-on: ubuntu-latest
151+
152+
steps:
153+
154+
- name: Checkout code
155+
uses: actions/checkout@v2
156+
157+
- name: Use Node.js
158+
uses: actions/setup-node@v2
159+
with:
160+
# node-version: ${{ matrix.node-version }}
161+
cache: "npm"
162+
cache-dependency-path: infra/staging/package-lock.json
163+
164+
# Install pulumi dependencies
165+
# Select the new pulumi stack
166+
- run: |
167+
npm install
168+
pulumi stack select -c gitcoin/staging/dev
169+
pulumi config -s gitcoin/staging/dev set aws:region us-west-2 --non-interactive
170+
working-directory: infra/staging
171+
env:
172+
PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_ACCESS_TOKEN }}
173+
174+
# Run pulumi actions
175+
- uses: pulumi/actions@v3
176+
id: pulumi
177+
with:
178+
command: up
179+
stack-name: gitcoin/staging/dev
180+
upsert: false
181+
work-dir: infra/staging
182+
env:
183+
PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_ACCESS_TOKEN }}
184+
PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }}
185+
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
186+
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
187+
AWS_REGION: ${{ secrets.AWS_REGION }}
188+
DB_NAME: ${{ secrets.DB_NAME }}
189+
DB_PASSWORD: ${{ secrets.DB_PASSWORD }}
190+
DB_USER: ${{ secrets.DB_USER }}
191+
DOCKER_GTC_WEB_IMAGE: ${{ needs.build-and-test.outputs.dockerTag }}
192+
DATADOG_KEY: ${{ secrets.DATADOG_KEY }}
193+
ROUTE_53_ZONE: ${{ secrets.ROUTE53_ZONE_ID }}
194+
DOMAIN: ${{ secrets.DOMAIN }}
195+
SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
196+
GITHUB_CLIENT_ID: ${{ secrets.GTC_GITHUB_CLIENT_ID }}
197+
GITHUB_CLIENT_SECRET: ${{ secrets.GTC_GITHUB_CLIENT_SECRET }}
198+
TEMP_DATABASE: ${{ secrets.TEMP_DATABASE }}
199+
200+
# The static files are already bundled and located in the folder /code/app/static in the container
201+
- name: Copy static files to bucket
202+
run: |
203+
mkdir static_files_to_deploy
204+
mkdir docker_bin
205+
206+
cat <<EOT >> docker_bin/static_files.sh
207+
#!/bin/bash
208+
cp -Rf /code/app/static/* /static_files_to_deploy/
209+
EOT
210+
211+
docker run -v $(pwd)/static_files_to_deploy:/static_files_to_deploy -v $(pwd)/docker_bin:/code/app/bin -e DATABASE_URL=${{ steps.pulumi.outputs.rdsConnectionUrl }} ${{ needs.build-and-test.outputs.dockerTag }} sh /code/app/bin/static_files.sh
212+
213+
echo "Syncing to bucket: ${{ steps.pulumi.outputs.bucketName }}"
214+
echo "Source folder: $(pwd)/static_files_to_deploy"
215+
216+
aws s3 sync $(pwd)/static_files_to_deploy s3://${{ steps.pulumi.outputs.bucketName }}/static --acl public-read --delete
217+
env:
218+
# We need AWS_EC2_METADATA_DISABLED, because: https://github.com/actions/checkout/issues/440
219+
AWS_EC2_METADATA_DISABLED: true
220+
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
221+
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
222+
BUNDLE_USE_CHECKSUM: 'false'
223+
224+

‎Dockerfile-prod

+119
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
FROM ubuntu:18.04 as build
2+
3+
ENV PYTHONUNBUFFERED 1
4+
ENV PYTHONDONTWRITEBYTECODE 1
5+
ENV DEBIAN_FRONTEND=noninteractive
6+
7+
# Define packages to be installed
8+
ARG PACKAGES="libpq-dev libxml2 libxslt1-dev libfreetype6 libjpeg-dev libmaxminddb-dev bash git tar gzip libmagic-dev build-essential python-dev libssl-dev python3-dev libsecp256k1-dev libsodium-dev python3-pip"
9+
ARG BUILD_DEPS="gcc g++ curl postgresql libxml2-dev libxslt-dev libfreetype6 libffi-dev libjpeg-dev autoconf automake libtool make dos2unix libvips libvips-dev"
10+
11+
# Install general dependencies.
12+
RUN apt-get update
13+
RUN apt-get install -y software-properties-common
14+
RUN add-apt-repository universe
15+
RUN apt-get update
16+
RUN apt-get install -y $PACKAGES
17+
RUN apt-get update --fix-missing
18+
RUN apt-get install -y $BUILD_DEPS --fix-missing
19+
20+
# Move to /code dir and copy in working dir content
21+
WORKDIR /code
22+
COPY dist/* ./
23+
24+
# GeoIP2 Data Files
25+
RUN mkdir -p /usr/share/GeoIP/ && \
26+
gunzip GeoLite2-City.mmdb.tar.gz && \
27+
gunzip GeoLite2-Country.mmdb.tar.gz && \
28+
tar -xvf GeoLite2-City.mmdb.tar && \
29+
tar -xvf GeoLite2-Country.mmdb.tar && \
30+
mv GeoLite2-City_20200128/*.mmdb /usr/share/GeoIP/ && \
31+
mv GeoLite2-Country_20200128/*.mmdb /usr/share/GeoIP/
32+
33+
# Upgrade package essentials.
34+
RUN pip3 install --upgrade pip==20.0.2 setuptools wheel dumb-init pipenv
35+
36+
# Install pip packages
37+
COPY requirements/ /code/
38+
RUN pip3 install --upgrade -r test.txt
39+
40+
# Copy over docker-command (start-up script)
41+
COPY bin/docker-command.bash /bin/docker-command.bash
42+
RUN dos2unix /bin/docker-command.bash
43+
44+
# Copy over code directory
45+
COPY app/ /code/app/
46+
47+
# Install yarn and set node version
48+
RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add -
49+
RUN echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list
50+
RUN apt-get update
51+
RUN apt-get install -y yarn
52+
RUN yarn global add n
53+
RUN n stable
54+
55+
COPY package.json /code/
56+
RUN cd /code && yarn install
57+
58+
COPY app/app/ci.env /code/app/app/.env
59+
WORKDIR /code/app
60+
RUN BUNDLE_USE_CHECKSUM=false python3 manage.py bundle
61+
RUN python3 manage.py collectstatic --disable-collectfast --noinput
62+
63+
###################################################################################
64+
# Creating the final container
65+
###################################################################################
66+
67+
FROM ubuntu:18.04
68+
69+
ENV PYTHONUNBUFFERED 1
70+
ENV PYTHONDONTWRITEBYTECODE 1
71+
ENV DEBIAN_FRONTEND=noninteractive
72+
73+
# Define packages to be installed
74+
# ARG PACKAGES="libpq-dev libxml2 libxslt1-dev libfreetype6 libjpeg-dev libmaxminddb-dev bash git tar gzip libmagic-dev build-essential python-dev libssl-dev python3-dev libsecp256k1-dev libsodium-dev python3-pip"
75+
ARG PACKAGES="git python3-pip libvips"
76+
# ARG BUILD_DEPS="gcc g++ curl postgresql libxml2-dev libxslt-dev libfreetype6 libffi-dev libjpeg-dev autoconf automake libtool make dos2unix libvips libvips-dev"
77+
78+
# Install general dependencies.
79+
RUN apt-get update
80+
RUN apt-get install -y software-properties-common
81+
RUN add-apt-repository universe
82+
RUN apt-get update
83+
RUN apt-get install -y $PACKAGES
84+
# RUN apt-get update --fix-missing
85+
# RUN apt-get install -y $BUILD_DEPS --fix-missing
86+
87+
# Move to /code dir and copy in working dir content
88+
WORKDIR /code
89+
90+
# Upgrade package essentials.
91+
RUN pip3 install --upgrade pip==20.0.2 setuptools wheel dumb-init pipenv
92+
93+
# Install pip packages
94+
COPY requirements/ /code/
95+
RUN pip3 install --upgrade -r test.txt
96+
97+
WORKDIR /code/app
98+
99+
# Init
100+
EXPOSE 9222
101+
ENTRYPOINT ["/usr/local/bin/dumb-init", "--"]
102+
CMD ["gunicorn", "-w", "1", "-b", "0.0.0.0:80", "app.wsgi:application", "--max-requests=200"]
103+
104+
# Copy over code directory
105+
COPY app/ /code/app/
106+
COPY --from=build /code/app/static /code/app/static
107+
COPY --from=build /usr/share/GeoIP /usr/share/GeoIP
108+
109+
# Tag
110+
ARG BUILD_DATETIME
111+
ARG SHA1
112+
LABEL co.gitcoin.description="Gitcoin web application image" \
113+
co.gitcoin.documentation="https://github.com/gitcoinco/web/blob/master/docs/RUNNING_LOCALLY_DOCKER.md" \
114+
co.gitcoin.licenses="AGPL-3.0" \
115+
co.gitcoin.image.revision=$SHA1 \
116+
co.gitcoin.image.vendor="Gitcoin" \
117+
co.gitcoin.image.source="https://github.com/gitcoinco/web" \
118+
co.gitcoin.image.title="Gitcoin Web" \
119+
co.gitcoin.image.created=$BUILD_DATETIME

‎app/assets/v2/js/status.js

+3-3
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ $(document).ready(function() {
3232
let selector = $('#bg-selector');
3333

3434
for (var i = 0; i < bgs.length; i++) {
35-
selector.append('<div data-bg-name="' + bgs[i] + '" class="bg-thumbnail"><img class="bg-icon" draggable="false" src="/static/status_backgrounds/' + bgs[i] + '-icon.png"/><div class="selector-bar d-none"></div></div>');
35+
selector.append('<div data-bg-name="' + bgs[i] + '" class="bg-thumbnail"><img class="bg-icon" draggable="false" src="' + static_url + 'status_backgrounds/' + bgs[i] + '-icon.png"/><div class="selector-bar d-none"></div></div>');
3636
}
3737

3838
function closeBackgroundDropdown(e) {
@@ -184,7 +184,7 @@ $(document).ready(function() {
184184
} else {
185185
$('#thumbnail-img').addClass('py-2 px-4');
186186
$('#thumbnail-img').css('width', '8rem');
187-
$('#thumbnail-img').attr('src', 'https://s.gitcoin.co/static/v2/images/team/gitcoinbot.c1e81ab42f13.png');
187+
$('#thumbnail-img').attr('src', static_url + 'v2/images/team/gitcoinbot.c1e81ab42f13.png');
188188
}
189189

190190
embedded_resource = url;
@@ -362,7 +362,7 @@ $(document).ready(function() {
362362

363363
let html = `
364364
<div data-gfx=` + item + ` id=video_container class="bg-lightblue p-2">
365-
<img src='/static/v2/images/` + item + `'>
365+
<img src='${static_url}v2/images/` + item + `'>
366366
</div>
367367
`;
368368

‎app/retail/templates/shared/activity.html

+3-1
Original file line numberDiff line numberDiff line change
@@ -486,7 +486,9 @@
486486
</div>
487487
{% if row.metadata.resource.type == 'background' and not for_email %}
488488
<div class="status_background_container">
489-
<img class="status_background" src="/static/status_backgrounds/{{row.metadata.resource.id}}.png" />
489+
{% with 'status_backgrounds/'|add:row.metadata.resource.id|add:'.png' as background_file %}
490+
<img class="status_background" src="{% static background_file %}" />
491+
{% endwith %}
490492
<div class="image-text-container">
491493
<p style="margin: 0pt;">
492494
{{row.metadata.title}}

0 commit comments

Comments
 (0)