[Flask Server] Use repo root as module root (#2217)

This is a better practice for reusing packages across apps and allows to
have both web and nl Flask apps in the same testing.

Repo module imports now are grouped together and much easier to
navigate.

Author: shifucun

__init__.py

@@ -19,6 +19,7 @@ ENV ENV=${ENV}
 
 WORKDIR /workspace
 
+COPY nl_app.py /workspace/nl_app.py
 COPY nl_server/requirements.txt /workspace/nl_server/requirements.txt
 # --no-cache-dir removes ~/.cache files, which can be a few GB.
 RUN pip3 install --no-cache-dir \
@@ -27,5 +28,5 @@ RUN pip3 install --no-cache-dir \
 COPY nl_server/. /workspace/nl_server/
 
 # Run server
-WORKDIR /workspace/nl_server
-CMD exec gunicorn --timeout 60 --bind :6060 main:app
+WORKDIR /workspace
+CMD exec gunicorn --timeout 60 --bind :6060 nl_app:app

build/nl_server/Dockerfile

@@ -19,6 +19,7 @@ ENV ENV=${ENV}
 
 WORKDIR /workspace
 
+COPY web_app.py /workspace/web_app.py
 COPY server/requirements.txt /workspace/server/requirements.txt
 # --no-cache-dir removes ~/.cache files, which can be a few GB.
 RUN pip3 install --no-cache-dir \
@@ -37,5 +38,5 @@ COPY static/. /workspace/static/
 RUN npm run-script build
 
 # Run server
-WORKDIR /workspace/server
-CMD exec gunicorn --timeout 60 --bind :8080 main:app
+WORKDIR /workspace
+CMD exec gunicorn --timeout 60 --bind :8080 web_app:app

build/web_server/Dockerfile

@@ -15,7 +15,7 @@
 
 import logging
 
-from __init__ import create_app
+from nl_server.__init__ import create_app
 
 logging.basicConfig(
     level=logging.INFO,

nl_server/main.py nl_app.py

@@ -16,10 +16,11 @@
 import os
 
 from flask import Flask
-import loader
-import routes
 import yaml
 
+import nl_server.loader as loader
+import nl_server.routes as routes
+
 
 def create_app():
   app = Flask(__name__)

nl_server/__init__.py

@@ -17,12 +17,13 @@
 from typing import Dict, List, Union
 
 from datasets import load_dataset
-import gcs
 from google.cloud import storage
 from sentence_transformers import SentenceTransformer
 from sentence_transformers.util import semantic_search
 import torch
 
+import nl_server.gcs as gcs
+
 TEMP_DIR = '/tmp/'
 MODEL_NAME = 'all-MiniLM-L6-v2'
 

nl_server/embeddings.py

@@ -15,8 +15,8 @@
 import logging
 import os
 
-from embeddings import Embeddings
-from ner_place_model import NERPlaces
+from nl_server.embeddings import Embeddings
+from nl_server.ner_place_model import NERPlaces
 
 nl_embeddings_cache_key = 'nl_embeddings'
 nl_ner_cache_key = 'nl_ner'

nl_server/loader.py

@@ -14,10 +14,11 @@
 
 import json
 
-import gcs
 from google.cloud import pubsub_v1
 from google.cloud import storage
-import loader
+
+from nl_server import gcs
+from nl_server import loader
 
 AUTOPUSH_FOLDER = 'autopush/'
 TOPIC_NAME = 'projects/datcom-204919/topics/nl-models-update'

nl_server/pubsub.py

@@ -17,13 +17,14 @@
 import unittest
 
 from diskcache import Cache
-from embeddings import Embeddings
-from loader import nl_cache_path
-from loader import nl_embeddings_cache_key
 from parameterized import parameterized
 from sklearn.metrics.pairwise import cosine_similarity
 import yaml
 
+from nl_server.embeddings import Embeddings
+from nl_server.loader import nl_cache_path
+from nl_server.loader import nl_embeddings_cache_key
+
 _root_dir = os.path.dirname(
     os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 

nl_server/tests/embeddings_test.py

@@ -16,11 +16,12 @@
 import unittest
 
 from diskcache import Cache
-from loader import nl_cache_path
-from loader import nl_ner_cache_key
-from ner_place_model import NERPlaces
 from parameterized import parameterized
 
+from nl_server.loader import nl_cache_path
+from nl_server.loader import nl_ner_cache_key
+from nl_server.ner_place_model import NERPlaces
+
 
 class TestNERPlaces(unittest.TestCase):
 

nl_server/tests/ner_place_model_test.py

@@ -30,6 +30,6 @@ python3 -m pip install --upgrade pip
 echo "nl_server custom requirements installation: starting."
 ./requirements_install.sh
 echo "nl_server custom requirements installation: done."
-
-python3 main.py $PORT
 cd ..
+
+python3 nl_app.py $PORT

run_nl_server.sh

@@ -81,7 +81,5 @@ echo "Starting localhost with FLASK_ENV='$FLASK_ENV' on port='$PORT'"
 
 python3 -m pip install --upgrade pip
 pip3 install -r server/requirements.txt -q
-cd server
-protoc -I=./config/ --python_out=./config ./config/subject_page.proto
-python3 main.py $PORT
-cd ..
+protoc -I=./server/config/ --python_out=./server/config ./server/config/subject_page.proto
+python3 web_app.py $PORT

run_server.sh

@@ -32,16 +32,16 @@ function setup_python {
 
   # Downloading the named-entity recognition (NER) library spacy and the large EN model
   # using the guidelines here: https://spacy.io/usage/models#production
-  # Unfortunately, pip is not able to recognize this data (as a library) as part of 
+  # Unfortunately, pip is not able to recognize this data (as a library) as part of
   # requirements.txt and will try to download a new version every single time.
   # Reason for doing this here is that if the library is already installed, no need
-  # to download > 560Mb file. 
+  # to download > 560Mb file.
   if python3 -c "import en_core_web_lg" &> /dev/null; then
       echo 'NER model (en_core_web_lg) already installed.'
   else
       echo 'Installing the NER model: en_core_web_lg'
       pip3 install $(spacy info en_core_web_lg --url)
-fi  
+fi
 }
 
 # Run test for client side code.
@@ -104,9 +104,7 @@ function run_npm_build () {
 function run_py_test {
   setup_python
   export FLASK_ENV=test
-  cd server
-  python3 -m pytest tests/ -s --ignore=sustainability
-  cd ..
+  python3 -m pytest server/tests/ -s --ignore=sustainability
 
   # TODO(beets): add tests for other private dc instances
   # export FLASK_ENV=test-sustainability
@@ -117,9 +115,9 @@ function run_py_test {
   echo "nl_server custom requirements installation: starting."
   ./requirements_install.sh
   echo "nl_server custom requirements installation: done."
-  python3 -m pytest tests/ -s
   cd ..
-  
+  python3 -m pytest nl_server/tests/ -s
+
   echo -e "#### Checking Python style"
   if ! yapf --recursive --diff --style='{based_on_style: google, indent_width: 2}' -p server/ nl_server/ tools/ -e=*pb2.py; then
     echo "Fix Python lint errors by running ./run_test.sh -f"
@@ -137,16 +135,14 @@ function run_py_test {
 function run_webdriver_test {
   printf '\n\e[1;35m%-6s\e[m\n\n' "!!! Have you generated the prod client packages? Run './run_test.sh -b' first to do so"
   setup_python
-  cd server
-  if [ ! -d dist  ]
+  if [ ! -d server/dist  ]
   then
     echo "no dist folder, please run ./run_test.sh -b to build js first."
     exit 1
   fi
   export FLASK_ENV=webdriver
   export GOOGLE_CLOUD_PROJECT=datcom-website-dev
-  python3 -m pytest -n 10 --reruns 3 webdriver_tests/tests/
-  cd ..
+  python3 -m pytest -n 10 --reruns 3 server/webdriver_tests/tests/
 }
 
 function run_all_tests {

run_test.sh

@@ -16,7 +16,6 @@
 import logging
 import os
 import tempfile
-import urllib.request
 
 import firebase_admin
 from firebase_admin import credentials
@@ -28,16 +27,17 @@
 from flask_babel import Babel
 from google.cloud import secretmanager
 from google_auth_oauthlib.flow import Flow
-import lib.config as libconfig
-from lib.disaster_dashboard import get_disaster_dashboard_data
-import lib.i18n as i18n
-import lib.util as libutil
 from opencensus.ext.flask.flask_middleware import FlaskMiddleware
 from opencensus.ext.stackdriver.trace_exporter import StackdriverExporter
 from opencensus.trace.propagation import google_cloud_format
 from opencensus.trace.samplers import AlwaysOnSampler
-from services.discovery import configure_endpoints_from_ingress
-from services.discovery import get_health_check_urls
+
+import server.lib.config as libconfig
+from server.lib.disaster_dashboard import get_disaster_dashboard_data
+import server.lib.i18n as i18n
+import server.lib.util as libutil
+from server.services.discovery import configure_endpoints_from_ingress
+from server.services.discovery import get_health_check_urls
 
 propagator = google_cloud_format.GoogleCloudFormatPropagator()
 
@@ -54,14 +54,14 @@ def createMiddleWare(app, exporter):
 
 def register_routes_base_dc(app):
   # apply the blueprints for all apps
-  from routes import dev
-  from routes import disease
-  from routes import import_wizard
-  from routes import placelist
-  from routes import protein
-  from routes import redirects
-  from routes import special_announcement
-  from routes import topic_page
+  from server.routes import dev
+  from server.routes import disease
+  from server.routes import import_wizard
+  from server.routes import placelist
+  from server.routes import protein
+  from server.routes import redirects
+  from server.routes import special_announcement
+  from server.routes import topic_page
   app.register_blueprint(dev.bp)
   app.register_blueprint(disease.bp)
   app.register_blueprint(placelist.bp)
@@ -70,9 +70,9 @@ def register_routes_base_dc(app):
   app.register_blueprint(special_announcement.bp)
   app.register_blueprint(topic_page.bp)
 
-  from routes.api import disease as disease_api
-  from routes.api import protein as protein_api
-  from routes.api.import_detection import detection as detection_api
+  from server.routes.api import disease as disease_api
+  from server.routes.api import protein as protein_api
+  from server.routes.api.import_detection import detection as detection_api
   app.register_blueprint(detection_api.bp)
   app.register_blueprint(disease_api.bp)
   app.register_blueprint(import_wizard.bp)
@@ -86,9 +86,9 @@ def register_routes_custom_dc(app):
 
 def register_routes_stanford_dc(app, is_test, is_local):
   # Install blueprints specific to Stanford DC
-  from routes import disasters
-  from routes import event
-  from routes.api import disaster_api
+  from server.routes import disasters
+  from server.routes import event
+  from server.routes.api import disaster_api
   app.register_blueprint(disasters.bp)
   app.register_blueprint(disaster_api.bp)
   app.register_blueprint(event.bp)
@@ -104,23 +104,23 @@ def register_routes_stanford_dc(app, is_test, is_local):
 
 
 def register_routes_admin(app):
-  from routes import user
+  from server.routes import user
   app.register_blueprint(user.bp)
-  from routes.api import user as user_api
+  from server.routes.api import user as user_api
   app.register_blueprint(user_api.bp)
 
 
 def register_routes_common(app):
   # apply the blueprints for main app
-  from routes import browser
-  from routes import factcheck
-  from routes import nl_interface
-  from routes import nl_interface_next
-  from routes import place
-  from routes import ranking
-  from routes import search
-  from routes import static
-  from routes import tools
+  from server.routes import browser
+  from server.routes import factcheck
+  from server.routes import nl_interface
+  from server.routes import nl_interface_next
+  from server.routes import place
+  from server.routes import ranking
+  from server.routes import search
+  from server.routes import static
+  from server.routes import tools
   app.register_blueprint(browser.bp)
   app.register_blueprint(nl_interface.bp)
   app.register_blueprint(nl_interface_next.bp)
@@ -130,22 +130,22 @@ def register_routes_common(app):
   app.register_blueprint(static.bp)
   app.register_blueprint(tools.bp)
   # TODO: Extract more out to base_dc
-  from routes.api import browser as browser_api
-  from routes.api import choropleth
-  from routes.api import csv
-  from routes.api import facets
-  from routes.api import landing_page
-  from routes.api import node
-  from routes.api import observation_dates
-  from routes.api import observation_existence
-  from routes.api import place as place_api
-  from routes.api import point
-  from routes.api import ranking as ranking_api
-  from routes.api import series
-  from routes.api import stats
-  from routes.api import translator
-  from routes.api import variable
-  from routes.api import variable_group
+  from server.routes.api import browser as browser_api
+  from server.routes.api import choropleth
+  from server.routes.api import csv
+  from server.routes.api import facets
+  from server.routes.api import landing_page
+  from server.routes.api import node
+  from server.routes.api import observation_dates
+  from server.routes.api import observation_existence
+  from server.routes.api import place as place_api
+  from server.routes.api import point
+  from server.routes.api import ranking as ranking_api
+  from server.routes.api import series
+  from server.routes.api import stats
+  from server.routes.api import translator
+  from server.routes.api import variable
+  from server.routes.api import variable_group
   app.register_blueprint(browser_api.bp)
   app.register_blueprint(choropleth.bp)
   app.register_blueprint(csv.bp)
@@ -190,7 +190,7 @@ def create_app():
     app.config['API_ROOT'] = 'http://127.0.0.1:8081'
 
   # Init extentions
-  from cache import cache
+  from server.cache import cache
 
   # For some instance with fast updated data, we may not want to use memcache.
   if app.config['USE_MEMCACHE']:
@@ -286,8 +286,8 @@ def create_app():
   if os.environ.get('ENABLE_MODEL') == 'true':
     libutil.check_backend_ready([app.config['NL_ROOT'] + '/healthz'])
     # Some specific imports for the NL Interface.
-    import lib.nl.training as libnl
-    import services.nl as nl
+    import server.lib.nl.training as libnl
+    import server.services.nl as nl
 
     # For the classification types available, check lib.training (libnl).
     classification_types = [

server/__init__.py

@@ -54,7 +54,7 @@ def map_config_string(env):
   env_list[index + 1] = env[index + 1].upper()
   env_list[0] = env[0].upper()
   env_updated = "".join(env_list).replace('-', '')
-  return 'configmodule.{}Config'.format(env_updated)
+  return 'server.configmodule.{}Config'.format(env_updated)
 
 
 def get_config():