Steve/161 better pytest

.github/workflows/main.yml

@@ -22,4 +22,4 @@ jobs:
       - name: Test with pytest
         working-directory: ./
         # Ideally we should not need all the environment variables to test.
-        run: pytest
+        run: pytest -m "not live"

.gitignore

@@ -291,6 +291,7 @@ Temporary Items
 .idea/**/usage.statistics.xml
 .idea/**/dictionaries
 .idea/**/shelf
+.run/
 
 # Generated files
 .idea/**/contentModel.xml
@@ -907,7 +908,6 @@ docs/_linkcheck/
 # Ignore sublime artifacts
 *.sublime-project
 *.sublime-workspace
-*.code-workspace
 ./sftp-config.json
 
 # Temp and scratch

api/pyproject.toml

@@ -11,6 +11,7 @@ dependencies = [
     "fastapi[all] == 0.85.0",
     "pandas == 1.5.1",
     "pre-commit == 2.20.0",
+    "pyodbc == 4.0.35",
     "psycopg2-binary == 2.9.5",
     "SQLAlchemy == 1.4.41",
     "sqlmodel == 0.0.8",
@@ -30,3 +31,7 @@ minversion = "7.1.3"
 testpaths = [
     "src/api/tests"
 ]
+markers = [
+    "smoke: subset of tests",
+    "live: tests that should only run in the live environment"
+]

api/src/api/demo/live.sql

@@ -0,0 +1,12 @@
+-- 2022-07-21
+-- Query by J Hunter
+-- see https://github.com/HYLODE/HyUi/issues/47#issuecomment-1160706270
+
+SELECT
+     pod.NAME AS pod_orc
+    ,c.OR_CASE_ID AS or_case_id
+    ,c.SURGERY_DATE AS SurgeryDateClarity
+FROM OR_CASE c
+INNER JOIN ZC_OR_POSTOP_DEST pod ON pod.POSTOP_DEST = c.POSTOP_DEST_C
+WHERE c.SURGERY_DATE >= GETDATE()
+AND c.SURGERY_DATE <= CONVERT(DATE,DATEADD(DAY, :days_ahead ,CURRENT_TIMESTAMP))

api/src/api/demo/mock.json

@@ -0,0 +1,26 @@
+{
+	"table": "OR_CASE",
+	"rows":
+	[
+		{
+			"pod_orc": "Day Surgery Ward",
+			"or_case_id": "473112",
+			"SurgeryDateClarity": "2022-11-25 00:00:00.000"
+		},
+		{
+			"pod_orc": "Day Surgery Ward",
+			"or_case_id": "597244",
+			"SurgeryDateClarity": "2022-11-25 00:00:00.000"
+		},
+		{
+			"pod_orc": "ITU/PACU Bed",
+			"or_case_id": "617016",
+			"SurgeryDateClarity": "2022-11-25 00:00:00.000"
+		},
+		{
+			"pod_orc": "Inpatient Ward",
+			"or_case_id": "719054",
+			"SurgeryDateClarity": "2022-11-25 00:00:00.000"
+		}
+	]
+}

api/src/api/demo/readme.md

@@ -0,0 +1 @@
+See [docs/developer/demo_vignette.md]() (path starting from the project root)

api/src/api/demo/router.py

@@ -0,0 +1,37 @@
+import json
+from pathlib import Path
+
+import pandas as pd
+from fastapi import APIRouter, Depends
+from sqlmodel import Session
+from sqlalchemy import text
+
+from api.db import get_clarity_session
+from models.demo import ClarityOrCase
+
+router = APIRouter(prefix="/demo")
+
+mock_router = APIRouter(prefix="/demo")
+
+
+@mock_router.get("/", response_model=list[ClarityOrCase])
+def get_mock_demo_rows():
+    """
+    Return mock data from adjacent mock.json file
+    """
+    with open(Path(__file__).parent / "mock.json", "r") as f:
+        mock_json = json.load(f)
+        mock_table = mock_json["rows"]
+    return [ClarityOrCase.parse_obj(row) for row in mock_table]
+
+
+@router.get("/", response_model=list[ClarityOrCase])
+def get_demo_rows(session: Session = Depends(get_clarity_session), days_ahead: int = 1):
+    """
+    Return mock data by running query in anger
+    response_model defines the data type (model) of the response
+    """
+    query = text((Path(__file__).parent / "live.sql").read_text())
+    params = {"days_ahead": days_ahead}
+    df = pd.read_sql(query, session.connection(), params=params)
+    return [ClarityOrCase.parse_obj(row) for row in df.to_dict(orient="records")]

api/src/api/hospital/router.py

@@ -2,7 +2,7 @@
 
 from api import wards
 
-from models.hosptial import BuildingDepartments
+from models.hospital import BuildingDepartments
 
 router = APIRouter(
     prefix="/hospital",

api/src/api/main.py

@@ -1,3 +1,7 @@
+"""
+Entry point and main file for the FastAPI backend
+"""
+
 from fastapi import FastAPI, APIRouter
 from fastapi.responses import ORJSONResponse
 
@@ -24,13 +28,20 @@
     router as hospital_router,
     mock_router as mock_hospital_router,
 )
+from api.demo.router import (
+    router as demo_router,
+    mock_router as mock_demo_router,
+)
 
 
 app = FastAPI(default_response_class=ORJSONResponse)
 mock_router = APIRouter(
     prefix="/mock",
 )
 
+app.include_router(demo_router)
+mock_router.include_router(mock_demo_router)
+
 app.include_router(hospital_router)
 mock_router.include_router(mock_hospital_router)
 

api/src/api/tests/test_demo.py

@@ -0,0 +1,33 @@
+import pytest
+
+from api.main import app
+from fastapi.testclient import TestClient
+
+from models.demo import ClarityOrCase
+
+client = TestClient(app)
+
+
+def test_get_mock_demo():
+    response = client.get("/mock/demo/", params={})
+    assert response.status_code == 200
+
+    demo_rows = [ClarityOrCase.parse_obj(row) for row in response.json()]
+    assert len(demo_rows) > 0
+
+
+@pytest.mark.live
+def test_get_demo():
+    """
+    Should only pass when run in the live environment hence the marker
+    i.e. when developing locally
+    `pytest -v -m 'not live'`
+
+    Should not fail - would suggest no operating in the next 3 days
+    :return:
+    """
+    response = client.get("/demo/", params={"days_ahead": 3})
+    assert response.status_code == 200
+
+    demo_rows = [ClarityOrCase.parse_obj(row) for row in response.json()]
+    assert len(demo_rows) > 0

docker/api/Dockerfile

@@ -35,7 +35,10 @@ COPY --chown=hyui:root ./models/pyproject.toml /app/models/
 RUN pip install --no-cache-dir /app/models
 
 COPY --chown=hyui:root ./api/pyproject.toml /app/api/
-RUN pip install --no-cache-dir /app/api
+# installs testing: useful for debugging
+RUN pip install --no-cache-dir /app/api[test]
+# without testing
+#RUN pip install --no-cache-dir /app/api
 
 COPY --chown=hyui:root ./models/src/models /app/models
 COPY --chown=hyui:root ./api/src/api /app/api

docs/developer/demo_vignette.qmd

@@ -0,0 +1,56 @@
+## Example of how to create a new endpoint
+
+1. Create a 'back end' endpoint using FastAPI
+2. Write a simple test that checks that route exists
+3. Create a 'front end' endpoint using Plotly Dash
+4. Write a simple integration test that checks that the page displays
+
+### 1. Back end FastAPI endpoint
+
+- Create a new 'python package' (directory containing an `__init__.py`) file under `api/src/api/`.
+- Create a `router.py` file within this package
+- Generate some fake/mock data ideally by hand. You might want to run the SQL script you're planning to use, then copy the column headings into an Excel file, and then create a couple of rows of fake data that you could/should save as a Python List of dictionaries or similar. You need to make this data available to your 'mock' endpoint.
+- create a mock and a non-mock endpoint in `router.py`
+- register these endpoints in `api/src/api/main.py`
+
+```python
+# At the top of the file
+from api.demo.router import (
+    router as demo_router,
+    mock_router as mock_demo_router,
+)
+
+# further imports etc. ...
+
+# declare the app
+app = FastAPI(default_response_class=ORJSONResponse)
+mock_router = APIRouter(
+    prefix="/mock",
+)
+
+# ...
+app.include_router(demo_router)
+mock_router.include_router(mock_demo_router)
+
+```
+### Test!
+
+
+### Plotly Dash Frontend
+
+- Create a new 'python package' (directory containing an `__init__.py`) file under `web/src/web/pages` (in this example named 'demo')
+- Create single page Plotly Dash application here
+- This will be automatically registered by the `web/src/web/main.py` module because of the `use_pages=True` argument passed to the Dash app
+- The page has 3 main components:
+  - A layout
+  - A data store
+  - A series of callbacks that fire as the user interacts with the data
+
+
+- Inspect at [http://localhost:8201/demo/demo]()
+
+
+## Ways of working
+
+- Pycharm
+  - setup the FastAPI and Plotly Dash configurations so that you can see the changes as you work

docs/developer/deployment.qmd

@@ -33,3 +33,11 @@ docker compose --project-name [project name] up
 ```sh
 docker compose --project-name [project name] down
 ```
+
+### An one-liner
+
+Replace _myname_ with anything that won't clash with other users (e.g. _your_ name!)
+
+```shell
+docker compose --project-name hyui-dev-myname up --build -d && docker compose -p hyui-dev-myname logs -f
+```

docs/developer/pycharm.qmd

@@ -0,0 +1,27 @@
+Notes on setting up PyCharm for development with specific reference to working on a Mac
+
+## Tips
+
+Set up run configurations
+
+- https://www.jetbrains.com/help/pycharm/run-debug-configuration.html
+- install the **EnvFile** plug-in so that the environment file can be read by your run configuration (see https://stackoverflow.com/a/42708476/992999)
+- set `OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES` as an environment variable in your run configuration to avoid [errors](https://stackoverflow.com/a/52230415/992999) on recent _macos_ installs
+- consider also specifying a 'pytest' configuration where you can store the arguments to prevent pytest running those tests that should only run in the live environment (assuming that you are developing locally)
+
+e.g. where you have decorated your test function
+
+```python
+@pytest.mark.live
+def test_get_demo():
+    response = client.get("/demo/", params={"days_ahead": 3})
+    assert response.status_code == 200
+```
+
+```shell
+pytest -v -m 'not live'
+```
+
+Example configurations here
+- [FastAPI](../snippets/HyUI-API%20FastAPI.run.xml)
+- [Plotly Dash](../snippets/HyUI-Web%20Plotly%20Dash.run.xml)

docs/developer/setup.qmd

@@ -104,14 +104,23 @@ Ensure you replace `.env` with the correct path to your development `.env` file.
 Sometimes it can be difficult to debug your staging services from within the NHS's firewall on live data. You can log into the `hyui-web` or `hyui-api` services from a terminal within the `HyUi` directory using the following:
 
 ```sh
-docker compose --project-name [project name] exec hyui-web bash
+docker compose --project-name [project name] exec web bash
 ```
 
 ```sh
-docker compose --project-name [project name] exec hyui-api bash
+docker compose --project-name [project name] exec api bash
 ```
 
 You can then change the code using vim or open a python console and debug from there. It may be useful to edit the `docker/api/Dockerfile` or `docker/web/Dockerfile` and add a `--reload` flag to the `ENTRYPOINT` so that code changes from within the container are reloaded automatically. `ENTRYPOINT ["uvicorn", "api.main:app", "--host", "0.0.0.0", "--reload"]` instead of `ENTRYPOINT ["uvicorn", "api.main:app", "--host", "0.0.0.0", "--reload"]`.
+You can also run `pytest` from within the docker container (if the `pip install` step enables the additonal options in the `pyproject.toml` file)
+
+e.g.
+```Dockerfile
+# without testing: standard set-up
+#RUN pip install --no-cache-dir /app/api
+# installs testing: useful for debugging
+RUN pip install --no-cache-dir /app/api[test]
+```
 
 If you prefer not to use vim then Visual Studio can SSH into the running containers somehow.
 

docs/snippets/HyUI-API FastAPI.run.xml

@@ -0,0 +1,36 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="HyUI-API FastAPI" type="PythonConfigurationType" factoryName="Python">
+    <module name="HyUi" />
+    <option name="INTERPRETER_OPTIONS" value="" />
+    <option name="PARENT_ENVS" value="true" />
+    <envs>
+      <env name="PYTHONUNBUFFERED" value="1" />
+      <env name="WEB_USERNAME" value="hyui" />
+    </envs>
+    <option name="SDK_HOME" value="" />
+    <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
+    <option name="IS_MODULE_SDK" value="true" />
+    <option name="ADD_CONTENT_ROOTS" value="true" />
+    <option name="ADD_SOURCE_ROOTS" value="true" />
+    <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
+    <EXTENSION ID="net.ashald.envfile">
+      <option name="IS_ENABLED" value="true" />
+      <option name="IS_SUBST" value="false" />
+      <option name="IS_PATH_MACRO_SUPPORTED" value="false" />
+      <option name="IS_IGNORE_MISSING_FILES" value="false" />
+      <option name="IS_ENABLE_EXPERIMENTAL_INTEGRATIONS" value="false" />
+      <ENTRIES>
+        <ENTRY IS_ENABLED="true" PARSER="runconfig" IS_EXECUTABLE="false" />
+        <ENTRY IS_ENABLED="true" PARSER="env" IS_EXECUTABLE="false" PATH=".env" />
+      </ENTRIES>
+    </EXTENSION>
+    <option name="SCRIPT_NAME" value="uvicorn" />
+    <option name="PARAMETERS" value="api.main:app --reload --port 8200" />
+    <option name="SHOW_COMMAND_LINE" value="false" />
+    <option name="EMULATE_TERMINAL" value="false" />
+    <option name="MODULE_MODE" value="true" />
+    <option name="REDIRECT_INPUT" value="false" />
+    <option name="INPUT_FILE" value="" />
+    <method v="2" />
+  </configuration>
+</component>

docs/snippets/HyUI-Web Plotly Dash.run.xml

@@ -0,0 +1,36 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="HyUI-Web Plotly Dash" type="PythonConfigurationType" factoryName="Python">
+    <module name="web" />
+    <option name="INTERPRETER_OPTIONS" value="" />
+    <option name="PARENT_ENVS" value="true" />
+    <envs>
+      <env name="PYTHONUNBUFFERED" value="1" />
+      <env name="OBJC_DISABLE_INITIALIZE_FORK_SAFETY" value="YES" />
+    </envs>
+    <option name="SDK_HOME" value="" />
+    <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/" />
+    <option name="IS_MODULE_SDK" value="true" />
+    <option name="ADD_CONTENT_ROOTS" value="true" />
+    <option name="ADD_SOURCE_ROOTS" value="true" />
+    <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
+    <EXTENSION ID="net.ashald.envfile">
+      <option name="IS_ENABLED" value="true" />
+      <option name="IS_SUBST" value="false" />
+      <option name="IS_PATH_MACRO_SUPPORTED" value="false" />
+      <option name="IS_IGNORE_MISSING_FILES" value="false" />
+      <option name="IS_ENABLE_EXPERIMENTAL_INTEGRATIONS" value="false" />
+      <ENTRIES>
+        <ENTRY IS_ENABLED="true" PARSER="runconfig" IS_EXECUTABLE="false" />
+        <ENTRY IS_ENABLED="true" PARSER="env" IS_EXECUTABLE="false" PATH=".env" />
+      </ENTRIES>
+    </EXTENSION>
+    <option name="SCRIPT_NAME" value="web.main" />
+    <option name="PARAMETERS" value="" />
+    <option name="SHOW_COMMAND_LINE" value="false" />
+    <option name="EMULATE_TERMINAL" value="false" />
+    <option name="MODULE_MODE" value="true" />
+    <option name="REDIRECT_INPUT" value="false" />
+    <option name="INPUT_FILE" value="" />
+    <method v="2" />
+  </configuration>
+</component>