Add flake detection mode for webdriver tests (datacommonsorg#4844)

- Also add a shared helper for clicking elements and use it to deflake a
test
- Document all of the above

run_test.sh

@@ -173,9 +173,17 @@ function run_webdriver_test {
   export FLASK_ENV=webdriver
   export ENABLE_MODEL=true
   export GOOGLE_CLOUD_PROJECT=datcom-website-dev
+  if [[ " ${extra_args[@]} " =~ " --flake-finder " ]]; then
+    export FLAKE_FINDER=true
+  fi
   source .env/bin/activate
   start_servers
-  python3 -m pytest -n auto --reruns 2 server/webdriver/tests/ ${@}
+  if [[ "$FLAKE_FINDER" == "true" ]]; then
+    python3 -m pytest -n auto server/webdriver/tests/ ${@}
+  else
+    # TODO: Stop using reruns once tests are deflaked.
+    python3 -m pytest -n auto --reruns 2 server/webdriver/tests/ ${@}
+  fi
   stop_servers
   deactivate
 }

server/requirements.txt

@@ -29,6 +29,7 @@ protobuf==4.25.3
 PyGithub==1.58.2
 pyOpenSSL==23.2.0
 python-dateutil==2.8.2
+pytest-flakefinder==1.1.0
 pytest-rerunfailures==10.2
 pytest-xdist==3.2.1
 PyYAML==6.0.1
@@ -38,4 +39,4 @@ selenium==4.21.0
 typing-extensions==4.10.0
 webdriver-manager==4.0.0
 Werkzeug==3.0.6
-wheel==0.38.1
+wheel==0.38.1

server/webdriver/README.md

@@ -2,9 +2,52 @@
 
 ## How To Run the Data Commons WebDriver Unit Tests
 
-Run the following command from the parent directory:
+Run the following command from the repo root:
 
-    ./run_tests.sh -w
+```bash
+./run_tests.sh -w
+```
+
+### Run a subset of tests
+
+#### All tests in a class
+
+```bash
+./run_Test.sh -w -k "TestClass"
+```
+
+#### Single method in a class
+
+```bash
+./run_test.sh -w -k "TestClass and test_method"
+```
+
+See also the [Pytest syntax reference](https://docs.pytest.org/en/stable/how-to/usage.html#specifying-which-tests-to-run).
+Note that assuming there are no duplicate test class names, this is the equivalent of
+
+```bash
+./run_test.sh -w path/to/file_with_test.py::TestClass::test_method
+```
+
+### Check if a test is flaky
+
+If you suspect a test is flaky (failing a small percent of runs consistently),
+you can use [pytest-flakefinder](https://pypi.org/project/pytest-flakefinder/)
+to run it many times at once.
+
+Run a test 50 times:
+
+```bash
+./run_test.sh -w --flake-finder -k "TestClass and test_method"
+```
+
+Run a test 100 times:
+
+```bash
+./run_test.sh -w --flake-finder --flake-runs=100 -k "TestClass and test_method"
+```
+
+If you find that a test fails (often due to a TimeoutException), try using one of the strategies below to make sure all required elements have loaded before asserting on them.
 
 ## Things To Note
 
@@ -48,7 +91,15 @@ In the example below, WebDriver will wait until the element contains `"Mountain
     'Mountain View')
     WebDriverWait(self.driver, SLEEP_SEC).until(element_present)
 
-### 3. Wait Until HTML Element Disappears
+### 3. Wait Until HTML Element is Clickable
+
+If a test flakes with an "element not interactable" error, you may need to wait for the expected condition `element_to_be_clickable`. You can combine the waiting step and the clicking step by passing an element locator the the shared helper `click_el`:
+
+    shared.click_el(self.driver,
+      (By.ID, 'Median_Income_Persondc/g/Demographics-Median_Income_Person'))
+    shared.click_el(self.driver, (By.CLASS, 'continue-button'))
+
+### 4. Wait Until HTML Element Disappears
 
 If you want to wait until an element disappears from the DOM, use `invisibility_of_element_located`.
 
@@ -60,8 +111,8 @@ WebDriver can wait for the 2nd element to disappear as follows:
     (By.CSS_SELECTOR,'.my-class:nth-child(2)'))
     WebDriverWait(self.driver, SLEEP_SEC).until(element_present)
 
-### 4. Wait Until Page Title Changes
+### 5. Wait Until Page Title Changes
 
 If you want to make sure that the site's title is correct, wait until the title contains your desired text. This can be used to check to see if an on-click event works as expected.
 
-    WebDriverWait(self.driver, SLEEP_SEC).until(EC.title_contains(TITLE_TEXT)
+    WebDriverWait(self.driver, SLEEP_SEC).until(EC.title_contains(TITLE_TEXT))

server/webdriver/shared.py

@@ -64,6 +64,18 @@ def click_sv_group(driver, svg_name):
       break
 
 
+def click_el(driver, element_locator):
+  """Waits for an element with the given locator to be clickable, then clicks it.
+
+  Returns the clicked element.
+  """
+  element_clickable = EC.element_to_be_clickable(element_locator)
+  WebDriverWait(driver, TIMEOUT).until(element_clickable)
+  element = driver.find_element(*element_locator)
+  element.click()
+  return element
+
+
 def select_source(driver, source_name, sv_dcid):
   """With the source selector modal open, choose the source with name
     source_name for variable with dcid sv_dcid"""

server/webdriver/shared_tests/vis_timeline_test.py

@@ -142,20 +142,21 @@ def test_manually_enter_options(self):
     self.driver.get(self.url_ + TIMELINE_URL.replace('#visType=timeline', ''))
 
     # Click the timeline tab
+    vis_types_clickable = EC.element_to_be_clickable(
+        (By.CLASS_NAME, 'vis-type-option'))
+    WebDriverWait(self.driver, self.TIMEOUT_SEC).until(vis_types_clickable)
     vis_type_options = self.driver.find_elements(By.CLASS_NAME,
                                                  'vis-type-option')
     for vis_type in vis_type_options:
       if 'Timeline' in vis_type.text:
         vis_type.click()
         break
-    page_header = self.driver.find_element(By.CSS_SELECTOR, '.info-content h3')
-    self.assertEqual(page_header.text, 'Timeline')
+    page_header_locator = (By.CSS_SELECTOR, '.info-content h3')
+    WebDriverWait(self.driver, self.TIMEOUT_SEC).until(
+        EC.text_to_be_present_in_element(page_header_locator, 'Timeline'))
 
     # Click the start button
-    element_present = EC.presence_of_element_located(
-        (By.CLASS_NAME, 'start-button'))
-    WebDriverWait(self.driver, self.TIMEOUT_SEC).until(element_present)
-    self.driver.find_element(By.CLASS_NAME, 'start-button').click()
+    shared.click_el(self.driver, (By.CLASS_NAME, 'start-button'))
 
     # Type california into the search box.
     element_present = EC.presence_of_element_located((By.ID, 'location-field'))
@@ -164,12 +165,8 @@ def test_manually_enter_options(self):
     search_box_input.send_keys(PLACE_SEARCH_CA)
 
     # Click on the first result.
-    element_present = EC.presence_of_element_located(
-        (By.CLASS_NAME, 'pac-item'))
-    WebDriverWait(self.driver, self.TIMEOUT_SEC).until(element_present)
-    first_result = self.driver.find_element(By.CSS_SELECTOR,
-                                            '.pac-item:nth-child(1)')
-    first_result.click()
+    first_result_locator = (By.XPATH, '(//*[contains(@class, "pac-item")])[1]')
+    shared.click_el(self.driver, first_result_locator)
 
     # Type USA into the search box after California has been selected.
     element_present = EC.presence_of_element_located(
@@ -179,45 +176,29 @@ def test_manually_enter_options(self):
     search_box_input.send_keys(PLACE_SEARCH_USA)
 
     # Click on the first result.
-    element_present = EC.presence_of_element_located(
-        (By.CLASS_NAME, 'pac-item'))
-    WebDriverWait(self.driver, self.TIMEOUT_SEC).until(element_present)
-    first_result = self.driver.find_element(By.CSS_SELECTOR,
-                                            '.pac-item:nth-child(1)')
-    first_result.click()
+    shared.click_el(self.driver, first_result_locator)
 
     # Click continue after USA has been selected.
     element_present = EC.text_to_be_present_in_element(
         (By.CLASS_NAME, 'place-selector-selections'),
         'United States of America')
     WebDriverWait(self.driver, self.TIMEOUT_SEC).until(element_present)
-    element_present = EC.presence_of_element_located(
-        (By.CLASS_NAME, 'continue-button'))
-    WebDriverWait(self.driver, self.TIMEOUT_SEC).until(element_present)
-    self.driver.find_element(By.CLASS_NAME, 'continue-button').click()
+    shared.click_el(self.driver, (By.CLASS_NAME, 'continue-button'))
 
     # Choose stat vars
     shared.wait_for_loading(self.driver)
     shared.click_sv_group(self.driver, "Demographics")
-    element_present = EC.presence_of_element_located(
+    shared.click_el(
+        self.driver,
         (By.ID, 'Median_Age_Persondc/g/Demographics-Median_Age_Person'))
-    WebDriverWait(self.driver, self.TIMEOUT_SEC).until(element_present)
-    self.driver.find_element(
-        By.ID, 'Median_Age_Persondc/g/Demographics-Median_Age_Person').click()
     shared.wait_for_loading(self.driver)
-    element_present = EC.presence_of_element_located(
+    shared.click_el(
+        self.driver,
         (By.ID, 'Median_Income_Persondc/g/Demographics-Median_Income_Person'))
-    WebDriverWait(self.driver, self.TIMEOUT_SEC).until(element_present)
-    self.driver.find_element(
-        By.ID,
-        'Median_Income_Persondc/g/Demographics-Median_Income_Person').click()
 
     # Click continue after selection is done loading.
     shared.wait_for_loading(self.driver)
-    element_present = EC.presence_of_element_located(
-        (By.CLASS_NAME, 'continue-button'))
-    WebDriverWait(self.driver, self.TIMEOUT_SEC).until(element_present)
-    self.driver.find_element(By.CLASS_NAME, 'continue-button').click()
+    shared.click_el(self.driver, (By.CLASS_NAME, 'continue-button'))
 
     # Assert chart is correct
     WebDriverWait(self.driver, self.TIMEOUT_SEC).until(shared.charts_rendered)