Merge branch 'master' into fix-sandbox-error

.github/dependabot.yml

@@ -7,15 +7,3 @@ updates:
         interval: "weekly"
     reviewers:
       - "openedx/arbi-bom"
-  - package-ecosystem: "github-actions"
-    directory: "/.github/actions/unit-tests/"
-    schedule:
-        interval: "weekly"
-    reviewers:
-      - "openedx/arbi-bom"
-  - package-ecosystem: "github-actions"
-    directory: "/.github/actions/verify-tests-count/"
-    schedule:
-        interval: "weekly"
-    reviewers:
-      - "openedx/arbi-bom"

.github/workflows/add-remove-label-on-comment.yml

@@ -17,3 +17,4 @@ on:
 jobs:
   add_remove_labels:
     uses: openedx/.github/.github/workflows/add-remove-label-on-comment.yml@master
+

.github/workflows/ci-static-analysis.yml

@@ -27,7 +27,7 @@ jobs:
 
       - name: Get pip cache dir
         id: pip-cache-dir
-        run: echo "::set-output name=dir::$(pip cache dir)"
+        run: echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
 
       - name: Cache pip dependencies
         id: cache-dependencies

.github/workflows/js-tests.yml

@@ -13,7 +13,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-20.04]
-        node-version: [18]
+        node-version: [18, 20]
         python-version:
           - "3.11"
 
@@ -49,7 +49,7 @@ jobs:
       - name: Get pip cache dir
         id: pip-cache-dir
         run: |
-          echo "::set-output name=dir::$(pip cache dir)"
+          echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
 
       - name: Cache pip dependencies
         id: cache-dependencies

.github/workflows/lint-imports.yml

@@ -28,7 +28,7 @@ jobs:
 
       - name: Get pip cache dir
         id: pip-cache-dir
-        run: echo "::set-output name=dir::$(pip cache dir)"
+        run: echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
 
       - name: Cache pip dependencies
         id: cache-dependencies

.github/workflows/migrations-check.yml

@@ -85,7 +85,7 @@ jobs:
       - name: Get pip cache dir
         id: pip-cache-dir
         run: |
-          echo "::set-output name=dir::$(pip cache dir)"
+          echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
 
       - name: Cache pip dependencies
         id: cache-dependencies

.github/workflows/pylint-checks.yml

@@ -18,7 +18,7 @@ jobs:
           - module-name: lms-2
             path: "--django-settings-module=lms.envs.test lms/djangoapps/gating/ lms/djangoapps/grades/ lms/djangoapps/instructor/ lms/djangoapps/instructor_analytics/ lms/djangoapps/discussion/ lms/djangoapps/edxnotes/ lms/djangoapps/email_marketing/ lms/djangoapps/experiments/ lms/djangoapps/instructor_task/ lms/djangoapps/learner_dashboard/ lms/djangoapps/learner_home/ lms/djangoapps/lms_initialization/ lms/djangoapps/lms_xblock/ lms/djangoapps/lti_provider/ lms/djangoapps/mailing/ lms/djangoapps/mobile_api/ lms/djangoapps/monitoring/ lms/djangoapps/ora_staff_grader/ lms/djangoapps/program_enrollments/ lms/djangoapps/rss_proxy lms/djangoapps/static_template_view/ lms/djangoapps/staticbook/ lms/djangoapps/support/ lms/djangoapps/survey/ lms/djangoapps/teams/ lms/djangoapps/tests/ lms/djangoapps/user_tours/ lms/djangoapps/verify_student/ lms/djangoapps/mfe_config_api/ lms/envs/ lms/lib/ lms/tests.py"
           - module-name: openedx-1
-            path: "--django-settings-module=lms.envs.test openedx/core/types/ openedx/core/djangoapps/ace_common/ openedx/core/djangoapps/agreements/ openedx/core/djangoapps/api_admin/ openedx/core/djangoapps/auth_exchange/ openedx/core/djangoapps/bookmarks/ openedx/core/djangoapps/cache_toolbox/ openedx/core/djangoapps/catalog/ openedx/core/djangoapps/ccxcon/ openedx/core/djangoapps/commerce/ openedx/core/djangoapps/common_initialization/ openedx/core/djangoapps/common_views/ openedx/core/djangoapps/config_model_utils/ openedx/core/djangoapps/content/ openedx/core/djangoapps/content_libraries/ openedx/core/djangoapps/content_staging/ openedx/core/djangoapps/contentserver/ openedx/core/djangoapps/cookie_metadata/ openedx/core/djangoapps/cors_csrf/ openedx/core/djangoapps/course_apps/ openedx/core/djangoapps/course_date_signals/ openedx/core/djangoapps/course_groups/ openedx/core/djangoapps/courseware_api/ openedx/core/djangoapps/crawlers/ openedx/core/djangoapps/credentials/ openedx/core/djangoapps/credit/ openedx/core/djangoapps/dark_lang/ openedx/core/djangoapps/debug/ openedx/core/djangoapps/demographics/ openedx/core/djangoapps/discussions/ openedx/core/djangoapps/django_comment_common/ openedx/core/djangoapps/embargo/ openedx/core/djangoapps/enrollments/ openedx/core/djangoapps/external_user_ids/ openedx/core/djangoapps/zendesk_proxy/ openedx/core/djangolib/ openedx/core/lib/ openedx/core/tests/ openedx/core/djangoapps/course_live/"
+            path: "--django-settings-module=lms.envs.test openedx/core/types/ openedx/core/djangoapps/ace_common/ openedx/core/djangoapps/agreements/ openedx/core/djangoapps/api_admin/ openedx/core/djangoapps/auth_exchange/ openedx/core/djangoapps/bookmarks/ openedx/core/djangoapps/cache_toolbox/ openedx/core/djangoapps/catalog/ openedx/core/djangoapps/ccxcon/ openedx/core/djangoapps/commerce/ openedx/core/djangoapps/common_initialization/ openedx/core/djangoapps/common_views/ openedx/core/djangoapps/config_model_utils/ openedx/core/djangoapps/content/ openedx/core/djangoapps/content_libraries/ openedx/core/djangoapps/content_staging/ openedx/core/djangoapps/contentserver/ openedx/core/djangoapps/cookie_metadata/ openedx/core/djangoapps/cors_csrf/ openedx/core/djangoapps/course_apps/ openedx/core/djangoapps/course_date_signals/ openedx/core/djangoapps/course_groups/ openedx/core/djangoapps/courseware_api/ openedx/core/djangoapps/crawlers/ openedx/core/djangoapps/credentials/ openedx/core/djangoapps/credit/ openedx/core/djangoapps/dark_lang/ openedx/core/djangoapps/debug/ openedx/core/djangoapps/discussions/ openedx/core/djangoapps/django_comment_common/ openedx/core/djangoapps/embargo/ openedx/core/djangoapps/enrollments/ openedx/core/djangoapps/external_user_ids/ openedx/core/djangoapps/zendesk_proxy/ openedx/core/djangolib/ openedx/core/lib/ openedx/core/tests/ openedx/core/djangoapps/course_live/"
           - module-name: openedx-2
             path: "--django-settings-module=lms.envs.test openedx/core/djangoapps/geoinfo/ openedx/core/djangoapps/header_control/ openedx/core/djangoapps/heartbeat/ openedx/core/djangoapps/lang_pref/ openedx/core/djangoapps/models/ openedx/core/djangoapps/monkey_patch/ openedx/core/djangoapps/oauth_dispatch/ openedx/core/djangoapps/olx_rest_api/ openedx/core/djangoapps/password_policy/ openedx/core/djangoapps/plugin_api/ openedx/core/djangoapps/plugins/ openedx/core/djangoapps/profile_images/ openedx/core/djangoapps/programs/ openedx/core/djangoapps/safe_sessions/ openedx/core/djangoapps/schedules/ openedx/core/djangoapps/service_status/ openedx/core/djangoapps/session_inactivity_timeout/ openedx/core/djangoapps/signals/ openedx/core/djangoapps/site_configuration/ openedx/core/djangoapps/system_wide_roles/ openedx/core/djangoapps/theming/ openedx/core/djangoapps/user_api/ openedx/core/djangoapps/user_authn/ openedx/core/djangoapps/util/ openedx/core/djangoapps/verified_track_content/ openedx/core/djangoapps/video_config/ openedx/core/djangoapps/video_pipeline/ openedx/core/djangoapps/waffle_utils/ openedx/core/djangoapps/xblock/ openedx/core/djangoapps/xmodule_django/ openedx/core/tests/ openedx/features/ openedx/testing/ openedx/tests/ openedx/core/djangoapps/learner_pathway/ openedx/core/djangoapps/notifications/ openedx/core/djangoapps/staticfiles/ openedx/core/djangoapps/content_tagging/"
           - module-name: common
@@ -44,7 +44,7 @@ jobs:
       - name: Get pip cache dir
         id: pip-cache-dir
         run: |
-          echo "::set-output name=dir::$(pip cache dir)"
+          echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
 
       - name: Cache pip dependencies
         id: cache-dependencies

.github/workflows/quality-checks.yml

@@ -16,7 +16,7 @@ jobs:
         os: [ubuntu-20.04]
         python-version:
           - "3.11"
-        node-version: [18]
+        node-version: [20]
 
     steps:
       - uses: actions/checkout@v4
@@ -45,7 +45,7 @@ jobs:
       - name: Get pip cache dir
         id: pip-cache-dir
         run: |
-          echo "::set-output name=dir::$(pip cache dir)"
+          echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
 
       - name: Cache pip dependencies
         id: cache-dependencies

.github/workflows/static-assets-check.yml

@@ -15,7 +15,7 @@ jobs:
         os: [ubuntu-20.04]
         python-version:
           - "3.11"
-        node-version: [18]
+        node-version: [18, 20]
         npm-version: [10.5.x]
         mongo-version:
           - "7.0"
@@ -58,7 +58,7 @@ jobs:
       - name: Get pip cache dir
         id: pip-cache-dir
         run: |
-          echo "::set-output name=dir::$(pip cache dir)"
+          echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
 
       - name: Cache pip dependencies
         id: cache-dependencies

.github/workflows/unit-test-shards.json

@@ -106,7 +106,6 @@
       "openedx/core/djangoapps/course_live/",
       "openedx/core/djangoapps/dark_lang/",
       "openedx/core/djangoapps/debug/",
-      "openedx/core/djangoapps/demographics/",
       "openedx/core/djangoapps/discussions/",
       "openedx/core/djangoapps/django_comment_common/",
       "openedx/core/djangoapps/embargo/",
@@ -187,7 +186,6 @@
       "openedx/core/djangoapps/credit/",
       "openedx/core/djangoapps/dark_lang/",
       "openedx/core/djangoapps/debug/",
-      "openedx/core/djangoapps/demographics/",
       "openedx/core/djangoapps/discussions/",
       "openedx/core/djangoapps/django_comment_common/",
       "openedx/core/djangoapps/embargo/",

.github/workflows/unit-tests.yml

@@ -99,8 +99,36 @@ jobs:
         run: |
           pip freeze
 
-      - name: Setup and run tests
-        uses: ./.github/actions/unit-tests
+      - name: set settings path
+        shell: bash
+        run: |
+          echo "settings_path=$(python scripts/unit_test_shards_parser.py --shard-name=${{ matrix.shard_name }} --output settings )" >> $GITHUB_ENV
+
+      - name: get unit tests for shard
+        shell: bash
+        run: |
+          echo "unit_test_paths=$(python scripts/unit_test_shards_parser.py --shard-name=${{ matrix.shard_name }} )" >> $GITHUB_ENV
+
+      - name: run tests
+        shell: bash
+        run: |
+          python -Wd -m pytest -p no:randomly --ds=${{ env.settings_path }} ${{ env.unit_test_paths }} --cov=.
+
+      - name: rename warnings json file
+        if: success()
+        shell: bash
+        run: |
+          cd test_root/log
+          mv pytest_warnings.json pytest_warnings_${{ matrix.shard_name }}.json
+
+      - name: save pytest warnings json file
+        if: success()
+        uses: actions/upload-artifact@v4
+        with:
+          name: pytest-warnings-json-${{ matrix.shard_name }}
+          path: |
+            test_root/log/pytest_warnings*.json
+          overwrite: true
 
       - name: Renaming coverage data file
         run: |
@@ -109,8 +137,8 @@ jobs:
       - name: Upload coverage
         uses: actions/upload-artifact@v4
         with:
-          name: coverage
-          path: reports/${{matrix.shard_name}}.coverage
+          name: coverage-${{ matrix.shard_name }}
+          path: reports/${{ matrix.shard_name }}.coverage
           overwrite: true
 
   collect-and-verify:
@@ -130,8 +158,49 @@ jobs:
         run: |
           make test-requirements
 
-      - name: verify unit tests count
-        uses: ./.github/actions/verify-tests-count
+      - name: collect tests from all modules
+        shell: bash
+        run: |
+          echo "root_cms_unit_tests_count=$(pytest --disable-warnings --collect-only --ds=cms.envs.test cms/ -q | head -n -2 | wc -l)" >> $GITHUB_ENV
+          echo "root_lms_unit_tests_count=$(pytest --disable-warnings --collect-only --ds=lms.envs.test lms/ openedx/ common/djangoapps/ xmodule/ pavelib/ -q | head -n -2 | wc -l)" >> $GITHUB_ENV
+
+      - name: get GHA unit test paths
+        shell: bash
+        run: |
+          echo "cms_unit_test_paths=$(python scripts/gha_unit_tests_collector.py --cms-only)" >> $GITHUB_ENV
+          echo "lms_unit_test_paths=$(python scripts/gha_unit_tests_collector.py --lms-only)" >> $GITHUB_ENV
+
+      - name: collect tests from GHA unit test shards
+        shell: bash
+        run: |
+          echo "cms_unit_tests_count=$(pytest --disable-warnings --collect-only --ds=cms.envs.test ${{ env.cms_unit_test_paths }} -q | head -n -2 | wc -l)" >> $GITHUB_ENV
+          echo "lms_unit_tests_count=$(pytest --disable-warnings --collect-only --ds=lms.envs.test ${{ env.lms_unit_test_paths }} -q | head -n -2 | wc -l)" >> $GITHUB_ENV
+
+      - name: add unit tests count
+        shell: bash
+        run: |
+          echo "root_all_unit_tests_count=$((${{ env.root_cms_unit_tests_count }}+${{ env.root_lms_unit_tests_count }}))" >> $GITHUB_ENV
+          echo "shards_all_unit_tests_count=$((${{ env.cms_unit_tests_count }}+${{ env.lms_unit_tests_count }}))" >> $GITHUB_ENV
+
+      - name: print unit tests count
+        shell: bash
+        run: |
+          echo CMS unit tests from root: ${{ env.root_cms_unit_tests_count }}
+          echo LMS unit tests from root: ${{ env.root_lms_unit_tests_count }}
+          echo CMS unit tests from shards: ${{ env.cms_unit_tests_count }}
+          echo LMS unit tests from shards: ${{ env.lms_unit_tests_count }}
+          echo All root unit tests count: ${{ env.root_all_unit_tests_count }}
+          echo All shards unit tests count: ${{ env.shards_all_unit_tests_count }}
+
+      - name: fail the check
+        shell: bash
+        if: ${{ env.root_all_unit_tests_count != env.shards_all_unit_tests_count }}
+        run: |
+          echo "::error title='Unit test modules in unit-test-shards.json (unit-tests.yml workflow) are outdated'::unit tests running in unit-tests
+          workflow don't match the count for unit tests for entire edx-platform suite, please update the unit-test-shards.json under .github/workflows
+          to add any missing apps and match the count. for more details please take a look at scripts/gha-shards-readme.md"
+          exit 1
+
 
   # This job aggregates test results. It's the required check for branch protection.
   # https://github.com/marketplace/actions/alls-green#why
@@ -156,7 +225,8 @@ jobs:
       - name: collect pytest warnings files
         uses: actions/download-artifact@v4
         with:
-          name: pytest-warnings-json
+          pattern: pytest-warnings-json-*
+          merge-multiple: true
           path: test_root/log
 
       - name: display structure of downloaded files
@@ -175,6 +245,24 @@ jobs:
             reports/pytest_warnings/warning_report_all.html
           overwrite: true
 
+  merge-artifacts:
+    runs-on: ubuntu-20.04
+    needs: [compile-warnings-report]
+    steps:
+      - name: Merge Pytest Warnings JSON Artifacts
+        uses: actions/upload-artifact/merge@v4
+        with:
+          name: pytest-warnings-json
+          pattern: pytest-warnings-json-*
+          delete-merged: true
+
+      - name: Merge Coverage Artifacts
+        uses: actions/upload-artifact/merge@v4
+        with:
+          name: coverage
+          pattern: coverage-*
+          delete-merged: true
+
   # Combine and upload coverage reports.
   coverage:
     if: (github.repository == 'edx/edx-platform-private') || (github.repository == 'openedx/edx-platform' && (startsWith(github.base_ref, 'open-release') == false))
@@ -196,7 +284,8 @@ jobs:
       - name: Download all artifacts
         uses: actions/download-artifact@v4
         with:
-          name: coverage
+          pattern: coverage-*
+          merge-multiple: true
           path: reports
 
       - name: Install Python dependencies