diff --git a/CHANGELOG.md b/CHANGELOG.md index e46a8521e..ed7fea45f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,16 @@ +## [1.9.0](https://github.com/w3c/aria-at-app/compare/v1.8.1...v1.9.0) (2024-10-09) + + +### Features + +* Vendor verification ([#1208](https://github.com/w3c/aria-at-app/issues/1208)) ([b8b13d4](https://github.com/w3c/aria-at-app/commit/b8b13d48a5e05e31613bfea54678a30b525d18c6)) + + +### Bug Fixes + +* Add tests for retry cancelled collection jobs ([#1214](https://github.com/w3c/aria-at-app/issues/1214)) ([34d7aac](https://github.com/w3c/aria-at-app/commit/34d7aac53492b5077fe487ef51e2a7bd5c536afc)) +* Assertion with EXCLUDE priority causing test result submit errors ([#1229](https://github.com/w3c/aria-at-app/issues/1229)) ([d7cc1fd](https://github.com/w3c/aria-at-app/commit/d7cc1fd6efe3a72b3df6158a5d1fb9580f3f6de2)) + ### [1.8.1](https://github.com/w3c/aria-at-app/compare/v1.8.0...v1.8.1) (2024-09-23) diff --git a/client/components/CandidateReview/TestPlans/index.jsx b/client/components/CandidateReview/TestPlans/index.jsx index d6760e1f9..fea18e87b 100644 --- a/client/components/CandidateReview/TestPlans/index.jsx +++ b/client/components/CandidateReview/TestPlans/index.jsx @@ -21,6 +21,7 @@ import ClippedProgressBar from '@components/common/ClippedProgressBar'; import { dates } from 'shared'; import './TestPlans.css'; import { calculations } from 'shared'; +import { UserPropType } from '../../common/proptypes'; const FullHeightContainer = styled(Container)` min-height: calc(100vh - 64px); @@ -177,7 +178,7 @@ const None = styled.span` } `; -const TestPlans = ({ testPlanVersions }) => { +const TestPlans = ({ testPlanVersions, me }) => { const [atExpandTableItems, setAtExpandTableItems] = useState({ 1: true, 2: true, @@ -503,20 +504,39 @@ const TestPlans = ({ testPlanVersions }) => { .filter(t => t.isCandidateReview === true) .filter(t => uniqueFilter(t, uniqueLinks, 'link')); + const canReview = + me.roles.includes('ADMIN') || + (me.roles.includes('VENDOR') && + me.company.ats.some(at => at.id === atId)); + + const getTitleEl = () => { + if (canReview) { + return ( + + {getTestPlanVersionTitle(testPlanVersion, { + includeVersionString: true + })}{' '} + ({testsCount} Test + {testsCount === 0 || testsCount > 1 ? `s` : ''}) + + ); + } + return ( + <> + {getTestPlanVersionTitle(testPlanVersion, { + includeVersionString: true + })}{' '} + ({testsCount} Test + {testsCount === 0 || testsCount > 1 ? `s` : ''}) + + ); + }; return ( dataExists && ( - - - {getTestPlanVersionTitle(testPlanVersion, { - includeVersionString: true - })}{' '} - ({testsCount} Test - {testsCount === 0 || testsCount > 1 ? `s` : ''}) - - + {getTitleEl()} {dates.convertDateToString( @@ -778,6 +798,7 @@ TestPlans.propTypes = { ) }) ).isRequired, + me: UserPropType.isRequired, triggerPageUpdate: PropTypes.func }; diff --git a/client/components/CandidateReview/index.jsx b/client/components/CandidateReview/index.jsx index ba0ded88f..db9d2aad9 100644 --- a/client/components/CandidateReview/index.jsx +++ b/client/components/CandidateReview/index.jsx @@ -3,12 +3,16 @@ import { useQuery } from '@apollo/client'; import PageStatus from '../common/PageStatus'; import TestPlans from './TestPlans'; import { CANDIDATE_REVIEW_PAGE_QUERY } from './queries'; +import { ME_QUERY } from '../App/queries'; const CandidateReview = () => { const { loading, data, error } = useQuery(CANDIDATE_REVIEW_PAGE_QUERY, { fetchPolicy: 'cache-and-network' }); + const { data: meData } = useQuery(ME_QUERY); + const { me } = meData; + if (error) { return ( { const testPlanVersions = data.testPlanVersions; - return ; + return ; }; export default CandidateReview; diff --git a/client/components/ConfirmAuth/index.jsx b/client/components/ConfirmAuth/index.jsx index c39bd0104..484bde1e7 100644 --- a/client/components/ConfirmAuth/index.jsx +++ b/client/components/ConfirmAuth/index.jsx @@ -1,15 +1,17 @@ import React from 'react'; -import { Navigate } from 'react-router-dom'; +import { Navigate, useParams } from 'react-router-dom'; import PropTypes from 'prop-types'; import { useQuery } from '@apollo/client'; import { ME_QUERY } from '../App/queries'; import { evaluateAuth } from '../../utils/evaluateAuth'; -const ConfirmAuth = ({ children, requiredPermission }) => { +const ConfirmAuth = ({ children, requiredPermission, requireVendorForAt }) => { const { data } = useQuery(ME_QUERY); + const { atId } = useParams(); const auth = evaluateAuth(data && data.me ? data.me : {}); const { roles, username, isAdmin, isSignedIn } = auth; + const company = data && data.me ? data.me.company : null; if (!username) return ; @@ -21,6 +23,14 @@ const ConfirmAuth = ({ children, requiredPermission }) => { return ; } + // Check if the user's company is the vendor for the associated At + if (requireVendorForAt && !isAdmin) { + const isVendorForAt = company && company.ats.some(at => at.id === atId); + if (!isVendorForAt) { + return ; + } + } + return children; }; @@ -29,7 +39,8 @@ ConfirmAuth.propTypes = { PropTypes.arrayOf(PropTypes.node), PropTypes.node ]).isRequired, - requiredPermission: PropTypes.string + requiredPermission: PropTypes.string, + requireVendorForAt: PropTypes.bool }; export default ConfirmAuth; diff --git a/client/components/DisclaimerInfo/index.jsx b/client/components/DisclaimerInfo/index.jsx index 5730e5433..79964d9c3 100644 --- a/client/components/DisclaimerInfo/index.jsx +++ b/client/components/DisclaimerInfo/index.jsx @@ -104,7 +104,6 @@ const DisclaimerInfo = ({ phase }) => {
setExpanded(!expanded)} diff --git a/client/components/ManageBotRunDialog/queries.js b/client/components/ManageBotRunDialog/queries.js index dba46068e..674c6856b 100644 --- a/client/components/ManageBotRunDialog/queries.js +++ b/client/components/ManageBotRunDialog/queries.js @@ -52,6 +52,9 @@ export const RETRY_CANCELED_COLLECTIONS = gql` retryCanceledCollections { id status + testStatus { + status + } } } } diff --git a/client/components/common/fragments/Me.js b/client/components/common/fragments/Me.js index 128e7072d..dcdf9adeb 100644 --- a/client/components/common/fragments/Me.js +++ b/client/components/common/fragments/Me.js @@ -6,6 +6,12 @@ const ME_FIELDS = gql` id username roles + company { + id + ats { + id + } + } } `; diff --git a/client/index.js b/client/index.js index 5d19d83a4..50c4ece39 100644 --- a/client/index.js +++ b/client/index.js @@ -51,7 +51,20 @@ window.signMeInAsTester = username => { }; window.signMeInAsVendor = username => { - return signMeInCommon({ username, roles: [{ name: 'VENDOR' }] }); + return signMeInCommon({ + username, + roles: [{ name: 'VENDOR' }], + company: { + id: '1', + name: 'vispero', + ats: [ + { + id: '1', + name: 'JAWS' + } + ] + } + }); }; window.startTestTransaction = async () => { diff --git a/client/routes/index.js b/client/routes/index.js index 89a1c600b..2f28f7afa 100644 --- a/client/routes/index.js +++ b/client/routes/index.js @@ -33,7 +33,7 @@ export default () => ( exact path="/candidate-test-plan/:testPlanVersionId/:atId" element={ - + } diff --git a/client/tests/__mocks__/GraphQLMocks/TestQueuePageAdminNotPopulatedMock.js b/client/tests/__mocks__/GraphQLMocks/TestQueuePageAdminNotPopulatedMock.js index 32c8de0cf..423268e4d 100644 --- a/client/tests/__mocks__/GraphQLMocks/TestQueuePageAdminNotPopulatedMock.js +++ b/client/tests/__mocks__/GraphQLMocks/TestQueuePageAdminNotPopulatedMock.js @@ -9,7 +9,12 @@ export default testQueuePageQuery => [ __typename: 'User', id: '1', username: 'foo-bar', - roles: ['ADMIN', 'TESTER'] + roles: ['ADMIN', 'TESTER'], + company: { + id: '1', + name: 'Company', + ats: [] + } }, users: [ { @@ -18,7 +23,12 @@ export default testQueuePageQuery => [ username: 'foo-bar', roles: ['ADMIN', 'TESTER'], isBot: false, - ats: [] + ats: [], + company: { + id: '1', + name: 'Company', + ats: [] + } }, { __typename: 'User', @@ -26,7 +36,12 @@ export default testQueuePageQuery => [ username: 'bar-foo', roles: ['TESTER'], isBot: false, - ats: [] + ats: [], + company: { + id: '1', + name: 'Company', + ats: [] + } }, { __typename: 'User', @@ -34,7 +49,12 @@ export default testQueuePageQuery => [ username: 'boo-far', roles: ['TESTER'], isBot: false, - ats: [] + ats: [], + company: { + id: '1', + name: 'Company', + ats: [] + } } ], ats: [], diff --git a/client/tests/__mocks__/GraphQLMocks/TestQueuePageTesterNotPopulatedMock.js b/client/tests/__mocks__/GraphQLMocks/TestQueuePageTesterNotPopulatedMock.js index 186f6ac4f..403698f61 100644 --- a/client/tests/__mocks__/GraphQLMocks/TestQueuePageTesterNotPopulatedMock.js +++ b/client/tests/__mocks__/GraphQLMocks/TestQueuePageTesterNotPopulatedMock.js @@ -9,7 +9,12 @@ export default testQueuePageQuery => [ __typename: 'User', id: '4', username: 'bar-foo', - roles: ['TESTER'] + roles: ['TESTER'], + company: { + id: '1', + name: 'Company', + ats: [] + } }, users: [ { @@ -18,7 +23,12 @@ export default testQueuePageQuery => [ username: 'foo-bar', roles: ['ADMIN', 'TESTER'], isBot: false, - ats: [] + ats: [], + company: { + id: '1', + name: 'Company', + ats: [] + } }, { __typename: 'User', @@ -26,7 +36,12 @@ export default testQueuePageQuery => [ username: 'bar-foo', roles: ['TESTER'], isBot: false, - ats: [] + ats: [], + company: { + id: '1', + name: 'Company', + ats: [] + } }, { __typename: 'User', @@ -34,7 +49,12 @@ export default testQueuePageQuery => [ username: 'boo-far', roles: ['TESTER'], isBot: false, - ats: [] + ats: [], + company: { + id: '1', + name: 'Company', + ats: [] + } } ], ats: [], diff --git a/client/tests/e2e/TestRun.e2e.test.js b/client/tests/e2e/TestRun.e2e.test.js index 3ec3aeb06..227ed3928 100644 --- a/client/tests/e2e/TestRun.e2e.test.js +++ b/client/tests/e2e/TestRun.e2e.test.js @@ -108,22 +108,24 @@ describe('Test Run when not signed in', () => { }); describe('Test Run when signed in as tester', () => { - const assignSelfAndNavigateToRun = async page => { - const modalDialogSectionButtonSelector = - 'button#disclosure-btn-modal-dialog-0'; - const modalDialogTableSelector = - 'table[aria-label="Reports for Modal Dialog Example V24.06.07 in draft phase"]'; + const assignSelfAndNavigateToRun = async ( + page, + { + testPlanSectionButtonSelector = 'button#disclosure-btn-modal-dialog-0', + testPlanTableSelector = 'table[aria-label="Reports for Modal Dialog Example V24.06.07 in draft phase"]' + } = {} + ) => { const startTestingButtonSelector = 'a[role="button"] ::-p-text(Start Testing)'; // Expand Modal Dialog's V24.06.07 section - await page.waitForSelector(modalDialogSectionButtonSelector); - await page.click(modalDialogSectionButtonSelector); + await page.waitForSelector(testPlanSectionButtonSelector); + await page.click(testPlanSectionButtonSelector); // Wait for the table to render - await page.waitForSelector(modalDialogTableSelector); + await page.waitForSelector(testPlanTableSelector); - await page.$eval(modalDialogTableSelector, el => { + await page.$eval(testPlanTableSelector, el => { // First button found on table would be 'Assign Yourself' el.querySelector('button').click(); }); @@ -161,6 +163,72 @@ describe('Test Run when signed in as tester', () => { await page.waitForNetworkIdle(); }; + const handlePageSubmit = async (page, { expectConflicts = true } = {}) => { + await page.waitForSelector('h1 ::-p-text(Test 1)'); + + // Confirm that submission cannot happen with empty form + // Specificity with selector because there's a 2nd 'hidden' button coming + // from the harness which is what is actually called for the submit event + const submitResultsButtonSelector = + 'button[class="btn btn-primary"] ::-p-text(Submit Results)'; + await page.waitForSelector(submitResultsButtonSelector); + await page.click(submitResultsButtonSelector); + await page.waitForNetworkIdle(); + await page.waitForSelector('::-p-text((required))'); + + // Should refocus on topmost output textarea on page + const activeElementAfterEmptySubmit = await page.evaluate(() => { + return { + id: document.activeElement.id, + nodeName: document.activeElement.nodeName.toLowerCase() + }; + }); + + // Input output for valid submission + await page.evaluate(() => { + const yesRadios = document.querySelectorAll( + 'input[data-testid^="radio-yes-"]' + ); + const noRadios = document.querySelectorAll( + 'input[data-testid^="radio-no-"]' + ); + const noUndesiredRadios = document.querySelectorAll( + 'input[id^="problem-"][id$="-true"]' + ); + const noOutputCheckboxes = document.querySelectorAll( + 'input[id^="no-output-checkbox"]' + ); + + yesRadios.forEach((radio, index) => { + if (index % 2 === 0) { + radio.click(); + } else { + noRadios[index].click(); + } + }); + + noUndesiredRadios.forEach(radio => { + radio.click(); + }); + + noOutputCheckboxes.forEach(checkbox => { + checkbox.click(); + }); + }); + // Submit valid form + await page.click(submitResultsButtonSelector); + await page.waitForNetworkIdle(); + if (expectConflicts) + await page.waitForSelector( + '::-p-text(This test has conflicting results)' + ); + await page.waitForSelector('h2 ::-p-text(Test Results)'); + await page.waitForSelector('button ::-p-text(Edit Results)'); + + expect(activeElementAfterEmptySubmit.id).toBe('speechoutput-0'); + expect(activeElementAfterEmptySubmit.nodeName).toBe('textarea'); + }; + it('self assigns tester on Test Queue page and opens test run', async () => { await getPage({ role: 'tester', url: '/test-queue' }, async page => { await assignSelfAndNavigateToRun(page); @@ -324,72 +392,25 @@ describe('Test Run when signed in as tester', () => { }); it('inputs results and successfully submits', async () => { - await getPage({ role: 'tester', url: '/test-queue' }, async page => { - await assignSelfAndNavigateToRun(page); - - await page.waitForSelector('h1 ::-p-text(Test 1)'); - - // Confirm that submission cannot happen with empty form - // Specificity with selector because there's a 2nd 'hidden' button coming - // from the harness which is what is actually called for the submit event - const submitResultsButtonSelector = - 'button[class="btn btn-primary"] ::-p-text(Submit Results)'; - await page.waitForSelector(submitResultsButtonSelector); - await page.click(submitResultsButtonSelector); - await page.waitForNetworkIdle(); - await page.waitForSelector('::-p-text((required))'); - - // Should refocus on topmost output textarea on page - const activeElementAfterEmptySubmit = await page.evaluate(() => { - return { - id: document.activeElement.id, - nodeName: document.activeElement.nodeName.toLowerCase() - }; - }); - - // Input output for valid submission - await page.evaluate(() => { - const yesRadios = document.querySelectorAll( - 'input[data-testid^="radio-yes-"]' - ); - const noRadios = document.querySelectorAll( - 'input[data-testid^="radio-no-"]' - ); - const noUndesiredRadios = document.querySelectorAll( - 'input[id^="problem-"][id$="-true"]' - ); - const noOutputCheckboxes = document.querySelectorAll( - 'input[id^="no-output-checkbox"]' - ); - - yesRadios.forEach((radio, index) => { - if (index % 2 === 0) { - radio.click(); - } else { - noRadios[index].click(); - } - }); - - noUndesiredRadios.forEach(radio => { - radio.click(); - }); - - noOutputCheckboxes.forEach(checkbox => { - checkbox.click(); + await getPage( + { role: 'tester', url: '/test-queue' }, + async (page, { baseUrl }) => { + await assignSelfAndNavigateToRun(page); + await handlePageSubmit(page); + + // Do the same for Color Viewer Slider which has specially handled + // exclusions; + // Excluded in tests.csv and re-included in *-commands.csv + await page.goto(`${baseUrl}/test-queue`); + await assignSelfAndNavigateToRun(page, { + testPlanSectionButtonSelector: + 'button#disclosure-btn-horizontal-slider-0', + testPlanTableSelector: + 'table[aria-label="Reports for Color Viewer Slider V24.06.26 in draft phase"]' }); - }); - // Submit valid form - await page.click(submitResultsButtonSelector); - await page.waitForNetworkIdle(); - await page.waitForSelector( - '::-p-text(This test has conflicting results)' - ); - await page.waitForSelector('h2 ::-p-text(Test Results)'); - await page.waitForSelector('button ::-p-text(Edit Results)'); - - expect(activeElementAfterEmptySubmit.id).toBe('speechoutput-0'); - expect(activeElementAfterEmptySubmit.nodeName).toBe('textarea'); - }); + await handlePageSubmit(page, { expectConflicts: false }); + } + ); }); it('opens popup with content after clicking "Open Test Page" button', async () => { diff --git a/client/tests/e2e/snapshots/saved/_candidate-review.html b/client/tests/e2e/snapshots/saved/_candidate-review.html index 4fb2ede26..068c908d6 100644 --- a/client/tests/e2e/snapshots/saved/_candidate-review.html +++ b/client/tests/e2e/snapshots/saved/_candidate-review.html @@ -127,7 +127,23 @@

Jul 6, 2022 Jan 2, 2023 - + + Changes requested for 1 test +
@@ -317,7 +333,23 @@

Review Status Summary

Modal Dialog Example V22.05.26 - + + Changes requested for 1 test + diff --git a/client/tests/e2e/snapshots/saved/_candidate-test-plan_24_1.html b/client/tests/e2e/snapshots/saved/_candidate-test-plan_24_1.html index 7c099f627..ac25ef7f2 100644 --- a/client/tests/e2e/snapshots/saved/_candidate-test-plan_24_1.html +++ b/client/tests/e2e/snapshots/saved/_candidate-test-plan_24_1.html @@ -116,12 +116,12 @@
    -
  1. +
  2. Open a Modal Dialog in reading mode + title="Changes Requested">
  3. -
    +
    +
    +

    + + JAWS Representative +

    +     +
    +

    @@ -682,7 +718,7 @@

    -
    +

    Test Review Options

      Radio Group Example Using aria-activedescendant - + @@ -276,11 +276,11 @@

      +
      @@ -377,7 +377,7 @@

      Test Plans Status Summary

      type="button" aria-pressed="false" class="css-14jv778 btn btn-secondary"> - R&D Complete (33) + R&D Complete (31)
    • @@ -385,7 +385,7 @@

      Test Plans Status Summary

      type="button" aria-pressed="false" class="css-14jv778 btn btn-secondary"> - In Draft Review (5) + In Draft Review (6)
    • @@ -401,7 +401,7 @@

      Test Plans Status Summary

      type="button" aria-pressed="false" class="css-14jv778 btn btn-secondary"> - Recommended Plans (1) + Recommended Plans (2)
    @@ -482,6 +482,121 @@

    Test Plans Status Summary

    + + Action Menu Button Example Using aria-activedescendant + + +
    + JAWS, NVDA and VoiceOver for macOS +
    + + +
    + Recommended +

    Since Sep 23, 2024

    +
    + + N/A + +
    + V24.09.18Review Completed Sep 22, 2024 +
    + + +
    + V24.09.18Review Completed Sep 23, 2024 +
    + + +
    + V24.09.18 +
    Approved Sep 23, 2024 +
    + + + Checkbox Example (Tri State)Test Plans Status Summary

- + Modal Dialog ExampleTest Plans Status Summary
0 Open - Issues1 Open + Issue None Yet - + Alert Example @@ -790,7 +905,65 @@

Test Plans Status Summary

Not Started None Yet - + + + Color Viewer Slider + + +
+ JAWS, NVDA and VoiceOver for macOS +
+ + +
+ Draft +

Review Started Oct 2, 2024

+
+ + N/A + +
+ V24.06.26 +
+
+ + Not Started + None Yet + + Command Button ExampleTest Plans Status Summary Not Started None Yet - + Select Only Combobox ExampleTest Plans Status Summary Not Started None Yet - + Toggle ButtonTest Plans Status Summary Not Started None Yet - - - Action Menu Button Example Using aria-activedescendant - - -
- JAWS, NVDA and VoiceOver for macOS -
- - -
- R&D -

Complete Sep 18, 2024

-
- - -
- V24.09.18 -
- - Not Started - Not Started - None Yet - - + Action Menu Button Example Using element.focus()Test Plans Status Summary Not Started None Yet - + Banner Landmark @@ -1226,7 +1344,7 @@

Test Plans Status Summary

Not Started None Yet - + Breadcrumb ExampleTest Plans Status Summary Not Started None Yet - + Checkbox Example (Two State)Test Plans Status Summary Not Started None Yet - - - Color Viewer Slider - - -
- JAWS, NVDA and VoiceOver for macOS -
- - -
- R&D -

Complete Jun 26, 2024

-
- - -
- V24.06.26 -
- - Not Started - Not Started - None Yet - + + + + + + + + + NVDA 2023.3.3 and Chrome for Checkbox Example (Mixed-State) | + ARIA-AT Reports + + + +
+
+ +
+
+
+

+ Checkbox Example (Mixed-State) with NVDA 2023.3.3 and + Chrome +

+ +

Introduction

+

+ This page shows detailed results for each test, including individual + commands that the tester entered into the AT, the output which was + captured from the AT in response, and whether that output was deemed + passing or failing for each of the assertions. The open test button + next to each test allows you to preview the test in your browser. +

+

Metadata

+ +

NVDA Versions Summary

+

+ The following table displays a summary of data for all versions of + NVDA that have been tested. +

+
+ + + + + + + + + + + + + + + + + + + + + + + +
VersionsMust-Have BehaviorsShould-Have BehaviorsMay-Have Behaviors
2023.3.3124 of 124 passed35 of 36 passedNone
2020.4124 of 124 passed36 of 36 passedNone
+
+

Results for NVDA 2023.3.3 and Chrome

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Test NameMust-Have BehaviorsShould-Have BehaviorsMay-Have Behaviors
+ Navigate forwards to a mixed checkbox in reading mode + 24 of 24 passed4 of 4 passedNone
+ Navigate forwards to a mixed checkbox in interaction + mode + 6 of 6 passed1 of 1 passedNone
+ Navigate backwards to a mixed checkbox in reading mode + 24 of 24 passed3 of 4 passedNone
+ Navigate backwards to a mixed checkbox in interaction + mode + 6 of 6 passed1 of 1 passedNone
+ Read information about a mixed checkbox in reading + mode + 8 of 8 passed6 of 6 passedNone
+ Read information about a mixed checkbox in interaction + mode + 8 of 8 passed6 of 6 passedNone
+ Operate a mixed checkbox in reading mode + 2 of 2 passed1 of 1 passedNone
+ Operate a mixed checkbox in interaction mode + 2 of 2 passed1 of 1 passedNone
+ Operate an unchecked checkbox in reading mode + 2 of 2 passed1 of 1 passedNone
+ Operate an unchecked checkbox in interaction mode + 2 of 2 passed1 of 1 passedNone
+ Navigate forwards out of a checkbox group in reading + mode + 16 of 16 passed4 of 4 passedNone
+ Navigate forwards out of a checkbox group in interaction + mode + 4 of 4 passed1 of 1 passedNone
+ Navigate backwards out of a checkbox group in reading + mode + 16 of 16 passed4 of 4 passedNone
+ Navigate backwards out of a checkbox group in interaction + mode + 4 of 4 passed1 of 1 passedNone
All Tests124 of 124 passed35 of 36 passedNone
+
+
+

+ Test 1: Navigate forwards to a mixed checkbox in reading + mode (28 passed, 0 failed) +
+
+ + The information in this report is generated from recommended + tests. Recommended ARIA-AT tests have been reviewed by + assistive technology developers and represent consensus + regarding: applicability and validity of the tests, and + accuracy of test results. +
+
+

+
+ Raise an IssueOpen Test +
+
+
+

Results for each command

+

X Results: 7 passed, 0 failed

+

NVDA Response:

+
+ automatically seeded sample output +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ X Results +
PriorityAssertionResult
MUSTRole 'group' is conveyedPassed
MUSTName 'Sandwich Condiments' is conveyedPassed
MUSTRole 'checkbox' is conveyedPassed
MUSTName ' All condiments' is conveyedPassed
MUSTState 'mixed' is conveyedPassed
MUST + Other behaviors that create severe negative impacts are + not exhibited + Passed
SHOULD + Other behaviors that create moderate negative impacts are + not exhibited + Passed
+
+ Other behaviors that create negative impact: None + +

F Results: 7 passed, 0 failed

+

NVDA Response:

+
+ automatically seeded sample output +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ F Results +
PriorityAssertionResult
MUSTRole 'group' is conveyedPassed
MUSTName 'Sandwich Condiments' is conveyedPassed
MUSTRole 'checkbox' is conveyedPassed
MUSTName ' All condiments' is conveyedPassed
MUSTState 'mixed' is conveyedPassed
MUST + Other behaviors that create severe negative impacts are + not exhibited + Passed
SHOULD + Other behaviors that create moderate negative impacts are + not exhibited + Passed
+
+ Other behaviors that create negative impact: None + +

Tab Results: 7 passed, 0 failed

+

NVDA Response:

+
+ automatically seeded sample output +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ Tab Results +
PriorityAssertionResult
MUSTRole 'group' is conveyedPassed
MUSTName 'Sandwich Condiments' is conveyedPassed
MUSTRole 'checkbox' is conveyedPassed
MUSTName ' All condiments' is conveyedPassed
MUSTState 'mixed' is conveyedPassed
MUST + Other behaviors that create severe negative impacts are + not exhibited + Passed
SHOULD + Other behaviors that create moderate negative impacts are + not exhibited + Passed
+
+ Other behaviors that create negative impact: None + +

+ Down Arrow then Down Arrow Results: 7 passed, 0 + failed +

+

NVDA Response:

+
+ automatically seeded sample output +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ Down Arrow then Down Arrow Results +
PriorityAssertionResult
MUSTRole 'group' is conveyedPassed
MUSTName 'Sandwich Condiments' is conveyedPassed
MUSTRole 'checkbox' is conveyedPassed
MUSTName ' All condiments' is conveyedPassed
MUSTState 'mixed' is conveyedPassed
MUST + Other behaviors that create severe negative impacts are + not exhibited + Passed
SHOULD + Other behaviors that create moderate negative impacts are + not exhibited + Passed
+
+ Other behaviors that create negative impact: None +
+
+

+ +

+
+ +
+
+
+

+ Test 2: Navigate forwards to a mixed checkbox in interaction + mode (7 passed, 0 failed) +
+
+ + The information in this report is generated from recommended + tests. Recommended ARIA-AT tests have been reviewed by + assistive technology developers and represent consensus + regarding: applicability and validity of the tests, and + accuracy of test results. +
+
+

+
+ Raise an IssueOpen Test +
+
+
+

Results for each command

+

Tab Results: 7 passed, 0 failed

+

NVDA Response:

+
+ automatically seeded sample output +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ Tab Results +
PriorityAssertionResult
MUSTRole 'group' is conveyedPassed
MUSTName 'Sandwich Condiments' is conveyedPassed
MUSTRole 'checkbox' is conveyedPassed
MUSTName ' All condiments' is conveyedPassed
MUSTState 'mixed' is conveyedPassed
MUST + Other behaviors that create severe negative impacts are + not exhibited + Passed
SHOULD + Other behaviors that create moderate negative impacts are + not exhibited + Passed
+
+ Other behaviors that create negative impact: None +
+
+

+ +

+
+ +
+
+
+

+ Test 3: Navigate backwards to a mixed checkbox in reading + mode (27 passed, 1 failed) +
+
+ + The information in this report is generated from recommended + tests. Recommended ARIA-AT tests have been reviewed by + assistive technology developers and represent consensus + regarding: applicability and validity of the tests, and + accuracy of test results. +
+
+

+
+ Raise an IssueOpen Test +
+
+
+

Results for each command

+

Shift+X Results: 6 passed, 1 failed

+

NVDA Response:

+
+ automatically seeded sample output +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ Shift+X Results +
PriorityAssertionResult
SHOULD + Other behaviors that create moderate negative impacts are + not exhibited + Failed
MUSTRole 'group' is conveyedPassed
MUSTName 'Sandwich Condiments' is conveyedPassed
MUSTRole 'checkbox' is conveyedPassed
MUSTName 'All condiments' is conveyedPassed
MUSTState 'mixed' is conveyedPassed
MUST + Other behaviors that create severe negative impacts are + not exhibited + Passed
+
+ Other behaviors that create negative impact: +
+ + + + + + + + + + + + + + + +
BehaviorDetailsImpact
OtherSeeded other unexpected behaviorMODERATE
+
+ +

Shift+F Results: 7 passed, 0 failed

+

NVDA Response:

+
+ automatically seeded sample output +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ Shift+F Results +
PriorityAssertionResult
MUSTRole 'group' is conveyedPassed
MUSTName 'Sandwich Condiments' is conveyedPassed
MUSTRole 'checkbox' is conveyedPassed
MUSTName 'All condiments' is conveyedPassed
MUSTState 'mixed' is conveyedPassed
MUST + Other behaviors that create severe negative impacts are + not exhibited + Passed
SHOULD + Other behaviors that create moderate negative impacts are + not exhibited + Passed
+
+ Other behaviors that create negative impact: None + +

Shift+Tab Results: 7 passed, 0 failed

+

NVDA Response:

+
+ automatically seeded sample output +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ Shift+Tab Results +
PriorityAssertionResult
MUSTRole 'group' is conveyedPassed
MUSTName 'Sandwich Condiments' is conveyedPassed
MUSTRole 'checkbox' is conveyedPassed
MUSTName 'All condiments' is conveyedPassed
MUSTState 'mixed' is conveyedPassed
MUST + Other behaviors that create severe negative impacts are + not exhibited + Passed
SHOULD + Other behaviors that create moderate negative impacts are + not exhibited + Passed
+
+ Other behaviors that create negative impact: None + +

+ Up Arrow then Up Arrow Results: 7 passed, 0 failed +

+

NVDA Response:

+
+ automatically seeded sample output +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ Up Arrow then Up Arrow Results +
PriorityAssertionResult
MUSTRole 'group' is conveyedPassed
MUSTName 'Sandwich Condiments' is conveyedPassed
MUSTRole 'checkbox' is conveyedPassed
MUSTName 'All condiments' is conveyedPassed
MUSTState 'mixed' is conveyedPassed
MUST + Other behaviors that create severe negative impacts are + not exhibited + Passed
SHOULD + Other behaviors that create moderate negative impacts are + not exhibited + Passed
+
+ Other behaviors that create negative impact: None +
+
+

+ +

+
+ +
+
+
+

+ Test 4: Navigate backwards to a mixed checkbox in interaction + mode (7 passed, 0 failed) +
+
+ + The information in this report is generated from recommended + tests. Recommended ARIA-AT tests have been reviewed by + assistive technology developers and represent consensus + regarding: applicability and validity of the tests, and + accuracy of test results. +
+
+

+
+ Raise an IssueOpen Test +
+
+
+

Results for each command

+

Shift+Tab Results: 7 passed, 0 failed

+

NVDA Response:

+
+ automatically seeded sample output +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ Shift+Tab Results +
PriorityAssertionResult
MUSTRole 'group' is conveyedPassed
MUSTName 'Sandwich Condiments' is conveyedPassed
MUSTRole 'checkbox' is conveyedPassed
MUSTName 'All condiments' is conveyedPassed
MUSTState 'mixed' is conveyedPassed
MUST + Other behaviors that create severe negative impacts are + not exhibited + Passed
SHOULD + Other behaviors that create moderate negative impacts are + not exhibited + Passed
+
+ Other behaviors that create negative impact: None +
+
+

+ +

+
+ +
+
+
+

+ Test 5: Read information about a mixed checkbox in reading + mode (14 passed, 0 failed) +
+
+ + The information in this report is generated from recommended + tests. Recommended ARIA-AT tests have been reviewed by + assistive technology developers and represent consensus + regarding: applicability and validity of the tests, and + accuracy of test results. +
+
+

+
+ Raise an IssueOpen Test +
+
+
+

Results for each command

+

Insert+Tab Results: 7 passed, 0 failed

+

NVDA Response:

+
+ automatically seeded sample output +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ Insert+Tab Results +
PriorityAssertionResult
MUSTRole 'checkbox' is conveyedPassed
MUSTName 'All condiments' is conveyedPassed
MUSTState 'mixed' is conveyedPassed
MUST + Other behaviors that create severe negative impacts are + not exhibited + Passed
SHOULDRole 'group' is conveyedPassed
SHOULDName 'Sandwich Condiments' is conveyedPassed
SHOULD + Other behaviors that create moderate negative impacts are + not exhibited + Passed
+
+ Other behaviors that create negative impact: None + +

Insert+Up Results: 7 passed, 0 failed

+

NVDA Response:

+
+ automatically seeded sample output +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ Insert+Up Results +
PriorityAssertionResult
MUSTRole 'checkbox' is conveyedPassed
MUSTName 'All condiments' is conveyedPassed
MUSTState 'mixed' is conveyedPassed
MUST + Other behaviors that create severe negative impacts are + not exhibited + Passed
SHOULDRole 'group' is conveyedPassed
SHOULDName 'Sandwich Condiments' is conveyedPassed
SHOULD + Other behaviors that create moderate negative impacts are + not exhibited + Passed
+
+ Other behaviors that create negative impact: None +
+
+

+ +

+
+ +
+
+
+

+ Test 6: Read information about a mixed checkbox in interaction + mode (14 passed, 0 failed) +
+
+ + The information in this report is generated from recommended + tests. Recommended ARIA-AT tests have been reviewed by + assistive technology developers and represent consensus + regarding: applicability and validity of the tests, and + accuracy of test results. +
+
+

+
+ Raise an IssueOpen Test +
+
+
+

Results for each command

+

Insert+Tab Results: 7 passed, 0 failed

+

NVDA Response:

+
+ automatically seeded sample output +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ Insert+Tab Results +
PriorityAssertionResult
MUSTRole 'checkbox' is conveyedPassed
MUSTName 'All condiments' is conveyedPassed
MUSTState 'mixed' is conveyedPassed
MUST + Other behaviors that create severe negative impacts are + not exhibited + Passed
SHOULDRole 'group' is conveyedPassed
SHOULDName 'Sandwich Condiments' is conveyedPassed
SHOULD + Other behaviors that create moderate negative impacts are + not exhibited + Passed
+
+ Other behaviors that create negative impact: None + +

Insert+Up Results: 7 passed, 0 failed

+

NVDA Response:

+
+ automatically seeded sample output +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ Insert+Up Results +
PriorityAssertionResult
MUSTRole 'checkbox' is conveyedPassed
MUSTName 'All condiments' is conveyedPassed
MUSTState 'mixed' is conveyedPassed
MUST + Other behaviors that create severe negative impacts are + not exhibited + Passed
SHOULDRole 'group' is conveyedPassed
SHOULDName 'Sandwich Condiments' is conveyedPassed
SHOULD + Other behaviors that create moderate negative impacts are + not exhibited + Passed
+
+ Other behaviors that create negative impact: None +
+
+

+ +

+
+ +
+
+
+

+ Test 7: Operate a mixed checkbox in reading + mode (3 passed, 0 failed) +
+
+ + The information in this report is generated from recommended + tests. Recommended ARIA-AT tests have been reviewed by + assistive technology developers and represent consensus + regarding: applicability and validity of the tests, and + accuracy of test results. +
+
+

+
+ Raise an IssueOpen Test +
+
+
+

Results for each command

+

Space Results: 3 passed, 0 failed

+

NVDA Response:

+
+ automatically seeded sample output +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
+ Space Results +
PriorityAssertionResult
MUSTChange in state, to 'checked', is conveyedPassed
MUST + Other behaviors that create severe negative impacts are + not exhibited + Passed
SHOULD + Other behaviors that create moderate negative impacts are + not exhibited + Passed
+
+ Other behaviors that create negative impact: None +
+
+

+ +

+
+ +
+
+
+

+ Test 8: Operate a mixed checkbox in interaction + mode (3 passed, 0 failed) +
+
+ + The information in this report is generated from recommended + tests. Recommended ARIA-AT tests have been reviewed by + assistive technology developers and represent consensus + regarding: applicability and validity of the tests, and + accuracy of test results. +
+
+

+
+ Raise an IssueOpen Test +
+
+
+

Results for each command

+

Space Results: 3 passed, 0 failed

+

NVDA Response:

+
+ automatically seeded sample output +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
+ Space Results +
PriorityAssertionResult
MUSTChange in state, to 'checked', is conveyedPassed
MUST + Other behaviors that create severe negative impacts are + not exhibited + Passed
SHOULD + Other behaviors that create moderate negative impacts are + not exhibited + Passed
+
+ Other behaviors that create negative impact: None +
+
+

+ +

+
+ +
+
+
+

+ Test 9: Operate an unchecked checkbox in reading + mode (3 passed, 0 failed) +
+
+ + The information in this report is generated from recommended + tests. Recommended ARIA-AT tests have been reviewed by + assistive technology developers and represent consensus + regarding: applicability and validity of the tests, and + accuracy of test results. +
+
+

+
+ Raise an IssueOpen Test +
+
+
+

Results for each command

+

Space Results: 3 passed, 0 failed

+

NVDA Response:

+
+ automatically seeded sample output +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
+ Space Results +
PriorityAssertionResult
MUSTChange in state, to 'mixed', is conveyedPassed
MUST + Other behaviors that create severe negative impacts are + not exhibited + Passed
SHOULD + Other behaviors that create moderate negative impacts are + not exhibited + Passed
+
+ Other behaviors that create negative impact: None +
+
+

+ +

+
+ +
+
+
+

+ Test 10: Operate an unchecked checkbox in interaction + mode (3 passed, 0 failed) +
+
+ + The information in this report is generated from recommended + tests. Recommended ARIA-AT tests have been reviewed by + assistive technology developers and represent consensus + regarding: applicability and validity of the tests, and + accuracy of test results. +
+
+

+
+ Raise an IssueOpen Test +
+
+
+

Results for each command

+

Space Results: 3 passed, 0 failed

+

NVDA Response:

+
+ automatically seeded sample output +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
+ Space Results +
PriorityAssertionResult
MUSTChange in state, to 'mixed', is conveyedPassed
MUST + Other behaviors that create severe negative impacts are + not exhibited + Passed
SHOULD + Other behaviors that create moderate negative impacts are + not exhibited + Passed
+
+ Other behaviors that create negative impact: None +
+
+

+ +

+
+ +
+
+
+

+ Test 11: Navigate forwards out of a checkbox group in reading + mode (20 passed, 0 failed) +
+
+ + The information in this report is generated from recommended + tests. Recommended ARIA-AT tests have been reviewed by + assistive technology developers and represent consensus + regarding: applicability and validity of the tests, and + accuracy of test results. +
+
+

+
+ Raise an IssueOpen Test +
+
+
+

Results for each command

+

Down Arrow Results: 5 passed, 0 failed

+

NVDA Response:

+
+ automatically seeded sample output +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ Down Arrow Results +
PriorityAssertionResult
MUSTGroup boundary is conveyedPassed
MUSTRole 'link' is conveyedPassed
MUSTName 'Navigate backwards from here' is conveyedPassed
MUST + Other behaviors that create severe negative impacts are + not exhibited + Passed
SHOULD + Other behaviors that create moderate negative impacts are + not exhibited + Passed
+
+ Other behaviors that create negative impact: None + +

U Results: 5 passed, 0 failed

+

NVDA Response:

+
+ automatically seeded sample output +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ U Results +
PriorityAssertionResult
MUSTGroup boundary is conveyedPassed
MUSTRole 'link' is conveyedPassed
MUSTName 'Navigate backwards from here' is conveyedPassed
MUST + Other behaviors that create severe negative impacts are + not exhibited + Passed
SHOULD + Other behaviors that create moderate negative impacts are + not exhibited + Passed
+
+ Other behaviors that create negative impact: None + +

K Results: 5 passed, 0 failed

+

NVDA Response:

+
+ automatically seeded sample output +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ K Results +
PriorityAssertionResult
MUSTGroup boundary is conveyedPassed
MUSTRole 'link' is conveyedPassed
MUSTName 'Navigate backwards from here' is conveyedPassed
MUST + Other behaviors that create severe negative impacts are + not exhibited + Passed
SHOULD + Other behaviors that create moderate negative impacts are + not exhibited + Passed
+
+ Other behaviors that create negative impact: None + +

Tab Results: 5 passed, 0 failed

+

NVDA Response:

+
+ automatically seeded sample output +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ Tab Results +
PriorityAssertionResult
MUSTGroup boundary is conveyedPassed
MUSTRole 'link' is conveyedPassed
MUSTName 'Navigate backwards from here' is conveyedPassed
MUST + Other behaviors that create severe negative impacts are + not exhibited + Passed
SHOULD + Other behaviors that create moderate negative impacts are + not exhibited + Passed
+
+ Other behaviors that create negative impact: None +
+
+

+ +

+
+ +
+
+
+

+ Test 12: Navigate forwards out of a checkbox group in interaction + mode (5 passed, 0 failed) +
+
+ + The information in this report is generated from recommended + tests. Recommended ARIA-AT tests have been reviewed by + assistive technology developers and represent consensus + regarding: applicability and validity of the tests, and + accuracy of test results. +
+
+

+
+ Raise an IssueOpen Test +
+
+
+

Results for each command

+

Tab Results: 5 passed, 0 failed

+

NVDA Response:

+
+ automatically seeded sample output +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ Tab Results +
PriorityAssertionResult
MUSTGroup boundary is conveyedPassed
MUSTRole 'link' is conveyedPassed
MUSTName 'Navigate backwards from here' is conveyedPassed
MUST + Other behaviors that create severe negative impacts are + not exhibited + Passed
SHOULD + Other behaviors that create moderate negative impacts are + not exhibited + Passed
+
+ Other behaviors that create negative impact: None +
+
+

+ +

+
+ +
+
+
+

+ Test 13: Navigate backwards out of a checkbox group in reading + mode (20 passed, 0 failed) +
+
+ + The information in this report is generated from recommended + tests. Recommended ARIA-AT tests have been reviewed by + assistive technology developers and represent consensus + regarding: applicability and validity of the tests, and + accuracy of test results. +
+
+

+
+ Raise an IssueOpen Test +
+
+
+

Results for each command

+

+ Up Arrow then Up Arrow Results: 5 passed, 0 failed +

+

NVDA Response:

+
+ automatically seeded sample output +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ Up Arrow then Up Arrow Results +
PriorityAssertionResult
MUSTGroup boundary is conveyedPassed
MUSTRole 'link' is conveyedPassed
MUSTName 'Navigate forwards from here' is conveyedPassed
MUST + Other behaviors that create severe negative impacts are + not exhibited + Passed
SHOULD + Other behaviors that create moderate negative impacts are + not exhibited + Passed
+
+ Other behaviors that create negative impact: None + +

Shift+K Results: 5 passed, 0 failed

+

NVDA Response:

+
+ automatically seeded sample output +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ Shift+K Results +
PriorityAssertionResult
MUSTGroup boundary is conveyedPassed
MUSTRole 'link' is conveyedPassed
MUSTName 'Navigate forwards from here' is conveyedPassed
MUST + Other behaviors that create severe negative impacts are + not exhibited + Passed
SHOULD + Other behaviors that create moderate negative impacts are + not exhibited + Passed
+
+ Other behaviors that create negative impact: None + +

Shift+U Results: 5 passed, 0 failed

+

NVDA Response:

+
+ automatically seeded sample output +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ Shift+U Results +
PriorityAssertionResult
MUSTGroup boundary is conveyedPassed
MUSTRole 'link' is conveyedPassed
MUSTName 'Navigate forwards from here' is conveyedPassed
MUST + Other behaviors that create severe negative impacts are + not exhibited + Passed
SHOULD + Other behaviors that create moderate negative impacts are + not exhibited + Passed
+
+ Other behaviors that create negative impact: None + +

Shift+Tab Results: 5 passed, 0 failed

+

NVDA Response:

+
+ automatically seeded sample output +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ Shift+Tab Results +
PriorityAssertionResult
MUSTGroup boundary is conveyedPassed
MUSTRole 'link' is conveyedPassed
MUSTName 'Navigate forwards from here' is conveyedPassed
MUST + Other behaviors that create severe negative impacts are + not exhibited + Passed
SHOULD + Other behaviors that create moderate negative impacts are + not exhibited + Passed
+
+ Other behaviors that create negative impact: None +
+
+

+ +

+
+ +
+
+
+

+ Test 14: Navigate backwards out of a checkbox group in interaction + mode (5 passed, 0 failed) +
+
+ + The information in this report is generated from recommended + tests. Recommended ARIA-AT tests have been reviewed by + assistive technology developers and represent consensus + regarding: applicability and validity of the tests, and + accuracy of test results. +
+
+

+
+ Raise an IssueOpen Test +
+
+
+

Results for each command

+

Shift+Tab Results: 5 passed, 0 failed

+

NVDA Response:

+
+ automatically seeded sample output +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ Shift+Tab Results +
PriorityAssertionResult
MUSTGroup boundary is conveyedPassed
MUSTRole 'link' is conveyedPassed
MUSTName 'Navigate forwards from here' is conveyedPassed
MUST + Other behaviors that create severe negative impacts are + not exhibited + Passed
SHOULD + Other behaviors that create moderate negative impacts are + not exhibited + Passed
+
+ Other behaviors that create negative impact: None +
+
+

+ +

+
+ +
+
+
+
+ +
+ + + diff --git a/client/tests/e2e/snapshots/saved/_reports.html b/client/tests/e2e/snapshots/saved/_reports.html index 21dc0c932..4097fc174 100644 --- a/client/tests/e2e/snapshots/saved/_reports.html +++ b/client/tests/e2e/snapshots/saved/_reports.html @@ -93,12 +93,98 @@

Support Levels

Test Plan + JAWS and Firefox JAWS and Chrome + NVDA and Firefox NVDA and Chrome + VoiceOver for macOS and Chrome VoiceOver for macOS and Safari + + + Action Menu Button Example Using aria-activedescendant + + +
+
+ 100% +
+
100%
+
+ + +
+
+ 100% +
+
100%
+
+ + +
+
+ 100% +
+
100%
+
+ + +
+
+ 100% +
+
100%
+
+ + +
+
+ 100% +
+
100%
+
+ + +
+
+ 100% +
+
100%
+
+ + Support Levels >Recommended + None
@@ -121,6 +208,7 @@

Support Levels

 + None
@@ -133,6 +221,7 @@

Support Levels

 + None
@@ -156,6 +245,7 @@

Support Levels

>Candidate + None
@@ -168,6 +258,7 @@

Support Levels

 + None
@@ -180,6 +271,7 @@

Support Levels

 + None
diff --git a/client/tests/e2e/snapshots/saved/_test-plan-report_1.html b/client/tests/e2e/snapshots/saved/_test-plan-report_1.html index 333d8d1e9..a4caaaa31 100644 --- a/client/tests/e2e/snapshots/saved/_test-plan-report_1.html +++ b/client/tests/e2e/snapshots/saved/_test-plan-report_1.html @@ -122,13 +122,13 @@
    -
  1. +
  2. Navigate forwards to a not pressed toggle button in reading mode + title="Has Conflicts">
    4. -
  4. +
  5. Navigate backwards to a pressed toggle button in reading mode + title="Has Conflicts">
    6. -
  6. +
  7. Navigate forwards to a pressed toggle button in interaction mode + title="Has Conflicts">
    8. -
  8. +
  9. Navigate backwards to a pressed toggle button in interaction mode + title="Has Conflicts">
+
@@ -1403,7 +1423,7 @@

Test Options

Test Options
-
+
+
+

+ Review Conflicts for "Navigate forwards to a not pressed + toggle button in reading mode" +

+
    +
  1. +

    + Assertion Results for "Down Arrow" Command and "Role + 'button' is conveyed" Assertion +

    +
      +
    • + Tester esmeralda-baggins recorded output + "automatically seeded sample output" and marked + assertion as passing. +
      • +
    • + Tester tom-proudfeet recorded output + "automatically seeded sample output" and marked + assertion as failing. +
      • +
    +
    2. +
  2. +

    + Assertion Results for "B" Command and "Role 'button' + is conveyed" Assertion +

    +
      +
    • + Tester esmeralda-baggins recorded output + "automatically seeded sample output" and marked + assertion as passing. +
      • +
    • + Tester tom-proudfeet recorded output + "automatically seeded sample output" and marked + assertion as failing. +
      • +
    +
    3. +
  3. +

    + Assertion Results for "F" Command and "Role 'button' + is conveyed" Assertion +

    +
      +
    • + Tester esmeralda-baggins recorded output + "automatically seeded sample output" and marked + assertion as passing. +
      • +
    • + Tester tom-proudfeet recorded output + "automatically seeded sample output" and marked + assertion as failing. +
      • +
    +
    4. +
  4. +

    + Assertion Results for "Tab" Command and "Role + 'button' is conveyed" Assertion +

    +
      +
    • + Tester esmeralda-baggins recorded output + "automatically seeded sample output" and marked + assertion as passing. +
      • +
    • + Tester tom-proudfeet recorded output + "automatically seeded sample output" and marked + assertion as failing. +
      • +
    +
    5. +
+
+
diff --git a/client/tests/e2e/snapshots/saved/_test-queue.html b/client/tests/e2e/snapshots/saved/_test-queue.html index 5bc5eac76..d375391b6 100644 --- a/client/tests/e2e/snapshots/saved/_test-queue.html +++ b/client/tests/e2e/snapshots/saved/_test-queue.html @@ -205,8 +205,12 @@

- @@ -505,6 +509,226 @@

+

Color Viewer Slider

+
+

+ +

+
+
+ View tests in V24.06.26 +
+
+
+ + + + + + + + + + + + + + + + + + + +
Assistive TechnologyBrowserTestersStatusActions
+
+ VoiceOver for macOS 11.6 (20G165) or later +
+ 		+
+ Safari Any version +
+ 		+
+ + +
+ + 		+
+
+
+
+
+ 0% complete by esmeralda-baggins  +
+ 		+
+ +
+ +
+ +
+
+
+
+

@@ -1673,6 +1897,15 @@

>esmeralda-baggins12 of 16 tests complete +
+
  • + tom-proudfeet13 of 16 tests complete +
    • @@ -1680,14 +1913,15 @@

    + style="clip-path: inset(0px 0px 0px 78%)">

    75% complete by esmeralda-baggins 78% complete by 2 testers with 14 conflicts diff --git a/client/tests/e2e/snapshots/utils.js b/client/tests/e2e/snapshots/utils.js index 9984eefc0..749a0963b 100644 --- a/client/tests/e2e/snapshots/utils.js +++ b/client/tests/e2e/snapshots/utils.js @@ -14,7 +14,8 @@ const snapshotRoutes = [ '/run/2', '/data-management/meter', '/candidate-test-plan/24/1', - '/test-queue/2/conflicts' + '/test-queue/2/conflicts', + '/report/67/targets/20' ]; async function cleanAndNormalizeSnapshot(page) { diff --git a/package.json b/package.json index db1886b91..927d4085b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "aria-at-app", - "version": "1.8.1", + "version": "1.9.0", "description": "Run ARIA-AT tests and report results", "main": "server/index.js", "private": true, diff --git a/server/controllers/AuthController.js b/server/controllers/AuthController.js index 18f36508a..52e46d32b 100644 --- a/server/controllers/AuthController.js +++ b/server/controllers/AuthController.js @@ -1,7 +1,15 @@ const { User } = require('../models'); -const { getOrCreateUser } = require('../models/services/UserService'); +const { + getOrCreateUser, + addUserVendor, + getUserById +} = require('../models/services/UserService'); const { GithubService } = require('../services'); const getUsersFromFile = require('../util/getUsersFromFile'); +const { + findVendorByName, + getOrCreateVendor +} = require('../models/services/VendorService'); const APP_SERVER = process.env.APP_SERVER; @@ -70,6 +78,40 @@ const oauthRedirectFromGithubController = async (req, res) => { transaction: req.transaction }); + if (roles.some(role => role.name === User.VENDOR)) { + const vendorEntry = vendors.find( + vendor => vendor.split('|')[0] === githubUsername + ); + if (vendorEntry) { + const [, companyName] = vendorEntry.split('|').trim(); + const vendor = await findVendorByName({ + name: companyName, + transaction: req.transaction + }); + if (vendor) { + await addUserVendor(user.id, vendor.id, { + transaction: req.transaction + }); + } else { + const vendor = await getOrCreateVendor({ + where: { name: companyName }, + transaction: req.transaction + }); + await addUserVendor(user.id, vendor.id, { + transaction: req.transaction + }); + } + // Fetch the user again with vendor and AT information + user = await getUserById({ + id: user.id, + vendorAttributes: ['id', 'name'], + atAttributes: ['id', 'name'], + includeVendorAts: true, + transaction: req.transaction + }); + } + } + req.session.user = user; return loginSucceeded(); diff --git a/server/controllers/FakeUserController.js b/server/controllers/FakeUserController.js index ab870ff03..ca64c337c 100644 --- a/server/controllers/FakeUserController.js +++ b/server/controllers/FakeUserController.js @@ -1,4 +1,12 @@ -const { getOrCreateUser } = require('../models/services/UserService'); +const { + getOrCreateUser, + addUserVendor, + getUserById +} = require('../models/services/UserService'); +const { + getOrCreateVendor, + updateVendorById +} = require('../models/services/VendorService'); const ALLOW_FAKE_USER = process.env.ALLOW_FAKE_USER === 'true'; @@ -25,6 +33,29 @@ const setFakeUserController = async (req, res) => { transaction: req.transaction }); + if (userToCreate.company) { + const [vendor] = await getOrCreateVendor({ + where: { name: userToCreate.company.name }, + transaction: req.transaction + }); + + await addUserVendor(user.id, vendor.id, { transaction: req.transaction }); + await updateVendorById({ + id: vendor.id, + ats: userToCreate.company.ats, + transaction: req.transaction + }); + + // Refresh user to include updated associations + user = await getUserById({ + id: user.id, + vendorAttributes: ['id', 'name'], + atAttributes: ['id', 'name'], + includeVendorAts: true, + transaction: req.transaction + }); + } + req.session.user = user; res.status(200).send(''); diff --git a/server/graphql-schema.js b/server/graphql-schema.js index dd70988cc..07295cb7d 100644 --- a/server/graphql-schema.js +++ b/server/graphql-schema.js @@ -59,6 +59,10 @@ const graphqlSchema = gql` The ATs the user has indicated they are able to test. """ ats: [At]! + """ + The vendor the user is associated with. + """ + company: Vendor } """ @@ -640,19 +644,26 @@ const graphqlSchema = gql` Some assertions are more akin to recommendations or best practices, and, while we want to record whether they are passing or failing, we do not want to count the entire test as failing when they fail. + + These definitions for MUST, SHOULD and MAY are described at + https://github.com/w3c/aria-at/wiki/Glossary#assertion-priority in more + detail """ enum AssertionPriority { """ - All required assertions must pass for the test to pass. This should be - considered as 'MUST Behaviors'. + The assertion is an absolute requirement for the test to pass. This should + be considered as 'MUST Behaviors'. """ MUST """ - This assertion is not considered when deciding if a test is passing. - This should be considered as 'SHOULD Behaviors'. + This assertion is a strongly recommended requirement. This should be + considered as 'SHOULD Behaviors'. """ SHOULD - # TODO Define MAY + """ + This assertion is truly optional. This should be considered as 'MAY + Behaviors'. + """ MAY """ This assertion should not be included in the test and should not be @@ -1256,6 +1267,18 @@ const graphqlSchema = gql` directory: String ): [TestPlanVersion]! """ + Get a vendor by ID. + """ + vendor(id: ID!): Vendor + """ + Get a vendor by name. + """ + vendorByName(name: String!): Vendor + """ + Get all vendors. + """ + vendors: [Vendor]! + """ Get a particular TestPlanVersion by ID. """ testPlanVersion(id: ID): TestPlanVersion @@ -1478,6 +1501,28 @@ const graphqlSchema = gql` retryCanceledCollections: CollectionJob! } + """ + Vendor company that makes an AT + """ + type Vendor { + """ + Postgres-provided numeric ID. + """ + id: ID! + """ + The name of the vendor company. + """ + name: String! + """ + The ATs associated with this vendor. + """ + ats: [At]! + """ + The users associated with this vendor. + """ + users: [User]! + } + type Mutation { """ Get the available mutations for the given AT. diff --git a/server/migrations/20240903164204-add-vendor-table.js b/server/migrations/20240903164204-add-vendor-table.js new file mode 100644 index 000000000..1eaaad688 --- /dev/null +++ b/server/migrations/20240903164204-add-vendor-table.js @@ -0,0 +1,55 @@ +'use strict'; + +module.exports = { + up: async (queryInterface, Sequelize) => { + await queryInterface.createTable('Vendor', { + id: { + type: Sequelize.INTEGER, + allowNull: false, + primaryKey: true, + autoIncrement: true + }, + name: { + type: Sequelize.TEXT, + allowNull: false, + unique: true + }, + createdAt: { + type: Sequelize.DATE, + allowNull: false + }, + updatedAt: { + type: Sequelize.DATE, + allowNull: false + } + }); + + await queryInterface.addColumn('User', 'vendorId', { + type: Sequelize.INTEGER, + allowNull: true, + references: { + model: 'Vendor', + key: 'id' + }, + onUpdate: 'CASCADE', + onDelete: 'SET NULL' + }); + + await queryInterface.addColumn('At', 'vendorId', { + type: Sequelize.INTEGER, + allowNull: true, + references: { + model: 'Vendor', + key: 'id' + }, + onUpdate: 'CASCADE', + onDelete: 'SET NULL' + }); + }, + + down: async queryInterface => { + await queryInterface.removeColumn('At', 'vendorId'); + await queryInterface.removeColumn('User', 'vendorId'); + await queryInterface.dropTable('Vendor'); + } +}; diff --git a/server/models/User.js b/server/models/User.js index 0ab297316..ae5d84972 100644 --- a/server/models/User.js +++ b/server/models/User.js @@ -37,6 +37,8 @@ module.exports = function (sequelize, DataTypes) { Model.ATS_ASSOCIATION = { through: 'UserAts', as: 'ats' }; Model.TEST_PLAN_RUN_ASSOCIATION = { as: 'testPlanRuns' }; + Model.VENDOR_ASSOCIATION = { as: 'company' }; + Model.associate = function (models) { Model.belongsToMany(models.Role, { ...Model.ROLE_ASSOCIATION, @@ -55,6 +57,11 @@ module.exports = function (sequelize, DataTypes) { foreignKey: 'testerUserId', sourceKey: 'id' }); + + Model.belongsTo(models.Vendor, { + ...Model.VENDOR_ASSOCIATION, + foreignKey: 'vendorId' + }); }; return Model; diff --git a/server/models/Vendor.js b/server/models/Vendor.js new file mode 100644 index 000000000..7507ecf47 --- /dev/null +++ b/server/models/Vendor.js @@ -0,0 +1,43 @@ +const MODEL_NAME = 'Vendor'; + +module.exports = function (sequelize, DataTypes) { + const Model = sequelize.define( + MODEL_NAME, + { + id: { + type: DataTypes.INTEGER, + allowNull: false, + primaryKey: true, + autoIncrement: true + }, + name: { + type: DataTypes.TEXT, + allowNull: false, + unique: true + } + }, + { + timestamps: true, + tableName: MODEL_NAME + } + ); + + Model.AT_ASSOCIATION = { as: 'ats' }; + Model.USER_ASSOCIATION = { as: 'users' }; + + Model.associate = function (models) { + Model.hasMany(models.At, { + as: 'ats', + foreignKey: 'vendorId', + sourceKey: 'id' + }); + + Model.hasMany(models.User, { + ...Model.USER_ASSOCIATION, + foreignKey: 'vendorId', + sourceKey: 'id' + }); + }; + + return Model; +}; diff --git a/server/models/services/CollectionJobService.js b/server/models/services/CollectionJobService.js index 8d640dc40..774a51e52 100644 --- a/server/models/services/CollectionJobService.js +++ b/server/models/services/CollectionJobService.js @@ -388,12 +388,13 @@ const getCollectionJobs = async ({ const triggerWorkflow = async (job, testIds, atVersion, { transaction }) => { const { testPlanVersion } = job.testPlanRun.testPlanReport; const { gitSha, directory } = testPlanVersion; + try { if (isGithubWorkflowEnabled()) { // TODO: pass the reduced list of testIds along / deal with them somehow await createGithubWorkflow({ job, directory, gitSha, atVersion }); } else { - await startCollectionJobSimulation(job, atVersion, transaction); + await startCollectionJobSimulation(job, testIds, atVersion, transaction); } } catch (error) { console.error(error); @@ -479,14 +480,12 @@ const retryCanceledCollections = async ({ collectionJob }, { transaction }) => { throw new Error('collectionJob is required to retry cancelled tests'); } - const cancelledTests = collectionJob.testPlanRun.testResults.filter( - testResult => - // Find tests that don't have complete output - !testResult?.scenarioResults?.every(scenario => scenario?.output !== null) + const cancelledTests = collectionJob.testStatus.filter( + testStatus => testStatus.status === COLLECTION_JOB_STATUS.CANCELLED ); const testPlanReport = await getTestPlanReportById({ - id: job.testPlanRun.testPlanReportId, + id: collectionJob.testPlanRun.testPlanReportId, transaction }); @@ -497,10 +496,22 @@ const retryCanceledCollections = async ({ collectionJob }, { transaction }) => { transaction ); - const testIds = cancelledTests.map(test => test.id); + const testIds = cancelledTests.map(test => test.testId); - const job = await getCollectionJobById({ + const job = await updateCollectionJobById({ id: collectionJob.id, + values: { status: COLLECTION_JOB_STATUS.QUEUED }, + transaction + }); + + await updateCollectionJobTestStatusByQuery({ + where: { + collectionJobId: job.id, + status: COLLECTION_JOB_STATUS.CANCELLED + }, + values: { + status: COLLECTION_JOB_STATUS.QUEUED + }, transaction }); @@ -652,6 +663,15 @@ const restartCollectionJob = async ({ id }, { transaction }) => { }, transaction }); + await updateCollectionJobTestStatusByQuery({ + where: { + collectionJobId: id + }, + values: { + status: COLLECTION_JOB_STATUS.QUEUED + }, + transaction + }); if (!job) { return null; @@ -669,7 +689,16 @@ const restartCollectionJob = async ({ id }, { transaction }) => { transaction ); - return triggerWorkflow(job, [], atVersion, { transaction }); + const tests = await runnableTestsResolver(testPlanReport, null, { + transaction + }); + + return triggerWorkflow( + job, + tests.map(test => test.id), + atVersion, + { transaction } + ); }; /** diff --git a/server/models/services/UserService.js b/server/models/services/UserService.js index a07019104..68a3f2e7b 100644 --- a/server/models/services/UserService.js +++ b/server/models/services/UserService.js @@ -41,6 +41,15 @@ const testPlanRunAssociation = testPlanRunAttributes => ({ attributes: testPlanRunAttributes }); +/** + * @param vendorAttributes - Vendor attributes + * @returns {{association: string, attributes: string[]}} + */ +const vendorAssociation = vendorAttributes => ({ + association: 'company', + attributes: vendorAttributes +}); + /** * You can pass any of the attribute arrays as '[]' to exclude that related association * @param {object} options @@ -58,16 +67,33 @@ const getUserById = async ({ roleAttributes = ROLE_ATTRIBUTES, atAttributes = AT_ATTRIBUTES, testPlanRunAttributes = TEST_PLAN_RUN_ATTRIBUTES, + vendorAttributes = [], + includeVendorAts = false, transaction }) => { + const include = [ + roleAssociation(roleAttributes), + atAssociation(atAttributes), + testPlanRunAssociation(testPlanRunAttributes) + ]; + + if (vendorAttributes.length > 0) { + const vendorInclude = vendorAssociation(vendorAttributes); + if (includeVendorAts) { + vendorInclude.include = [ + { + association: 'ats', + attributes: atAttributes + } + ]; + } + include.push(vendorInclude); + } + return ModelService.getById(User, { id, attributes: userAttributes, - include: [ - roleAssociation(roleAttributes), - atAssociation(atAttributes), - testPlanRunAssociation(testPlanRunAttributes) - ], + include, transaction }); }; @@ -89,6 +115,7 @@ const getUserByUsername = async ({ roleAttributes = ROLE_ATTRIBUTES, atAttributes = AT_ATTRIBUTES, testPlanRunAttributes = TEST_PLAN_RUN_ATTRIBUTES, + vendorAttributes = [], transaction }) => { return ModelService.getByQuery(User, { @@ -97,7 +124,8 @@ const getUserByUsername = async ({ include: [ roleAssociation(roleAttributes), atAssociation(atAttributes), - testPlanRunAssociation(testPlanRunAttributes) + testPlanRunAssociation(testPlanRunAttributes), + vendorAssociation(vendorAttributes) ], transaction }); @@ -126,6 +154,7 @@ const getUsers = async ({ roleAttributes = ROLE_ATTRIBUTES, atAttributes = AT_ATTRIBUTES, testPlanRunAttributes = TEST_PLAN_RUN_ATTRIBUTES, + vendorAttributes = [], pagination = {}, transaction }) => { @@ -141,7 +170,8 @@ const getUsers = async ({ include: [ roleAssociation(roleAttributes), atAssociation(atAttributes), - testPlanRunAssociation(testPlanRunAttributes) + testPlanRunAssociation(testPlanRunAttributes), + vendorAssociation(vendorAttributes) ], pagination, transaction @@ -247,6 +277,7 @@ const createUser = async ({ roleAttributes = ROLE_ATTRIBUTES, atAttributes = AT_ATTRIBUTES, testPlanRunAttributes = TEST_PLAN_RUN_ATTRIBUTES, + vendorAttributes = [], transaction }) => { const userResult = await ModelService.create(User, { @@ -262,7 +293,8 @@ const createUser = async ({ include: [ roleAssociation(roleAttributes), atAssociation(atAttributes), - testPlanRunAssociation(testPlanRunAttributes) + testPlanRunAssociation(testPlanRunAttributes), + vendorAssociation(vendorAttributes) ], transaction }); @@ -286,6 +318,7 @@ const updateUserById = async ({ roleAttributes = ROLE_ATTRIBUTES, atAttributes = AT_ATTRIBUTES, testPlanRunAttributes = TEST_PLAN_RUN_ATTRIBUTES, + vendorAttributes = [], transaction }) => { await ModelService.update(User, { @@ -300,7 +333,8 @@ const updateUserById = async ({ include: [ roleAssociation(roleAttributes), atAssociation(atAttributes), - testPlanRunAssociation(testPlanRunAttributes) + testPlanRunAssociation(testPlanRunAttributes), + vendorAssociation(vendorAttributes) ], transaction }); @@ -412,6 +446,7 @@ const getOrCreateUser = async ({ roleAttributes = ROLE_ATTRIBUTES, atAttributes = AT_ATTRIBUTES, testPlanRunAttributes = TEST_PLAN_RUN_ATTRIBUTES, + vendorAttributes = [], transaction }) => { const accumulatedResults = await ModelService.nestedGetOrCreate({ @@ -443,6 +478,7 @@ const getOrCreateUser = async ({ roleAttributes, atAttributes, testPlanRunAttributes, + vendorAttributes, transaction }); @@ -480,6 +516,22 @@ const removeUserRole = async (userId, roleName, { transaction }) => { }); }; +/** + * @param {number} userId - id of the User that the vendor will be added to + * @param {number} vendorId - id of the Vendor that the User will be associated with + * @param {object} options + * @param {*} options.transaction - Sequelize transaction + * @returns {Promise<*>} + */ +const addUserVendor = async (userId, vendorId, { transaction }) => { + // Set the company Vendor association for the user + return ModelService.update(User, { + where: { id: userId }, + values: { vendorId }, + transaction + }); +}; + module.exports = { // Basic CRUD getUserById, @@ -497,5 +549,6 @@ module.exports = { // Custom Functions addUserRole, - removeUserRole + removeUserRole, + addUserVendor }; diff --git a/server/models/services/VendorService.js b/server/models/services/VendorService.js new file mode 100644 index 000000000..3997545aa --- /dev/null +++ b/server/models/services/VendorService.js @@ -0,0 +1,195 @@ +const ModelService = require('./ModelService'); +const { + VENDOR_ATTRIBUTES, + AT_ATTRIBUTES, + USER_ATTRIBUTES +} = require('./helpers'); +const { Vendor } = require('../'); +const { Op } = require('sequelize'); + +// Association helpers + +const atAssociation = atAttributes => ({ + association: 'ats', + attributes: atAttributes +}); + +const userAssociation = userAttributes => ({ + association: 'users', + attributes: userAttributes +}); + +/** + * @param {object} options + * @param {number} options.id - id of Vendor to be retrieved + * @param {string[]} options.vendorAttributes - Vendor attributes to be returned in the result + * @param {string[]} options.atAttributes - AT attributes to be returned in the result + * @param {string[]} options.userAttributes - User attributes to be returned in the result + * @param {*} options.transaction - Sequelize transaction + * @returns {Promise<*>} + */ +const findVendorById = async ({ + id, + vendorAttributes = VENDOR_ATTRIBUTES, + atAttributes = AT_ATTRIBUTES, + userAttributes = USER_ATTRIBUTES, + transaction +}) => { + return ModelService.getById(Vendor, { + id, + attributes: vendorAttributes, + include: [atAssociation(atAttributes), userAssociation(userAttributes)], + transaction + }); +}; + +/** + * @param {object} options + * @param {string} options.name - name of Vendor to be retrieved + * @param {string[]} options.vendorAttributes - Vendor attributes to be returned in the result + * @param {string[]} options.atAttributes - AT attributes to be returned in the result + * @param {string[]} options.userAttributes - User attributes to be returned in the result + * @param {*} options.transaction - Sequelize transaction + * @returns {Promise<*>} + */ +const findVendorByName = async ({ + name, + vendorAttributes = VENDOR_ATTRIBUTES, + atAttributes = AT_ATTRIBUTES, + userAttributes = USER_ATTRIBUTES, + transaction +}) => { + return ModelService.getByQuery(Vendor, { + where: { name }, + attributes: vendorAttributes, + include: [atAssociation(atAttributes), userAssociation(userAttributes)], + transaction + }); +}; + +/** + * @param {object} options + * @param {string|any} options.search - use this to combine with {@param filter} to be passed to Sequelize's where clause + * @param {object} options.where - use this define conditions to be passed to Sequelize's where clause + * @param {string[]} options.vendorAttributes - Vendor attributes to be returned in the result + * @param {string[]} options.atAttributes - AT attributes to be returned in the result + * @param {string[]} options.userAttributes - User attributes to be returned in the result + * @param {object} options.pagination - pagination options for query + * @param {*} options.transaction - Sequelize transaction + * @returns {Promise<*>} + */ +const findAllVendors = async ({ + search, + where = {}, + vendorAttributes = VENDOR_ATTRIBUTES, + atAttributes = AT_ATTRIBUTES, + userAttributes = USER_ATTRIBUTES, + pagination = {}, + transaction +}) => { + const searchQuery = search ? `%${search}%` : ''; + if (searchQuery) { + where = { ...where, name: { [Op.iLike]: searchQuery } }; + } + + return ModelService.get(Vendor, { + where, + attributes: vendorAttributes, + include: [atAssociation(atAttributes), userAssociation(userAttributes)], + pagination, + transaction + }); +}; + +/** + * @param {object} options + * @param {object} options.where - conditions to find or create the Vendor + * @param {string[]} options.vendorAttributes - Vendor attributes to be returned in the result + * @param {*} options.transaction - Sequelize transaction + * @returns {Promise<[*, boolean]>} + */ +const getOrCreateVendor = async ({ + where, + vendorAttributes = VENDOR_ATTRIBUTES, + transaction +}) => { + const [vendor, created] = await Vendor.findOrCreate({ + where, + attributes: vendorAttributes, + transaction + }); + return [vendor, created]; +}; + +/** + * @param {object} options + * @param {object} options.values - values to be used to create the Vendor + * @param {string[]} options.vendorAttributes - Vendor attributes to be returned in the result + * @param {*} options.transaction - Sequelize transaction + * @returns {Promise<*>} + */ +const createVendor = async ({ + values, + vendorAttributes = VENDOR_ATTRIBUTES, + transaction +}) => { + const vendorResult = await ModelService.create(Vendor, { + values, + transaction + }); + const { id } = vendorResult; + + return ModelService.getById(Vendor, { + id, + attributes: vendorAttributes, + transaction + }); +}; + +/** + * @param {object} options + * @param {number} options.id - id of the Vendor record to be updated + * @param {object} options.values - values to be used to update columns for the record + * @param {string[]} options.vendorAttributes - Vendor attributes to be returned in the result + * @param {*} options.transaction - Sequelize transaction + * @returns {Promise<*>} + */ +const updateVendorById = async ({ + id, + values, + vendorAttributes = VENDOR_ATTRIBUTES, + transaction +}) => { + await ModelService.update(Vendor, { + where: { id }, + values, + transaction + }); + + return ModelService.getById(Vendor, { + id, + attributes: vendorAttributes, + transaction + }); +}; + +/** + * @param {object} options + * @param {number} options.id - id of the Vendor record to be removed + * @param {boolean} options.truncate - Sequelize specific deletion options that could be passed + * @param {*} options.transaction - Sequelize transaction + * @returns {Promise} + */ +const removeVendorById = async ({ id, truncate = false, transaction }) => { + return ModelService.removeById(Vendor, { id, truncate, transaction }); +}; + +module.exports = { + findVendorById, + findVendorByName, + findAllVendors, + getOrCreateVendor, + createVendor, + updateVendorById, + removeVendorById +}; diff --git a/server/models/services/helpers.js b/server/models/services/helpers.js index 402787495..54a0930ca 100644 --- a/server/models/services/helpers.js +++ b/server/models/services/helpers.js @@ -12,7 +12,8 @@ const { UserRoles, UserAts, CollectionJob, - CollectionJobTestStatus + CollectionJobTestStatus, + Vendor } = require('../index'); /** @@ -43,5 +44,6 @@ module.exports = { COLLECTION_JOB_ATTRIBUTES: getSequelizeModelAttributes(CollectionJob), COLLECTION_JOB_TEST_STATUS_ATTRIBUTES: getSequelizeModelAttributes( CollectionJobTestStatus - ) + ), + VENDOR_ATTRIBUTES: getSequelizeModelAttributes(Vendor) }; diff --git a/server/resolvers/TestResultOperations/saveTestResultCommon.js b/server/resolvers/TestResultOperations/saveTestResultCommon.js index 3a6f908ba..c6312bbf4 100644 --- a/server/resolvers/TestResultOperations/saveTestResultCommon.js +++ b/server/resolvers/TestResultOperations/saveTestResultCommon.js @@ -60,7 +60,7 @@ const saveTestResultCommon = async ({ } if (isSubmit) { - assertTestResultIsValid(newTestResult); + assertTestResultIsValid(newTestResult, test.assertions); newTestResult.completedAt = new Date(); } else { newTestResult.completedAt = null; @@ -115,15 +115,22 @@ const saveTestResultCommon = async ({ return populateData({ testResultId }, { context }); }; -const assertTestResultIsValid = newTestResult => { +const assertTestResultIsValid = (newTestResult, assertions = []) => { let failed = false; const checkAssertionResult = assertionResult => { + const isExcluded = assertions.find( + assertion => + assertion.id === assertionResult.assertionId && + assertion.priority === 'EXCLUDE' + ); + if ( assertionResult.passed === null || assertionResult.passed === undefined ) { - failed = true; + if (isExcluded) assertionResult.passed = false; + else failed = true; } }; diff --git a/server/resolvers/index.js b/server/resolvers/index.js index 58bfd1d80..7642ad198 100644 --- a/server/resolvers/index.js +++ b/server/resolvers/index.js @@ -10,6 +10,9 @@ const testPlanVersion = require('./testPlanVersionResolver'); const testPlanVersions = require('./testPlanVersionsResolver'); const testPlanRun = require('./testPlanRunResolver'); const testPlanRuns = require('./testPlanRunsResolver'); +const vendors = require('./vendorsResolver'); +const vendor = require('./vendorResolver'); +const vendorByName = require('./vendorByNameResolver'); const createTestPlanReport = require('./createTestPlanReportResolver'); const addViewer = require('./addViewerResolver'); const mutateAt = require('./mutateAtResolver'); @@ -64,7 +67,10 @@ const resolvers = { populateData, collectionJob, collectionJobs, - collectionJobByTestPlanRunId + collectionJobByTestPlanRunId, + vendors, + vendor, + vendorByName }, Mutation: { at: mutateAt, diff --git a/server/resolvers/vendorByNameResolver.js b/server/resolvers/vendorByNameResolver.js new file mode 100644 index 000000000..fcc6cb6ba --- /dev/null +++ b/server/resolvers/vendorByNameResolver.js @@ -0,0 +1,7 @@ +const { findVendorByName } = require('../models/services/VendorService'); + +const vendorByNameResolver = async (_, { name }, { transaction }) => { + return findVendorByName({ name, transaction }); +}; + +module.exports = vendorByNameResolver; diff --git a/server/resolvers/vendorResolver.js b/server/resolvers/vendorResolver.js new file mode 100644 index 000000000..a52a0b411 --- /dev/null +++ b/server/resolvers/vendorResolver.js @@ -0,0 +1,8 @@ +const { findVendorById } = require('../models/services/VendorService'); + +const vendorResolver = async (_, { id }, { transaction }) => { + // No auth since the vendors.txt file is public + return findVendorById({ id, transaction }); +}; + +module.exports = vendorResolver; diff --git a/server/resolvers/vendorsResolver.js b/server/resolvers/vendorsResolver.js new file mode 100644 index 000000000..25c9412f1 --- /dev/null +++ b/server/resolvers/vendorsResolver.js @@ -0,0 +1,8 @@ +const { findAllVendors } = require('../models/services/VendorService'); + +const vendorsResolver = async (_, __, { transaction }) => { + // No auth since the vendors.txt file is public + return findAllVendors({ transaction }); +}; + +module.exports = vendorsResolver; diff --git a/server/scripts/populate-test-data/index.js b/server/scripts/populate-test-data/index.js index 1d43c0d3c..92da3f641 100644 --- a/server/scripts/populate-test-data/index.js +++ b/server/scripts/populate-test-data/index.js @@ -1,7 +1,10 @@ /* eslint-disable no-console */ const fs = require('fs'); const { sequelize } = require('../../models'); -const populateFakeTestResults = require('./populateFakeTestResults'); +const { + populateFakeTestResults, + FAKE_RESULT_CONFLICTS_OPTIONS +} = require('./populateFakeTestResults'); const populateTestDatabase = async transaction => { const testDataScript = fs.readFileSync( @@ -11,23 +14,8 @@ const populateTestDatabase = async transaction => { await sequelize.query(testDataScript, { transaction }); - await populateFakeTestResults( - 1, - [ - 'completeAndPassing', - 'incompleteAndPassing', - 'incompleteAndFailingDueToNoOutputAssertions', - 'incompleteAndEmpty', - null, - 'completeAndFailingDueToIncorrectAssertions', - 'completeAndFailingDueToNoOutputAssertions', - 'completeAndFailingDueToUnexpectedBehaviors', - 'completeAndPassing' - ], - { transaction } - ); - - // Report 2 contains conflicts which is convenient to have in sample data + // Convenient way to generate conflicts between TestPlanRuns attached to + // the same TestPlanReport const conflicts = [ ['completeAndPassing', 'completeAndFailingDueToIncorrectAssertions'], [ @@ -41,6 +29,24 @@ const populateTestDatabase = async transaction => { ] ]; + await populateFakeTestResults( + 1, + [ + 'completeAndPassing', + 'incompleteAndPassing', + 'incompleteAndFailingDueToNoOutputAssertions', + 'incompleteAndEmpty', + null, + 'completeAndFailingDueToIncorrectAssertions', + 'completeAndFailingDueToNoOutputAssertions', + 'completeAndFailingDueToUnexpectedBehaviors' + ], + { + transaction, + numFakeTestResultConflicts: FAKE_RESULT_CONFLICTS_OPTIONS.ALL + } + ); + await populateFakeTestResults( 2, [ @@ -51,7 +57,7 @@ const populateTestDatabase = async transaction => { conflicts[1][0], conflicts[2][0], conflicts[3][0], - 'incompleteAndPassing' + 'incompleteAndFailingDueToUnexpectedBehaviors' ], { transaction } ); @@ -66,10 +72,7 @@ const populateTestDatabase = async transaction => { conflicts[1][1], conflicts[2][1], conflicts[3][1], - 'incompleteAndFailingDueToUnexpectedBehaviors', - 'completeAndPassing', - 'completeAndPassing', - 'completeAndPassing' + 'incompleteAndFailingDueToUnexpectedBehaviors' ], { transaction } ); @@ -81,13 +84,7 @@ const populateTestDatabase = async transaction => { 'completeAndPassing', 'completeAndFailingDueToIncorrectAssertions', 'completeAndFailingDueToNoOutputAssertions', - 'completeAndFailingDueToUnexpectedBehaviors', - 'completeAndPassing', - 'completeAndPassing', - 'completeAndPassing', - 'completeAndPassing', - 'completeAndPassing', - 'completeAndPassing' + 'completeAndFailingDueToUnexpectedBehaviors' ], { transaction } ); @@ -101,8 +98,7 @@ const populateTestDatabase = async transaction => { 'completeAndPassing', 'completeAndFailingDueToNoOutputAssertions', 'completeAndPassing', - 'completeAndFailingDueToUnexpectedBehaviors', - 'completeAndPassing' + 'completeAndFailingDueToUnexpectedBehaviors' ], { transaction } ); @@ -115,8 +111,7 @@ const populateTestDatabase = async transaction => { 'completeAndFailingDueToIncorrectAssertions', 'completeAndPassing', 'completeAndFailingDueToIncorrectAssertions', - 'completeAndFailingDueToUnexpectedBehaviors', - 'completeAndPassing' + 'completeAndFailingDueToUnexpectedBehaviors' ], { transaction } ); @@ -129,103 +124,98 @@ const populateTestDatabase = async transaction => { 'completeAndPassing', 'completeAndFailingDueToIncorrectAssertions', 'completeAndPassing', - 'completeAndFailingDueToNoOutputAssertions', - 'completeAndPassing' + 'completeAndFailingDueToNoOutputAssertions' ], { transaction } ); - await populateFakeTestResults( - 8, - [ - 'completeAndPassing', - 'completeAndPassing', - 'completeAndPassing', - 'completeAndPassing', - 'completeAndPassing', - 'completeAndPassing', - 'completeAndPassing', - 'completeAndPassing' - ], - { transaction } - ); + await populateFakeTestResults(8, ['completeAndPassing'], { transaction }); - await populateFakeTestResults( - 9, - [ - 'completeAndPassing', - 'completeAndPassing', - 'completeAndPassing', - 'completeAndPassing', - 'completeAndPassing', - 'completeAndPassing', - 'completeAndPassing', - 'completeAndPassing' - ], - { transaction } - ); + await populateFakeTestResults(9, ['completeAndPassing'], { transaction }); + + await populateFakeTestResults(10, ['completeAndPassing'], { transaction }); + + await populateFakeTestResults(11, ['completeAndPassing'], { transaction }); + + await populateFakeTestResults(12, ['completeAndPassing'], { transaction }); + + await populateFakeTestResults(13, ['completeAndPassing'], { transaction }); + + await populateFakeTestResults(14, ['completeAndPassing'], { + transaction + }); + + await populateFakeTestResults(15, ['completeAndPassing'], { + transaction + }); + + await populateFakeTestResults(16, ['completeAndPassing'], { + transaction + }); + + await populateFakeTestResults(17, ['completeAndPassing'], { + transaction + }); + + await populateFakeTestResults(18, ['completeAndPassing'], { + transaction + }); + + await populateFakeTestResults(19, ['completeAndPassing'], { + transaction + }); + // Slightly different from older already recommended TestPlanRun 13 to get + // multiple results shown for: + // Recommended Report + Checkbox Example (Mixed-State) + NVDA + Chrome await populateFakeTestResults( - 10, + 20, [ 'completeAndPassing', 'completeAndPassing', - 'completeAndPassing', - 'completeAndPassing', - 'completeAndPassing', - 'completeAndPassing', - 'completeAndPassing' + 'completeAndFailingDueToUnexpectedBehaviors' ], { transaction } ); + // Conflicting with TestPlanRun 1 await populateFakeTestResults( - 11, + 21, [ + 'completeAndFailingDueToIncorrectAssertions', + 'incompleteAndPassing', + 'incompleteAndFailingDueToNoOutputAssertions', + 'incompleteAndEmpty', 'completeAndPassing', - 'completeAndPassing', - 'completeAndPassing', - 'completeAndPassing', - 'completeAndPassing', - 'completeAndPassing', - 'completeAndPassing' + 'completeAndFailingDueToUnexpectedBehaviors' ], - { transaction } - ); - - await populateFakeTestResults( - 12, - ['completeAndPassing', 'completeAndPassing', 'completeAndPassing'], - { transaction } - ); - - await populateFakeTestResults( - 13, - ['completeAndPassing', 'completeAndPassing', 'completeAndPassing'], - { transaction } + { + transaction, + numFakeTestResultConflicts: FAKE_RESULT_CONFLICTS_OPTIONS.ALL + } ); - await populateFakeTestResults(14, new Array(16).fill('completeAndPassing'), { + await populateFakeTestResults(22, ['completeAndPassing'], { transaction }); - await populateFakeTestResults(15, new Array(8).fill('completeAndPassing'), { + await populateFakeTestResults(23, ['completeAndPassing'], { transaction }); - await populateFakeTestResults(16, new Array(3).fill('completeAndPassing'), { + await populateFakeTestResults(24, ['completeAndPassing'], { transaction }); - await populateFakeTestResults(17, new Array(3).fill('completeAndPassing'), { + await populateFakeTestResults(25, ['completeAndPassing'], { transaction }); - await populateFakeTestResults(18, new Array(3).fill('completeAndPassing'), { + await populateFakeTestResults(26, ['completeAndPassing'], { transaction }); - await populateFakeTestResults(19, new Array(12).fill('completeAndPassing'), { + await populateFakeTestResults(27, ['completeAndPassing'], { transaction }); diff --git a/server/scripts/populate-test-data/pg_dump_test_data.sql b/server/scripts/populate-test-data/pg_dump_test_data.sql index 7768e440f..84db3ef54 100644 --- a/server/scripts/populate-test-data/pg_dump_test_data.sql +++ b/server/scripts/populate-test-data/pg_dump_test_data.sql @@ -83,6 +83,16 @@ INSERT INTO "TestPlanReport" (id, "testPlanVersionId", "createdAt", "markedFinal INSERT INTO "TestPlanReport" (id, "testPlanVersionId", "createdAt", "markedFinalAt", "atId", "minimumAtVersionId", "browserId", "vendorReviewStatus") VALUES (17, get_test_plan_version_id(text 'Command Button Example', '2'), '2023-12-13 14:18:23.602-05', '2023-12-14', 2, 2, 2, 'READY'); INSERT INTO "TestPlanReport" (id, "testPlanVersionId", "createdAt", "markedFinalAt", "atId", "minimumAtVersionId", "browserId", "vendorReviewStatus") VALUES (18, get_test_plan_version_id(text 'Command Button Example', '2'), '2023-12-13 14:18:23.602-05', '2023-12-14', 3, 3, 3, 'READY'); INSERT INTO "TestPlanReport" (id, "testPlanVersionId", "createdAt", "atId", "minimumAtVersionId", "browserId", "vendorReviewStatus") VALUES (19, get_test_plan_version_id(text 'Modal Dialog Example', '2'), '2024-05-14 14:18:23.602-05', 2, 2, 2, 'READY'); +INSERT INTO "TestPlanReport" (id, "testPlanVersionId", "createdAt", "markedFinalAt", "atId", "exactAtVersionId", "browserId", "vendorReviewStatus") VALUES (20, get_test_plan_version_id(text 'Checkbox Example (Mixed-State)', '1'), '2021-05-14 14:18:23.602-05', '2022-08-07', 2, 4, 2, 'READY'); +-- Fully recommended TestPlanVersion [START] +INSERT INTO "TestPlanReport" (id, "testPlanVersionId", "createdAt", "markedFinalAt", "atId", "exactAtVersionId", "browserId", "vendorReviewStatus") VALUES (21, get_test_plan_version_id(text 'Action Menu Button Example Using aria-activedescendant', '2'), '2024-09-20 14:18:23.602-05', '2024-09-21', 1, 1, 1, 'READY'); +INSERT INTO "TestPlanReport" (id, "testPlanVersionId", "createdAt", "markedFinalAt", "atId", "exactAtVersionId", "browserId", "vendorReviewStatus") VALUES (22, get_test_plan_version_id(text 'Action Menu Button Example Using aria-activedescendant', '2'), '2024-09-20 14:18:23.602-05', '2024-09-21', 1, 1, 2, 'READY'); +INSERT INTO "TestPlanReport" (id, "testPlanVersionId", "createdAt", "markedFinalAt", "atId", "exactAtVersionId", "browserId", "vendorReviewStatus") VALUES (23, get_test_plan_version_id(text 'Action Menu Button Example Using aria-activedescendant', '2'), '2024-09-20 14:18:23.602-05', '2024-09-21', 2, 4, 1, 'READY'); +INSERT INTO "TestPlanReport" (id, "testPlanVersionId", "createdAt", "markedFinalAt", "atId", "exactAtVersionId", "browserId", "vendorReviewStatus") VALUES (24, get_test_plan_version_id(text 'Action Menu Button Example Using aria-activedescendant', '2'), '2024-09-20 14:18:23.602-05', '2024-09-21', 2, 4, 2, 'READY'); +INSERT INTO "TestPlanReport" (id, "testPlanVersionId", "createdAt", "markedFinalAt", "atId", "exactAtVersionId", "browserId", "vendorReviewStatus") VALUES (25, get_test_plan_version_id(text 'Action Menu Button Example Using aria-activedescendant', '2'), '2024-09-20 14:18:23.602-05', '2024-09-21', 3, 5, 2, 'READY'); +INSERT INTO "TestPlanReport" (id, "testPlanVersionId", "createdAt", "markedFinalAt", "atId", "exactAtVersionId", "browserId", "vendorReviewStatus") VALUES (26, get_test_plan_version_id(text 'Action Menu Button Example Using aria-activedescendant', '2'), '2024-09-20 14:18:23.602-05', '2024-09-21', 3, 5, 3, 'READY'); +--- Fully recommended TestPlanVersion [END] +INSERT INTO "TestPlanReport" (id, "testPlanVersionId", "createdAt", "atId", "minimumAtVersionId", "browserId", "vendorReviewStatus") VALUES (27, get_test_plan_version_id(text 'Color Viewer Slider', '2'), '2024-10-02 14:18:23.602-05', 3, 3, 3, 'READY'); -- -- Data for Name: TestPlanVersion; Type: TABLE DATA; Schema: public; Owner: atr @@ -91,10 +101,12 @@ INSERT INTO "TestPlanReport" (id, "testPlanVersionId", "createdAt", "atId", "min UPDATE "TestPlanVersion" SET "phase" = 'DRAFT', "draftPhaseReachedAt" = '2022-07-06' WHERE id = get_test_plan_version_id(text 'Toggle Button', '1'); UPDATE "TestPlanVersion" SET "phase" = 'DRAFT', "draftPhaseReachedAt" = '2022-07-06' WHERE id = get_test_plan_version_id(text 'Select Only Combobox Example', '1'); UPDATE "TestPlanVersion" SET "phase" = 'CANDIDATE', "draftPhaseReachedAt" = '2022-07-06', "candidatePhaseReachedAt" = '2022-07-06', "recommendedPhaseTargetDate" = '2023-01-02' WHERE id = get_test_plan_version_id(text 'Modal Dialog Example', '1'); -UPDATE "TestPlanVersion" SET "phase" = 'RECOMMENDED', "candidatePhaseReachedAt" = '2022-07-06', "recommendedPhaseTargetDate" = '2023-01-02', "recommendedPhaseReachedAt" = '2023-01-03' WHERE id = get_test_plan_version_id(text 'Checkbox Example (Mixed-State)', '1'); +UPDATE "TestPlanVersion" SET "phase" = 'RECOMMENDED', "draftPhaseReachedAt" = '2022-07-07', "candidatePhaseReachedAt" = '2022-07-06', "recommendedPhaseTargetDate" = '2023-01-02', "recommendedPhaseReachedAt" = '2023-01-03' WHERE id = get_test_plan_version_id(text 'Checkbox Example (Mixed-State)', '1'); UPDATE "TestPlanVersion" SET "phase" = 'DRAFT', "draftPhaseReachedAt" = '2022-07-06' WHERE id = get_test_plan_version_id(text 'Alert Example', '1'); UPDATE "TestPlanVersion" SET "phase" = 'DRAFT', "draftPhaseReachedAt" = '2023-12-14' WHERE id = get_test_plan_version_id(text 'Command Button Example', '2'); UPDATE "TestPlanVersion" SET "phase" = 'DRAFT', "draftPhaseReachedAt" = '2024-05-14' WHERE id = get_test_plan_version_id(text 'Modal Dialog Example', '2'); +UPDATE "TestPlanVersion" SET "phase" = 'RECOMMENDED', "draftPhaseReachedAt" = '2024-09-20', "candidatePhaseReachedAt" = '2024-09-22', "recommendedPhaseTargetDate" = '2025-03-21', "recommendedPhaseReachedAt" = '2024-09-23' WHERE id = get_test_plan_version_id(text 'Action Menu Button Example Using aria-activedescendant', '2'); +UPDATE "TestPlanVersion" SET "phase" = 'DRAFT', "draftPhaseReachedAt" = '2024-10-02' WHERE id = get_test_plan_version_id(text 'Color Viewer Slider', '2'); -- -- Data for Name: User; Type: TABLE DATA; Schema: public; Owner: atr @@ -102,13 +114,17 @@ UPDATE "TestPlanVersion" SET "phase" = 'DRAFT', "draftPhaseReachedAt" = '2024-05 INSERT INTO "User" (id, username, "createdAt", "updatedAt") VALUES (1, 'esmeralda-baggins', '2021-05-14 13:57:16.232-05', '2021-05-14 13:57:20.473-05'); INSERT INTO "User" (id, username, "createdAt", "updatedAt") VALUES (2, 'tom-proudfeet', '2021-05-14 13:57:16.232-05', '2021-05-14 13:57:20.473-05'); +INSERT INTO "User" (id, username, "createdAt", "updatedAt") VALUES (3, 'foo-bar', '2021-05-14 13:57:16.232-05', '2021-05-14 13:57:20.473-05'); INSERT INTO "UserAts" ("userId", "atId") VALUES (1, 1); INSERT INTO "UserAts" ("userId", "atId") VALUES (1, 2); INSERT INTO "UserAts" ("userId", "atId") VALUES (2, 3); +INSERT INTO "UserAts" ("userId", "atId") VALUES (3, 1); +INSERT INTO "UserAts" ("userId", "atId") VALUES (3, 2); +INSERT INTO "UserAts" ("userId", "atId") VALUES (3, 3); INSERT INTO "UserRoles" ("userId", "roleName") VALUES (1, 'ADMIN'); INSERT INTO "UserRoles" ("userId", "roleName") VALUES (1, 'TESTER'); INSERT INTO "UserRoles" ("userId", "roleName") VALUES (2, 'TESTER'); - +INSERT INTO "UserRoles" ("userId", "roleName") VALUES (3, 'TESTER'); -- -- Data for Name: TestPlanRun; Type: TABLE DATA; Schema: public; Owner: atr @@ -133,6 +149,15 @@ INSERT INTO "TestPlanRun" (id, "testerUserId", "testPlanReportId", "testResults" INSERT INTO "TestPlanRun" (id, "testerUserId", "testPlanReportId", "testResults") VALUES (17, 1, 17, '[]'); INSERT INTO "TestPlanRun" (id, "testerUserId", "testPlanReportId", "testResults") VALUES (18, 1, 18, '[]'); INSERT INTO "TestPlanRun" (id, "testerUserId", "testPlanReportId", "testResults") VALUES (19, 1, 19, '[]'); +INSERT INTO "TestPlanRun" (id, "testerUserId", "testPlanReportId", "testResults") VALUES (20, 1, 20, '[]'); +INSERT INTO "TestPlanRun" (id, "testerUserId", "testPlanReportId", "testResults") VALUES (21, 2, 1, '[]'); +INSERT INTO "TestPlanRun" (id, "testerUserId", "testPlanReportId", "testResults") VALUES (22, 2, 21, '[]'); +INSERT INTO "TestPlanRun" (id, "testerUserId", "testPlanReportId", "testResults") VALUES (23, 2, 22, '[]'); +INSERT INTO "TestPlanRun" (id, "testerUserId", "testPlanReportId", "testResults") VALUES (24, 2, 23, '[]'); +INSERT INTO "TestPlanRun" (id, "testerUserId", "testPlanReportId", "testResults") VALUES (25, 2, 24, '[]'); +INSERT INTO "TestPlanRun" (id, "testerUserId", "testPlanReportId", "testResults") VALUES (26, 2, 25, '[]'); +INSERT INTO "TestPlanRun" (id, "testerUserId", "testPlanReportId", "testResults") VALUES (27, 2, 26, '[]'); +INSERT INTO "TestPlanRun" (id, "testerUserId", "testPlanReportId", "testResults") VALUES (28, 1, 27, '[]'); -- -- Data for Name: CollectionJob; Type: TABLE DATA; Schema: public; Owner: atr diff --git a/server/scripts/populate-test-data/populateFakeTestResults.js b/server/scripts/populate-test-data/populateFakeTestResults.js index 28b7c2a94..dad1be2f2 100644 --- a/server/scripts/populate-test-data/populateFakeTestResults.js +++ b/server/scripts/populate-test-data/populateFakeTestResults.js @@ -5,31 +5,57 @@ const { } = require('../../models/services/BrowserService'); const { query, mutate } = require('../../tests/util/graphql-test-utilities'); +const FAKE_RESULT_CONFLICTS_OPTIONS = { + SINGLE: 1, + ALL: Infinity +}; + +/** + * + * @param {number} testPlanRunId + * @param {string['completeAndPassing' | + * 'completeAndFailingDueToIncorrectAssertions' | + * 'completeAndFailingDueToNoOutputAssertions' | + * 'completeAndFailingDueToUnexpectedBehaviors' | + * 'completeAndFailingDueToMultiple' | + * 'incompleteAndEmpty' | + * 'incompleteAndPassing' | + * 'incompleteAndFailingDueToIncorrectAssertions' | + * 'incompleteAndFailingDueToNoOutputAssertions' | + * 'incompleteAndFailingDueToUnexpectedBehaviors' | + * 'incompleteAndFailingDueToMultiple']} fakeTestResultTypes + * @param {import('sequelize').Transaction} transaction + * @param {number} numFakeTestResultConflicts + * @returns {Promise} + */ const populateFakeTestResults = async ( testPlanRunId, fakeTestResultTypes, - { transaction } + { + transaction, + numFakeTestResultConflicts = FAKE_RESULT_CONFLICTS_OPTIONS.SINGLE + } ) => { const { populateData: { testPlanReport } } = await query( gql` - query { - populateData(locationOfData: { testPlanRunId: ${testPlanRunId} }) { - testPlanReport { - at { - id - } - browser { - id - } - runnableTests { - id - } - } - } + query { + populateData(locationOfData: { testPlanRunId: ${testPlanRunId} }) { + testPlanReport { + at { + id + } + browser { + id + } + runnableTests { + id } - `, + } + } + } + `, { transaction } ); @@ -64,6 +90,7 @@ const populateFakeTestResults = async ( index, fakeTestResultType: 'failingDueToIncorrectAssertions', submit: true, + numFakeTestResultConflicts, transaction }); break; @@ -74,6 +101,7 @@ const populateFakeTestResults = async ( index, fakeTestResultType: 'failingDueToNoOutputAssertions', submit: true, + numFakeTestResultConflicts, transaction }); break; @@ -84,6 +112,7 @@ const populateFakeTestResults = async ( index, fakeTestResultType: 'failingDueToUnexpectedBehaviors', submit: true, + numFakeTestResultConflicts, transaction }); break; @@ -94,6 +123,7 @@ const populateFakeTestResults = async ( index, fakeTestResultType: 'failingDueToMultiple', submit: true, + numFakeTestResultConflicts, transaction }); break; @@ -124,6 +154,7 @@ const populateFakeTestResults = async ( index, fakeTestResultType: 'failingDueToIncorrectAssertions', submit: false, + numFakeTestResultConflicts, transaction }); break; @@ -134,6 +165,7 @@ const populateFakeTestResults = async ( index, fakeTestResultType: 'failingDueToNoOutputAssertions', submit: false, + numFakeTestResultConflicts, transaction }); break; @@ -144,6 +176,7 @@ const populateFakeTestResults = async ( index, fakeTestResultType: 'failingDueToUnexpectedBehaviors', submit: false, + numFakeTestResultConflicts, transaction }); break; @@ -154,6 +187,7 @@ const populateFakeTestResults = async ( index, fakeTestResultType: 'failingDueToMultiple', submit: false, + numFakeTestResultConflicts, transaction }); break; @@ -190,6 +224,7 @@ const getFake = async ({ index, fakeTestResultType, submit, + numFakeTestResultConflicts, transaction }) => { const testId = testPlanReport.runnableTests[index].id; @@ -211,28 +246,28 @@ const getFake = async ({ } } = await mutate( gql` - mutation { - testPlanRun(id: ${testPlanRunId}) { - findOrCreateTestResult( - testId: "${testId}", - atVersionId: "${atVersion.id}", - browserVersionId: "${browserVersion.id}" - ) { - testResult { - id - scenarioResults { - id - output - assertionResults { - id - passed - } - } - } - } + mutation { + testPlanRun(id: ${testPlanRunId}) { + findOrCreateTestResult( + testId: "${testId}", + atVersionId: "${atVersion.id}", + browserVersionId: "${browserVersion.id}" + ) { + testResult { + id + scenarioResults { + id + output + assertionResults { + id + passed } + } } - `, + } + } + } + `, { transaction } ); @@ -255,60 +290,68 @@ const getFake = async ({ const testResult = getPassing(); - switch (fakeTestResultType) { - case 'empty': - return; - case 'passing': - break; - case 'failingDueToIncorrectAssertions': - testResult.scenarioResults[0].assertionResults[0].passed = false; - testResult.scenarioResults[0].assertionResults[0].failedReason = - 'INCORRECT_OUTPUT'; - break; - case 'failingDueToNoOutputAssertions': - testResult.scenarioResults[0].assertionResults[0].passed = false; - testResult.scenarioResults[0].assertionResults[0].failedReason = - 'NO_OUTPUT'; - break; - case 'failingDueToUnexpectedBehaviors': - testResult.scenarioResults[0].unexpectedBehaviors.push({ - id: 'OTHER', - impact: 'MODERATE', - details: 'Seeded other unexpected behavior' - }); - break; - case 'failingDueToMultiple': - testResult.scenarioResults[0].assertionResults[0].passed = false; - testResult.scenarioResults[0].assertionResults[0].failedReason = - 'INCORRECT_OUTPUT'; - testResult.scenarioResults[0].unexpectedBehaviors.push({ - id: 'EXCESSIVELY_VERBOSE', - impact: 'MODERATE', - details: 'N/A' - }); - testResult.scenarioResults[0].unexpectedBehaviors.push({ - id: 'OTHER', - impact: 'SEVERE', - details: 'Seeded other unexpected behavior' - }); - break; - default: - throw new Error(); + const applyResult = (scenarioResult, type) => { + switch (type) { + case 'failingDueToIncorrectAssertions': + scenarioResult.assertionResults[0].passed = false; + scenarioResult.assertionResults[0].failedReason = 'INCORRECT_OUTPUT'; + break; + case 'failingDueToNoOutputAssertions': + scenarioResult.assertionResults[0].passed = false; + scenarioResult.assertionResults[0].failedReason = 'NO_OUTPUT'; + break; + case 'failingDueToUnexpectedBehaviors': + scenarioResult.unexpectedBehaviors.push({ + id: 'OTHER', + impact: 'MODERATE', + details: 'Seeded other unexpected behavior' + }); + break; + case 'failingDueToMultiple': + scenarioResult.assertionResults[0].passed = false; + scenarioResult.assertionResults[0].failedReason = 'INCORRECT_OUTPUT'; + scenarioResult.unexpectedBehaviors.push( + { id: 'EXCESSIVELY_VERBOSE', impact: 'MODERATE', details: 'N/A' }, + { + id: 'OTHER', + impact: 'SEVERE', + details: 'Seeded other unexpected behavior' + } + ); + break; + default: + throw new Error(); + } + }; + + if (fakeTestResultType === 'empty') { + return; + } else if (fakeTestResultType === 'passing') { + // Do nothing + } else { + const setResult = scenarioResult => + applyResult(scenarioResult, fakeTestResultType); + + if (numFakeTestResultConflicts > 0) { + testResult.scenarioResults + .slice(0, numFakeTestResultConflicts) + .forEach(setResult); + } } const persistTestResult = submit ? 'submitTestResult' : 'saveTestResult'; await mutate( gql` - mutation PersistTestResult($input: TestResultInput!) { - testResult(id: "${testResult.id}") { - ${persistTestResult}(input: $input) { - locationOfData - } - } - } - `, + mutation PersistTestResult($input: TestResultInput!) { + testResult(id: "${testResult.id}") { + ${persistTestResult}(input: $input) { + locationOfData + } + } + } + `, { variables: { input: testResult }, transaction } ); }; -module.exports = populateFakeTestResults; +module.exports = { populateFakeTestResults, FAKE_RESULT_CONFLICTS_OPTIONS }; diff --git a/server/seeders/20240903165406-add-vendors.js b/server/seeders/20240903165406-add-vendors.js new file mode 100644 index 000000000..5041d66f3 --- /dev/null +++ b/server/seeders/20240903165406-add-vendors.js @@ -0,0 +1,64 @@ +'use strict'; + +const { VENDOR_NAME_TO_AT_MAPPING } = require('../util/constants'); +const getUsersFromFile = require('../util/getUsersFromFile'); + +module.exports = { + async up(queryInterface, Sequelize) { + const vendorLines = await getUsersFromFile('vendors.txt'); + + for (const line of vendorLines) { + const [username, companyName] = line.split('|'); + if (username && companyName) { + // Create vendor if it doesn't exist + const [vendor] = await queryInterface.sequelize.query( + `INSERT INTO "Vendor" (name, "createdAt", "updatedAt") + VALUES (:name, NOW(), NOW()) + ON CONFLICT (name) DO UPDATE SET name = :name + RETURNING id`, + { + replacements: { name: companyName }, + type: Sequelize.QueryTypes.INSERT + } + ); + + // Associate user with vendor if user exists + await queryInterface.sequelize.query( + `UPDATE "User" SET "vendorId" = :vendorId + WHERE username = :username`, + { + replacements: { vendorId: vendor[0].id, username }, + type: Sequelize.QueryTypes.UPDATE + } + ); + + if (VENDOR_NAME_TO_AT_MAPPING[companyName]) { + for (const atName of VENDOR_NAME_TO_AT_MAPPING[companyName]) { + await queryInterface.sequelize.query( + `UPDATE "At" SET "vendorId" = :vendorId + WHERE name = :atName`, + { + replacements: { vendorId: vendor[0].id, atName }, + type: Sequelize.QueryTypes.UPDATE + } + ); + } + } + } + } + }, + + async down(queryInterface, Sequelize) { + const vendorLines = await getUsersFromFile('vendors.txt'); + + await queryInterface.sequelize.query( + `DELETE FROM "Vendor" WHERE name IN (:vendorNames)`, + { + replacements: { + vendorNames: vendorLines.map(line => line.split('|')[1]) + }, + type: Sequelize.QueryTypes.DELETE + } + ); + } +}; diff --git a/server/tests/integration/automation-scheduler.test.js b/server/tests/integration/automation-scheduler.test.js index 4ec8e4be3..60a53b191 100644 --- a/server/tests/integration/automation-scheduler.test.js +++ b/server/tests/integration/automation-scheduler.test.js @@ -1,7 +1,8 @@ const startSupertestServer = require('../util/api-server'); const automationRoutes = require('../../routes/automation'); const { - setupMockAutomationSchedulerServer + setupMockAutomationSchedulerServer, + startCollectionJobSimulation } = require('../util/mock-automation-scheduler-server'); const db = require('../../models/index'); const { query, mutate } = require('../util/graphql-test-utilities'); @@ -197,6 +198,22 @@ const restartCollectionJobByMutation = async (jobId, { transaction }) => { transaction } ); +const retryCanceledCollectionJobByMutation = async (jobId, { transaction }) => + await mutate( + ` + mutation { + collectionJob(id: "${jobId}") { + retryCanceledCollections { + id + status + testStatus { + status + } + } + } + } `, + { transaction } + ); const cancelCollectionJobByMutation = async (jobId, { transaction }) => await mutate( ` @@ -237,6 +254,18 @@ describe('Automation controller', () => { expect(storedJob.status).toEqual('QUEUED'); expect(storedJob.testPlanRun.testPlanReport.id).toEqual(testPlanReportId); expect(storedJob.testPlanRun.testResults.length).toEqual(0); + const collectionJob = await getCollectionJobById({ + id: storedJob.id, + transaction + }); + const tests = + collectionJob.testPlanRun.testPlanReport.testPlanVersion.tests.filter( + test => test.at.key === 'voiceover_macos' + ); + // check testIds - order doesn't matter, so we sort them + expect( + startCollectionJobSimulation.lastCallParams.testIds.sort() + ).toEqual(tests.map(test => test.id).sort()); }); }); @@ -260,6 +289,105 @@ describe('Automation controller', () => { }); }); + it('should retry a cancelled job with only remaining tests', async () => { + await apiServer.sessionAgentDbCleaner(async transaction => { + const { scheduleCollectionJob: job } = + await scheduleCollectionJobByMutation({ transaction }); + const collectionJob = await getCollectionJobById({ + id: job.id, + transaction + }); + const secret = await getJobSecret(collectionJob.id, { transaction }); + + // start "RUNNING" the job + const response1 = await sessionAgent + .post(`/api/jobs/${collectionJob.id}`) + .send({ status: 'RUNNING' }) + .set('x-automation-secret', secret) + .set('x-transaction-id', transaction.id); + expect(response1.statusCode).toBe(200); + + // simulate a response for a test + const automatedTestResponse = 'AUTOMATED TEST RESPONSE'; + const ats = await AtLoader().getAll({ transaction }); + const browsers = await BrowserLoader().getAll({ transaction }); + const at = ats.find( + at => at.id === collectionJob.testPlanRun.testPlanReport.at.id + ); + const browser = browsers.find( + browser => + browser.id === collectionJob.testPlanRun.testPlanReport.browser.id + ); + const { tests } = + collectionJob.testPlanRun.testPlanReport.testPlanVersion; + const selectedTestIndex = 2; + + const selectedTest = tests[selectedTestIndex]; + const selectedTestRowNumber = selectedTest.rowNumber; + + const numberOfScenarios = selectedTest.scenarios.filter( + scenario => scenario.atId === at.id + ).length; + const response2 = await sessionAgent + .post(`/api/jobs/${collectionJob.id}/test/${selectedTestRowNumber}`) + .send({ + capabilities: { + atName: at.name, + atVersion: at.atVersions[0].name, + browserName: browser.name, + browserVersion: browser.browserVersions[0].name + }, + responses: new Array(numberOfScenarios).fill(automatedTestResponse) + }) + .set('x-automation-secret', secret) + .set('x-transaction-id', transaction.id); + expect(response2.statusCode).toBe(200); + // cancel the job + const { + collectionJob: { cancelCollectionJob: cancelledCollectionJob } + } = await cancelCollectionJobByMutation(collectionJob.id, { + transaction + }); + + // check canceled status + + expect(cancelledCollectionJob.status).toEqual('CANCELLED'); + const { collectionJob: storedCollectionJob } = await getTestCollectionJob( + collectionJob.id, + { transaction } + ); + expect(storedCollectionJob.status).toEqual('CANCELLED'); + for (const test of storedCollectionJob.testStatus) { + const expectedStatus = + test.test.id == selectedTest.id ? 'COMPLETED' : 'CANCELLED'; + expect(test.status).toEqual(expectedStatus); + } + + // retry job + const data = await retryCanceledCollectionJobByMutation( + collectionJob.id, + { transaction } + ); + expect(data.collectionJob.retryCanceledCollections.status).toBe('QUEUED'); + const { collectionJob: restartedCollectionJob } = + await getTestCollectionJob(collectionJob.id, { transaction }); + // check restarted status + expect(restartedCollectionJob.status).toEqual('QUEUED'); + for (const test of restartedCollectionJob.testStatus) { + const expectedStatus = + test.test.id == selectedTest.id ? 'COMPLETED' : 'QUEUED'; + expect(test.status).toEqual(expectedStatus); + } + const expectedTests = tests.filter( + test => test.at.key === 'voiceover_macos' && test.id != selectedTest.id + ); + // check testIds - order doesn't matter, so we sort them + expect( + startCollectionJobSimulation.lastCallParams.testIds.sort() + ).toEqual(expectedTests.map(test => test.id).sort()); + }); + }); + it('should gracefully reject request to cancel a job that does not exist', async () => { await dbCleaner(async transaction => { expect.assertions(1); // Make sure an assertion is made diff --git a/server/tests/integration/dataManagement.test.js b/server/tests/integration/dataManagement.test.js index e076903f9..6fe4c3920 100644 --- a/server/tests/integration/dataManagement.test.js +++ b/server/tests/integration/dataManagement.test.js @@ -34,75 +34,75 @@ const testPlanVersionsQuery = ({ }) => { return query( gql` - query { - testPlanVersions( - directory: ${directory} - phases: ${phases} - ) { + query { + testPlanVersions( + directory: ${directory} + phases: ${phases} + ) { + id + phase + gitSha + metadata + testPlan { + directory + } + testPlanReports { + id + markedFinalAt + at { + id + } + browser { + id + } + draftTestPlanRuns { + testResults { + id + completedAt + test { + id + rowNumber + title + ats { + id + name + } + scenarios { id - phase - gitSha - metadata - testPlan { - directory + commands { + id + text } - testPlanReports { - id - markedFinalAt - at { - id - } - browser { - id - } - draftTestPlanRuns { - testResults { - id - completedAt - test { - id - rowNumber - title - ats { - id - name - } - scenarios { - id - commands { - id - text - } - } - assertions { - id - priority - text - } - } - scenarioResults { - output - assertionResults { - id - assertion { - text - } - passed - } - scenario { - id - commands { - id - text - } - } - } - } - } + } + assertions { + id + priority + text + } + } + scenarioResults { + output + assertionResults { + id + assertion { + text } + passed + } + scenario { + id + commands { + id + text + } + } } + } } - `, + } + } + } + `, { transaction } ); }; @@ -114,87 +114,87 @@ const updateVersionToPhaseQuery = ( ) => { return mutate( gql` - mutation { - testPlanVersion(id: ${testPlanVersionId}) { - updatePhase( - phase: ${phase} - testPlanVersionDataToIncludeId: ${ - testPlanVersionDataToIncludeId ?? null + mutation { + testPlanVersion(id: ${testPlanVersionId}) { + updatePhase( + phase: ${phase} + testPlanVersionDataToIncludeId: ${ + testPlanVersionDataToIncludeId ?? null + } + ) { + testPlanVersion { + id + phase + gitSha + metadata + testPlan { + directory + } + testPlanReports { + id + markedFinalAt + at { + id + } + minimumAtVersion { + id + } + exactAtVersion { + id + } + browser { + id + } + draftTestPlanRuns { + testResults { + id + completedAt + test { + id + rowNumber + title + ats { + id + name + } + scenarios { + id + commands { + id + text } - ) { - testPlanVersion { - id - phase - gitSha - metadata - testPlan { - directory - } - testPlanReports { - id - markedFinalAt - at { - id - } - minimumAtVersion { - id - } - exactAtVersion { - id - } - browser { - id - } - draftTestPlanRuns { - testResults { - id - completedAt - test { - id - rowNumber - title - ats { - id - name - } - scenarios { - id - commands { - id - text - } - } - assertions { - id - priority - text - } - } - scenarioResults { - output - assertionResults { - id - assertion { - text - } - passed - } - scenario { - id - commands { - id - text - } - } - } - } - } - } + } + assertions { + id + priority + text + } + } + scenarioResults { + output + assertionResults { + id + assertion { + text } + passed + } + scenario { + id + commands { + id + text + } + } } + } } + } } - `, + } + } + } + `, { transaction } ); }; @@ -208,32 +208,32 @@ const createTestPlanReportQuery = ( ) => { return mutate( gql` - mutation { - createTestPlanReport(input: { - testPlanVersionId: ${testPlanVersionId} - atId: ${atId} - minimumAtVersionId: ${minimumAtVersionId} - browserId: ${browserId} - }) { - testPlanReport { - id - at { - id - } - browser { - id - } - } - testPlanVersion { - id - phase - testPlanReports { - id - } - } - } + mutation { + createTestPlanReport(input: { + testPlanVersionId: ${testPlanVersionId} + atId: ${atId} + minimumAtVersionId: ${minimumAtVersionId} + browserId: ${browserId} + }) { + testPlanReport { + id + at { + id } - `, + browser { + id + } + } + testPlanVersion { + id + phase + testPlanReports { + id + } + } + } + } + `, { transaction } ); }; @@ -241,17 +241,17 @@ const createTestPlanReportQuery = ( const markAsFinal = (testPlanReportId, { transaction }) => { return mutate( gql` - mutation { - testPlanReport(id: ${testPlanReportId}) { - markAsFinal { - testPlanReport { - id - markedFinalAt - } - } - } + mutation { + testPlanReport(id: ${testPlanReportId}) { + markAsFinal { + testPlanReport { + id + markedFinalAt } - `, + } + } + } + `, { transaction } ); }; @@ -306,8 +306,12 @@ const getTestableTestPlanVersions = async ({ // remove any newer version that deprecated it await rawQuery( ` - delete from "TestPlanReport" where "testPlanVersionId" = ${newTestPlanVersion.id}; - delete from "TestPlanVersion" where id = ${newTestPlanVersion.id};`, + delete + from "TestPlanReport" + where "testPlanVersionId" = ${newTestPlanVersion.id}; + delete + from "TestPlanVersion" + where id = ${newTestPlanVersion.id};`, { transaction } ); @@ -405,15 +409,15 @@ describe('data management', () => { const previous = await query( gql` - query { - testPlanVersion(id: ${testPlanVersionId}) { - phase - testPlanReports { - id - } - } - } - `, + query { + testPlanVersion(id: ${testPlanVersionId}) { + phase + testPlanReports { + id + } + } + } + `, { transaction } ); const previousPhase = previous.testPlanVersion.phase; @@ -435,12 +439,12 @@ describe('data management', () => { const testPlanReportsToMarkAsFinalResult = await query( gql` - query { - testPlanReports(testPlanVersionId: ${testPlanVersionId}) { - id - } - } - `, + query { + testPlanReports(testPlanVersionId: ${testPlanVersionId}) { + id + } + } + `, { transaction } ); diff --git a/server/tests/integration/graphql.test.js b/server/tests/integration/graphql.test.js index 7ffbf8c69..6c498662d 100644 --- a/server/tests/integration/graphql.test.js +++ b/server/tests/integration/graphql.test.js @@ -153,6 +153,7 @@ describe('graphql', () => { ['CollectionJob', 'testPlanRun'], ['CollectionJob', 'externalLogsUrl'], ['CollectionJob', 'testStatus'], + ['User', 'company'], // These interact with Response Scheduler API // which is mocked in other tests. ['Mutation', 'scheduleCollectionJob'], @@ -242,6 +243,10 @@ describe('graphql', () => { username roles isBot + company { + id + name + } } me { __typename @@ -252,6 +257,10 @@ describe('graphql', () => { id name } + company { + id + name + } } collectionJob(id: 1) { __typename @@ -271,6 +280,18 @@ describe('graphql', () => { id status } + vendors { + id + name + } + vendor(id: 1) { + id + name + } + vendorByName(name: "apple") { + id + name + } v2TestPlanVersion: testPlanVersion(id: 80) { __typename id diff --git a/server/tests/integration/testQueue.test.js b/server/tests/integration/testQueue.test.js index b74b9062e..42c598471 100644 --- a/server/tests/integration/testQueue.test.js +++ b/server/tests/integration/testQueue.test.js @@ -85,19 +85,19 @@ describe('test queue', () => { await dbCleaner(async transaction => { // A1 const testReportId = '1'; - const newTesterId = '2'; + const newTesterId = '3'; const previous = await query( gql` - query { - testPlanReport(id: ${testReportId}) { - draftTestPlanRuns { - tester { - id - } - } - } - } - `, + query { + testPlanReport(id: ${testReportId}) { + draftTestPlanRuns { + tester { + id + } + } + } + } + `, { transaction } ); const previousTesterIds = previous.testPlanReport.draftTestPlanRuns.map( @@ -107,30 +107,30 @@ describe('test queue', () => { // A2 const result = await mutate( gql` - mutation { - testPlanReport(id: ${testReportId}) { - assignTester(userId: ${newTesterId}) { - testPlanReport { - draftTestPlanRuns { - tester { - id - username - } - } - } - } - } + mutation { + testPlanReport(id: ${testReportId}) { + assignTester(userId: ${newTesterId}) { + testPlanReport { + draftTestPlanRuns { + tester { + id + username } - `, + } + } + } + } + } + `, { transaction } ); // prettier-ignore const resultTesterIds = result - .testPlanReport - .assignTester - .testPlanReport - .draftTestPlanRuns.map(testPlanRun => testPlanRun.tester.id); + .testPlanReport + .assignTester + .testPlanReport + .draftTestPlanRuns.map(testPlanRun => testPlanRun.tester.id); // A3 expect(previousTesterIds).not.toEqual( @@ -147,16 +147,16 @@ describe('test queue', () => { const previousTesterId = '1'; const previous = await query( gql` - query { - testPlanReport(id: ${testPlanReportId}) { - draftTestPlanRuns { - tester { - id - } - } - } - } - `, + query { + testPlanReport(id: ${testPlanReportId}) { + draftTestPlanRuns { + tester { + id + } + } + } + } + `, { transaction } ); const previousTesterIds = previous.testPlanReport.draftTestPlanRuns.map( @@ -166,29 +166,29 @@ describe('test queue', () => { // A2 const result = await mutate( gql` - mutation { - testPlanReport(id: ${testPlanReportId}) { - deleteTestPlanRun(userId: ${previousTesterId}) { - testPlanReport { - draftTestPlanRuns { - tester { - username - } - } - } - } - } + mutation { + testPlanReport(id: ${testPlanReportId}) { + deleteTestPlanRun(userId: ${previousTesterId}) { + testPlanReport { + draftTestPlanRuns { + tester { + username } - `, + } + } + } + } + } + `, { transaction } ); // prettier-ignore const resultTesterIds = result - .testPlanReport - .deleteTestPlanRun - .testPlanReport - .draftTestPlanRuns.map((run) => run.tester.id); + .testPlanReport + .deleteTestPlanRun + .testPlanReport + .draftTestPlanRuns.map((run) => run.tester.id); // A3 expect(previousTesterIds).toEqual( @@ -280,29 +280,29 @@ describe('test queue', () => { // A2 const result = await mutate( gql` - mutation { - createTestPlanReport(input: { - testPlanVersionId: ${testPlanVersionId} - atId: ${atId} - minimumAtVersionId: ${minimumAtVersionId} - browserId: ${browserId} - }) { - testPlanReport { - id - at { - id - } - browser { - id - } - } - testPlanVersion { - id - phase - } - } - } - `, + mutation { + createTestPlanReport(input: { + testPlanVersionId: ${testPlanVersionId} + atId: ${atId} + minimumAtVersionId: ${minimumAtVersionId} + browserId: ${browserId} + }) { + testPlanReport { + id + at { + id + } + browser { + id + } + } + testPlanVersion { + id + phase + } + } + } + `, { transaction } ); const { testPlanReport, testPlanVersion } = result.createTestPlanReport; @@ -333,39 +333,39 @@ describe('test queue', () => { const testPlanReportId = '4'; const queryBefore = await query( gql` - query { - testPlanReport(id: ${testPlanReportId}) { - id - draftTestPlanRuns { - id - } - } - } - `, + query { + testPlanReport(id: ${testPlanReportId}) { + id + draftTestPlanRuns { + id + } + } + } + `, { transaction } ); const { draftTestPlanRuns } = queryBefore.testPlanReport; await mutate( gql` - mutation { - testPlanReport(id: ${testPlanReportId}) { - deleteTestPlanReport - } - } - `, + mutation { + testPlanReport(id: ${testPlanReportId}) { + deleteTestPlanReport + } + } + `, { transaction } ); const queryAfter = await query( gql` - query { - testPlanReport(id: ${testPlanReportId}) { - id - } - testPlanRun(id: ${draftTestPlanRuns[0].id}) { - id - } - } - `, + query { + testPlanReport(id: ${testPlanReportId}) { + id + } + testPlanRun(id: ${draftTestPlanRuns[0].id}) { + id + } + } + `, { transaction } ); @@ -381,45 +381,45 @@ describe('test queue', () => { const result = await query( gql` - query { - testPlanReport(id: ${conflictingReportId}) { - conflicts { - source { - test { - title - rowNumber - } - scenario { - commands { - text - } - } - assertion { - text - } - } - conflictingResults { - testPlanRun { - id - tester { - username - } - } - scenarioResult { - output - unexpectedBehaviors { - text - details - } - } - assertionResult { - passed - } - } - } - } + query { + testPlanReport(id: ${conflictingReportId}) { + conflicts { + source { + test { + title + rowNumber + } + scenario { + commands { + text + } + } + assertion { + text + } + } + conflictingResults { + testPlanRun { + id + tester { + username + } } - `, + scenarioResult { + output + unexpectedBehaviors { + text + details + } + } + assertionResult { + passed + } + } + } + } + } + `, { transaction: false } ); diff --git a/server/tests/models/At.spec.js b/server/tests/models/At.spec.js index e9d8c7101..8ea1ddc65 100644 --- a/server/tests/models/At.spec.js +++ b/server/tests/models/At.spec.js @@ -8,6 +8,7 @@ const { const AtModel = require('../../models/At'); const AtVersionModel = require('../../models/AtVersion'); const BrowserModel = require('../../models/Browser'); +const VendorModel = require('../../models/Vendor'); describe('AtModel', () => { // A1 @@ -26,18 +27,27 @@ describe('AtModel', () => { // A1 const AT_VERSION_ASSOCIATION = { as: 'atVersions' }; const BROWSER_ASSOCIATION = { through: 'AtBrowsers', as: 'browsers' }; + const VENDOR_ASSOCIATION = { as: 'vendor' }; // A2 beforeAll(() => { Model.hasMany(AtVersionModel, AT_VERSION_ASSOCIATION); Model.hasMany(BrowserModel, BROWSER_ASSOCIATION); + Model.belongsTo(VendorModel, VENDOR_ASSOCIATION); }); it('defined a hasMany association with AtVersion', () => { // A3 expect(Model.hasMany).toHaveBeenCalledWith( AtVersionModel, - expect.objectContaining(Model.AT_VERSION_ASSOCIATION) + expect.objectContaining(AT_VERSION_ASSOCIATION) + ); + }); + + it('defined a belongsTo association with Vendor', () => { + expect(Model.belongsTo).toHaveBeenCalledWith( + VendorModel, + expect.objectContaining(VENDOR_ASSOCIATION) ); }); }); diff --git a/server/tests/models/User.spec.js b/server/tests/models/User.spec.js index b67c776e0..927229709 100644 --- a/server/tests/models/User.spec.js +++ b/server/tests/models/User.spec.js @@ -8,6 +8,7 @@ const { const UserModel = require('../../models/User'); const RoleModel = require('../../models/Role'); const TestPlanRunModel = require('../../models/TestPlanRun'); +const VendorModel = require('../../models/Vendor'); describe('UserModel Schema Checks', () => { // A1 @@ -28,11 +29,13 @@ describe('UserModel Schema Checks', () => { // A1 const ROLE_ASSOCIATION = { through: 'UserRoles', as: 'roles' }; const TEST_PLAN_RUN_ASSOCIATION = { as: 'testPlanRuns' }; + const VENDOR_ASSOCIATION = { as: 'company' }; // A2 beforeAll(() => { Model.belongsToMany(RoleModel, ROLE_ASSOCIATION); Model.hasMany(TestPlanRunModel, TEST_PLAN_RUN_ASSOCIATION); + Model.belongsTo(VendorModel, VENDOR_ASSOCIATION); }); // A3 @@ -49,5 +52,12 @@ describe('UserModel Schema Checks', () => { expect.objectContaining(Model.TEST_PLAN_RUN_ASSOCIATION) ); }); + + it('defined a belongsTo association with Vendor', () => { + expect(Model.belongsTo).toHaveBeenCalledWith( + VendorModel, + expect.objectContaining(Model.VENDOR_ASSOCIATION) + ); + }); }); }); diff --git a/server/tests/models/Vendor.spec.js b/server/tests/models/Vendor.spec.js new file mode 100644 index 000000000..e177ae98d --- /dev/null +++ b/server/tests/models/Vendor.spec.js @@ -0,0 +1,45 @@ +const { + sequelize, + dataTypes, + checkModelName, + checkPropertyExists +} = require('sequelize-test-helpers'); + +const VendorModel = require('../../models/Vendor'); +const AtModel = require('../../models/At'); +const UserModel = require('../../models/User'); + +describe('VendorModel', () => { + const Model = VendorModel(sequelize, dataTypes); + const modelInstance = new Model(); + + checkModelName(Model)('Vendor'); + + describe('properties', () => { + ['name'].forEach(checkPropertyExists(modelInstance)); + }); + + describe('associations', () => { + const AT_ASSOCIATION = { as: 'ats' }; + const USER_ASSOCIATION = { as: 'users' }; + + beforeAll(() => { + Model.hasMany(AtModel, AT_ASSOCIATION); + Model.hasMany(UserModel, USER_ASSOCIATION); + }); + + it('defined a hasMany association with At', () => { + expect(Model.hasMany).toHaveBeenCalledWith( + AtModel, + expect.objectContaining(Model.AT_ASSOCIATION) + ); + }); + + it('defined a hasMany association with User', () => { + expect(Model.hasMany).toHaveBeenCalledWith( + UserModel, + expect.objectContaining(Model.USER_ASSOCIATION) + ); + }); + }); +}); diff --git a/server/tests/models/services/TestPlanRunService.test.js b/server/tests/models/services/TestPlanRunService.test.js index 92a85ae56..8cc6fc462 100644 --- a/server/tests/models/services/TestPlanRunService.test.js +++ b/server/tests/models/services/TestPlanRunService.test.js @@ -103,7 +103,7 @@ describe('TestPlanRunModel Data Checks', () => { it('should create testPlanRun if none exists for tester', async () => { await dbCleaner(async transaction => { const _testPlanReportId = 1; - const _testerUserId = 2; + const _testerUserId = 3; const testPlanRuns = await TestPlanRunService.getTestPlanRuns({ transaction diff --git a/server/tests/models/services/VendorService.test.js b/server/tests/models/services/VendorService.test.js new file mode 100644 index 000000000..089c0068e --- /dev/null +++ b/server/tests/models/services/VendorService.test.js @@ -0,0 +1,74 @@ +const { sequelize } = require('../../../models'); +const VendorService = require('../../../models/services/VendorService'); +const randomStringGenerator = require('../../util/random-character-generator'); +const dbCleaner = require('../../util/db-cleaner'); + +afterAll(async () => { + await sequelize.close(); +}); + +describe('VendorService', () => { + it('should create and retrieve a vendor', async () => { + await dbCleaner(async transaction => { + const vendorName = randomStringGenerator(); + + const createdVendor = await VendorService.createVendor({ + values: { name: vendorName }, + transaction + }); + + expect(createdVendor).toHaveProperty('id'); + expect(createdVendor.name).toBe(vendorName); + + const retrievedVendor = await VendorService.findVendorById({ + id: createdVendor.id, + transaction + }); + + expect(retrievedVendor).toMatchObject(createdVendor); + }); + }); + + it('should update a vendor', async () => { + await dbCleaner(async transaction => { + const vendorName = randomStringGenerator(); + const updatedName = randomStringGenerator(); + + const createdVendor = await VendorService.createVendor({ + values: { name: vendorName }, + transaction + }); + + const updatedVendor = await VendorService.updateVendorById({ + id: createdVendor.id, + values: { name: updatedName }, + transaction + }); + + expect(updatedVendor.name).toBe(updatedName); + }); + }); + + it('should delete a vendor', async () => { + await dbCleaner(async transaction => { + const vendorName = randomStringGenerator(); + + const createdVendor = await VendorService.createVendor({ + values: { name: vendorName }, + transaction + }); + + await VendorService.removeVendorById({ + id: createdVendor.id, + transaction + }); + + const deletedVendor = await VendorService.findVendorById({ + id: createdVendor.id, + transaction + }); + + expect(deletedVendor).toBeNull(); + }); + }); +}); diff --git a/server/tests/util/mock-automation-scheduler-server.js b/server/tests/util/mock-automation-scheduler-server.js index b92b6890d..232fa7b4a 100644 --- a/server/tests/util/mock-automation-scheduler-server.js +++ b/server/tests/util/mock-automation-scheduler-server.js @@ -136,10 +136,18 @@ const simulateResultCompletion = async ( } }; -const startCollectionJobSimulation = async (job, atVersion, transaction) => { +const startCollectionJobSimulation = async ( + job, + testIds, + atVersion, + transaction +) => { if (!mockSchedulerEnabled) throw new Error('mock scheduler is not enabled'); if (process.env.ENVIRONMENT === 'test') { // stub behavior in test suite + + startCollectionJobSimulation.lastCallParams = { job, testIds, atVersion }; + return { status: COLLECTION_JOB_STATUS.QUEUED }; } else { const { data } = await apolloServer.executeOperation( diff --git a/server/util/constants.js b/server/util/constants.js index d6c90d026..27c126e41 100644 --- a/server/util/constants.js +++ b/server/util/constants.js @@ -9,7 +9,14 @@ const AT_VERSIONS_SUPPORTED_BY_COLLECTION_JOBS = { NVDA: ['2024.1', '2023.3.3', '2023.3'] }; +const VENDOR_NAME_TO_AT_MAPPING = { + vispero: ['JAWS'], + nvAccess: ['NVDA'], + apple: ['VoiceOver for macOS'] +}; + module.exports = { NO_OUTPUT_STRING, - AT_VERSIONS_SUPPORTED_BY_COLLECTION_JOBS + AT_VERSIONS_SUPPORTED_BY_COLLECTION_JOBS, + VENDOR_NAME_TO_AT_MAPPING }; diff --git a/server/util/getAtVersionWithRequirements.js b/server/util/getAtVersionWithRequirements.js index eb72940d2..0af679f31 100644 --- a/server/util/getAtVersionWithRequirements.js +++ b/server/util/getAtVersionWithRequirements.js @@ -1,5 +1,6 @@ const { Op } = require('sequelize'); const { getAtVersions } = require('../models/services/AtService'); +const { AT_VERSIONS_SUPPORTED_BY_COLLECTION_JOBS } = require('./constants'); const getAtVersionWithRequirements = async ( atId, @@ -33,11 +34,16 @@ const getAtVersionWithRequirements = async ( transaction }); - const latestSupportedAtVersion = matchingAtVersions.find(async atv => { - // supportedByAutomation is a computed graphql field, - // so we need to compute directly here - return atv.supportedByAutomation; - }); + const supportedVersions = + AT_VERSIONS_SUPPORTED_BY_COLLECTION_JOBS[ + matchingAtVersions[0]?.at?.name + ] || []; + + const latestSupportedAtVersion = matchingAtVersions.find( + atv => + supportedVersions.includes(atv.name) && + new Date(atv.releasedAt) >= new Date(minimumAtVersion.releasedAt) + ); if (!latestSupportedAtVersion) { throw new Error( diff --git a/vendors.txt b/vendors.txt index a48d48696..d98840a59 100644 --- a/vendors.txt +++ b/vendors.txt @@ -4,4 +4,5 @@ RFischer-FS|vispero stevefaulkner|vispero BrettLewisVispero|vispero -cookiecrook|apple \ No newline at end of file +cookiecrook|apple +twilco|apple