Skip to content

Commit

Permalink
feat(cli): execute one test from a test file (#338)
Browse files Browse the repository at this point in the history
Resolves #281 

Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
Co-authored-by: Razvan Marescu <[email protected]>
  • Loading branch information
3 people authored Feb 28, 2025
1 parent 1dba67e commit 6218f66
Show file tree
Hide file tree
Showing 12 changed files with 1,307 additions and 568 deletions.
7 changes: 4 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -190,9 +190,10 @@ shortest(`
### Running tests

```bash
pnpm shortest # Run all tests
pnpm shortest __tests__/login.test.ts # Run specific test
pnpm shortest --headless # Run in headless mode using CLI
pnpm shortest # Run all tests
pnpm shortest login.test.ts # Run specific tests from a file
pnpm shortest login.test.ts:23 # Run specific test from a file using a line number
pnpm shortest --headless # Run in headless mode using
```

You can find example tests in the [`examples`](./examples) directory.
Expand Down
7 changes: 3 additions & 4 deletions examples/api-assert-bearer.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,9 @@ import { shortest } from "@antiwork/shortest";
import { ALLOWED_TEST_BEARER, TESTING_API_BASE_URI } from "@/lib/constants";

// @note you should be authenticated in Clerk to run this test
shortest(`
Test the API POST endpoint ${TESTING_API_BASE_URI}/assert-bearer with body { "flagged": "false" } without providing a bearer token.
Expect the response to indicate that the token is missing
`);
shortest(
`Test the API POST endpoint ${TESTING_API_BASE_URI}/assert-bearer with body { "flagged": "false" } without providing a bearer token.`,
).expect("Expect the response to indicate that the token is missing");

shortest(`
Test the API POST endpoint ${TESTING_API_BASE_URI}/assert-bearer with body { "flagged": "true" } and the bearer token ${ALLOWED_TEST_BEARER}.
Expand Down
5 changes: 3 additions & 2 deletions packages/shortest/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -188,8 +188,9 @@ shortest(`

```bash
pnpm shortest # Run all tests
pnpm shortest login.test.ts # Run specific test
pnpm shortest --headless # Run in headless mode using cli
pnpm shortest login.test.ts # Run specific tests from a file
pnpm shortest login.test.ts:23 # Run specific test from a file using a line number
pnpm shortest --headless # Run in headless mode using
```

You can find example tests in the [`examples`](./examples) directory.
Expand Down
10 changes: 7 additions & 3 deletions packages/shortest/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,9 @@
"prepublishOnly": "pnpm build",
"postinstall": "node -e \"if (process.platform !== 'win32') { try { require('child_process').execSync('chmod +x dist/cli/bin.js') } catch (_) {} }\"",
"build:types": "tsup src/index.ts --dts-only --format esm --outDir dist",
"build:js": "esbuild src/index.ts --bundle --platform=node --format=esm --outfile=dist/index.js --external:esbuild --external:punycode --external:playwright --external:expect --external:dotenv --external:ai --external:@ai-sdk/*",
"build:cjs": "esbuild src/index.ts --bundle --platform=node --format=cjs --outfile=dist/index.cjs --external:esbuild --external:punycode --external:playwright --external:expect --external:dotenv --external:ai --external:@ai-sdk/*",
"build:cli": "esbuild src/cli/bin.ts --bundle --platform=node --format=esm --outdir=dist/cli --metafile=dist/meta-cli.json --external:fsevents --external:chokidar --external:glob --external:esbuild --external:events --external:path --external:fs --external:util --external:stream --external:os --external:assert --external:url --external:playwright --external:expect --external:dotenv --external:otplib --external:picocolors --external:punycode --external:https --external:http --external:net --external:tls --external:crypto --external:mailosaur --external:ai --external:@ai-sdk/*",
"build:js": "esbuild src/index.ts --bundle --platform=node --format=esm --outfile=dist/index.js --external:esbuild --external:punycode --external:playwright --external:expect --external:dotenv --external:ai --external:@ai-sdk/* --external:@babel/* --external:tty",
"build:cjs": "esbuild src/index.ts --bundle --platform=node --format=cjs --outfile=dist/index.cjs --external:esbuild --external:punycode --external:playwright --external:expect --external:dotenv --external:ai --external:@ai-sdk/* --external:@babel/* --external:tty",
"build:cli": "esbuild src/cli/bin.ts --bundle --platform=node --format=esm --outdir=dist/cli --metafile=dist/meta-cli.json --external:fsevents --external:chokidar --external:glob --external:esbuild --external:events --external:path --external:fs --external:util --external:stream --external:os --external:assert --external:url --external:playwright --external:expect --external:dotenv --external:otplib --external:picocolors --external:punycode --external:https --external:http --external:net --external:tls --external:crypto --external:mailosaur --external:ai --external:@ai-sdk/* --external:@babel/* --external:tty --external:debug",
"dev": "pnpm build --watch",
"test:unit": "npx vitest run",
"test:unit:watch": "npx vitest --watch",
Expand Down Expand Up @@ -61,6 +61,10 @@
"@ai-sdk/anthropic": "^1.1.9",
"@ai-sdk/provider": "^1.0.8",
"ai": "^4.1.45",
"@babel/parser": "^7.26.9",
"@babel/traverse": "^7.26.9",
"@babel/types": "^7.26.9",
"@types/babel__traverse": "^7.20.6",
"dotenv": "^16.4.5",
"esbuild": "^0.20.1",
"expect": "^29.7.0",
Expand Down
14 changes: 12 additions & 2 deletions packages/shortest/src/cli/bin.ts
Original file line number Diff line number Diff line change
Expand Up @@ -175,8 +175,15 @@ const main = async () => {
const baseUrl = args
.find((arg) => arg.startsWith("--target="))
?.split("=")[1];
const testPattern = args.find((arg) => !arg.startsWith("--"));
let testPattern = args.find((arg) => !arg.startsWith("--"));
const noCache = args.includes("--no-cache");
let lineNumber: number | undefined;

if (testPattern?.includes(":")) {
const [file, line] = testPattern.split(":");
testPattern = file;
lineNumber = parseInt(line, 10);
}

const cliOptions: CLIOptions = {
headless,
Expand All @@ -194,7 +201,10 @@ const main = async () => {
log.trace("Initializing TestRunner");
const runner = new TestRunner(process.cwd(), config);
await runner.initialize();
const success = await runner.execute(config.testPattern);
const success = await runner.execute(
testPattern ?? config.testPattern,
lineNumber,
);
process.exitCode = success ? 0 : 1;
} catch (error: any) {
log.trace("Handling error for TestRunner");
Expand Down
84 changes: 75 additions & 9 deletions packages/shortest/src/core/runner/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,10 @@ import { BrowserTool } from "@/browser/core/browser-tool";
import { BrowserManager } from "@/browser/manager";
import { TestCache } from "@/cache";
import { TestCompiler } from "@/core/compiler";
import {
EXPRESSION_PLACEHOLDER,
parseShortestTestFile,
} from "@/core/runner/test-file-parser";
import { TestReporter } from "@/core/runner/test-reporter";
import { getLogger, Log } from "@/log";
import {
Expand Down Expand Up @@ -281,16 +285,78 @@ export class TestRunner {
};
}

private async executeTestFile(file: string) {
private async filterTestsByLineNumber(
tests: TestFunction[],
file: string,
lineNumber: number,
): Promise<TestFunction[]> {
const testLocations = parseShortestTestFile(file);
const escapeRegex = (str: string) =>
str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");

const filteredTests = tests.filter((test) => {
const testNameNormalized = test.name.trim();
let testLocation = testLocations.find(
(location) => location.testName === testNameNormalized,
);

if (!testLocation) {
testLocation = testLocations.find((location) => {
const TEMP_TOKEN = "##PLACEHOLDER##";
let pattern = location.testName.replace(
new RegExp(escapeRegex(EXPRESSION_PLACEHOLDER), "g"),
TEMP_TOKEN,
);

pattern = escapeRegex(pattern);
pattern = pattern.replace(new RegExp(TEMP_TOKEN, "g"), ".*");
const regex = new RegExp(`^${pattern}$`);

return regex.test(testNameNormalized);
});
}

if (!testLocation) {
return false;
}

const isInRange =
lineNumber >= testLocation.startLine &&
lineNumber <= testLocation.endLine;
return isInRange;
});

return filteredTests;
}

private async executeTestFile(file: string, lineNumber?: number) {
try {
this.log.trace("Executing test file", { file, lineNumber });
const registry = (global as any).__shortest__.registry;
registry.tests.clear();
registry.currentFileTests = [];

const filePathWithoutCwd = file.replace(this.cwd + "/", "");
const compiledPath = await this.compiler.compileFile(file);

this.log.trace("Importing compiled file", { compiledPath });
await import(pathToFileURL(compiledPath).href);
let testsToRun = registry.currentFileTests;

if (lineNumber) {
testsToRun = await this.filterTestsByLineNumber(
registry.currentFileTests,
file,
lineNumber,
);
if (testsToRun.length === 0) {
this.reporter.error(
"Test Discovery",
`No test found at line ${lineNumber} in ${filePathWithoutCwd}`,
);
process.exit(1);
}
}

let context;
try {
Expand All @@ -309,14 +375,11 @@ export class TestRunner {
await hook(testContext);
}

this.reporter.onFileStart(
filePathWithoutCwd,
registry.currentFileTests.length,
);
this.reporter.onFileStart(filePathWithoutCwd, testsToRun.length);

// Execute tests in order they were defined
this.log.info(`Running ${registry.currentFileTests.length} test(s)`);
for (const test of registry.currentFileTests) {
this.log.info(`Running ${testsToRun.length} test(s)`);
for (const test of testsToRun) {
// Execute beforeEach hooks with shared context
for (const hook of registry.beforeEachFns) {
await hook(testContext);
Expand Down Expand Up @@ -363,12 +426,15 @@ export class TestRunner {
}
}

async execute(testPattern: string): Promise<boolean> {
async execute(testPattern: string, lineNumber?: number): Promise<boolean> {
this.log.trace("Finding test files", { testPattern });

const files = await glob(testPattern, {
cwd: this.cwd,
absolute: true,
});
this.log.trace("Found test files", { files });

if (files.length === 0) {
this.reporter.error(
"Test Discovery",
Expand All @@ -382,7 +448,7 @@ export class TestRunner {

this.reporter.onRunStart(files.length);
for (const file of files) {
await this.executeTestFile(file);
await this.executeTestFile(file, lineNumber);
}
this.reporter.onRunEnd();

Expand Down
163 changes: 163 additions & 0 deletions packages/shortest/src/core/runner/test-file-parser.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,163 @@
import { readFileSync } from "fs";
import * as parser from "@babel/parser";
import type { NodePath } from "@babel/traverse";
import traverse from "@babel/traverse";
import type * as t from "@babel/types";
import * as babelTypes from "@babel/types";
import { z } from "zod";
import { getLogger } from "@/log";

export const EXPRESSION_PLACEHOLDER = "${...}";

export const TestLocationSchema = z.object({
testName: z.string(),
startLine: z.number().int().positive(),
endLine: z.number().int().positive(),
});
export type TestLocation = z.infer<typeof TestLocationSchema>;

const TestLocationsSchema = z.array(TestLocationSchema);

export const parseShortestTestFile = (filePath: string): TestLocation[] => {
const log = getLogger();
try {
log.setGroup("File Parser");

const TemplateElementSchema = z.object({
value: z.object({
cooked: z.string().optional(),
raw: z.string().optional(),
}),
});
type TemplateElement = z.infer<typeof TemplateElementSchema>;

const StringLiteralSchema = z.object({
type: z.literal("StringLiteral"),
value: z.string(),
});

const TemplateLiteralSchema = z.object({
type: z.literal("TemplateLiteral"),
quasis: z.array(TemplateElementSchema),
});

const fileContent = readFileSync(filePath, "utf8");
const ast = parser.parse(fileContent, {
sourceType: "module",
plugins: [
"typescript",
"objectRestSpread",
"optionalChaining",
"nullishCoalescingOperator",
],
});

const testLocations: TestLocation[] = [];

const testCallsByLine = new Map<
number,
{ name: string; node: NodePath<t.CallExpression> }
>();

traverse(ast, {
CallExpression(path: NodePath<t.CallExpression>) {
const node = path.node;

if (
!node.type ||
node.type !== "CallExpression" ||
!node.callee ||
node.callee.type !== "Identifier" ||
node.callee.name !== "shortest"
) {
return;
}

const args = node.arguments || [];
if (args.length === 0) return;

const firstArg = args[0];
let testName = "";

if (babelTypes.isStringLiteral(firstArg)) {
const parsed = StringLiteralSchema.parse(firstArg);
testName = parsed.value;
} else if (babelTypes.isTemplateLiteral(firstArg)) {
const parsed = TemplateLiteralSchema.parse(firstArg);
testName = parsed.quasis
.map(
(quasi: TemplateElement, i: number, arr: TemplateElement[]) => {
const str = quasi.value.cooked || quasi.value.raw || "";
return i < arr.length - 1 ? str + EXPRESSION_PLACEHOLDER : str;
},
)
.join("")
.replace(/\s+/g, " ")
.trim();
} else {
return;
}

const startLine = node.loc?.start?.line || 0;
testCallsByLine.set(startLine, {
name: testName,
node: path,
});
},
});

const sortedStartLines = Array.from(testCallsByLine.keys()).sort(
(a, b) => a - b,
);

for (let i = 0; i < sortedStartLines.length; i++) {
const currentLine = sortedStartLines[i];
const nextLine = sortedStartLines[i + 1] || Number.MAX_SAFE_INTEGER;
const { name, node } = testCallsByLine.get(currentLine)!;

let path = node;
let endLine = path.node.loc?.end?.line || 0;

let currentPath: NodePath<t.Node> = path;
while (
currentPath.parentPath &&
(currentPath.parentPath.node.loc?.end?.line ?? 0) < nextLine
) {
const parentType = currentPath.parentPath.node.type;

if (parentType === "ExpressionStatement") {
endLine = currentPath.parentPath.node.loc?.end?.line || endLine;
break;
}

if (
parentType === "CallExpression" ||
parentType === "MemberExpression"
) {
currentPath = currentPath.parentPath;
endLine = Math.max(endLine, currentPath.node.loc?.end?.line || 0);
} else {
endLine = Math.max(
endLine,
currentPath.parentPath.node.loc?.end?.line || endLine,
);
break;
}
}
endLine = Math.min(endLine, nextLine - 1);

const testLocation = TestLocationSchema.parse({
testName: name,
startLine: currentLine,
endLine,
});
testLocations.push(testLocation);
}

log.trace("Test locations", { filePath, testLocations });

return TestLocationsSchema.parse(testLocations);
} finally {
log.resetGroup();
}
};
Loading

0 comments on commit 6218f66

Please sign in to comment.