Skip to content

Commit

Permalink
Remove docker and user tests, related infra (microsoft#53118)
Browse files Browse the repository at this point in the history
  • Loading branch information
@jakebailey
jakebailey authored Mar 7, 2023
1 parent 967911c commit 6a39f91
Show file tree
Hide file tree
Showing 247 changed files with 4 additions and 27,605 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/accept-baselines-fix-lints.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ jobs:
git config user.email "[email protected]"
git config user.name "TypeScript Bot"
npm install
git rm -r --quiet tests/baselines/reference :^tests/baselines/reference/docker :^tests/baselines/reference/user
git rm -r --quiet tests/baselines/reference
npx hereby runtests-parallel --ci --fix || true
npx hereby baseline-accept
git add ./src
Expand Down
23 changes: 1 addition & 22 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,6 @@ tests/webTestServer.js.map
tests/webhost/*.d.ts
tests/webhost/webtsc.js
tests/cases/**/*.js
!tests/cases/docker/*.js/
tests/cases/**/*.js.map
*.config
scripts/eslint/built/
Expand All @@ -59,29 +58,9 @@ internal/
yarn.lock
yarn-error.log
.parallelperf.*
tests/cases/user/*/package-lock.json
tests/cases/user/*/node_modules/
tests/cases/user/*/**/*.js
tests/cases/user/*/**/*.js.map
tests/cases/user/*/**/*.d.ts
!tests/cases/user/zone.js/
!tests/cases/user/bignumber.js/
!tests/cases/user/discord.js/
tests/baselines/reference/dt
.failed-tests
TEST-results.xml
package-lock.json
tests/cases/user/npm/npm
tests/cases/user/TypeScript-React-Starter/TypeScript-React-Starter
tests/cases/user/TypeScript-Node-Starter/TypeScript-Node-Starter
tests/cases/user/TypeScript-React-Native-Starter/TypeScript-React-Native-Starter
tests/cases/user/TypeScript-Vue-Starter/TypeScript-Vue-Starter
tests/cases/user/TypeScript-WeChat-Starter/TypeScript-WeChat-Starter
tests/cases/user/create-react-app/create-react-app
tests/cases/user/fp-ts/fp-ts
tests/cases/user/webpack/webpack
tests/cases/user/puppeteer/puppeteer
tests/cases/user/axios-src/axios-src
tests/cases/user/prettier/prettier
.eslintcache
*v8.log
*v8.log
5 changes: 1 addition & 4 deletions CONTRIBUTING.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,10 +81,7 @@ hereby tests # Build the test infrastructure using the built compile
hereby runtests # Run tests using the built compiler and test infrastructure.
# You can override the specific suite runner used or specify a test for this command.
# Use --tests=<testPath> for a specific test and/or --runner=<runnerName> for a specific suite.
# Valid runners include conformance, compiler, fourslash, project, user, and docker
# The user and docker runners are extended test suite runners - the user runner
# works on disk in the tests/cases/user directory, while the docker runner works in containers.
# You'll need to have the docker executable in your system path for the docker runner to work.
# Valid runners include conformance, compiler, fourslash, and project
hereby runtests-parallel # Like runtests, but split across multiple threads. Uses a number of threads equal to the system
# core count by default. Use --workers=<number> to adjust this.
hereby baseline-accept # This replaces the baseline test results with the results obtained from hereby runtests.
Expand Down
2 changes: 1 addition & 1 deletion src/harness/runnerbase.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import {
} from "./_namespaces/Harness";
import * as ts from "./_namespaces/ts";

export type TestRunnerKind = CompilerTestKind | FourslashTestKind | "project" | "rwc" | "test262" | "user" | "dt" | "docker";
export type TestRunnerKind = CompilerTestKind | FourslashTestKind | "project" | "rwc" | "test262" | "dt";
export type CompilerTestKind = "conformance" | "compiler";
export type FourslashTestKind = "fourslash" | "fourslash-shims" | "fourslash-shims-pp" | "fourslash-server";

Expand Down
202 changes: 0 additions & 202 deletions src/testRunner/externalCompileRunner.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,180 +113,6 @@ abstract class ExternalCompileRunnerBase extends RunnerBase {
}
}

export class UserCodeRunner extends ExternalCompileRunnerBase {
readonly testDir = "tests/cases/user/";
kind(): TestRunnerKind {
return "user";
}
report(result: ExecResult) {
// eslint-disable-next-line no-null/no-null
return result.status === 0 && !result.stdout.length && !result.stderr.length ? null : `Exit Code: ${result.status}
Standard output:
${sortErrors(stripAbsoluteImportPaths(result.stdout.toString().replace(/\r\n/g, "\n")))}
Standard error:
${stripAbsoluteImportPaths(result.stderr.toString().replace(/\r\n/g, "\n"))}`;
}
}

export class DockerfileRunner extends ExternalCompileRunnerBase {
readonly testDir = "tests/cases/docker/";
kind(): TestRunnerKind {
return "docker";
}
override initializeTests(): void {
// Read in and evaluate the test list
const testList = this.tests && this.tests.length ? this.tests : this.getTestFiles();

// eslint-disable-next-line @typescript-eslint/no-this-alias
const cls = this;
describe(`${this.kind()} code samples`, function (this: Mocha.Suite) {
this.timeout(cls.timeout); // 20 minutes
before(() => {
// cached because workspace is hashed to determine cacheability
cls.exec("docker", ["build", ".", "-t", "typescript/typescript", "-f", cls.testDir + "Dockerfile"], {
cwd: IO.getWorkspaceRoot(),
env: { ...process.env, DOCKER_BUILDKIT: "1" }, // We need buildkit to allow Dockerfile.dockerignore to work.
});
});
for (const test of testList) {
const directory = typeof test === "string" ? test : test.file;
const cwd = path.join(IO.getWorkspaceRoot(), cls.testDir, directory);
it(`should build ${directory} successfully`, () => {
const imageName = `tstest/${directory}`;
cls.exec("docker", ["build", "--no-cache", ".", "-t", imageName], { cwd }); // --no-cache so the latest version of the repos referenced is always fetched
const cp: typeof import("child_process") = require("child_process");
Baseline.runBaseline(`${cls.kind()}/${directory}.log`, cls.report(cp.spawnSync(`docker`, ["run", imageName], { cwd, timeout: cls.timeout, shell: true })));
});
}
});
}

private timeout = 1_200_000; // 20 minutes;
private exec(command: string, args: string[], options: { cwd: string; env?: NodeJS.ProcessEnv }): void {
const cp: typeof import("child_process") = require("child_process");
const stdio = isWorker ? "pipe" : "inherit";
const res = cp.spawnSync(isWorker ? `${command} 2>&1` : command, args, { timeout: this.timeout, shell: true, stdio, ...options });
if (res.status !== 0) {
throw new Error(`${command} ${args.join(" ")} for ${options.cwd} failed: ${res.stdout && res.stdout.toString()}`);
}
}
report(result: ExecResult) {
// eslint-disable-next-line no-null/no-null
return result.status === 0 && !result.stdout.length && !result.stderr.length ? null : `Exit Code: ${result.status}
Standard output:
${sanitizeDockerfileOutput(result.stdout.toString())}
Standard error:
${sanitizeDockerfileOutput(result.stderr.toString())}`;
}
}

function sanitizeDockerfileOutput(result: string): string {
return [
normalizeNewlines,
stripANSIEscapes,
stripRushStageNumbers,
stripWebpackHash,
sanitizeVersionSpecifiers,
sanitizeTimestamps,
sanitizeSizes,
sanitizeUnimportantGulpOutput,
stripAbsoluteImportPaths,
].reduce((result, f) => f(result), result);
}

function normalizeNewlines(result: string): string {
return result.replace(/\r\n/g, "\n");
}

function stripANSIEscapes(result: string): string {
return result.replace(/\x1b\[[0-9;]*[a-zA-Z]/g, "");
}

function stripRushStageNumbers(result: string): string {
return result.replace(/\d+ of \d+:/g, "XX of XX:");
}

function stripWebpackHash(result: string): string {
return result.replace(/Hash: \w+/g, "Hash: [redacted]");
}

function sanitizeSizes(result: string): string {
return result.replace(/\d+(\.\d+)? ((Ki|M)B|bytes)/g, "X KiB");
}

/**
* Gulp's output order within a `parallel` block is nondeterministic (and there's no way to configure it to execute in series),
* so we purge as much of the gulp output as we can
*/
function sanitizeUnimportantGulpOutput(result: string): string {
return result.replace(/^.*(\] (Starting)|(Finished)).*$/gm, "") // "gulp" task start/end messages (nondeterministic order)
.replace(/^.*(\] . (finished)|(started)).*$/gm, "") // "just" task start/end messages (nondeterministic order)
.replace(/^.*\] Respawned to PID: \d+.*$/gm, "") // PID of child is OS and system-load dependent (likely stableish in a container but still dangerous)
.replace(/\n+/g, "\n")
.replace(/\/tmp\/yarn--.*?\/node/g, "");
}

function sanitizeTimestamps(result: string): string {
return result.replace(/\[\d?\d:\d\d:\d\d (A|P)M\]/g, "[XX:XX:XX XM]")
.replace(/\[\d?\d:\d\d:\d\d\]/g, "[XX:XX:XX]")
.replace(/\/\d+-\d+-[\d_TZ]+-debug.log/g, "\/XXXX-XX-XXXXXXXXX-debug.log")
.replace(/\d+\/\d+\/\d+ \d+:\d+:\d+ (AM|PM)/g, "XX/XX/XX XX:XX:XX XM")
.replace(/\d+(\.\d+)? sec(onds?)?/g, "? seconds")
.replace(/\d+(\.\d+)? min(utes?)?/g, "")
.replace(/\d+(\.\d+)? ?m?s/g, "?s")
.replace(/ \(\?s\)/g, "");
}

function sanitizeVersionSpecifiers(result: string): string {
return result
.replace(/\d+.\d+.\d+-insiders.\d\d\d\d\d\d\d\d/g, "X.X.X-insiders.xxxxxxxx")
.replace(/Rush Multi-Project Build Tool (\d+)\.\d+\.\d+/g, "Rush Multi-Project Build Tool $1.X.X")
.replace(/([@v\()])\d+\.\d+\.\d+/g, "$1X.X.X")
.replace(/webpack (\d+)\.\d+\.\d+/g, "webpack $1.X.X")
.replace(/Webpack version: (\d+)\.\d+\.\d+/g, "Webpack version: $1.X.X");
}

/**
* Import types and some other error messages use absolute paths in errors as they have no context to be written relative to;
* This is problematic for error baselines, so we grep for them and strip them out.
*/
function stripAbsoluteImportPaths(result: string) {
const workspaceRegexp = new RegExp(IO.getWorkspaceRoot().replace(/\\/g, "\\\\"), "g");
return result
.replace(/import\(".*?\/tests\/cases\/user\//g, `import("/`)
.replace(/Module '".*?\/tests\/cases\/user\//g, `Module '"/`)
.replace(workspaceRegexp, "../../..");
}

function sortErrors(result: string) {
return ts.flatten(splitBy(result.split("\n"), s => /^\S+/.test(s)).sort(compareErrorStrings)).join("\n");
}

const errorRegexp = /^(.+\.[tj]sx?)\((\d+),(\d+)\)(: error TS.*)/;
function compareErrorStrings(a: string[], b: string[]) {
ts.Debug.assertGreaterThanOrEqual(a.length, 1);
ts.Debug.assertGreaterThanOrEqual(b.length, 1);
const matchA = a[0].match(errorRegexp);
if (!matchA) {
return -1;
}
const matchB = b[0].match(errorRegexp);
if (!matchB) {
return 1;
}
const [, errorFileA, lineNumberStringA, columnNumberStringA, remainderA] = matchA;
const [, errorFileB, lineNumberStringB, columnNumberStringB, remainderB] = matchB;
return ts.comparePathsCaseSensitive(errorFileA, errorFileB) ||
ts.compareValues(parseInt(lineNumberStringA), parseInt(lineNumberStringB)) ||
ts.compareValues(parseInt(columnNumberStringA), parseInt(columnNumberStringB)) ||
ts.compareStringsCaseSensitive(remainderA, remainderB) ||
ts.compareStringsCaseSensitive(a.slice(1).join("\n"), b.slice(1).join("\n"));
}

export class DefinitelyTypedRunner extends ExternalCompileRunnerBase {
readonly testDir = "../DefinitelyTyped/types/";
override workingDirectory = this.testDir;
Expand All @@ -304,31 +130,3 @@ Standard error:
${result.stderr.toString().replace(/\r\n/g, "\n")}`;
}
}

/**
* Split an array into multiple arrays whenever `isStart` returns true.
* @example
* splitBy([1,2,3,4,5,6], isOdd)
* ==> [[1, 2], [3, 4], [5, 6]]
* where
* const isOdd = n => !!(n % 2)
*/
function splitBy<T>(xs: T[], isStart: (x: T) => boolean): T[][] {
const result = [];
let group: T[] = [];
for (const x of xs) {
if (isStart(x)) {
if (group.length) {
result.push(group);
}
group = [x];
}
else {
group.push(x);
}
}
if (group.length) {
result.push(group);
}
return result;
}
18 changes: 0 additions & 18 deletions src/testRunner/runner.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ import {
CompilerBaselineRunner,
CompilerTestType,
DefinitelyTypedRunner,
DockerfileRunner,
FourSlashRunner,
GeneratedFourslashRunner,
IO,
Expand All @@ -14,7 +13,6 @@ import {
setShards,
Test262BaselineRunner,
TestRunnerKind,
UserCodeRunner,
} from "./_namespaces/Harness";
import * as project from "./_namespaces/project";
import * as RWC from "./_namespaces/RWC";
Expand Down Expand Up @@ -80,12 +78,8 @@ export function createRunner(kind: TestRunnerKind): RunnerBase {
return new RWC.RWCRunner();
case "test262":
return new Test262BaselineRunner();
case "user":
return new UserCodeRunner();
case "dt":
return new DefinitelyTypedRunner();
case "docker":
return new DockerfileRunner();
}
return ts.Debug.fail(`Unknown runner kind ${kind}`);
}
Expand Down Expand Up @@ -223,15 +217,9 @@ function handleTestConfig() {
case "test262":
runners.push(new Test262BaselineRunner());
break;
case "user":
runners.push(new UserCodeRunner());
break;
case "dt":
runners.push(new DefinitelyTypedRunner());
break;
case "docker":
runners.push(new DockerfileRunner());
break;
}
}
}
Expand All @@ -250,12 +238,6 @@ function handleTestConfig() {
runners.push(new FourSlashRunner(FourSlash.FourSlashTestType.ShimsWithPreprocess));
runners.push(new FourSlashRunner(FourSlash.FourSlashTestType.Server));
// runners.push(new GeneratedFourslashRunner());

// CRON-only tests
if (process.env.TRAVIS_EVENT_TYPE === "cron") {
runners.push(new UserCodeRunner());
runners.push(new DockerfileRunner());
}
}
if (runUnitTests === undefined) {
runUnitTests = runners.length !== 1; // Don't run unit tests when running only one runner if unit tests were not explicitly asked for
Expand Down
Loading

0 comments on commit 6a39f91

Please sign in to comment.