Source code
Revision control
Copy as Markdown
Other Tools
#! /usr/bin/env -S node
/**
* @license
* Copyright 2022 Google Inc.
* SPDX-License-Identifier: Apache-2.0
*/
import {spawn} from 'node:child_process';
import {randomUUID} from 'node:crypto';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import {globSync} from 'glob';
import yargs from 'yargs';
import {hideBin} from 'yargs/helpers';
import {
zPlatform,
zTestSuiteFile,
type MochaResults,
type TestExpectation,
type TestSuite,
type TestSuiteFile,
} from './types.js';
import {
extendProcessEnv,
filterByParameters,
filterByPlatform,
getExpectationUpdates,
getSuggestionsForAction,
printSuggestions,
readJSON,
writeJSON,
type RecommendedExpectation,
} from './utils.js';
const {
_: mochaArgs,
testSuite: testSuiteId,
saveStatsTo,
cdpTests: includeCdpTests,
suggestions: provideSuggestions,
coverage: useCoverage,
minTests,
shard,
reporter,
printMemory,
ignoreUnexpectedlyPassing,
} = yargs(hideBin(process.argv))
.parserConfiguration({'unknown-options-as-args': true})
.scriptName('@puppeteer/mocha-runner')
.option('coverage', {
boolean: true,
default: false,
})
.option('suggestions', {
boolean: true,
default: true,
})
.option('cdp-tests', {
boolean: true,
default: true,
})
.option('save-stats-to', {
string: true,
requiresArg: true,
})
.option('min-tests', {
number: true,
default: 0,
requiresArg: true,
})
.option('test-suite', {
string: true,
requiresArg: true,
})
.option('shard', {
string: true,
requiresArg: true,
})
.option('reporter', {
string: true,
requiresArg: true,
})
.option('print-memory', {
boolean: true,
default: false,
})
.option('ignore-unexpectedly-passing', {
boolean: true,
default: false,
})
.parseSync();
function getApplicableTestSuites(parsedSuitesFile: TestSuiteFile): TestSuite[] {
let applicableSuites: TestSuite[] = [];
if (!testSuiteId) {
applicableSuites = parsedSuitesFile.testSuites;
} else {
const testSuite = parsedSuitesFile.testSuites.find(suite => {
return suite.id === testSuiteId;
});
if (!testSuite) {
console.error(`Test suite ${testSuiteId} is not defined`);
process.exit(1);
}
applicableSuites = [testSuite];
}
return applicableSuites;
}
async function main() {
let statsPath = saveStatsTo;
if (statsPath && statsPath.includes('INSERTID')) {
statsPath = statsPath.replace(/INSERTID/gi, randomUUID());
}
const platform = zPlatform.parse(os.platform());
const expectations = readJSON(
path.join(process.cwd(), 'test', 'TestExpectations.json'),
) as TestExpectation[];
const parsedSuitesFile = zTestSuiteFile.parse(
readJSON(path.join(process.cwd(), 'test', 'TestSuites.json')),
);
const applicableSuites = getApplicableTestSuites(parsedSuitesFile);
console.log('Planning to run the following test suites', applicableSuites);
if (statsPath) {
console.log('Test stats will be saved to', statsPath);
}
let fail = false;
const recommendations: RecommendedExpectation[] = [];
try {
for (const suite of applicableSuites) {
const parameters = suite.parameters;
const applicableExpectations = filterByParameters(
filterByPlatform(expectations, platform),
parameters,
).reverse();
// Add more logging when the GitHub Action Debugging option is set
const githubActionDebugging = process.env['RUNNER_DEBUG']
? {
DEBUG: 'puppeteer:*',
EXTRA_LAUNCH_OPTIONS: JSON.stringify({
dumpio: true,
extraPrefsFirefox: {
'remote.log.level': 'Trace',
},
}),
}
: {};
const env = extendProcessEnv([
...parameters.map(param => {
return parsedSuitesFile.parameterDefinitions[param];
}),
{
PUPPETEER_SKIPPED_TEST_CONFIG: JSON.stringify(
applicableExpectations.map(ex => {
return {
testIdPattern: ex.testIdPattern,
skip: ex.expectations.includes('SKIP'),
};
}),
),
},
githubActionDebugging,
]);
const tmpDir = fs.mkdtempSync(
path.join(os.tmpdir(), 'puppeteer-test-runner-'),
);
const tmpFilename = statsPath
? statsPath
: path.join(tmpDir, 'output.json');
console.log('Running', JSON.stringify(parameters), tmpFilename);
const args = [
'-u',
path.join(__dirname, 'interface.js'),
'-R',
!reporter ? path.join(__dirname, 'reporter.js') : reporter,
'-O',
`output=${tmpFilename}`,
'-n',
'trace-warnings',
];
if (printMemory) {
args.push('-n', 'expose-gc');
}
const specPattern = 'test/build/**/*.spec.js';
const specs = globSync(specPattern, {
ignore: !includeCdpTests ? 'test/build/cdp/**/*.spec.js' : undefined,
}).sort((a, b) => {
return a.localeCompare(b);
});
if (shard) {
// Shard ID is 1-based.
const [shardId, shards] = shard.split('-').map(s => {
return Number(s);
}) as [number, number];
const argsLength = args.length;
for (let i = 0; i < specs.length; i++) {
if (i % shards === shardId - 1) {
args.push(specs[i]!);
}
}
if (argsLength === args.length) {
throw new Error('Shard did not result in any test files');
}
console.log(
`Running shard ${shardId}-${shards}. Picked ${
args.length - argsLength
} files out of ${specs.length}.`,
);
} else {
args.push(...specs);
}
const handle = spawn(
'npx',
[
...(useCoverage
? ['c8', '--check-coverage', '--lines', '90', 'npx']
: []),
'mocha',
...mochaArgs.map(String),
...args,
],
{
shell: true,
cwd: process.cwd(),
stdio: 'inherit',
env,
},
);
await new Promise<void>((resolve, reject) => {
handle.on('error', err => {
reject(err);
});
handle.on('close', () => {
resolve();
});
});
try {
const results = (() => {
try {
return readJSON(tmpFilename) as MochaResults;
} catch (cause) {
throw new Error('Test results are not found', {
cause,
});
}
})();
console.log('Finished', JSON.stringify(parameters));
const updates = getExpectationUpdates(
results,
applicableExpectations,
{
platforms: [os.platform()],
parameters,
},
ignoreUnexpectedlyPassing,
);
const totalTests = results.stats.tests;
results.parameters = parameters;
results.platform = platform;
results.date = new Date().toISOString();
if (updates.length > 0) {
fail = true;
recommendations.push(...updates);
results.updates = updates;
writeJSON(tmpFilename, results);
} else {
if (!shard && totalTests < minTests) {
fail = true;
console.log(
`Test run matches expectations but the number of discovered tests is too low (expected: ${minTests}, actual: ${totalTests}).`,
);
writeJSON(tmpFilename, results);
continue;
}
console.log('Test run matches expectations');
writeJSON(tmpFilename, results);
continue;
}
} catch (err) {
fail = true;
console.error(err);
}
}
} catch (err) {
fail = true;
console.error(err);
} finally {
const added = getSuggestionsForAction(recommendations, 'add');
const removed = getSuggestionsForAction(recommendations, 'remove');
const updated = getSuggestionsForAction(recommendations, 'update');
if (!!provideSuggestions) {
printSuggestions(
added,
'Add the following to TestExpectations.json to ignore the error:',
true,
);
printSuggestions(
removed,
'Remove the following from the TestExpectations.json to ignore the error:',
);
printSuggestions(
updated,
'Update the following expectations in the TestExpectations.json to ignore the error:',
true,
);
}
const unexpected = added.length + removed.length + updated.length;
console.log(
fail && Boolean(unexpected)
? `Run failed: ${unexpected} unexpected result(s).`
: `Run succeeded.`,
);
process.exit(fail ? 1 : 0);
}
}
main().catch(error => {
console.error(error);
process.exit(1);
});