/*
This file is part of GNU Taler
(C) 2021 Taler Systems S.A.
GNU Taler is free software; you can redistribute it and/or modify it under the
terms of the GNU General Public License as published by the Free Software
Foundation; either version 3, or (at your option) any later version.
GNU Taler is distributed in the hope that it will be useful, but WITHOUT ANY
WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
A PARTICULAR PURPOSE. See the GNU General Public License for more details.
You should have received a copy of the GNU General Public License along with
GNU Taler; see the file COPYING. If not, see
*/
import {
CancellationToken,
Logger,
minimatch,
setGlobalLogLevelFromString,
} from "@gnu-taler/taler-util";
import * as child_process from "child_process";
import * as fs from "fs";
import * as os from "os";
import * as path from "path";
import url from "url";
import {
GlobalTestState,
runCommand,
runTestWithState,
shouldLingerInTest,
TestRunResult,
} from "../harness/harness.js";
import { spawnSync } from "child_process";
import { runAgeRestrictionsMerchantTest } from "./test-age-restrictions-merchant.js";
import { runBankApiTest } from "./test-bank-api.js";
import { runClaimLoopTest } from "./test-claim-loop.js";
import { runClauseSchnorrTest } from "./test-clause-schnorr.js";
import { runDenomUnofferedTest } from "./test-denom-unoffered.js";
import { runDepositTest } from "./test-deposit.js";
import { runExchangeManagementTest } from "./test-exchange-management.js";
import { runExchangeTimetravelTest } from "./test-exchange-timetravel.js";
import { runFeeRegressionTest } from "./test-fee-regression.js";
import { runForcedSelectionTest } from "./test-forced-selection.js";
import { runLibeufinApiBankaccountTest } from "./test-libeufin-api-bankaccount.js";
import { runLibeufinApiBankconnectionTest } from "./test-libeufin-api-bankconnection.js";
import { runLibeufinApiFacadeTest } from "./test-libeufin-api-facade.js";
import { runLibeufinApiFacadeBadRequestTest } from "./test-libeufin-api-facade-bad-request.js";
import { runLibeufinApiPermissionsTest } from "./test-libeufin-api-permissions.js";
import { runLibeufinApiSandboxCamtTest } from "./test-libeufin-api-sandbox-camt.js";
import { runLibeufinApiSandboxTransactionsTest } from "./test-libeufin-api-sandbox-transactions.js";
import { runLibeufinApiSchedulingTest } from "./test-libeufin-api-scheduling.js";
import { runLibeufinApiUsersTest } from "./test-libeufin-api-users.js";
import { runLibeufinBadGatewayTest } from "./test-libeufin-bad-gateway.js";
import { runLibeufinBasicTest } from "./test-libeufin-basic.js";
import { runLibeufinC5xTest } from "./test-libeufin-c5x.js";
import { runLibeufinAnastasisFacadeTest } from "./test-libeufin-facade-anastasis.js";
import { runLibeufinKeyrotationTest } from "./test-libeufin-keyrotation.js";
import { runLibeufinNexusBalanceTest } from "./test-libeufin-nexus-balance.js";
import { runLibeufinRefundTest } from "./test-libeufin-refund.js";
import { runLibeufinRefundMultipleUsersTest } from "./test-libeufin-refund-multiple-users.js";
import { runLibeufinSandboxWireTransferCliTest } from "./test-libeufin-sandbox-wire-transfer-cli.js";
import { runLibeufinTutorialTest } from "./test-libeufin-tutorial.js";
import { runMerchantExchangeConfusionTest } from "./test-merchant-exchange-confusion.js";
import { runMerchantInstancesTest } from "./test-merchant-instances.js";
import { runMerchantInstancesDeleteTest } from "./test-merchant-instances-delete.js";
import { runMerchantInstancesUrlsTest } from "./test-merchant-instances-urls.js";
import { runMerchantLongpollingTest } from "./test-merchant-longpolling.js";
import { runMerchantRefundApiTest } from "./test-merchant-refund-api.js";
import { runMerchantSpecPublicOrdersTest } from "./test-merchant-spec-public-orders.js";
import { runPayPaidTest } from "./test-pay-paid.js";
import { runPaymentTest } from "./test-payment.js";
import { runPaymentClaimTest } from "./test-payment-claim.js";
import { runPaymentFaultTest } from "./test-payment-fault.js";
import { runPaymentForgettableTest } from "./test-payment-forgettable.js";
import { runPaymentIdempotencyTest } from "./test-payment-idempotency.js";
import { runPaymentMultipleTest } from "./test-payment-multiple.js";
import { runPaymentTransientTest } from "./test-payment-transient.js";
import { runPaymentZeroTest } from "./test-payment-zero.js";
import { runPaywallFlowTest } from "./test-paywall-flow.js";
import { runPeerToPeerPullTest } from "./test-peer-to-peer-pull.js";
import { runPeerToPeerPushTest } from "./test-peer-to-peer-push.js";
import { runRefundTest } from "./test-refund.js";
import { runRefundAutoTest } from "./test-refund-auto.js";
import { runRefundGoneTest } from "./test-refund-gone.js";
import { runRefundIncrementalTest } from "./test-refund-incremental.js";
import { runRevocationTest } from "./test-revocation.js";
import { runTimetravelAutorefreshTest } from "./test-timetravel-autorefresh.js";
import { runTimetravelWithdrawTest } from "./test-timetravel-withdraw.js";
import { runTippingTest } from "./test-tipping.js";
import { runWalletBackupBasicTest } from "./test-wallet-backup-basic.js";
import { runWalletBackupDoublespendTest } from "./test-wallet-backup-doublespend.js";
import { runWalletDblessTest } from "./test-wallet-dbless.js";
import { runWallettestingTest } from "./test-wallettesting.js";
import { runWithdrawalAbortBankTest } from "./test-withdrawal-abort-bank.js";
import { runWithdrawalBankIntegratedTest } from "./test-withdrawal-bank-integrated.js";
import { runWithdrawalFakebankTest } from "./test-withdrawal-fakebank.js";
import { runWithdrawalManualTest } from "./test-withdrawal-manual.js";
import { runAgeRestrictionsPeerTest } from "./test-age-restrictions-peer.js";
import { runWalletNotificationsTest } from "./test-wallet-notifications.js";
import { runAgeRestrictionsMixedMerchantTest } from "./test-age-restrictions-mixed-merchant.js";
import { runWalletCryptoWorkerTest } from "./test-wallet-cryptoworker.js";
import { runWithdrawalHugeTest } from "./test-withdrawal-huge.js";
import { runKycTest } from "./test-kyc.js";
import { runPaymentAbortTest } from "./test-payment-abort.js";
import { runWithdrawalFeesTest } from "./test-withdrawal-fees.js";
import { runWalletBalanceTest } from "./test-wallet-balance.js";
import { runPaymentTemplateTest } from "./test-payment-template.js";
import { runExchangeDepositTest } from "./test-exchange-deposit.js";
import { runPeerRepairTest } from "./test-peer-repair.js";
import { runPaymentShareTest } from "./test-payment-share.js";
import { runSimplePaymentTest } from "./test-simple-payment.js";
import { runTermOfServiceFormatTest } from "./test-tos-format.js";
import { runExchangePurseTest } from "./test-exchange-purse.js";
import { getSharedTestDir } from "../harness/helpers.js";
import { runStoredBackupsTest } from "./test-stored-backups.js";
import { runPaymentExpiredTest } from "./test-payment-expired.js";
/**
* Test runner.
*/
const logger = new Logger("testrunner.ts");
/**
* Spec for one test.
*/
interface TestMainFunction {
(t: GlobalTestState): Promise;
timeoutMs?: number;
excludeByDefault?: boolean;
experimental?: boolean;
suites?: string[];
}
const allTests: TestMainFunction[] = [
runAgeRestrictionsMerchantTest,
runAgeRestrictionsMixedMerchantTest,
runAgeRestrictionsPeerTest,
runBankApiTest,
runClaimLoopTest,
runClauseSchnorrTest,
runDenomUnofferedTest,
runDepositTest,
runSimplePaymentTest,
runExchangeManagementTest,
runExchangeTimetravelTest,
runFeeRegressionTest,
runForcedSelectionTest,
runKycTest,
runExchangePurseTest,
runExchangeDepositTest,
runLibeufinAnastasisFacadeTest,
runLibeufinApiBankaccountTest,
runLibeufinApiBankconnectionTest,
runLibeufinApiFacadeBadRequestTest,
runLibeufinApiFacadeTest,
runLibeufinApiPermissionsTest,
runLibeufinApiSandboxCamtTest,
runLibeufinApiSandboxTransactionsTest,
runLibeufinApiSchedulingTest,
runLibeufinApiUsersTest,
runLibeufinBadGatewayTest,
runLibeufinBasicTest,
runLibeufinC5xTest,
runLibeufinKeyrotationTest,
runLibeufinNexusBalanceTest,
runLibeufinRefundMultipleUsersTest,
runLibeufinRefundTest,
runLibeufinSandboxWireTransferCliTest,
runLibeufinTutorialTest,
runMerchantExchangeConfusionTest,
runMerchantInstancesDeleteTest,
runMerchantInstancesTest,
runMerchantInstancesUrlsTest,
runMerchantLongpollingTest,
runMerchantRefundApiTest,
runMerchantSpecPublicOrdersTest,
runPaymentClaimTest,
runPaymentFaultTest,
runPaymentForgettableTest,
runPaymentIdempotencyTest,
runPaymentMultipleTest,
runPaymentTest,
runPaymentShareTest,
runPaymentTemplateTest,
runPaymentAbortTest,
runPaymentTransientTest,
runPaymentZeroTest,
runPayPaidTest,
runPeerRepairTest,
runWalletBalanceTest,
runPaywallFlowTest,
runPeerToPeerPullTest,
runPeerToPeerPushTest,
runRefundAutoTest,
runRefundGoneTest,
runRefundIncrementalTest,
runRefundTest,
runRevocationTest,
runWithdrawalManualTest,
runTimetravelAutorefreshTest,
runTimetravelWithdrawTest,
runTippingTest,
runWalletBackupBasicTest,
runWalletBackupDoublespendTest,
runWalletNotificationsTest,
runWalletCryptoWorkerTest,
runWalletDblessTest,
runWallettestingTest,
runWithdrawalAbortBankTest,
runWithdrawalBankIntegratedTest,
runWithdrawalFakebankTest,
runWithdrawalFeesTest,
runWithdrawalHugeTest,
runTermOfServiceFormatTest,
runStoredBackupsTest,
runPaymentExpiredTest,
];
export interface TestRunSpec {
includePattern?: string;
suiteSpec?: string;
dryRun?: boolean;
failFast?: boolean;
waitOnFail?: boolean;
includeExperimental: boolean;
noTimeout: boolean;
verbosity: number;
}
export interface TestInfo {
name: string;
suites: string[];
excludeByDefault: boolean;
experimental: boolean;
}
function updateCurrentSymlink(testDir: string): void {
const currLink = path.join(
os.tmpdir(),
`taler-integrationtests-${os.userInfo().username}-current`,
);
try {
fs.unlinkSync(currLink);
} catch (e) {
// Ignore
}
try {
fs.symlinkSync(testDir, currLink);
} catch (e) {
console.log(e);
// Ignore
}
}
export function getTestName(tf: TestMainFunction): string {
const res = tf.name.match(/run([a-zA-Z0-9]*)Test/);
if (!res) {
throw Error("invalid test name, must be 'run${NAME}Test'");
}
return res[1]
.replace(/[a-z0-9][A-Z]/g, (x) => {
return x[0] + "-" + x[1];
})
.toLowerCase();
}
interface RunTestChildInstruction {
testName: string;
testRootDir: string;
}
function purgeSharedTestEnvironment() {
const rmRes = spawnSync("rm", ["-rf", `${getSharedTestDir()}`]);
if (rmRes.status != 0) {
logger.warn("can't delete shared test directory");
}
const psqlRes = spawnSync("psql", ["-Aqtl"], {
encoding: "utf-8",
});
if (psqlRes.status != 0) {
logger.warn("could not list available postgres databases");
return;
}
if (psqlRes.output[1]!!.indexOf("taler-integrationtest-shared") >= 0) {
const dropRes = spawnSync("dropdb", ["taler-integrationtest-shared"], {
encoding: "utf-8",
});
if (dropRes.status != 0) {
logger.warn("could not drop taler-integrationtest-shared database");
return;
}
}
}
export async function runTests(spec: TestRunSpec) {
if (!process.env.TALER_HARNESS_KEEP) {
logger.info("purging shared test environment");
purgeSharedTestEnvironment();
} else {
logger.info("keeping shared test environment");
}
const testRootDir = fs.mkdtempSync(
path.join(os.tmpdir(), "taler-integrationtests-"),
);
updateCurrentSymlink(testRootDir);
console.log(`testsuite root directory: ${testRootDir}`);
const testResults: TestRunResult[] = [];
let currentChild: child_process.ChildProcess | undefined;
const handleSignal = (s: NodeJS.Signals) => {
console.log(`received signal ${s} in test parent`);
if (currentChild) {
currentChild.kill("SIGTERM");
}
reportAndQuit(testRootDir, testResults, true);
};
process.on("SIGINT", (s) => handleSignal(s));
process.on("SIGTERM", (s) => handleSignal(s));
//process.on("unhandledRejection", handleSignal);
//process.on("uncaughtException", handleSignal);
let suites: Set | undefined;
if (spec.suiteSpec) {
suites = new Set(spec.suiteSpec.split(",").map((x) => x.trim()));
}
for (const [n, testCase] of allTests.entries()) {
const testName = getTestName(testCase);
if (spec.includePattern && !minimatch(testName, spec.includePattern)) {
continue;
}
if (suites) {
const ts = new Set(testCase.suites ?? []);
const intersection = new Set([...suites].filter((x) => ts.has(x)));
if (intersection.size === 0) {
continue;
}
} else {
if (testCase.excludeByDefault) {
continue;
}
if (testCase.experimental && !spec.includeExperimental) {
continue;
}
}
if (spec.dryRun) {
console.log(`dry run: would run test ${testName}`);
continue;
}
const testInstr: RunTestChildInstruction = {
testName,
testRootDir,
};
const myFilename = url.fileURLToPath(import.meta.url);
currentChild = child_process.fork(myFilename, ["__TWCLI_TESTWORKER"], {
env: {
TWCLI_RUN_TEST_INSTRUCTION: JSON.stringify(testInstr),
...process.env,
},
stdio: ["pipe", "pipe", "pipe", "ipc"],
});
const testDir = path.join(testRootDir, testName);
fs.mkdirSync(testDir, { recursive: true });
const harnessLogFilename = path.join(testRootDir, testName, "harness.log");
const harnessLogStream = fs.createWriteStream(harnessLogFilename);
if (spec.verbosity > 0) {
currentChild.stderr?.pipe(process.stderr);
currentChild.stdout?.pipe(process.stdout);
}
currentChild.stdout?.pipe(harnessLogStream);
currentChild.stderr?.pipe(harnessLogStream);
// Default timeout when the test doesn't override it.
let defaultTimeout = 60000;
const overrideDefaultTimeout = process.env.TALER_TEST_TIMEOUT;
if (overrideDefaultTimeout) {
defaultTimeout = Number.parseInt(overrideDefaultTimeout, 10) * 1000;
}
// Set the timeout to at least be the default timeout.
const testTimeoutMs = testCase.timeoutMs
? Math.max(testCase.timeoutMs, defaultTimeout)
: defaultTimeout;
if (spec.noTimeout) {
console.log(`running ${testName}, no timeout`);
} else {
console.log(`running ${testName} with timeout ${testTimeoutMs}ms`);
}
const token = spec.noTimeout
? CancellationToken.CONTINUE
: CancellationToken.timeout(testTimeoutMs).token;
const resultPromise: Promise = new Promise(
(resolve, reject) => {
let msg: TestRunResult | undefined;
currentChild!.on("message", (m) => {
if (token.isCancelled) {
return;
}
msg = m as TestRunResult;
});
currentChild!.on("exit", (code, signal) => {
if (token.isCancelled) {
return;
}
logger.info(`process exited code=${code} signal=${signal}`);
if (signal) {
reject(new Error(`test worker exited with signal ${signal}`));
} else if (code != 0) {
reject(new Error(`test worker exited with code ${code}`));
} else if (!msg) {
reject(
new Error(
`test worker exited without giving back the test results`,
),
);
} else {
resolve(msg);
}
});
currentChild!.on("error", (err) => {
if (token.isCancelled) {
return;
}
reject(err);
});
},
);
let result: TestRunResult;
try {
result = await token.racePromise(resultPromise);
if (result.status === "fail" && spec.failFast) {
logger.error("test failed and failing fast, exit!");
throw Error("exit on fail fast");
}
} catch (e: any) {
console.error(`test ${testName} timed out`);
if (token.isCancelled) {
result = {
status: "fail",
reason: "timeout",
timeSec: testTimeoutMs / 1000,
name: testName,
};
currentChild.kill("SIGTERM");
} else {
throw Error(e);
}
}
harnessLogStream.close();
console.log(`parent: got result ${JSON.stringify(result)}`);
testResults.push(result);
}
reportAndQuit(testRootDir, testResults);
}
export function reportAndQuit(
testRootDir: string,
testResults: TestRunResult[],
interrupted: boolean = false,
): never {
let numTotal = 0;
let numFail = 0;
let numSkip = 0;
let numPass = 0;
for (const result of testResults) {
numTotal++;
if (result.status === "fail") {
numFail++;
} else if (result.status === "skip") {
numSkip++;
} else if (result.status === "pass") {
numPass++;
}
}
const resultsFile = path.join(testRootDir, "results.json");
fs.writeFileSync(
path.join(testRootDir, "results.json"),
JSON.stringify({ testResults, interrupted }, undefined, 2),
);
if (interrupted) {
console.log("test suite was interrupted");
}
console.log(`See ${resultsFile} for details`);
console.log(`Skipped: ${numSkip}/${numTotal}`);
console.log(`Failed: ${numFail}/${numTotal}`);
console.log(`Passed: ${numPass}/${numTotal}`);
if (interrupted) {
process.exit(3);
} else if (numPass < numTotal - numSkip) {
process.exit(1);
} else {
process.exit(0);
}
}
export function getTestInfo(): TestInfo[] {
return allTests.map((x) => ({
name: getTestName(x),
suites: x.suites ?? [],
excludeByDefault: x.excludeByDefault ?? false,
experimental: x.experimental ?? false,
}));
}
const runTestInstrStr = process.env["TWCLI_RUN_TEST_INSTRUCTION"];
if (runTestInstrStr && process.argv.includes("__TWCLI_TESTWORKER")) {
// Test will call taler-wallet-cli, so we must not propagate this variable.
delete process.env["TWCLI_RUN_TEST_INSTRUCTION"];
const { testRootDir, testName } = JSON.parse(
runTestInstrStr,
) as RunTestChildInstruction;
process.on("disconnect", () => {
logger.trace("got disconnect from parent");
process.exit(3);
});
const runTest = async () => {
let testMain: TestMainFunction | undefined;
for (const t of allTests) {
if (getTestName(t) === testName) {
testMain = t;
break;
}
}
if (!process.send) {
logger.error("can't communicate with parent");
process.exit(2);
}
if (!testMain) {
logger.info(`test ${testName} not found`);
process.exit(2);
}
const testDir = path.join(testRootDir, testName);
logger.info(`running test ${testName}`);
const gc = new GlobalTestState({
testDir,
});
const testResult = await runTestWithState(gc, testMain, testName);
logger.info(`done test ${testName}: ${testResult.status}`);
process.send(testResult);
};
runTest()
.then(() => {
logger.trace(`test ${testName} finished in worker`);
if (shouldLingerInTest()) {
logger.trace("lingering ...");
return;
}
process.exit(0);
})
.catch((e) => {
logger.error(e);
process.exit(1);
});
}