path: root/packages/taler-harness/src/integrationtests/testrunner.ts

/*
 This file is part of GNU Taler
 (C) 2021 Taler Systems S.A.

 GNU Taler is free software; you can redistribute it and/or modify it under the
 terms of the GNU General Public License as published by the Free Software
 Foundation; either version 3, or (at your option) any later version.

 GNU Taler is distributed in the hope that it will be useful, but WITHOUT ANY
 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
 A PARTICULAR PURPOSE.  See the GNU General Public License for more details.

 You should have received a copy of the GNU General Public License along with
 GNU Taler; see the file COPYING.  If not, see <http://www.gnu.org/licenses/>
 */

import { CancellationToken, Logger, minimatch } from "@gnu-taler/taler-util";
import * as child_process from "child_process";
import { spawnSync } from "child_process";
import * as fs from "fs";
import * as os from "os";
import * as path from "path";
import url from "url";
import {
  GlobalTestState,
  TestRunResult,
  runTestWithState,
  shouldLingerInTest,
} from "../harness/harness.js";
import { getSharedTestDir } from "../harness/helpers.js";
import { runAccountRestrictionsTest } from "./test-account-restrictions.js";
import { runAgeRestrictionsDepositTest } from "./test-age-restrictions-deposit.js";
import { runAgeRestrictionsMerchantTest } from "./test-age-restrictions-merchant.js";
import { runAgeRestrictionsMixedMerchantTest } from "./test-age-restrictions-mixed-merchant.js";
import { runAgeRestrictionsPeerTest } from "./test-age-restrictions-peer.js";
import { runBankApiTest } from "./test-bank-api.js";
import { runClaimLoopTest } from "./test-claim-loop.js";
import { runClauseSchnorrTest } from "./test-clause-schnorr.js";
import { runCurrencyScopeTest } from "./test-currency-scope.js";
import { runDenomLostTest } from "./test-denom-lost.js";
import { runDenomUnofferedTest } from "./test-denom-unoffered.js";
import { runDepositTest } from "./test-deposit.js";
import { runExchangeDepositTest } from "./test-exchange-deposit.js";
import { runExchangeManagementFaultTest } from "./test-exchange-management-fault.js";
import { runExchangeManagementTest } from "./test-exchange-management.js";
import { runExchangeMasterPubChangeTest } from "./test-exchange-master-pub-change.js";
import { runExchangePurseTest } from "./test-exchange-purse.js";
import { runExchangeTimetravelTest } from "./test-exchange-timetravel.js";
import { runFeeRegressionTest } from "./test-fee-regression.js";
import { runForcedSelectionTest } from "./test-forced-selection.js";
import { runKycBalanceWithdrawalTest } from "./test-kyc-balance-withdrawal.js";
import { runKycDepositAggregateTest } from "./test-kyc-deposit-aggregate.js";
import { runKycDepositDepositKyctransferTest } from "./test-kyc-deposit-deposit-kyctransfer.js";
import { runKycDepositDepositTest } from "./test-kyc-deposit-deposit.js";
import { runKycExchangeWalletTest } from "./test-kyc-exchange-wallet.js";
import { runKycFormWithdrawalTest } from "./test-kyc-form-withdrawal.js";
import { runKycMerchantAggregateTest } from "./test-kyc-merchant-aggregate.js";
import { runKycMerchantDepositTest } from "./test-kyc-merchant-deposit.js";
import { runKycNewMeasureTest } from "./test-kyc-new-measure.js";
import { runKycPeerPullTest } from "./test-kyc-peer-pull.js";
import { runKycPeerPushTest } from "./test-kyc-peer-push.js";
import { runKycThresholdWithdrawalTest } from "./test-kyc-threshold-withdrawal.js";
import { runKycTest } from "./test-kyc.js";
import { runLibeufinBankTest } from "./test-libeufin-bank.js";
import { runMerchantCategoriesTest } from "./test-merchant-categories.js";
import { runMerchantExchangeConfusionTest } from "./test-merchant-exchange-confusion.js";
import { runMerchantInstancesDeleteTest } from "./test-merchant-instances-delete.js";
import { runMerchantInstancesUrlsTest } from "./test-merchant-instances-urls.js";
import { runMerchantInstancesTest } from "./test-merchant-instances.js";
import { runMerchantLongpollingTest } from "./test-merchant-longpolling.js";
import { runMerchantRefundApiTest } from "./test-merchant-refund-api.js";
import { runMerchantSpecPublicOrdersTest } from "./test-merchant-spec-public-orders.js";
import { runMultiExchangeTest } from "./test-multiexchange.js";
import { runOtpTest } from "./test-otp.js";
import { runPayPaidTest } from "./test-pay-paid.js";
import { runPaymentAbortTest } from "./test-payment-abort.js";
import { runPaymentClaimTest } from "./test-payment-claim.js";
import { runPaymentDeletedTest } from "./test-payment-deleted.js";
import { runPaymentExpiredTest } from "./test-payment-expired.js";
import { runPaymentFaultTest } from "./test-payment-fault.js";
import { runPaymentForgettableTest } from "./test-payment-forgettable.js";
import { runPaymentIdempotencyTest } from "./test-payment-idempotency.js";
import { runPaymentMultipleTest } from "./test-payment-multiple.js";
import { runPaymentShareTest } from "./test-payment-share.js";
import { runPaymentTemplateTest } from "./test-payment-template.js";
import { runPaymentTransientTest } from "./test-payment-transient.js";
import { runPaymentZeroTest } from "./test-payment-zero.js";
import { runPaymentTest } from "./test-payment.js";
import { runPaywallFlowTest } from "./test-paywall-flow.js";
import { runPeerPullLargeTest } from "./test-peer-pull-large.js";
import { runPeerPushLargeTest } from "./test-peer-push-large.js";
import { runPeerRepairTest } from "./test-peer-repair.js";
import { runPeerToPeerPullTest } from "./test-peer-to-peer-pull.js";
import { runPeerToPeerPushTest } from "./test-peer-to-peer-push.js";
import { runRefundAutoTest } from "./test-refund-auto.js";
import { runRefundGoneTest } from "./test-refund-gone.js";
import { runRefundIncrementalTest } from "./test-refund-incremental.js";
import { runRefundTest } from "./test-refund.js";
import { runRepurchaseTest } from "./test-repurchase.js";
import { runRevocationTest } from "./test-revocation.js";
import { runSimplePaymentTest } from "./test-simple-payment.js";
import { runStoredBackupsTest } from "./test-stored-backups.js";
import { runTimetravelAutorefreshTest } from "./test-timetravel-autorefresh.js";
import { runTimetravelWithdrawTest } from "./test-timetravel-withdraw.js";
import { runTermOfServiceFormatTest } from "./test-tos-format.js";
import { runWalletBackupBasicTest } from "./test-wallet-backup-basic.js";
import { runWalletBackupDoublespendTest } from "./test-wallet-backup-doublespend.js";
import { runWalletBalanceNotificationsTest } from "./test-wallet-balance-notifications.js";
import { runWalletBalanceZeroTest } from "./test-wallet-balance-zero.js";
import { runWalletBalanceTest } from "./test-wallet-balance.js";
import { runWalletBlockedDepositTest } from "./test-wallet-blocked-deposit.js";
import { runWalletBlockedPayMerchantTest } from "./test-wallet-blocked-pay-merchant.js";
import { runWalletBlockedPayPeerPullTest } from "./test-wallet-blocked-pay-peer-pull.js";
import { runWalletBlockedPayPeerPushTest } from "./test-wallet-blocked-pay-peer-push.js";
import { runWalletCliTerminationTest } from "./test-wallet-cli-termination.js";
import { runWalletConfigTest } from "./test-wallet-config.js";
import { runWalletCryptoWorkerTest } from "./test-wallet-cryptoworker.js";
import { runWalletDblessTest } from "./test-wallet-dbless.js";
import { runWalletDd48Test } from "./test-wallet-dd48.js";
import { runWalletDenomExpireTest } from "./test-wallet-denom-expire.js";
import { runWalletDevExperimentsTest } from "./test-wallet-dev-experiments.js";
import { runWalletExchangeUpdateTest } from "./test-wallet-exchange-update.js";
import { runWalletGenDbTest } from "./test-wallet-gendb.js";
import { runWalletInsufficientBalanceTest } from "./test-wallet-insufficient-balance.js";
import { runWalletNetworkAvailabilityTest } from "./test-wallet-network-availability.js";
import { runWalletNotificationsTest } from "./test-wallet-notifications.js";
import { runWalletObservabilityTest } from "./test-wallet-observability.js";
import { runWalletRefreshErrorsTest } from "./test-wallet-refresh-errors.js";
import { runWalletRefreshTest } from "./test-wallet-refresh.js";
import { runWalletWirefeesTest } from "./test-wallet-wirefees.js";
import { runWallettestingTest } from "./test-wallettesting.js";
import { runWithdrawalAbortBankTest } from "./test-withdrawal-abort-bank.js";
import { runWithdrawalAmountTest } from "./test-withdrawal-amount.js";
import { runWithdrawalBankIntegratedTest } from "./test-withdrawal-bank-integrated.js";
import { runWithdrawalConversionTest } from "./test-withdrawal-conversion.js";
import { runWithdrawalExternalTest } from "./test-withdrawal-external.js";
import { runWithdrawalFakebankTest } from "./test-withdrawal-fakebank.js";
import { runWithdrawalFeesTest } from "./test-withdrawal-fees.js";
import { runWithdrawalFlexTest } from "./test-withdrawal-flex.js";
import { runWithdrawalHandoverTest } from "./test-withdrawal-handover.js";
import { runWithdrawalHugeTest } from "./test-withdrawal-huge.js";
import { runWithdrawalIdempotentTest } from "./test-withdrawal-idempotent.js";
import { runWithdrawalManualTest } from "./test-withdrawal-manual.js";
import { runWithdrawalPrepareTest } from "./test-withdrawal-prepare.js";

/**
 * Test runner.
 */
const logger = new Logger("testrunner.ts");

/**
 * Spec for one test.
 */
interface TestMainFunction {
  (t: GlobalTestState): Promise<void>;
  timeoutMs?: number;
  experimental?: boolean;
  suites?: string[];
}

const allTests: TestMainFunction[] = [
  runAgeRestrictionsMerchantTest,
  runAgeRestrictionsMixedMerchantTest,
  runAgeRestrictionsPeerTest,
  runAgeRestrictionsDepositTest,
  runBankApiTest,
  runClaimLoopTest,
  runClauseSchnorrTest,
  runDenomUnofferedTest,
  runDepositTest,
  runSimplePaymentTest,
  runExchangeManagementFaultTest,
  runExchangeTimetravelTest,
  runFeeRegressionTest,
  runForcedSelectionTest,
  runKycTest,
  runExchangePurseTest,
  runExchangeDepositTest,
  runMerchantExchangeConfusionTest,
  runMerchantInstancesDeleteTest,
  runMerchantInstancesTest,
  runMerchantInstancesUrlsTest,
  runMerchantLongpollingTest,
  runMerchantRefundApiTest,
  runMerchantSpecPublicOrdersTest,
  runPaymentClaimTest,
  runPaymentFaultTest,
  runPaymentForgettableTest,
  runPaymentIdempotencyTest,
  runPaymentMultipleTest,
  runPaymentTest,
  runPaymentShareTest,
  runPaymentTemplateTest,
  runPaymentAbortTest,
  runPaymentTransientTest,
  runPaymentZeroTest,
  runPayPaidTest,
  runPeerRepairTest,
  runMultiExchangeTest,
  runWalletBalanceTest,
  runPaywallFlowTest,
  runPeerToPeerPullTest,
  runPeerToPeerPushTest,
  runRefundAutoTest,
  runRefundGoneTest,
  runRefundIncrementalTest,
  runRefundTest,
  runRevocationTest,
  runWithdrawalManualTest,
  runTimetravelAutorefreshTest,
  runTimetravelWithdrawTest,
  runWalletBackupBasicTest,
  runWalletBackupDoublespendTest,
  runWalletNotificationsTest,
  runWalletCryptoWorkerTest,
  runWalletDblessTest,
  runWallettestingTest,
  runWithdrawalAbortBankTest,
  // runWithdrawalNotifyBeforeTxTest,
  runWithdrawalBankIntegratedTest,
  runWithdrawalFakebankTest,
  runWithdrawalFeesTest,
  runWithdrawalConversionTest,
  runWithdrawalHugeTest,
  runTermOfServiceFormatTest,
  runStoredBackupsTest,
  runPaymentExpiredTest,
  runWalletGenDbTest,
  runLibeufinBankTest,
  runPaymentDeletedTest,
  runWalletDd48Test,
  runCurrencyScopeTest,
  runWalletRefreshTest,
  runWalletCliTerminationTest,
  runOtpTest,
  runWalletBalanceNotificationsTest,
  runExchangeManagementTest,
  runWalletConfigTest,
  runWalletObservabilityTest,
  runWalletDevExperimentsTest,
  runWalletBalanceZeroTest,
  runWalletInsufficientBalanceTest,
  runWalletWirefeesTest,
  runDenomLostTest,
  runWalletDenomExpireTest,
  runWalletBlockedDepositTest,
  runWalletBlockedPayMerchantTest,
  runWalletBlockedPayPeerPushTest,
  runWalletBlockedPayPeerPullTest,
  runWalletExchangeUpdateTest,
  runWalletRefreshErrorsTest,
  runWalletNetworkAvailabilityTest,
  runPeerPullLargeTest,
  runPeerPushLargeTest,
  runWithdrawalHandoverTest,
  runWithdrawalAmountTest,
  runWithdrawalFlexTest,
  runExchangeMasterPubChangeTest,
  runMerchantCategoriesTest,
  runWithdrawalExternalTest,
  runWithdrawalIdempotentTest,
  runKycThresholdWithdrawalTest,
  runKycExchangeWalletTest,
  runKycPeerPushTest,
  runKycPeerPullTest,
  runKycDepositAggregateTest,
  runKycFormWithdrawalTest,
  runKycBalanceWithdrawalTest,
  runKycNewMeasureTest,
  runKycDepositDepositTest,
  runKycMerchantDepositTest,
  runKycMerchantAggregateTest,
  runKycDepositDepositKyctransferTest,
  runWithdrawalPrepareTest,
  runAccountRestrictionsTest,
  runRepurchaseTest,
];

export interface TestRunSpec {
  includePattern?: string;
  suiteSpec?: string;
  dryRun?: boolean;
  failFast?: boolean;
  waitOnFail?: boolean;
  includeExperimental: boolean;
  noTimeout: boolean;
  verbosity: number;
}

export interface TestInfo {
  name: string;
  suites: string[];
  experimental: boolean;
}

function updateCurrentSymlink(testDir: string): void {
  const currLink = path.join(
    os.tmpdir(),
    `taler-integrationtests-${os.userInfo().username}-current`,
  );
  try {
    fs.unlinkSync(currLink);
  } catch (e) {
    // Ignore
  }
  try {
    fs.symlinkSync(testDir, currLink);
  } catch (e) {
    console.log(e);
    // Ignore
  }
}

export function getTestName(tf: TestMainFunction): string {
  const res = tf.name.match(/run([a-zA-Z0-9]*)Test/);
  if (!res) {
    throw Error("invalid test name, must be 'run${NAME}Test'");
  }
  return res[1]
    .replace(/[a-z0-9][A-Z]/g, (x) => {
      return x[0] + "-" + x[1];
    })
    .toLowerCase();
}

interface RunTestChildInstruction {
  testName: string;
  testRootDir: string;
}

function purgeSharedTestEnvironment() {
  const rmRes = spawnSync("rm", ["-rf", `${getSharedTestDir()}`]);
  if (rmRes.status != 0) {
    logger.warn("can't delete shared test directory");
  }
  const psqlRes = spawnSync("psql", ["-Aqtl"], {
    encoding: "utf-8",
  });
  if (psqlRes.status != 0) {
    logger.warn("could not list available postgres databases");
    return;
  }
  if (psqlRes.output[1]!!.indexOf("taler-integrationtest-shared") >= 0) {
    const dropRes = spawnSync("dropdb", ["taler-integrationtest-shared"], {
      encoding: "utf-8",
    });
    if (dropRes.status != 0) {
      logger.warn("could not drop taler-integrationtest-shared database");
      return;
    }
  }
}

export async function runTests(spec: TestRunSpec) {
  if (!process.env.TALER_HARNESS_KEEP) {
    logger.info("purging shared test environment");
    purgeSharedTestEnvironment();
  } else {
    logger.info("keeping shared test environment");
  }

  const testRootDir = fs.mkdtempSync(
    path.join(os.tmpdir(), "taler-integrationtests-"),
  );
  updateCurrentSymlink(testRootDir);
  console.log(`testsuite root directory: ${testRootDir}`);

  const testResults: TestRunResult[] = [];

  let currentChild: child_process.ChildProcess | undefined;

  const handleSignal = (s: NodeJS.Signals) => {
    console.log(`received signal ${s} in test parent`);
    if (currentChild) {
      currentChild.kill("SIGTERM");
    }
    reportAndQuit(testRootDir, testResults, true);
  };

  process.on("SIGINT", (s) => handleSignal(s));
  process.on("SIGTERM", (s) => handleSignal(s));
  //process.on("unhandledRejection", handleSignal);
  //process.on("uncaughtException", handleSignal);

  let suites: Set<string> | undefined;

  if (spec.suiteSpec) {
    suites = new Set(spec.suiteSpec.split(",").map((x) => x.trim()));
  }

  for (const [n, testCase] of allTests.entries()) {
    const testName = getTestName(testCase);
    if (spec.includePattern && !minimatch(testName, spec.includePattern)) {
      continue;
    }

    if (testCase.experimental && !spec.includeExperimental) {
      continue;
    }

    if (suites) {
      const ts = new Set(testCase.suites ?? []);
      const intersection = new Set([...suites].filter((x) => ts.has(x)));
      if (intersection.size === 0) {
        continue;
      }
    }

    if (spec.dryRun) {
      console.log(`dry run: would run test ${testName}`);
      continue;
    }

    const testInstr: RunTestChildInstruction = {
      testName,
      testRootDir,
    };

    const myFilename = url.fileURLToPath(import.meta.url);

    currentChild = child_process.fork(myFilename, ["__TWCLI_TESTWORKER"], {
      env: {
        TWCLI_RUN_TEST_INSTRUCTION: JSON.stringify(testInstr),
        ...process.env,
      },
      stdio: ["pipe", "pipe", "pipe", "ipc"],
    });

    const testDir = path.join(testRootDir, testName);
    fs.mkdirSync(testDir, { recursive: true });

    const harnessLogFilename = path.join(testRootDir, testName, "harness.log");
    const harnessLogStream = fs.createWriteStream(harnessLogFilename);

    if (spec.verbosity > 0) {
      currentChild.stderr?.pipe(process.stderr);
      currentChild.stdout?.pipe(process.stdout);
    }

    currentChild.stdout?.pipe(harnessLogStream);
    currentChild.stderr?.pipe(harnessLogStream);

    // Default timeout when the test doesn't override it.
    let defaultTimeout = 60000;
    const overrideDefaultTimeout = process.env.TALER_TEST_TIMEOUT;
    if (overrideDefaultTimeout) {
      defaultTimeout = Number.parseInt(overrideDefaultTimeout, 10) * 1000;
    }

    // Set the timeout to at least be the default timeout.
    const testTimeoutMs = testCase.timeoutMs
      ? Math.max(testCase.timeoutMs, defaultTimeout)
      : defaultTimeout;

    if (spec.noTimeout) {
      console.log(`running ${testName}, no timeout`);
    } else {
      console.log(`running ${testName} with timeout ${testTimeoutMs}ms`);
    }

    const token = spec.noTimeout
      ? CancellationToken.CONTINUE
      : CancellationToken.timeout(testTimeoutMs).token;

    const resultPromise: Promise<TestRunResult> = new Promise(
      (resolve, reject) => {
        let msg: TestRunResult | undefined;
        currentChild!.on("message", (m) => {
          if (token.isCancelled) {
            return;
          }
          msg = m as TestRunResult;
        });
        currentChild!.on("exit", (code, signal) => {
          if (token.isCancelled) {
            return;
          }
          logger.info(`process exited code=${code} signal=${signal}`);
          if (signal) {
            reject(new Error(`test worker exited with signal ${signal}`));
          } else if (code != 0) {
            reject(new Error(`test worker exited with code ${code}`));
          } else if (!msg) {
            reject(
              new Error(
                `test worker exited without giving back the test results`,
              ),
            );
          } else {
            resolve(msg);
          }
        });
        currentChild!.on("error", (err) => {
          if (token.isCancelled) {
            return;
          }
          reject(err);
        });
      },
    );

    let result: TestRunResult;

    try {
      result = await token.racePromise(resultPromise);
      if (result.status === "fail" && spec.failFast) {
        logger.error("test failed and failing fast, exit!");
        throw Error("exit on fail fast");
      }
    } catch (e: any) {
      console.error(`test ${testName} timed out`);
      if (token.isCancelled) {
        result = {
          status: "fail",
          reason: "timeout",
          timeSec: testTimeoutMs / 1000,
          name: testName,
        };
        currentChild.kill("SIGTERM");
      } else {
        throw Error(e);
      }
    }

    harnessLogStream.close();

    console.log(`parent: got result ${JSON.stringify(result)}`);

    testResults.push(result);
  }

  reportAndQuit(testRootDir, testResults);
}

export function reportAndQuit(
  testRootDir: string,
  testResults: TestRunResult[],
  interrupted: boolean = false,
): never {
  let numTotal = 0;
  let numFail = 0;
  let numSkip = 0;
  let numPass = 0;

  for (const result of testResults) {
    numTotal++;
    if (result.status === "fail") {
      numFail++;
    } else if (result.status === "skip") {
      numSkip++;
    } else if (result.status === "pass") {
      numPass++;
    }
  }

  const resultsFile = path.join(testRootDir, "results.json");
  fs.writeFileSync(
    path.join(testRootDir, "results.json"),
    JSON.stringify({ testResults, interrupted }, undefined, 2),
  );
  if (interrupted) {
    console.log("test suite was interrupted");
  }
  console.log(`See ${resultsFile} for details`);
  console.log(`Skipped: ${numSkip}/${numTotal}`);
  console.log(`Failed: ${numFail}/${numTotal}`);
  console.log(`Passed: ${numPass}/${numTotal}`);

  if (interrupted) {
    process.exit(3);
  } else if (numPass < numTotal - numSkip) {
    process.exit(1);
  } else {
    process.exit(0);
  }
}

export function getTestInfo(): TestInfo[] {
  return allTests.map((x) => ({
    name: getTestName(x),
    suites: x.suites ?? [],
    experimental: x.experimental ?? false,
  }));
}

const runTestInstrStr = process.env["TWCLI_RUN_TEST_INSTRUCTION"];
if (runTestInstrStr && process.argv.includes("__TWCLI_TESTWORKER")) {
  // Test will call taler-wallet-cli, so we must not propagate this variable.
  delete process.env["TWCLI_RUN_TEST_INSTRUCTION"];
  const { testRootDir, testName } = JSON.parse(
    runTestInstrStr,
  ) as RunTestChildInstruction;

  process.on("disconnect", () => {
    logger.trace("got disconnect from parent");
    process.exit(3);
  });

  const runTest = async () => {
    let testMain: TestMainFunction | undefined;
    for (const t of allTests) {
      if (getTestName(t) === testName) {
        testMain = t;
        break;
      }
    }

    if (!process.send) {
      logger.error("can't communicate with parent");
      process.exit(2);
    }

    if (!testMain) {
      logger.info(`test ${testName} not found`);
      process.exit(2);
    }

    const testDir = path.join(testRootDir, testName);
    logger.info(`running test ${testName}`);
    const gc = new GlobalTestState({
      testDir,
    });
    const testResult = await runTestWithState(gc, testMain, testName);
    logger.info(`done test ${testName}: ${testResult.status}`);
    process.send(testResult);
  };

  runTest()
    .then(() => {
      logger.trace(`test ${testName} finished in worker`);
      if (shouldLingerInTest()) {
        logger.trace("lingering ...");
        return;
      }
      process.exit(0);
    })
    .catch((e) => {
      logger.error(e);
      process.exit(1);
    });
}