From 6fb2b29bcb66c63c1181e0e7a687ef7d271c1ef4 Mon Sep 17 00:00:00 2001 From: siiky Date: Mon, 10 Mar 2025 14:16:42 +0000 Subject: [PATCH] feat: start re-working stress testing --- lib/middlewares/populateDeviceId.js | 4 + tests/stress/child.js | 87 ------- tests/stress/cli.js | 59 +++++ tests/stress/consts.js | 10 + tests/stress/db.js | 73 ++++++ tests/stress/env.js | 93 ++++++++ tests/stress/index.js | 82 +++++-- tests/stress/load-tx-dummy-data.js | 47 ---- tests/stress/machines.js | 92 ++++++++ tests/stress/queries-performance-analyzer.js | 231 ------------------- tests/stress/scripts/create-machines.sh | 19 +- tests/stress/scripts/index.js | 32 --- tests/stress/server.js | 59 +++++ tests/stress/test-server.js | 7 - tests/stress/utils/index.js | 5 - tests/stress/utils/init-cert.js | 12 - tests/stress/utils/save-config.js | 3 - tools/lamassu-server-stress-testing | 2 + 18 files changed, 454 insertions(+), 463 deletions(-) delete mode 100644 tests/stress/child.js create mode 100644 tests/stress/cli.js create mode 100644 tests/stress/consts.js create mode 100644 tests/stress/db.js create mode 100644 tests/stress/env.js delete mode 100644 tests/stress/load-tx-dummy-data.js create mode 100644 tests/stress/machines.js delete mode 100644 tests/stress/queries-performance-analyzer.js delete mode 100644 tests/stress/scripts/index.js create mode 100644 tests/stress/server.js delete mode 100644 tests/stress/test-server.js delete mode 100644 tests/stress/utils/index.js delete mode 100644 tests/stress/utils/init-cert.js delete mode 100644 tests/stress/utils/save-config.js create mode 100755 tools/lamassu-server-stress-testing diff --git a/lib/middlewares/populateDeviceId.js b/lib/middlewares/populateDeviceId.js index dd2fd9e2..e406578d 100644 --- a/lib/middlewares/populateDeviceId.js +++ b/lib/middlewares/populateDeviceId.js @@ -3,6 +3,8 @@ const crypto = require('crypto') const logger = require('../logger') +const IS_STRESS_TESTING = process.env.LAMASSU_STRESS_TESTING === "YES" + function sha256 (buf) { if (!buf) return null const hash = crypto.createHash('sha256') @@ -14,6 +16,8 @@ function sha256 (buf) { const populateDeviceId = function (req, res, next) { const deviceId = _.isFunction(req.connection.getPeerCertificate) ? sha256(req.connection.getPeerCertificate()?.raw) + : IS_STRESS_TESTING + ? 'placeholder' /* TODO: req... ? */ : null if (!deviceId) return res.status(500).json({ error: 'Unable to find certificate' }) diff --git a/tests/stress/child.js b/tests/stress/child.js deleted file mode 100644 index c6a7d9a1..00000000 --- a/tests/stress/child.js +++ /dev/null @@ -1,87 +0,0 @@ -const https = require('https') -const path = require('path') -const pify = require('pify') -const fs = pify(require('fs')) -const uuid = require('uuid') -const _ = require('lodash/fp') -const { PerformanceObserver, performance } = require('perf_hooks') - -const utils = require('./utils') -const variables = require('./utils/variables') - -var certificate = {} -var connectionInfo = {} - -const getCert = machineIndex => { - const key = fs.readFile(path.resolve(__dirname, 'machines', `${machineIndex}`, 'client.key')) - const cert = fs.readFile(path.resolve(__dirname, 'machines', `${machineIndex}`, 'client.pem')) - - return Promise.all([key, cert]).then(([key, cert]) => { - return { key, cert } - }).catch(err => { - console.error('The following error when reading the certificate: ', err) - return null - }) -} - -const getConnectionInfo = machineIndex => { - return fs.readFile(path.resolve(__dirname, 'machines', `${machineIndex}`, 'connection_info.json')) -} - -let counter = 0 -const requestTimes = [] -let latestResponseTime = 0 - -const request = (machineIndex, pid) => { - performance.mark('A') - https.get({ - hostname: 'localhost', - port: 3000, - path: '/poll?state=chooseCoin&model=unknown&version=7.5.0-beta.0&idle=true&pid=' + pid + '&sn=' + counter, - method: 'GET', - key: certificate.key, - cert: certificate.cert, - ca: connectionInfo.ca, - headers: { - date: new Date().toISOString(), - 'request-id': uuid.v4() - } - }, res => { - res.on('data', (d) => { - performance.mark('B') - performance.measure('A to B', 'A', 'B') - console.log(`Machine ${machineIndex} || Avg request response time: ${_.mean(requestTimes).toFixed(3)} || Latest response time: ${latestResponseTime.toFixed(3)}`) - process.send({ message: Buffer.from(d).toString() }) - }) - }) - - counter++ -} - -const obs = new PerformanceObserver((items) => { - latestResponseTime = items.getEntries()[0].duration - requestTimes.push(latestResponseTime) - performance.clearMarks() -}) -obs.observe({ entryTypes: ['measure'] }) - -process.on('message', async (msg) => { - console.log('Message from parent:', msg) - - const promises = [getCert(msg.machineIndex), getConnectionInfo(msg.machineIndex)] - Promise.all(promises).then(values => { - certificate = values[0] - connectionInfo = JSON.parse(values[1]) - }).catch(err => { - console.error('The following error occurred during certificate parsing: ', err) - }) - - if (msg.hasVariance) await new Promise(resolve => setTimeout(resolve, utils.randomIntFromInterval(1, variables.POLLING_INTERVAL))) - const pid = uuid.v4() - request(msg.machineIndex, pid) - - setInterval(() => { - const pid = uuid.v4() - request(msg.machineIndex, pid) - }, 5000) -}) diff --git a/tests/stress/cli.js b/tests/stress/cli.js new file mode 100644 index 00000000..10ce1c14 --- /dev/null +++ b/tests/stress/cli.js @@ -0,0 +1,59 @@ +const trimStart = (s, c) => { + let idx = 0 + while (idx < s.length && s[idx] === c) + idx++ + return idx > 0 ? s.substring(idx) : s +} + +const optkey = (opt) => trimStart(opt, '-') + +const parse = (default_options, option_arities) => (args) => { + const positional = [] + const parsed_options = {} + + for (let i = 0; i < args.length;) { + const arg = args[i] + i++ + + const arity = option_arities[arg] + if (typeof(arity) === 'number') { + if (arity+i > args.length) + return [`${arg}: not enough arguments.`, parsed_options, positional] + + const opt = optkey(arg) + switch (arity) { + case 0: parsed_options[opt] = true; break + case 1: parsed_options[opt] = args[i]; break + default: parsed_options[opt] = args.slice(i, i+arity); break + } + i += arity + } else { + positional.push(arg) + } + } + + const options = Object.assign({}, default_options, parsed_options) + return [null, options, positional] +} + +const help = ({ grammar, usage }) => () => { + if (usage) console.log(usage) + grammar.forEach( + ([optargs, optdesc, def]) => { + const deftext = def ? ` (default: ${def})` : "" + console.log(`\t${optargs.join(' ')}\t${optdesc}${deftext}`) + } + ) +} + +const CLI = ({ grammar, usage }) => { + const details = grammar.map(([[opt, ...optargs], optdesc, def]) => [opt, optargs.length, def]) + const option_arities = Object.fromEntries(details.map(([opt, arity, _def]) => [opt, arity])) + const default_options = Object.fromEntries(details.map(([opt, _arity, def]) => [optkey(opt), def])) + return { + parse: parse(default_options, option_arities), + help: help({ grammar, usage }), + } +} + +module.exports = CLI diff --git a/tests/stress/consts.js b/tests/stress/consts.js new file mode 100644 index 00000000..6ccb8eb8 --- /dev/null +++ b/tests/stress/consts.js @@ -0,0 +1,10 @@ +const EXIT = { + OK: 0, + EXCEPTION: 1, + UNKNOWN: 2, + BADARGS: 3, +} + +module.exports = { + EXIT, +} diff --git a/tests/stress/db.js b/tests/stress/db.js new file mode 100644 index 00000000..77aaff0b --- /dev/null +++ b/tests/stress/db.js @@ -0,0 +1,73 @@ +const cp = require('node:child_process') +const path = require('node:path') + +require('../../lib/environment-helper') +const db = require('../../lib/db') + +const { EXIT } = require('./consts') +const CLI = require('./cli') + +const help_message = "Setup the DB according to the previously defined environment." + +const cli = CLI({ + grammar: [ + [["--help"], "Show this help message"], + ], +}) + +const help = (exit_code) => { + console.log("Usage: lamassu-server-stress-testing db ARGS...") + console.log(help_message) + cli.help() + return exit_code +} + +const migrate = async () => { + const lamassu_migrate_path = path.join(__dirname, "../../bin/lamassu-migrate") + const { stdout, stderr, status, signal, error } = cp.spawnSync(lamassu_migrate_path, [], { + cwd: process.cwd(), + encoding: 'utf8', + }) + + if (typeof(status) !== 'number' || typeof(signal) === 'string' || error) { + console.error("stdout:", stdout) + console.error("stderr:", stderr) + console.error("status:", status) + console.error("signal:", signal) + console.error("error:", error) + return EXIT.EXCEPTION + } else { + return EXIT.OK + } +} + +const configure = async () => { + const config = '{"config":{"triggersConfig_expirationTime":"Forever","triggersConfig_automation":"Automatic","locale_timezone":"Pacific/Honolulu","cashIn_cashboxReset":"Manual","notifications_email_security":true,"notifications_sms_security":true,"notifications_notificationCenter_security":true,"wallets_advanced_feeMultiplier":"1","wallets_advanced_cryptoUnits":"full","wallets_advanced_allowTransactionBatching":false,"wallets_advanced_id":"c5e3e61e-71b2-4200-9851-0142db7e6797","triggersConfig_customerAuthentication":"SMS","commissions_cashOutFixedFee":1,"machineScreens_rates_active":true,"wallets_BTC_zeroConfLimit":0,"wallets_BTC_coin":"BTC","wallets_BTC_wallet":"mock-wallet","wallets_BTC_ticker":"mock-ticker","wallets_BTC_exchange":"mock-exchange","wallets_BTC_zeroConf":"none","locale_id":"5f18e5ae-4a5d-45b2-8184-a6d69f4cc237","locale_country":"US","locale_fiatCurrency":"USD","locale_languages":["en-US","ja-JP","de-DE"],"locale_cryptoCurrencies":["BTC"],"commissions_minimumTx":2,"commissions_fixedFee":3,"commissions_cashOut":4,"commissions_cashIn":5,"commissions_id":"06d11aaa-34e5-45ab-956b-9728ccfd9330"}}' + await db.none("INSERT INTO user_config (type, data, created, valid, schema_version) VALUES ('config', $1, now(), 't', 2)", [config]) + return EXIT.OK +} + +const run = async (args) => { + const [err, options, positional] = cli.parse(args) + if (err) { + console.error(err) + return help(EXIT.BADARGS) + } + + if (options.help) + return help(EXIT.OK) + + const funcs = [migrate, configure] + for (let func of funcs) { + const exit_code = await func() + if (exit_code !== EXIT.OK) + return exit_code + } + + return EXIT.OK +} + +module.exports = { + help_message, + run, +} diff --git a/tests/stress/env.js b/tests/stress/env.js new file mode 100644 index 00000000..0d06ba3c --- /dev/null +++ b/tests/stress/env.js @@ -0,0 +1,93 @@ +const fs = require('node:fs') +const os = require('node:os') +const path = require('node:path') + +const dotenv = require('dotenv') + +const { EXIT } = require('./consts') +const CLI = require('./cli') + +const help_message = "Produce an .env file appropriate for stress testing." + +const cli = CLI({ + grammar: [ + [["--help"], "Show this help message"], + [["--inenv", "ENV"], "Environment file path to read", ".env"], + [["--outenv", "ENV"], "Environment file path to write", ".stress.env"], + [["--dbuser", "DBUSER"], "Database username", "postgres"], + [["--dbpass", "DBPASS"], "Database password", "postgres123"], + [["--dbhost", "DBHOST"], "Database hostname", "localhost"], + [["--dbport", "DBPORT"], "Database port", "5432"], + [["--dbname", "DBNAME"], "Database name", "lamassu_stress"], + ], +}) + +const help = (exit_code) => { + console.log("Usage: lamassu-server-stress-testing env ARGS...") + console.log(help_message) + cli.help() + return exit_code +} + +const env_read = (path) => { + const envstr = fs.readFileSync(path, { encoding: 'utf8' }) + return dotenv.parse(envstr) + //const entries = envstr + // .split(os.EOL) + // .flatMap((line) => { + // line = line.trimStart() + // const i = line.indexOf('=') + // + // if (line.startsWith('#') || i <= 0) + // return [] + // + // const varname = line.substring(0, i) + // const value = line.substring(i + 1) + // return [[varname, value]] + // }) + //return Object.fromEntries(entries) +} + +const env_write = (envvars, path) => { + const envcontent = Object.entries(envvars) + .map(([varname, value]) => [varname, value].join('=')) + .join(os.EOL) + os.EOL + fs.writeFileSync(path, envcontent) +} + +const run = async (args) => { + const [err, options, positional] = cli.parse(args) + if (err) { + console.error(err) + return help(EXIT.BADARGS) + } + + if (options.help) + return help(EXIT.OK) + + if (positional.length > 0) { + console.error("Unknown arguments:", positional) + return help(EXIT.BADARGS) + } + + const inenvpath = path.resolve(process.cwd(), options.inenv ?? ".env") + const inenvvars = env_read(inenvpath) + + const outenvpath = path.resolve(process.cwd(), options.outenv ?? ".stress.env") + const outenvvars = { + ...inenvvars, + POSTGRES_USER: options.dbuser ?? "postgres", + POSTGRES_PASSWORD: options.dbpass ?? "postgres123", + POSTGRES_HOST: options.dbhost ?? "localhost", + POSTGRES_PORT: options.dbport ?? "5432", + POSTGRES_DB: options.dbname ?? "lamassu_stress", + } + env_write(outenvvars, outenvpath) + + return EXIT.OK +} + +module.exports = { + help_message, + run, +} diff --git a/tests/stress/index.js b/tests/stress/index.js index afe6de90..2d6acefc 100644 --- a/tests/stress/index.js +++ b/tests/stress/index.js @@ -1,35 +1,69 @@ - -const { fork } = require('child_process') const minimist = require('minimist') -const cmd = require('./scripts') -const variables = require('./utils/variables') +const { EXIT } = require('./consts') -function createMachines (numberOfMachines) { - return cmd.execCommand( - `bash ./scripts/create-machines.sh ${numberOfMachines} ${variables.SERVER_CERT_PATH} ${variables.MACHINE_PATH}` +const SUBCMDS = { + env: require('./env'), + db: require('./db'), + machines: require('./machines'), + server: require('./server'), +} + +const README = ` +This program will help you set the lamassu-server up for stress testing. This +short introduction is meant only as a quickstart guide; the subcommands may +support more options beyond those shown here. Use the --help flag for details +on each subcommand. + +First of all, you need to create a suitable .env file. With the following +commands, .env.bak will be used as a starting point, and the result will be +saved in .env. This is to avoid losing the real configurations. + +$ cp .env .env.bak +$ lamassu-server-stress-testing env --inenv .env.bak --outenv .env + +The database chosen in the command above (by default lamassu_stress) +must be initialized, and must have a bare-bones configuration. The following +command does that: + +$ lamassu-server-stress-testing db + +You also need to create fake machines that will be used later in the actual +stress tests (including certificates, and pairing each to the server). The +following command creates 10 fake machines, and saves their data in +path/to/stress/data/. path/to/real/machine/code/ is the path to the root of the +machine's code. + +$ lamassu-server-stress-testing machines -n 10 --fake_data_dir path/to/stress/data/ --machine path/to/real/machine/code/ +`; + +const help = (exit_code) => { + console.log("Usage: lamassu-server-stress-testing SUBCMD ARGS...",) + console.log("Where SUBCMD is one of the following:") + Object.entries(SUBCMDS).forEach( + ([subcmd, { help_message }]) => { + console.log(`\t${subcmd}\t${help_message ?? ''}`) + } ) + + console.log(README) + + return exit_code } -function startServer () { - const forked = fork('test-server.js') - forked.send('start') -} +const main = async (args) => { + try { + const subcmd = SUBCMDS[args[0]] -async function run (args = minimist(process.argv.slice(2))) { - const NUMBER_OF_MACHINES = args._[0] - const HAS_VARIANCE = args.v || false + const exit_code = (args.length === 0) ? help(EXIT.OK) : + (!subcmd) ? help(EXIT.BADARGS) : + await subcmd.run(args.slice(1)) - await createMachines(NUMBER_OF_MACHINES) - startServer() - - for (let i = 1; i <= NUMBER_OF_MACHINES; i++) { - const forked = fork('child.js') - forked.send({ machineIndex: i, hasVariance: HAS_VARIANCE }) - forked.on('message', msg => { - console.log(`Machine ${i} || ${msg}`) - }) + process.exit(exit_code ?? EXIT.UNKNOWN) + } catch (err) { + console.error(err) + process.exit(EXIT.EXCEPTION) } } -run() +module.exports = main diff --git a/tests/stress/load-tx-dummy-data.js b/tests/stress/load-tx-dummy-data.js deleted file mode 100644 index 69021b2c..00000000 --- a/tests/stress/load-tx-dummy-data.js +++ /dev/null @@ -1,47 +0,0 @@ -const db = require('../../lib/db') - -const loadDummyTxData = () => { - const sql = ` - CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; - - INSERT INTO customers - VALUES ('99ac9999-9999-99e9-9999-9f99a9999999', null, null, null, null, null, null, - 'load_test_customers', null, null, null, null, null, null, '2021-04-16 10:51:38', - 'automatic', null, 'automatic', null, 'automatic', null, 'automatic', null, 'automatic', - null, 'automatic', null, null, null, null, null, null, 'automatic', null, null, - null, null, null, null, null, null, null, null, null, null) - ON CONFLICT DO NOTHING; - - INSERT INTO cash_in_txs - SELECT uuid_generate_v4(), md5(random()::text), md5(random()::text), i::integer, 'BTC', - i::integer, 'EUR', null, null, null, null, now() - random() * INTERVAL '2 days', random() > 0.5, - random() > 0.5, random() > 0.5, now() - random() * INTERVAL '2 days', null, random() > 0.5, - random() > 0.5, i::integer, i::integer, 1, '99ac9999-9999-99e9-9999-9f99a9999999', - 6, random() > 0.5, random() * (0.9-0.1) + 0.1::int, i::integer, random() > 0.5, null, null, false, - null, null, null - FROM generate_series(1, 5000000) as t(i); - - INSERT INTO cash_out_txs - SELECT uuid_generate_v4(), md5(random()::text), md5(random()::text), i::integer, 'BTC', - i::integer, 'EUR', 'confirmed', random() > 0.5, random() > 0.5, random() > 0.5, - null, null, now() - random() * INTERVAL '2 days', now() - random() * INTERVAL '2 days', null, - random() > 0.5, random() > 0.5, random() > 0.5, 0, 1, 20, 50, null, '99ac9999-9999-99e9-9999-9f99a9999999', - random() * (40-1) + 1::int, now() - random() * INTERVAL '2 days', random() > 0.5, null, - random() * (0.9-0.1) + 0.1::int, i::integer, i::integer, null, null, null, null, null, null, null, null - FROM generate_series(1, 5000000) as t(i); - - INSERT INTO logs - SELECT uuid_generate_v4(), md5(random()::text), 'info', now() - random() * INTERVAL '2 days', - 'message', now() - random() * INTERVAL '2 days',0 - FROM generate_series(1, 5000000) as t(i); - - INSERT INTO bills - SELECT uuid_generate_v4(), i::integer, 'USD', '3d92c323-58c6-4172-9f30-91b80f0c653c', - i::integer, '2021-04-16 11:51:38', 'BTC', i::integer - FROM generate_series(1, 5000000) as t(i); - - ` - db.none(sql) -} - -loadDummyTxData() diff --git a/tests/stress/machines.js b/tests/stress/machines.js new file mode 100644 index 00000000..11d5e334 --- /dev/null +++ b/tests/stress/machines.js @@ -0,0 +1,92 @@ +const cp = require('node:child_process') +const fs = require('node:fs') +const path = require('node:path') + +const { EXIT } = require('./consts') +const CLI = require('./cli') + +const help_message = "Setup fake machines to be used as stress test clients." + +const cli = CLI({ + grammar: [ + [["--help"], "Show this help message"], + [["--machine", "PATH"], "Path to the machine's source code root"], + [["--fake_data_dir", "PATH"], "Where to save the fake machines' data"], + [["-n", "NUMBER"], "Number of fake machines to create"], + ], +}) + +const help = (exit_code) => { + console.log("Usage: lamassu-server-stress-testing machines ARGS...") + console.log(help_message) + cli.help() + return exit_code +} + +const create_fake_machine = async (gencerts_path, fake_data_dir, i) => + new Promise((resolve, reject) => { + const machine_data_dir = path.join(fake_data_dir, i.toString()) + fs.mkdirSync(machine_data_dir, { recursive: true, mode: 0o750 }) + + console.log("Creating fake machine number", i) + const gc = cp.fork(gencerts_path, [machine_data_dir], { + cwd: process.cwd(), + encoding: 'utf8', + }) + + gc.on('error', (error) => { + console.log(error) + resolve(EXIT.EXCEPTION) + }) + + gc.on('exit', (code, signal) => { + console.error("lamassu-server code:", code) + console.error("lamassu-server signal:", signal) + resolve(typeof(code) === 'number' ? code : EXIT.EXCEPTION) + }) + }) + +const create_fake_machines = async ({ machine, fake_data_dir, n }) => { + n = parseInt(n) + if (Number.isNaN(n) || n <= 0) { + console.error("Expected n to be a positive number, got", n) + return help(EXIT.BADARGS) + } + + /* TODO: Remove all data of previous machines? */ + //fs.rmSync(fake_data_dir, { recursive: true, force: true }) + + /* Create the root data directory */ + fs.mkdirSync(fake_data_dir, { recursive: true, mode: 0o750 }) + + const gencerts_path = path.join(machine, "tools", "generate-certificates") + let exit_code = EXIT.OK + for (let i = 0; i < n && exit_code === EXIT.OK; i++) + exit_code = await create_fake_machine(gencerts_path, fake_data_dir, i) + + return exit_code +} + +const run = async (args) => { + const [err, options, positional] = cli.parse(args) + if (err) { + console.error(err) + return help(EXIT.BADARGS) + } + + if (options.help) + return help(EXIT.OK) + + const missing_options = ["n", "machine", "fake_data_dir"].filter((opt) => !options[opt]) + if (missing_options.length > 0) { + console.error("The following options are required:", missing_options.join(", ")) + return help(EXIT.BADARGS) + } + + return await create_fake_machines(options) +} + +module.exports = { + help_message, + run, +} diff --git a/tests/stress/queries-performance-analyzer.js b/tests/stress/queries-performance-analyzer.js deleted file mode 100644 index d235ff62..00000000 --- a/tests/stress/queries-performance-analyzer.js +++ /dev/null @@ -1,231 +0,0 @@ -const db = require('../../lib/db') -const Pgp = require('pg-promise')() -const _ = require('lodash/fp') -const cashInTx = require('../../lib/cash-in/cash-in-tx') -const { CASH_OUT_TRANSACTION_STATES, REDEEMABLE_AGE } = require('../../lib/cash-out/cash-out-helper') - -const TX_PASSTHROUGH_ERROR_CODES = ['operatorCancel', 'scoreThresholdReached'] - -function filterTransaction () { - const sql = `EXPLAIN ANALYZE - SELECT DISTINCT * FROM ( - SELECT 'type' AS type, 'Cash In' AS value UNION - SELECT 'type' AS type, 'Cash Out' AS value UNION - SELECT 'machine' AS type, name AS value FROM devices d INNER JOIN cash_in_txs t ON d.device_id = t.device_id UNION - SELECT 'machine' AS type, name AS value FROM devices d INNER JOIN cash_out_txs t ON d.device_id = t.device_id UNION - SELECT 'customer' AS type, concat(id_card_data::json->>'firstName', ' ', id_card_data::json->>'lastName') AS value - FROM customers c INNER JOIN cash_in_txs t ON c.id = t.customer_id - WHERE c.id_card_data::json->>'firstName' IS NOT NULL or c.id_card_data::json->>'lastName' IS NOT NULL UNION - SELECT 'customer' AS type, concat(id_card_data::json->>'firstName', ' ', id_card_data::json->>'lastName') AS value - FROM customers c INNER JOIN cash_out_txs t ON c.id = t.customer_id - WHERE c.id_card_data::json->>'firstName' IS NOT NULL or c.id_card_data::json->>'lastName' IS NOT NULL UNION - SELECT 'fiat' AS type, fiat_code AS value FROM cash_in_txs UNION - SELECT 'fiat' AS type, fiat_code AS value FROM cash_out_txs UNION - SELECT 'crypto' AS type, crypto_code AS value FROM cash_in_txs UNION - SELECT 'crypto' AS type, crypto_code AS value FROM cash_out_txs UNION - SELECT 'address' AS type, to_address AS value FROM cash_in_txs UNION - SELECT 'address' AS type, to_address AS value FROM cash_out_txs UNION - SELECT 'status' AS type, ${cashInTx.TRANSACTION_STATES} AS value FROM cash_in_txs UNION - SELECT 'status' AS type, ${CASH_OUT_TRANSACTION_STATES} AS value FROM cash_out_txs - ) f` - return db.any(sql) -} - -function filterCustomer () { - const sql = `EXPLAIN ANALYZE - SELECT DISTINCT * FROM ( - SELECT 'phone' AS type, phone AS value FROM customers WHERE phone IS NOT NULL UNION - SELECT 'name' AS type, id_card_data::json->>'firstName' AS value FROM customers WHERE id_card_data::json->>'firstName' IS NOT NULL AND id_card_data::json->>'lastName' IS NULL UNION - SELECT 'name' AS type, id_card_data::json->>'lastName' AS value FROM customers WHERE id_card_data::json->>'firstName' IS NULL AND id_card_data::json->>'lastName' IS NOT NULL UNION - SELECT 'name' AS type, concat(id_card_data::json->>'firstName', ' ', id_card_data::json->>'lastName') AS value FROM customers WHERE id_card_data::json->>'firstName' IS NOT NULL AND id_card_data::json->>'lastName' IS NOT NULL UNION - SELECT 'address' as type, id_card_data::json->>'address' AS value FROM customers WHERE id_card_data::json->>'address' IS NOT NULL UNION - SELECT 'id' AS type, id_card_data::json->>'documentNumber' AS value FROM customers WHERE id_card_data::json->>'documentNumber' IS NOT NULL - ) f` - return db.any(sql) -} - -function getCustomerById (id) { - const passableErrorCodes = _.map(Pgp.as.text, TX_PASSTHROUGH_ERROR_CODES).join(',') - - const sql = `EXPLAIN ANALYZE - select id, authorized_override, days_suspended, is_suspended, front_camera_path, front_camera_override, - phone, sms_override, id_card_data, id_card_data_override, id_card_data_expiration, - id_card_photo_path, id_card_photo_override, us_ssn, us_ssn_override, sanctions, sanctions_at, - sanctions_override, total_txs, total_spent, created as last_active, fiat as last_tx_fiat, - fiat_code as last_tx_fiat_code, tx_class as last_tx_class, subscriber_info - from ( - select c.id, c.authorized_override, - greatest(0, date_part('day', c.suspended_until - now())) as days_suspended, - c.suspended_until > now() as is_suspended, - c.front_camera_path, c.front_camera_override, - c.phone, c.sms_override, c.id_card_data, c.id_card_data_override, c.id_card_data_expiration, - c.id_card_photo_path, c.id_card_photo_override, c.us_ssn, c.us_ssn_override, c.sanctions, - c.sanctions_at, c.sanctions_override, c.subscriber_info, t.tx_class, t.fiat, t.fiat_code, t.created, - row_number() over (partition by c.id order by t.created desc) as rn, - sum(case when t.id is not null then 1 else 0 end) over (partition by c.id) as total_txs, - sum(case when error_code is null or error_code not in ($1^) then t.fiat else 0 end) over (partition by c.id) as total_spent - from customers c left outer join ( - select 'cashIn' as tx_class, id, fiat, fiat_code, created, customer_id, error_code - from cash_in_txs where send_confirmed = true union - select 'cashOut' as tx_class, id, fiat, fiat_code, created, customer_id, error_code - from cash_out_txs where confirmed_at is not null) t on c.id = t.customer_id - where c.id = $2 - ) as cl where rn = 1` - return db.any(sql, [passableErrorCodes, id]) -} - -function simpleGetMachineLogs (deviceId, from = new Date(0).toISOString(), until = new Date().toISOString(), limit = null, offset = 0) { - const sql = `EXPLAIN ANALYZE - select id, log_level, timestamp, message from logs - where device_id=$1 - and timestamp >= $2 - and timestamp <= $3 - order by timestamp desc, serial desc - limit $4 - offset $5` - return db.any(sql, [ deviceId, from, until, limit, offset ]) -} - -function batchCashIn ( - from = new Date(0).toISOString(), - until = new Date().toISOString(), - limit = null, - offset = 0, - id = null, - txClass = null, - machineName = null, - customerName = null, - fiatCode = null, - cryptoCode = null, - toAddress = null, - status = null, - simplified = false -) { - const cashInSql = `EXPLAIN ANALYZE - SELECT 'cashIn' AS tx_class, txs.*, - c.phone AS customer_phone, - c.id_card_data_number AS customer_id_card_data_number, - c.id_card_data_expiration AS customer_id_card_data_expiration, - c.id_card_data AS customer_id_card_data, - concat(c.id_card_data::json->>'firstName', ' ', c.id_card_data::json->>'lastName') AS customer_name, - c.front_camera_path AS customer_front_camera_path, - c.id_card_photo_path AS customer_id_card_photo_path, - ((NOT txs.send_confirmed) AND (txs.created <= now() - interval $1)) AS expired - FROM (SELECT *, ${cashInTx.TRANSACTION_STATES} AS txStatus FROM cash_in_txs) AS txs - LEFT OUTER JOIN customers c ON txs.customer_id = c.id - INNER JOIN devices d ON txs.device_id = d.device_id - WHERE txs.created >= $2 AND txs.created <= $3 ${ - id !== null ? `AND txs.device_id = $6` : `` -} - AND ($7 is null or $7 = 'Cash In') - AND ($8 is null or d.name = $8) - AND ($9 is null or concat(c.id_card_data::json->>'firstName', ' ', c.id_card_data::json->>'lastName') = $9) - AND ($10 is null or txs.fiat_code = $10) - AND ($11 is null or txs.crypto_code = $11) - AND ($12 is null or txs.to_address = $12) - AND ($13 is null or txs.txStatus = $13) - AND (fiat > 0) - ORDER BY created DESC limit $4 offset $5` - - return db.any(cashInSql, [cashInTx.PENDING_INTERVAL, from, until, limit, offset, id, txClass, machineName, customerName, fiatCode, cryptoCode, toAddress, status]) -} - -function batchCashOut ( - from = new Date(0).toISOString(), - until = new Date().toISOString(), - limit = null, - offset = 0, - id = null, - txClass = null, - machineName = null, - customerName = null, - fiatCode = null, - cryptoCode = null, - toAddress = null, - status = null, - simplified = false -) { - const cashOutSql = `EXPLAIN ANALYZE - SELECT 'cashOut' AS tx_class, - txs.*, - actions.tx_hash, - c.phone AS customer_phone, - c.id_card_data_number AS customer_id_card_data_number, - c.id_card_data_expiration AS customer_id_card_data_expiration, - c.id_card_data AS customer_id_card_data, - concat(c.id_card_data::json->>'firstName', ' ', c.id_card_data::json->>'lastName') AS customer_name, - c.front_camera_path AS customer_front_camera_path, - c.id_card_photo_path AS customer_id_card_photo_path, - (extract(epoch FROM (now() - greatest(txs.created, txs.confirmed_at))) * 1000) >= $1 AS expired - FROM (SELECT *, ${CASH_OUT_TRANSACTION_STATES} AS txStatus FROM cash_out_txs) txs - INNER JOIN cash_out_actions actions ON txs.id = actions.tx_id - AND actions.action = 'provisionAddress' - LEFT OUTER JOIN customers c ON txs.customer_id = c.id - INNER JOIN devices d ON txs.device_id = d.device_id - WHERE txs.created >= $2 AND txs.created <= $3 ${ - id !== null ? `AND txs.device_id = $6` : `` -} - AND ($7 is null or $7 = 'Cash Out') - AND ($8 is null or d.name = $8) - AND ($9 is null or concat(c.id_card_data::json->>'firstName', ' ', c.id_card_data::json->>'lastName') = $9) - AND ($10 is null or txs.fiat_code = $10) - AND ($11 is null or txs.crypto_code = $11) - AND ($12 is null or txs.to_address = $12) - AND ($13 is null or txs.txStatus = $13) - AND (fiat > 0) - ORDER BY created DESC limit $4 offset $5` - - return db.any(cashOutSql, [REDEEMABLE_AGE, from, until, limit, offset, id, txClass, machineName, customerName, fiatCode, cryptoCode, toAddress, status]) -} - -function getTx (txId, txClass) { - const cashInSql = `EXPLAIN ANALYZE - select 'cashIn' as tx_class, txs.*, - ((not txs.send_confirmed) and (txs.created <= now() - interval $1)) as expired - from cash_in_txs as txs - where txs.id=$2` - - const cashOutSql = `EXPLAIN ANALYZE - select 'cashOut' as tx_class, - txs.*, - (extract(epoch from (now() - greatest(txs.created, txs.confirmed_at))) * 1000) >= $2 as expired - from cash_out_txs txs - where txs.id=$1` - - return txClass === 'cashIn' - ? db.any(cashInSql, [cashInTx.PENDING_INTERVAL, txId]) - : db.any(cashOutSql, [txId, REDEEMABLE_AGE]) -} - -function getTxAssociatedData (txId, txClass) { - const billsSql = `EXPLAIN ANALYZE select 'bills' as bills, b.* from bills b where cash_in_txs_id = $1` - const actionsSql = `EXPLAIN ANALYZE select 'cash_out_actions' as cash_out_actions, actions.* from cash_out_actions actions where tx_id = $1` - - return txClass === 'cashIn' - ? db.any(billsSql, [txId]) - : db.any(actionsSql, [txId]) -} - -const run = () => { - const deviceId = '7526924341dc4a57f02b6411a85923de' // randomly generated by the load script - const customerId = '99ac9999-9999-99e9-9999-9f99a9999999' // hardcoded on the current load script - const cashOutTxId = 'c402a7ae-b8f7-4781-8080-1e9ab76d62b5' // randomly generated by the load script - const cashInTxId = '4d8d89f4-7d77-4d30-87e8-be9de05deea7' // randomly generated by the load script - - const getExecutionTime = _.compose(_.get('QUERY PLAN'), _.last) - Promise.all([filterCustomer(), filterTransaction(), getCustomerById(customerId), simpleGetMachineLogs(deviceId), batchCashIn(), batchCashOut(), - getTx(cashInTxId, 'cashIn'), getTx(cashOutTxId, 'cashOut'), getTxAssociatedData(cashInTxId, 'cashIn'), getTxAssociatedData(cashOutTxId, 'cashOut')]) - .then(([filterCustomer, filterTransaction, getCustomerById, logs, batchCashIn, batchCashOut, getTxCashOut, getTxCashIn, - getTxAssociatedDataCashIn, getTxAssociatedDataCashOut]) => { - console.log(`filterCustomer => ${getExecutionTime(filterCustomer)}`) - console.log(`filterTransaction => ${getExecutionTime(filterTransaction)}`) - console.log(`getCustomerById => ${getExecutionTime(getCustomerById)}`) - console.log(`batchCashOut + batchCashIn => ${getExecutionTime(batchCashOut) + ' + ' + getExecutionTime(batchCashIn)} `) - console.log(`getTx (cash-out) => ${getExecutionTime(getTxCashOut)}`) - console.log(`getTx (cash-in) => ${getExecutionTime(getTxCashIn)}`) - console.log(`getTxAssociatedData (cash-in) => ${getExecutionTime(getTxAssociatedDataCashIn)}`) - console.log(`getTxAssociatedDataCashOut (cash-out) => ${getExecutionTime(getTxAssociatedDataCashOut)}`) - }) -} - -run() diff --git a/tests/stress/scripts/create-machines.sh b/tests/stress/scripts/create-machines.sh index 9c7ac003..9c997e3d 100644 --- a/tests/stress/scripts/create-machines.sh +++ b/tests/stress/scripts/create-machines.sh @@ -6,19 +6,12 @@ if [ $# -eq 0 ] echo "usage: ./build-machines [number_of_machines] /path/to/server/cert/lamassu_op_root_ca.pem /path/to/machine/" && exit 1 fi -case $1 in - ''|*[!0-9]*) echo "usage: ./build-machines [number_of_machines] /path/to/server/cert/lamassu_op_root_ca.pem /path/to/machine/" && exit 1;; -esac - SERVER_CERT=$(perl -pe 's/\n/\\n/' < $2) if [ -z "$SERVER_CERT" ] then echo "Lamassu-op-root-ca.pem is empty" && exit 1 fi -# Remove old folders -rm -rf ./machines/* - # Create stress database sudo -u postgres psql postgres -c "drop database if exists lamassu_stress" sudo -u postgres psql postgres -c "create database lamassu_stress with template lamassu" @@ -33,16 +26,12 @@ do cp "$3"/data/client.sample.pem ./machines/$NUMBER/ cp "$3"/data/client.sample.key ./machines/$NUMBER/ - - cat > ./machines/$NUMBER/connection_info.json << EOL - {"host":"localhost","ca":"$SERVER_CERT"} -EOL - - echo 'Generating certs...' - node ./utils/init-cert.js $NUMBER + cat > ./machines/$NUMBER/connection_info.json << EOF +{"host":"localhost","ca":"$SERVER_CERT"} +EOF # Get device_id - DEVICE_ID=`openssl x509 -outform der -in ./machines/$NUMBER/client.pem | sha256sum | cut -d " " -f 1` + DEVICE_ID=`openssl x509 -outform der -in ./machines/$NUMBER/client.pem | sha256sum | cut -d ' ' -f 1` # Update db config NEW_CONFIG=$(node ./utils/save-config.js $NUMBER $DEVICE_ID) diff --git a/tests/stress/scripts/index.js b/tests/stress/scripts/index.js deleted file mode 100644 index 991b2d19..00000000 --- a/tests/stress/scripts/index.js +++ /dev/null @@ -1,32 +0,0 @@ -const exec = require('child_process').exec - -/** - * Execute simple shell command (async wrapper). - * @param {String} cmd - * @return {Object} { stdout: String, stderr: String } - */ -function execCommand (cmd) { - return new Promise(function (resolve, reject) { - const proc = exec(cmd, (err, stdout, stderr) => { - if (err) { - reject(err) - } else { - resolve({ stdout, stderr }) - } - }) - - proc.stdout.on('data', data => { - console.log(data) - }) - - proc.stderr.on('data', data => { - console.log(data) - }) - - proc.on('exit', code => { - console.log('child process exited with code ' + code.toString()) - }) - }) -} - -module.exports = { execCommand } diff --git a/tests/stress/server.js b/tests/stress/server.js new file mode 100644 index 00000000..ae066a6a --- /dev/null +++ b/tests/stress/server.js @@ -0,0 +1,59 @@ +const cp = require('node:child_process') +const path = require('node:path') + +const { EXIT } = require('./consts') +const CLI = require('./cli') + +const help_message = "Start the server configured for stress testing." + +const cli = CLI({ + grammar: [ + [["--help"], "Show this help message"], + ], +}) + +const help = (exit_code) => { + console.log("Usage: lamassu-server-stress-testing server ARGS...") + console.log(help_message) + cli.help() + return exit_code +} + +const start_server = (args) => + new Promise((resolve, reject) => { + const lamassu_server = path.join(__dirname, "../../bin/lamassu-server") + const ls = cp.fork(lamassu_server, args, { + cwd: process.cwd(), + encoding: 'utf8', + env: { LAMASSU_STRESS_TESTING: "YES" }, + }) + + ls.on('error', (error) => { + console.log(error) + resolve(EXIT.EXCEPTION) + }) + + ls.on('exit', (code, signal) => { + console.error("lamassu-server code:", code) + console.error("lamassu-server signal:", signal) + resolve(typeof(code) === 'number' ? code : EXIT.EXCEPTION) + }) + }) + +const run = async (args) => { + const [err, options, positional] = cli.parse(args) + if (err) { + console.error(err) + return help(EXIT.BADARGS) + } + + if (options.help) + return help(EXIT.OK) + + return await start_server(positional) +} + +module.exports = { + help_message, + run, +} diff --git a/tests/stress/test-server.js b/tests/stress/test-server.js deleted file mode 100644 index 51569f7e..00000000 --- a/tests/stress/test-server.js +++ /dev/null @@ -1,7 +0,0 @@ -const cmd = require('./scripts') - -process.on('message', async (msg) => { - console.log('Message from parent:', msg) - - await cmd.execCommand(`node --prof LAMASSU_DB=STRESS_TEST ../../bin/lamassu-server`) -}) diff --git a/tests/stress/utils/index.js b/tests/stress/utils/index.js deleted file mode 100644 index e2445705..00000000 --- a/tests/stress/utils/index.js +++ /dev/null @@ -1,5 +0,0 @@ -function randomIntFromInterval (min, max) { - return Math.floor(Math.random() * (max - min + 1) + min) -} - -module.exports = { randomIntFromInterval } diff --git a/tests/stress/utils/init-cert.js b/tests/stress/utils/init-cert.js deleted file mode 100644 index cbfef975..00000000 --- a/tests/stress/utils/init-cert.js +++ /dev/null @@ -1,12 +0,0 @@ -const path = require('path') -const variables = require('./variables') -const { init } = require(`../${variables.MACHINE_PATH}/lib/pairing`) - -const number = process.argv[2] - -const certPath = { - cert: path.resolve(process.cwd(), 'machines', number, 'client.pem'), - key: path.resolve(process.cwd(), 'machines', number, 'client.key') -} - -init(certPath) diff --git a/tests/stress/utils/save-config.js b/tests/stress/utils/save-config.js deleted file mode 100644 index 9a39db0a..00000000 --- a/tests/stress/utils/save-config.js +++ /dev/null @@ -1,3 +0,0 @@ -const config = require('./default-config.json') - -console.log(JSON.stringify(config)) diff --git a/tools/lamassu-server-stress-testing b/tools/lamassu-server-stress-testing new file mode 100755 index 00000000..d58bfddc --- /dev/null +++ b/tools/lamassu-server-stress-testing @@ -0,0 +1,2 @@ +#!/usr/bin/env node +require('../tests/stress')(process.argv.slice(2))