From c8adaabf852367675e3bb9afc2012127f01aae3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9rgio=20Salgado?= Date: Fri, 21 May 2021 16:21:54 +0100 Subject: [PATCH 1/9] feat: add batch_id to cash_in_txs feat: bitcoind sendmany request feat: check if wallet supports transaction batching --- lib/plugins/wallet/bitcoind/bitcoind.js | 16 ++++++++++++++ lib/wallet.js | 5 +++++ .../1621556014244-add-btc-tx-batching.js | 22 +++++++++++++++++++ 3 files changed, 43 insertions(+) create mode 100644 migrations/1621556014244-add-btc-tx-batching.js diff --git a/lib/plugins/wallet/bitcoind/bitcoind.js b/lib/plugins/wallet/bitcoind/bitcoind.js index 7f9c802a..d374b0fd 100644 --- a/lib/plugins/wallet/bitcoind/bitcoind.js +++ b/lib/plugins/wallet/bitcoind/bitcoind.js @@ -79,6 +79,22 @@ function sendCoins (account, tx, settings, operatorId, feeMultiplier) { }) } +function sendCoinsBatch (account, txs, cryptoCode) { + return checkCryptoCode(cryptoCode) + .then(() => { + const txAddressAmountPairs = _.map(tx => [tx.address, tx.cryptoAtoms.shift(-unitScale).toFixed(8)], txs) + return Promise.all([JSON.stringify(_.fromPairs(txAddressAmountPairs))]) + }) + .then(([obj]) => fetch('sendmany', ['', obj])) + .then(res => ({ + txid: res.txid + })) + .catch(err => { + if (err.code === -6) throw new E.InsufficientFundsError() + throw err + }) +} + function newAddress (account, info, tx, settings, operatorId) { return checkCryptoCode(info.cryptoCode) .then(() => fetch('getnewaddress')) diff --git a/lib/wallet.js b/lib/wallet.js index a6999ede..6ec32192 100644 --- a/lib/wallet.js +++ b/lib/wallet.js @@ -210,6 +210,11 @@ function isStrictAddress (settings, cryptoCode, toAddress) { }) } +function supportsBatching (settings, cryptoCode) { + return fetchWallet(settings, cryptoCode) + .then(r => _.isFunction(r.wallet.sendCoinsBatch)) +} + const coinFilter = ['ETH'] const balance = (settings, cryptoCode) => { diff --git a/migrations/1621556014244-add-btc-tx-batching.js b/migrations/1621556014244-add-btc-tx-batching.js new file mode 100644 index 00000000..4750b0fb --- /dev/null +++ b/migrations/1621556014244-add-btc-tx-batching.js @@ -0,0 +1,22 @@ +var db = require('./db') + +exports.up = function (next) { + var sql = [ + `CREATE TYPE transaction_batch_status AS ENUM('open', 'failed', 'sent')`, + `CREATE TABLE transaction_batches ( + id UUID PRIMARY KEY, + crypto_code TEXT NOT NULL, + status transaction_batch_status NOT NULL DEFAULT 'open', + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + closed_at TIMESTAMPTZ, + error_message TEXT + )`, + `ALTER TABLE cash_in_txs ADD COLUMN batch_id REFERENCES transaction_batches(id)` + ] + + db.multi(sql, next) +} + +exports.down = function (next) { + next() +} From 73c0d09198e6890ea81673039cc20db56214354e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9rgio=20Salgado?= Date: Mon, 24 May 2021 02:53:36 +0100 Subject: [PATCH 2/9] feat: add transaction batching module feat: plugin sendCoins batching support feat: batching processing on poller feat: mock-wallet batching fix: bitcoin tx batching fix: transaction batching db table --- lib/plugins.js | 10 +++ lib/plugins/wallet/bitcoind/bitcoind.js | 12 ++- lib/plugins/wallet/mock-wallet/mock-wallet.js | 25 ++++++ lib/poller.js | 3 + lib/tx-batching.js | 84 +++++++++++++++++++ lib/wallet.js | 22 ++++- .../1621556014244-add-btc-tx-batching.js | 4 +- 7 files changed, 154 insertions(+), 6 deletions(-) create mode 100644 lib/tx-batching.js diff --git a/lib/plugins.js b/lib/plugins.js index a0ae0b8a..7f0ed759 100644 --- a/lib/plugins.js +++ b/lib/plugins.js @@ -22,6 +22,7 @@ const machineLoader = require('./machine-loader') const customers = require('./customers') const commissionMath = require('./commission-math') const loyalty = require('./loyalty') +const transactionBatching = require('./tx-batching') const { cassetteMaxCapacity, CASH_OUT_DISPENSE_READY, CONFIRMATION_CODE } = require('./constants') @@ -277,6 +278,15 @@ function plugins (settings, deviceId) { } function sendCoins (tx) { + if (wallet.supportsBatching(settings, tx.cryptoCode)) { + return transactionBatching.addTransactionToBatch(tx) + .then(() => ({ + batched: true, + sendPending: false, + error: null, + errorCode: null + })) + } return wallet.sendCoins(settings, tx) } diff --git a/lib/plugins/wallet/bitcoind/bitcoind.js b/lib/plugins/wallet/bitcoind/bitcoind.js index d374b0fd..516e7f39 100644 --- a/lib/plugins/wallet/bitcoind/bitcoind.js +++ b/lib/plugins/wallet/bitcoind/bitcoind.js @@ -81,13 +81,18 @@ function sendCoins (account, tx, settings, operatorId, feeMultiplier) { function sendCoinsBatch (account, txs, cryptoCode) { return checkCryptoCode(cryptoCode) + .then(() => calculateFeeDiscount(feeMultiplier)) + .then(newFee => fetch('settxfee', [newFee])) .then(() => { const txAddressAmountPairs = _.map(tx => [tx.address, tx.cryptoAtoms.shift(-unitScale).toFixed(8)], txs) return Promise.all([JSON.stringify(_.fromPairs(txAddressAmountPairs))]) }) .then(([obj]) => fetch('sendmany', ['', obj])) - .then(res => ({ - txid: res.txid + .then((txId) => fetch('gettransaction', [txId])) + .then((res) => _.pick(['fee', 'txid'], res)) + .then((pickedObj) => ({ + fee: BN(pickedObj.fee).abs().shift(unitScale).round(), + txid: pickedObj.txid })) .catch(err => { if (err.code === -6) throw new E.InsufficientFundsError() @@ -173,5 +178,6 @@ module.exports = { newFunding, cryptoNetwork, fetchRBF, - estimateFee + estimateFee, + sendCoinsBatch } diff --git a/lib/plugins/wallet/mock-wallet/mock-wallet.js b/lib/plugins/wallet/mock-wallet/mock-wallet.js index 75c3c340..4a945869 100644 --- a/lib/plugins/wallet/mock-wallet/mock-wallet.js +++ b/lib/plugins/wallet/mock-wallet/mock-wallet.js @@ -1,3 +1,5 @@ +const _ = require('lodash/fp') + const BN = require('../../../bn') const E = require('../../../error') const { utils: coinUtils } = require('lamassu-coins') @@ -57,7 +59,29 @@ function sendCoins (account, tx, settings, operatorId) { }) } +<<<<<<< HEAD function newAddress (account, info, tx, settings, operatorId) { +======= +function sendCoinsBatch (account, txs, cryptoCode) { + sendCount = sendCount + txs.length + return new Promise((resolve, reject) => { + setTimeout(() => { + const cryptoSum = _.reduce((acc, value) => acc.add(value.crypto_atoms), BN(0), txs) + if (isInsufficient(cryptoSum, cryptoCode)) { + console.log('[%s] DEBUG: Mock wallet insufficient funds: %s', + cryptoCode, cryptoSum.toString()) + return reject(new E.InsufficientFundsError()) + } + + console.log('[%s] DEBUG: Mock wallet sending %s cryptoAtoms in a batch', + cryptoCode, cryptoSum.toString()) + return resolve({ txid: '', fee: BN(0) }) + }, 2000) + }) +} + +function newAddress () { +>>>>>>> feat: add transaction batching module t0 = Date.now() return Promise.resolve('') } @@ -93,6 +117,7 @@ function getStatus (account, tx, requested, settings, operatorId) { module.exports = { NAME, balance, + sendCoinsBatch, sendCoins, newAddress, getStatus, diff --git a/lib/poller.js b/lib/poller.js index e7c9948b..1c05bef1 100644 --- a/lib/poller.js +++ b/lib/poller.js @@ -17,6 +17,7 @@ const NodeCache = require('node-cache') const util = require('util') const db = require('./db') const state = require('./middlewares/state') +const batching = require('./tx-batching') const INCOMING_TX_INTERVAL = 30 * T.seconds const LIVE_INCOMING_TX_INTERVAL = 5 * T.seconds @@ -31,6 +32,7 @@ const SANCTIONS_INITIAL_DOWNLOAD_INTERVAL = 5 * T.minutes const SANCTIONS_UPDATE_INTERVAL = 1 * T.day const RADAR_UPDATE_INTERVAL = 5 * T.minutes const PRUNE_MACHINES_HEARTBEAT = 1 * T.day +const TRANSACTION_BATCH_LIFECYCLE = 20 * T.minutes const CHECK_NOTIFICATION_INTERVAL = 20 * T.seconds const PENDING_INTERVAL = 10 * T.seconds @@ -214,6 +216,7 @@ function doPolling (schema) { } addToQueue(cashOutTx.monitorUnnotified, UNNOTIFIED_INTERVAL, schema, QUEUE.FAST, settings) addToQueue(cashInTx.monitorPending, PENDING_INTERVAL, schema, QUEUE.FAST, settings) + addToQueue(batching.processBatches, UNNOTIFIED_INTERVAL, schema, QUEUE.FAST, settings, TRANSACTION_BATCH_LIFECYCLE) addToQueue(pi().sweepHd, SWEEP_HD_INTERVAL, schema, QUEUE.FAST, settings) addToQueue(pi().pong, PONG_INTERVAL, schema, QUEUE.FAST) addToQueue(pi().clearOldLogs, LOGS_CLEAR_INTERVAL, schema, QUEUE.SLOW) diff --git a/lib/tx-batching.js b/lib/tx-batching.js new file mode 100644 index 00000000..93cd5eb9 --- /dev/null +++ b/lib/tx-batching.js @@ -0,0 +1,84 @@ +const _ = require('lodash/fp') +const pgp = require('pg-promise')() +const uuid = require('uuid') + +const db = require('./db') +const wallet = require('./wallet') + +function createTransactionBatch (cryptoCode) { + const sql = `INSERT INTO transaction_batches (id, crypto_code) VALUES ($1, $2) RETURNING *` + + return db.one(sql, [uuid.v4(), cryptoCode]) +} + +function closeTransactionBatch (batch) { + const sql = `UPDATE transaction_batches SET status='ready', closed_at=now() WHERE id=$1` + + return db.none(sql, [batch.id]) +} + +function confirmSentBatch (batch) { + const sql = `UPDATE transaction_batches SET status='sent', error_message=NULL WHERE id=$1` + + return db.none(sql, [batch.id]) +} + +function setErroredBatch (batch, errorMsg) { + const sql = `UPDATE transaction_batches SET status='failed', error_message=$1 WHERE id=$2` + + return db.none(sql, [errorMsg, batch.id]) +} + +function addTransactionToBatch (tx) { + const sql = `SELECT * FROM transaction_batches WHERE crypto_code=$1 AND status='open' ORDER BY created_at DESC LIMIT 1` + const sql2 = `UPDATE cash_in_txs SET batch_id=$1 WHERE id=$2` + + return db.oneOrNone(sql, [tx.cryptoCode]) + .then(batch => { + if (_.isNil(batch)) + return createTransactionBatch(tx.cryptoCode) + return Promise.resolve(batch) + }) + .then(batch => db.none(sql2, [batch.id, tx.id])) +} + +function getBatchTransactions (batch) { + const sql = `SELECT * FROM cash_in_txs WHERE batch_id=$1` + return db.manyOrNone(sql, [batch.id]) +} + +function getBatchesByStatus (statuses) { + const sql = `SELECT *, EXTRACT(EPOCH FROM (now() - created_at)) as time_elapsed FROM transaction_batches WHERE status in ($1^)` + + return db.manyOrNone(sql, [_.map(pgp.as.text, statuses).join(',')]) +} + +function submitBatch (settings, batch) { + getBatchTransactions(batch) + .then(txs => { + wallet.sendCoinsBatch(settings, txs, batch.crypto_code) + .then(() => confirmSentBatch(batch)) + .catch(err => setErroredBatch(batch, err.message)) + }) +} + +function processBatches (settings, lifecycle) { + getBatchesByStatus(['open', 'failed']) + .then(batches => { + _.each(batch => { + const elapsedMS = batch.time_elapsed * 1000 + + if (elapsedMS >= lifecycle) { + return closeTransactionBatch(batch) + .then(() => submitBatch(settings, batch)) + } + }, batches) + }) +} + +module.exports = { + createTransactionBatch, + closeTransactionBatch, + addTransactionToBatch, + processBatches +} diff --git a/lib/wallet.js b/lib/wallet.js index 6ec32192..d1d3db82 100644 --- a/lib/wallet.js +++ b/lib/wallet.js @@ -75,6 +75,24 @@ function sendCoins (settings, tx) { }) } +function sendCoinsBatch (settings, txs, cryptoCode) { + return fetchWallet(settings, cryptoCode) + .then(r => { + return r.wallet.sendCoinsBatch(r.account, txs, cryptoCode) + .then(res => { + mem.clear(module.exports.balance) + return res + }) + }) + .catch(err => { + if (err.name === INSUFFICIENT_FUNDS_NAME) { + throw httpError(INSUFFICIENT_FUNDS_NAME, INSUFFICIENT_FUNDS_CODE) + } + + throw err + }) +} + function newAddress (settings, info, tx) { const walletAddressPromise = fetchWallet(settings, info.cryptoCode) .then(r => r.wallet.newAddress(r.account, info, tx, settings, r.operatorId)) @@ -235,11 +253,13 @@ const balanceFiltered = mem(_balance, { module.exports = { balance, sendCoins, + sendCoinsBatch, newAddress, getStatus, isStrictAddress, sweep, isHd, newFunding, - cryptoNetwork + cryptoNetwork, + supportsBatching } diff --git a/migrations/1621556014244-add-btc-tx-batching.js b/migrations/1621556014244-add-btc-tx-batching.js index 4750b0fb..69eafc24 100644 --- a/migrations/1621556014244-add-btc-tx-batching.js +++ b/migrations/1621556014244-add-btc-tx-batching.js @@ -2,7 +2,7 @@ var db = require('./db') exports.up = function (next) { var sql = [ - `CREATE TYPE transaction_batch_status AS ENUM('open', 'failed', 'sent')`, + `CREATE TYPE transaction_batch_status AS ENUM('open', 'ready', 'failed', 'sent')`, `CREATE TABLE transaction_batches ( id UUID PRIMARY KEY, crypto_code TEXT NOT NULL, @@ -11,7 +11,7 @@ exports.up = function (next) { closed_at TIMESTAMPTZ, error_message TEXT )`, - `ALTER TABLE cash_in_txs ADD COLUMN batch_id REFERENCES transaction_batches(id)` + `ALTER TABLE cash_in_txs ADD COLUMN batch_id UUID REFERENCES transaction_batches(id)` ] db.multi(sql, next) From 8289c55acfacade7a2a76629717a4dd41500cdfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9rgio=20Salgado?= Date: Mon, 24 May 2021 16:16:51 +0100 Subject: [PATCH 3/9] fix: check for wallet tx batching support --- lib/plugins.js | 32 +++++++++++-------- .../wallet/bitcoincashd/bitcoincashd.js | 8 ++++- lib/plugins/wallet/bitcoind/bitcoind.js | 8 ++++- lib/plugins/wallet/bitgo/bitgo.js | 8 ++++- lib/plugins/wallet/dashd/dashd.js | 8 ++++- lib/plugins/wallet/geth/base.js | 8 ++++- lib/plugins/wallet/litecoind/litecoind.js | 8 ++++- lib/plugins/wallet/mock-wallet/mock-wallet.js | 13 +++++--- lib/plugins/wallet/zcashd/zcashd.js | 8 ++++- lib/wallet.js | 2 +- 10 files changed, 77 insertions(+), 26 deletions(-) diff --git a/lib/plugins.js b/lib/plugins.js index 7f0ed759..89c6a018 100644 --- a/lib/plugins.js +++ b/lib/plugins.js @@ -238,6 +238,7 @@ function plugins (settings, deviceId) { const pingPromise = recordPing(deviceTime, machineVersion, machineModel) const currentConfigVersionPromise = fetchCurrentConfigVersion() const currentAvailablePromoCodes = loyalty.getNumberOfAvailablePromoCodes() + const supportsBatchingPromise = cryptoCodes.map(c => wallet.supportsBatching(settings, c)) const timezoneObj = { utcOffset: timezone[0], dstOffset: timezone[1] } const promises = [ @@ -246,6 +247,7 @@ function plugins (settings, deviceId) { currentConfigVersionPromise, timezoneObj ].concat( + supportsBatchingPromise, tickerPromises, balancePromises, testnetPromises, @@ -258,9 +260,10 @@ function plugins (settings, deviceId) { const configVersion = arr[2] const tz = arr[3] const cryptoCodesCount = cryptoCodes.length - const tickers = arr.slice(4, cryptoCodesCount + 4) - const balances = arr.slice(cryptoCodesCount + 4, 2 * cryptoCodesCount + 4) - const testNets = arr.slice(2 * cryptoCodesCount + 4, arr.length - 2) + const batchableCoins = arr.slice(4, cryptoCodesCount + 4) + const tickers = arr.slice(cryptoCodesCount + 4, 2 * cryptoCodesCount + 4) + const balances = arr.slice(2 * cryptoCodesCount + 4, 3 * cryptoCodesCount + 4) + const testNets = arr.slice(3 * cryptoCodesCount + 4, arr.length - 1) const coinParams = _.zip(cryptoCodes, testNets) const coinsWithoutRate = _.map(mapCoinSettings, coinParams) const areThereAvailablePromoCodes = arr[arr.length - 1] > 0 @@ -278,16 +281,19 @@ function plugins (settings, deviceId) { } function sendCoins (tx) { - if (wallet.supportsBatching(settings, tx.cryptoCode)) { - return transactionBatching.addTransactionToBatch(tx) - .then(() => ({ - batched: true, - sendPending: false, - error: null, - errorCode: null - })) - } - return wallet.sendCoins(settings, tx) + return wallet.supportsBatching(settings, tx.cryptoCode) + .then(supportsBatching => { + if (supportsBatching) { + return transactionBatching.addTransactionToBatch(tx) + .then(() => ({ + batched: true, + sendPending: false, + error: null, + errorCode: null + })) + } + return wallet.sendCoins(settings, tx) + }) } function recordPing (deviceTime, version, model) { diff --git a/lib/plugins/wallet/bitcoincashd/bitcoincashd.js b/lib/plugins/wallet/bitcoincashd/bitcoincashd.js index 3d08156c..14d628ce 100644 --- a/lib/plugins/wallet/bitcoincashd/bitcoincashd.js +++ b/lib/plugins/wallet/bitcoincashd/bitcoincashd.js @@ -129,11 +129,17 @@ function cryptoNetwork (account, cryptoCode, settings, operatorId) { .then(() => parseInt(rpcConfig.port, 10) === 18332 ? 'test' : 'main') } +function supportsBatching (account, cryptoCode) { + return checkCryptoCode(cryptoCode) + .then(() => _.isFunction(sendCoinsBatch)) +} + module.exports = { balance, sendCoins, newAddress, getStatus, newFunding, - cryptoNetwork + cryptoNetwork, + supportsBatching } diff --git a/lib/plugins/wallet/bitcoind/bitcoind.js b/lib/plugins/wallet/bitcoind/bitcoind.js index 516e7f39..7ccb88ea 100644 --- a/lib/plugins/wallet/bitcoind/bitcoind.js +++ b/lib/plugins/wallet/bitcoind/bitcoind.js @@ -170,6 +170,11 @@ function fetchRBF (txId) { }) } +function supportsBatching (account, cryptoCode) { + return checkCryptoCode(cryptoCode) + .then(() => _.isFunction(sendCoinsBatch)) +} + module.exports = { balance, sendCoins, @@ -179,5 +184,6 @@ module.exports = { cryptoNetwork, fetchRBF, estimateFee, - sendCoinsBatch + sendCoinsBatch, + supportsBatching } diff --git a/lib/plugins/wallet/bitgo/bitgo.js b/lib/plugins/wallet/bitgo/bitgo.js index 448e2f84..02abaaa6 100644 --- a/lib/plugins/wallet/bitgo/bitgo.js +++ b/lib/plugins/wallet/bitgo/bitgo.js @@ -157,6 +157,11 @@ function cryptoNetwork (account, cryptoCode, settings, operatorId) { .then(() => account.environment === 'test' ? 'test' : 'main') } +function supportsBatching (account, cryptoCode) { + return checkCryptoCode(cryptoCode) + .then(() => _.isFunction(sendCoinsBatch)) +} + module.exports = { NAME, balance, @@ -164,5 +169,6 @@ module.exports = { newAddress, getStatus, newFunding, - cryptoNetwork + cryptoNetwork, + supportsBatching } diff --git a/lib/plugins/wallet/dashd/dashd.js b/lib/plugins/wallet/dashd/dashd.js index db356210..78fa15c1 100644 --- a/lib/plugins/wallet/dashd/dashd.js +++ b/lib/plugins/wallet/dashd/dashd.js @@ -112,10 +112,16 @@ function newFunding (account, cryptoCode, settings, operatorId) { })) } +function supportsBatching (account, cryptoCode) { + return checkCryptoCode(cryptoCode) + .then(() => _.isFunction(sendCoinsBatch)) +} + module.exports = { balance, sendCoins, newAddress, getStatus, - newFunding + newFunding, + supportsBatching } diff --git a/lib/plugins/wallet/geth/base.js b/lib/plugins/wallet/geth/base.js index 8b4cdab0..0f5329fc 100644 --- a/lib/plugins/wallet/geth/base.js +++ b/lib/plugins/wallet/geth/base.js @@ -29,7 +29,8 @@ module.exports = { newFunding, privateKey, isStrictAddress, - connect + connect, + supportsBatching } function connect (url) { @@ -222,3 +223,8 @@ function newFunding (account, cryptoCode, settings, operatorId) { })) }) } + +function supportsBatching (account, cryptoCode) { + return checkCryptoCode(cryptoCode) + .then(() => _.isFunction(sendCoinsBatch)) +} diff --git a/lib/plugins/wallet/litecoind/litecoind.js b/lib/plugins/wallet/litecoind/litecoind.js index 38373e3b..25d744d0 100644 --- a/lib/plugins/wallet/litecoind/litecoind.js +++ b/lib/plugins/wallet/litecoind/litecoind.js @@ -112,10 +112,16 @@ function newFunding (account, cryptoCode, settings, operatorId) { })) } +function supportsBatching (account, cryptoCode) { + return checkCryptoCode(cryptoCode) + .then(() => _.isFunction(sendCoinsBatch)) +} + module.exports = { balance, sendCoins, newAddress, getStatus, - newFunding + newFunding, + supportsBatching } diff --git a/lib/plugins/wallet/mock-wallet/mock-wallet.js b/lib/plugins/wallet/mock-wallet/mock-wallet.js index 4a945869..31fa8f40 100644 --- a/lib/plugins/wallet/mock-wallet/mock-wallet.js +++ b/lib/plugins/wallet/mock-wallet/mock-wallet.js @@ -3,8 +3,10 @@ const _ = require('lodash/fp') const BN = require('../../../bn') const E = require('../../../error') const { utils: coinUtils } = require('lamassu-coins') +const consoleLogLevel = require('console-log-level') const NAME = 'FakeWallet' +const BATCHABLE_COINS = ['BTC'] const SECONDS = 1000 const PUBLISH_TIME = 3 * SECONDS @@ -59,9 +61,6 @@ function sendCoins (account, tx, settings, operatorId) { }) } -<<<<<<< HEAD -function newAddress (account, info, tx, settings, operatorId) { -======= function sendCoinsBatch (account, txs, cryptoCode) { sendCount = sendCount + txs.length return new Promise((resolve, reject) => { @@ -81,7 +80,6 @@ function sendCoinsBatch (account, txs, cryptoCode) { } function newAddress () { ->>>>>>> feat: add transaction batching module t0 = Date.now() return Promise.resolve('') } @@ -114,6 +112,10 @@ function getStatus (account, tx, requested, settings, operatorId) { return Promise.resolve({status: 'confirmed'}) } +function supportsBatching (account, cryptoCode) { + return Promise.resolve(_.includes(cryptoCode, BATCHABLE_COINS)) +} + module.exports = { NAME, balance, @@ -121,5 +123,6 @@ module.exports = { sendCoins, newAddress, getStatus, - newFunding + newFunding, + supportsBatching } diff --git a/lib/plugins/wallet/zcashd/zcashd.js b/lib/plugins/wallet/zcashd/zcashd.js index d35099a9..b564449f 100644 --- a/lib/plugins/wallet/zcashd/zcashd.js +++ b/lib/plugins/wallet/zcashd/zcashd.js @@ -138,10 +138,16 @@ function newFunding (account, cryptoCode, settings, operatorId) { })) } +function supportsBatching (account, cryptoCode) { + return checkCryptoCode(cryptoCode) + .then(() => _.isFunction(sendCoinsBatch)) +} + module.exports = { balance, sendCoins, newAddress, getStatus, - newFunding + newFunding, + supportsBatching } diff --git a/lib/wallet.js b/lib/wallet.js index d1d3db82..a9fb15a6 100644 --- a/lib/wallet.js +++ b/lib/wallet.js @@ -230,7 +230,7 @@ function isStrictAddress (settings, cryptoCode, toAddress) { function supportsBatching (settings, cryptoCode) { return fetchWallet(settings, cryptoCode) - .then(r => _.isFunction(r.wallet.sendCoinsBatch)) + .then(r => r.wallet.supportsBatching(settings, cryptoCode)) } const coinFilter = ['ETH'] From 00c38ea721180ad6ddbd940c0a7460f8ea70f2d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9rgio=20Salgado?= Date: Mon, 24 May 2021 19:32:45 +0100 Subject: [PATCH 4/9] feat: add batch_time and batched to cash_in_txs feat: add batched flag to clear tx to send feat: add batch information to front-end feat: change transaction sending info on batch sending fix: batching support function added to all wallet plugins fix: mock-wallet batching check feat: send machine information about batchable coins --- lib/cash-in/cash-in-low.js | 6 ++-- lib/cash-in/cash-in-tx.js | 30 +++++++++++++------ .../graphql/types/transaction.type.js | 2 ++ lib/plugins.js | 5 ++-- lib/tx-batching.js | 12 +++++--- .../1621556014244-add-btc-tx-batching.js | 4 ++- .../src/pages/Transactions/Transactions.js | 2 ++ .../src/pages/Transactions/helper.js | 1 + 8 files changed, 43 insertions(+), 19 deletions(-) diff --git a/lib/cash-in/cash-in-low.js b/lib/cash-in/cash-in-low.js index 42f8d3bd..1b7ddf07 100644 --- a/lib/cash-in/cash-in-low.js +++ b/lib/cash-in/cash-in-low.js @@ -102,7 +102,7 @@ function diff (oldTx, newTx) { } function ensureRatchet (oldField, newField, fieldKey) { - const monotonic = ['cryptoAtoms', 'fiat', 'cashInFeeCrypto', 'send', 'sendConfirmed', 'operatorCompleted', 'timedout', 'txVersion'] + const monotonic = ['cryptoAtoms', 'fiat', 'cashInFeeCrypto', 'send', 'sendConfirmed', 'operatorCompleted', 'timedout', 'txVersion', 'batched'] const free = ['sendPending', 'error', 'errorCode', 'customerId'] if (_.isNil(oldField)) return true @@ -138,11 +138,11 @@ function nilEqual (a, b) { function isClearToSend (oldTx, newTx) { const now = Date.now() - return newTx.send && + return (newTx.send || newTx.batched) && (!oldTx || (!oldTx.sendPending && !oldTx.sendConfirmed)) && (newTx.created > now - PENDING_INTERVAL_MS) } function isFinalTxStage (txChanges) { - return txChanges.send + return txChanges.send || txChanges.batched } diff --git a/lib/cash-in/cash-in-tx.js b/lib/cash-in/cash-in-tx.js index f76c6fab..6f932bd5 100644 --- a/lib/cash-in/cash-in-tx.js +++ b/lib/cash-in/cash-in-tx.js @@ -120,15 +120,27 @@ function postProcess (r, pi, isBlacklisted, addressReuse, failedWalletScore) { if (!cashInLow.isClearToSend(r.dbTx, r.tx)) return Promise.resolve({}) return pi.sendCoins(r.tx) - .then(txObj => ({ - txHash: txObj.txid, - fee: txObj.fee, - sendConfirmed: true, - sendTime: 'now()^', - sendPending: false, - error: null, - errorCode: null - })) + .then(txObj => { + if (txObj.batched) { + return { + batched: true, + batchTime: 'now()^', + sendPending: true, + error: null, + errorCode: null + } + } + + return { + txHash: txObj.txid, + fee: txObj.fee, + sendConfirmed: true, + sendTime: 'now()^', + sendPending: false, + error: null, + errorCode: null + } + }) .catch(err => { // Important: We don't know what kind of error this is // so not safe to assume that funds weren't sent. diff --git a/lib/new-admin/graphql/types/transaction.type.js b/lib/new-admin/graphql/types/transaction.type.js index f4c86e3e..c121d0bd 100644 --- a/lib/new-admin/graphql/types/transaction.type.js +++ b/lib/new-admin/graphql/types/transaction.type.js @@ -45,6 +45,8 @@ const typeDef = gql` discount: Int txCustomerPhotoPath: String txCustomerPhotoAt: Date + batched: Boolean + batchTime: Date } type Filter { diff --git a/lib/plugins.js b/lib/plugins.js index 89c6a018..fe7bd663 100644 --- a/lib/plugins.js +++ b/lib/plugins.js @@ -260,7 +260,8 @@ function plugins (settings, deviceId) { const configVersion = arr[2] const tz = arr[3] const cryptoCodesCount = cryptoCodes.length - const batchableCoins = arr.slice(4, cryptoCodesCount + 4) + const batchableCoinsRes = arr.slice(4, cryptoCodesCount + 4) + const batchableCoins = batchableCoinsRes.map(it => ({ batchable: it })) const tickers = arr.slice(cryptoCodesCount + 4, 2 * cryptoCodesCount + 4) const balances = arr.slice(2 * cryptoCodesCount + 4, 3 * cryptoCodesCount + 4) const testNets = arr.slice(3 * cryptoCodesCount + 4, arr.length - 1) @@ -272,7 +273,7 @@ function plugins (settings, deviceId) { cassettes, rates: buildRates(tickers), balances: buildBalances(balances), - coins: _.zipWith(_.assign, coinsWithoutRate, tickers), + coins: _.zipWith(_.assign, _.zipWith(_.assign, coinsWithoutRate, tickers), batchableCoins), configVersion, areThereAvailablePromoCodes, timezone: tz diff --git a/lib/tx-batching.js b/lib/tx-batching.js index 93cd5eb9..fef98b93 100644 --- a/lib/tx-batching.js +++ b/lib/tx-batching.js @@ -2,6 +2,7 @@ const _ = require('lodash/fp') const pgp = require('pg-promise')() const uuid = require('uuid') +const BN = require('./bn') const db = require('./db') const wallet = require('./wallet') @@ -17,10 +18,13 @@ function closeTransactionBatch (batch) { return db.none(sql, [batch.id]) } -function confirmSentBatch (batch) { - const sql = `UPDATE transaction_batches SET status='sent', error_message=NULL WHERE id=$1` +function confirmSentBatch (batch, tx) { + return db.tx(t => { + const q1 = t.none(`UPDATE transaction_batches SET status='sent', error_message=NULL WHERE id=$1`, [batch.id]) + const q2 = t.none(`UPDATE cash_in_txs SET tx_hash=$1, fee=$2, send=true, send_confirmed=true, send_time=now(), send_pending=false, error=NULL, error_code=NULL WHERE batch_id=$3`, [tx.txid, tx.fee.toString(), batch.id]) - return db.none(sql, [batch.id]) + return t.batch([q1, q2]) + }) } function setErroredBatch (batch, errorMsg) { @@ -57,7 +61,7 @@ function submitBatch (settings, batch) { getBatchTransactions(batch) .then(txs => { wallet.sendCoinsBatch(settings, txs, batch.crypto_code) - .then(() => confirmSentBatch(batch)) + .then(res => confirmSentBatch(batch, res)) .catch(err => setErroredBatch(batch, err.message)) }) } diff --git a/migrations/1621556014244-add-btc-tx-batching.js b/migrations/1621556014244-add-btc-tx-batching.js index 69eafc24..fdb92702 100644 --- a/migrations/1621556014244-add-btc-tx-batching.js +++ b/migrations/1621556014244-add-btc-tx-batching.js @@ -11,7 +11,9 @@ exports.up = function (next) { closed_at TIMESTAMPTZ, error_message TEXT )`, - `ALTER TABLE cash_in_txs ADD COLUMN batch_id UUID REFERENCES transaction_batches(id)` + `ALTER TABLE cash_in_txs ADD COLUMN batch_id UUID REFERENCES transaction_batches(id)`, + `ALTER TABLE cash_in_txs ADD COLUMN batched BOOLEAN NOT NULL DEFAULT false`, + `ALTER TABLE cash_in_txs ADD COLUMN batch_time TIMESTAMPTZ` ] db.multi(sql, next) diff --git a/new-lamassu-admin/src/pages/Transactions/Transactions.js b/new-lamassu-admin/src/pages/Transactions/Transactions.js index 049aa210..64cfbac4 100644 --- a/new-lamassu-admin/src/pages/Transactions/Transactions.js +++ b/new-lamassu-admin/src/pages/Transactions/Transactions.js @@ -110,6 +110,8 @@ const GET_TRANSACTIONS = gql` discount customerId isAnonymous + batched + batchTime } } ` diff --git a/new-lamassu-admin/src/pages/Transactions/helper.js b/new-lamassu-admin/src/pages/Transactions/helper.js index 749782db..5e90626d 100644 --- a/new-lamassu-admin/src/pages/Transactions/helper.js +++ b/new-lamassu-admin/src/pages/Transactions/helper.js @@ -11,6 +11,7 @@ const getCashInStatus = it => { if (it.hasError) return 'Error' if (it.sendConfirmed) return 'Sent' if (it.expired) return 'Expired' + if (it.batched) return 'Batched' return 'Pending' } From e6be999f0295f54299a49985749eab8886b6186c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9rgio=20Salgado?= Date: Mon, 7 Jun 2021 18:17:06 +0100 Subject: [PATCH 5/9] fix: supportsBatching function --- lib/plugins/wallet/bitcoincashd/bitcoincashd.js | 4 +++- lib/plugins/wallet/bitcoind/bitcoind.js | 4 +++- lib/plugins/wallet/bitgo/bitgo.js | 4 +++- lib/plugins/wallet/dashd/dashd.js | 4 +++- lib/plugins/wallet/geth/base.js | 4 +++- lib/plugins/wallet/litecoind/litecoind.js | 4 +++- lib/plugins/wallet/zcashd/zcashd.js | 4 +++- 7 files changed, 21 insertions(+), 7 deletions(-) diff --git a/lib/plugins/wallet/bitcoincashd/bitcoincashd.js b/lib/plugins/wallet/bitcoincashd/bitcoincashd.js index 14d628ce..5d5f187d 100644 --- a/lib/plugins/wallet/bitcoincashd/bitcoincashd.js +++ b/lib/plugins/wallet/bitcoincashd/bitcoincashd.js @@ -10,6 +10,8 @@ const cryptoRec = coinUtils.getCryptoCurrency('BCH') const configPath = coinUtils.configPath(cryptoRec, options.blockchainDir) const unitScale = cryptoRec.unitScale +const SUPPORTS_BATCHING = false + function rpcConfig () { try { const config = jsonRpc.parseConf(configPath) @@ -131,7 +133,7 @@ function cryptoNetwork (account, cryptoCode, settings, operatorId) { function supportsBatching (account, cryptoCode) { return checkCryptoCode(cryptoCode) - .then(() => _.isFunction(sendCoinsBatch)) + .then(() => SUPPORTS_BATCHING) } module.exports = { diff --git a/lib/plugins/wallet/bitcoind/bitcoind.js b/lib/plugins/wallet/bitcoind/bitcoind.js index 7ccb88ea..7aaeec1f 100644 --- a/lib/plugins/wallet/bitcoind/bitcoind.js +++ b/lib/plugins/wallet/bitcoind/bitcoind.js @@ -9,6 +9,8 @@ const { utils: coinUtils } = require('lamassu-coins') const cryptoRec = coinUtils.getCryptoCurrency('BTC') const unitScale = cryptoRec.unitScale + +const SUPPORTS_BATCHING = true const rpcConfig = jsonRpc.rpcConfig(cryptoRec) function fetch (method, params) { @@ -172,7 +174,7 @@ function fetchRBF (txId) { function supportsBatching (account, cryptoCode) { return checkCryptoCode(cryptoCode) - .then(() => _.isFunction(sendCoinsBatch)) + .then(() => SUPPORTS_BATCHING) } module.exports = { diff --git a/lib/plugins/wallet/bitgo/bitgo.js b/lib/plugins/wallet/bitgo/bitgo.js index 02abaaa6..56192de9 100644 --- a/lib/plugins/wallet/bitgo/bitgo.js +++ b/lib/plugins/wallet/bitgo/bitgo.js @@ -14,6 +14,8 @@ const NAME = 'BitGo' const SUPPORTED_COINS = ['BTC', 'ZEC', 'LTC', 'BCH', 'DASH'] const BCH_CODES = ['BCH', 'TBCH'] +const SUPPORTS_BATCHING = false + function buildBitgo (account) { const env = account.environment === 'test' ? 'test' : 'prod' return new BitGo.BitGo({ accessToken: account.token.trim(), env, userAgent: userAgent }) @@ -159,7 +161,7 @@ function cryptoNetwork (account, cryptoCode, settings, operatorId) { function supportsBatching (account, cryptoCode) { return checkCryptoCode(cryptoCode) - .then(() => _.isFunction(sendCoinsBatch)) + .then(() => SUPPORTS_BATCHING) } module.exports = { diff --git a/lib/plugins/wallet/dashd/dashd.js b/lib/plugins/wallet/dashd/dashd.js index 78fa15c1..74217b2b 100644 --- a/lib/plugins/wallet/dashd/dashd.js +++ b/lib/plugins/wallet/dashd/dashd.js @@ -9,6 +9,8 @@ const E = require('../../../error') const cryptoRec = coinUtils.getCryptoCurrency('DASH') const unitScale = cryptoRec.unitScale + +const SUPPORTS_BATCHING = false const rpcConfig = jsonRpc.rpcConfig(cryptoRec) function fetch (method, params) { @@ -114,7 +116,7 @@ function newFunding (account, cryptoCode, settings, operatorId) { function supportsBatching (account, cryptoCode) { return checkCryptoCode(cryptoCode) - .then(() => _.isFunction(sendCoinsBatch)) + .then(() => SUPPORTS_BATCHING) } module.exports = { diff --git a/lib/plugins/wallet/geth/base.js b/lib/plugins/wallet/geth/base.js index 0f5329fc..e85ab8cc 100644 --- a/lib/plugins/wallet/geth/base.js +++ b/lib/plugins/wallet/geth/base.js @@ -17,6 +17,8 @@ const paymentPrefixPath = "m/44'/60'/0'/0'" const defaultPrefixPath = "m/44'/60'/1'/0'" let lastUsedNonces = {} +const SUPPORTS_BATCHING = false + module.exports = { NAME, balance, @@ -226,5 +228,5 @@ function newFunding (account, cryptoCode, settings, operatorId) { function supportsBatching (account, cryptoCode) { return checkCryptoCode(cryptoCode) - .then(() => _.isFunction(sendCoinsBatch)) + .then(() => SUPPORTS_BATCHING) } diff --git a/lib/plugins/wallet/litecoind/litecoind.js b/lib/plugins/wallet/litecoind/litecoind.js index 25d744d0..5017252a 100644 --- a/lib/plugins/wallet/litecoind/litecoind.js +++ b/lib/plugins/wallet/litecoind/litecoind.js @@ -9,6 +9,8 @@ const E = require('../../../error') const cryptoRec = coinUtils.getCryptoCurrency('LTC') const unitScale = cryptoRec.unitScale + +const SUPPORTS_BATCHING = false const rpcConfig = jsonRpc.rpcConfig(cryptoRec) function fetch (method, params) { @@ -114,7 +116,7 @@ function newFunding (account, cryptoCode, settings, operatorId) { function supportsBatching (account, cryptoCode) { return checkCryptoCode(cryptoCode) - .then(() => _.isFunction(sendCoinsBatch)) + .then(() => SUPPORTS_BATCHING) } module.exports = { diff --git a/lib/plugins/wallet/zcashd/zcashd.js b/lib/plugins/wallet/zcashd/zcashd.js index b564449f..4544f65d 100644 --- a/lib/plugins/wallet/zcashd/zcashd.js +++ b/lib/plugins/wallet/zcashd/zcashd.js @@ -10,6 +10,8 @@ const E = require('../../../error') const cryptoRec = coinUtils.getCryptoCurrency('ZEC') const unitScale = cryptoRec.unitScale +const SUPPORTS_BATCHING = false + const rpcConfig = jsonRpc.rpcConfig(cryptoRec) function fetch (method, params) { @@ -140,7 +142,7 @@ function newFunding (account, cryptoCode, settings, operatorId) { function supportsBatching (account, cryptoCode) { return checkCryptoCode(cryptoCode) - .then(() => _.isFunction(sendCoinsBatch)) + .then(() => SUPPORTS_BATCHING) } module.exports = { From bef24cf3a3192c5b4df9c48f0903663ff7c10588 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9rgio=20Salgado?= Date: Wed, 9 Jun 2021 14:39:36 +0100 Subject: [PATCH 6/9] fix: remove settings parameter from individual wallet interfaces --- lib/plugins/wallet/bitcoincashd/bitcoincashd.js | 2 +- lib/plugins/wallet/bitcoind/bitcoind.js | 2 +- lib/plugins/wallet/bitgo/bitgo.js | 2 +- lib/plugins/wallet/dashd/dashd.js | 2 +- lib/plugins/wallet/geth/base.js | 2 +- lib/plugins/wallet/litecoind/litecoind.js | 2 +- lib/plugins/wallet/mock-wallet/mock-wallet.js | 2 +- lib/plugins/wallet/zcashd/zcashd.js | 2 +- lib/wallet.js | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/plugins/wallet/bitcoincashd/bitcoincashd.js b/lib/plugins/wallet/bitcoincashd/bitcoincashd.js index 5d5f187d..7ba5871e 100644 --- a/lib/plugins/wallet/bitcoincashd/bitcoincashd.js +++ b/lib/plugins/wallet/bitcoincashd/bitcoincashd.js @@ -131,7 +131,7 @@ function cryptoNetwork (account, cryptoCode, settings, operatorId) { .then(() => parseInt(rpcConfig.port, 10) === 18332 ? 'test' : 'main') } -function supportsBatching (account, cryptoCode) { +function supportsBatching (cryptoCode) { return checkCryptoCode(cryptoCode) .then(() => SUPPORTS_BATCHING) } diff --git a/lib/plugins/wallet/bitcoind/bitcoind.js b/lib/plugins/wallet/bitcoind/bitcoind.js index 7aaeec1f..a5d5256d 100644 --- a/lib/plugins/wallet/bitcoind/bitcoind.js +++ b/lib/plugins/wallet/bitcoind/bitcoind.js @@ -172,7 +172,7 @@ function fetchRBF (txId) { }) } -function supportsBatching (account, cryptoCode) { +function supportsBatching (cryptoCode) { return checkCryptoCode(cryptoCode) .then(() => SUPPORTS_BATCHING) } diff --git a/lib/plugins/wallet/bitgo/bitgo.js b/lib/plugins/wallet/bitgo/bitgo.js index 56192de9..34621840 100644 --- a/lib/plugins/wallet/bitgo/bitgo.js +++ b/lib/plugins/wallet/bitgo/bitgo.js @@ -159,7 +159,7 @@ function cryptoNetwork (account, cryptoCode, settings, operatorId) { .then(() => account.environment === 'test' ? 'test' : 'main') } -function supportsBatching (account, cryptoCode) { +function supportsBatching (cryptoCode) { return checkCryptoCode(cryptoCode) .then(() => SUPPORTS_BATCHING) } diff --git a/lib/plugins/wallet/dashd/dashd.js b/lib/plugins/wallet/dashd/dashd.js index 74217b2b..79c773b7 100644 --- a/lib/plugins/wallet/dashd/dashd.js +++ b/lib/plugins/wallet/dashd/dashd.js @@ -114,7 +114,7 @@ function newFunding (account, cryptoCode, settings, operatorId) { })) } -function supportsBatching (account, cryptoCode) { +function supportsBatching (cryptoCode) { return checkCryptoCode(cryptoCode) .then(() => SUPPORTS_BATCHING) } diff --git a/lib/plugins/wallet/geth/base.js b/lib/plugins/wallet/geth/base.js index e85ab8cc..26b3a50d 100644 --- a/lib/plugins/wallet/geth/base.js +++ b/lib/plugins/wallet/geth/base.js @@ -226,7 +226,7 @@ function newFunding (account, cryptoCode, settings, operatorId) { }) } -function supportsBatching (account, cryptoCode) { +function supportsBatching (cryptoCode) { return checkCryptoCode(cryptoCode) .then(() => SUPPORTS_BATCHING) } diff --git a/lib/plugins/wallet/litecoind/litecoind.js b/lib/plugins/wallet/litecoind/litecoind.js index 5017252a..1b4c6236 100644 --- a/lib/plugins/wallet/litecoind/litecoind.js +++ b/lib/plugins/wallet/litecoind/litecoind.js @@ -114,7 +114,7 @@ function newFunding (account, cryptoCode, settings, operatorId) { })) } -function supportsBatching (account, cryptoCode) { +function supportsBatching (cryptoCode) { return checkCryptoCode(cryptoCode) .then(() => SUPPORTS_BATCHING) } diff --git a/lib/plugins/wallet/mock-wallet/mock-wallet.js b/lib/plugins/wallet/mock-wallet/mock-wallet.js index 31fa8f40..a7a9493c 100644 --- a/lib/plugins/wallet/mock-wallet/mock-wallet.js +++ b/lib/plugins/wallet/mock-wallet/mock-wallet.js @@ -112,7 +112,7 @@ function getStatus (account, tx, requested, settings, operatorId) { return Promise.resolve({status: 'confirmed'}) } -function supportsBatching (account, cryptoCode) { +function supportsBatching (cryptoCode) { return Promise.resolve(_.includes(cryptoCode, BATCHABLE_COINS)) } diff --git a/lib/plugins/wallet/zcashd/zcashd.js b/lib/plugins/wallet/zcashd/zcashd.js index 4544f65d..b37d0dd3 100644 --- a/lib/plugins/wallet/zcashd/zcashd.js +++ b/lib/plugins/wallet/zcashd/zcashd.js @@ -140,7 +140,7 @@ function newFunding (account, cryptoCode, settings, operatorId) { })) } -function supportsBatching (account, cryptoCode) { +function supportsBatching (cryptoCode) { return checkCryptoCode(cryptoCode) .then(() => SUPPORTS_BATCHING) } diff --git a/lib/wallet.js b/lib/wallet.js index a9fb15a6..6ed54f63 100644 --- a/lib/wallet.js +++ b/lib/wallet.js @@ -230,7 +230,7 @@ function isStrictAddress (settings, cryptoCode, toAddress) { function supportsBatching (settings, cryptoCode) { return fetchWallet(settings, cryptoCode) - .then(r => r.wallet.supportsBatching(settings, cryptoCode)) + .then(r => r.wallet.supportsBatching(cryptoCode)) } const coinFilter = ['ETH'] From f6e793ea4d8b0ea38377be0cdecbe5740935f5ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9rgio=20Salgado?= Date: Wed, 24 Nov 2021 15:34:00 +0000 Subject: [PATCH 7/9] fix: usage of .plus function in BN calculation fix: move batching database writing to transactions --- lib/plugins/wallet/mock-wallet/mock-wallet.js | 2 +- lib/tx-batching.js | 85 ++++++++++--------- 2 files changed, 46 insertions(+), 41 deletions(-) diff --git a/lib/plugins/wallet/mock-wallet/mock-wallet.js b/lib/plugins/wallet/mock-wallet/mock-wallet.js index a7a9493c..f8eb7a7f 100644 --- a/lib/plugins/wallet/mock-wallet/mock-wallet.js +++ b/lib/plugins/wallet/mock-wallet/mock-wallet.js @@ -65,7 +65,7 @@ function sendCoinsBatch (account, txs, cryptoCode) { sendCount = sendCount + txs.length return new Promise((resolve, reject) => { setTimeout(() => { - const cryptoSum = _.reduce((acc, value) => acc.add(value.crypto_atoms), BN(0), txs) + const cryptoSum = _.reduce((acc, value) => acc.plus(value.crypto_atoms), BN(0), txs) if (isInsufficient(cryptoSum, cryptoCode)) { console.log('[%s] DEBUG: Mock wallet insufficient funds: %s', cryptoCode, cryptoSum.toString()) diff --git a/lib/tx-batching.js b/lib/tx-batching.js index fef98b93..0e7ce6ee 100644 --- a/lib/tx-batching.js +++ b/lib/tx-batching.js @@ -12,27 +12,6 @@ function createTransactionBatch (cryptoCode) { return db.one(sql, [uuid.v4(), cryptoCode]) } -function closeTransactionBatch (batch) { - const sql = `UPDATE transaction_batches SET status='ready', closed_at=now() WHERE id=$1` - - return db.none(sql, [batch.id]) -} - -function confirmSentBatch (batch, tx) { - return db.tx(t => { - const q1 = t.none(`UPDATE transaction_batches SET status='sent', error_message=NULL WHERE id=$1`, [batch.id]) - const q2 = t.none(`UPDATE cash_in_txs SET tx_hash=$1, fee=$2, send=true, send_confirmed=true, send_time=now(), send_pending=false, error=NULL, error_code=NULL WHERE batch_id=$3`, [tx.txid, tx.fee.toString(), batch.id]) - - return t.batch([q1, q2]) - }) -} - -function setErroredBatch (batch, errorMsg) { - const sql = `UPDATE transaction_batches SET status='failed', error_message=$1 WHERE id=$2` - - return db.none(sql, [errorMsg, batch.id]) -} - function addTransactionToBatch (tx) { const sql = `SELECT * FROM transaction_batches WHERE crypto_code=$1 AND status='open' ORDER BY created_at DESC LIMIT 1` const sql2 = `UPDATE cash_in_txs SET batch_id=$1 WHERE id=$2` @@ -46,38 +25,64 @@ function addTransactionToBatch (tx) { .then(batch => db.none(sql2, [batch.id, tx.id])) } -function getBatchTransactions (batch) { - const sql = `SELECT * FROM cash_in_txs WHERE batch_id=$1` - return db.manyOrNone(sql, [batch.id]) +function closeTransactionBatch (t, batch) { + const sql = `UPDATE transaction_batches SET status='ready', closed_at=now() WHERE id=$1` + + return t.none(sql, [batch.id]) } -function getBatchesByStatus (statuses) { +function confirmSentBatch (t, batch, tx) { + return t.none(`UPDATE transaction_batches SET status='sent', error_message=NULL WHERE id=$1`, [batch.id]) + .then(() => + t.none(`UPDATE cash_in_txs SET tx_hash=$1, fee=$2, send=true, send_confirmed=true, send_time=now(), send_pending=false, error=NULL, error_code=NULL WHERE batch_id=$3`, [tx.txid, tx.fee.toString(), batch.id]) + ) +} + +function setErroredBatch (t, batch, errorMsg) { + const sql = `UPDATE transaction_batches SET status='failed', error_message=$1 WHERE id=$2` + + return t.none(sql, [errorMsg, batch.id]) +} + +function getBatchTransactions (t, batch) { + const sql = `SELECT * FROM cash_in_txs WHERE batch_id=$1` + return t.manyOrNone(sql, [batch.id]) +} + +function getBatchesByStatus (t, statuses) { const sql = `SELECT *, EXTRACT(EPOCH FROM (now() - created_at)) as time_elapsed FROM transaction_batches WHERE status in ($1^)` - return db.manyOrNone(sql, [_.map(pgp.as.text, statuses).join(',')]) + return t.manyOrNone(sql, [_.map(pgp.as.text, statuses).join(',')]) } -function submitBatch (settings, batch) { - getBatchTransactions(batch) +function submitBatch (t, settings, batch) { + getBatchTransactions(t, batch) .then(txs => { wallet.sendCoinsBatch(settings, txs, batch.crypto_code) - .then(res => confirmSentBatch(batch, res)) - .catch(err => setErroredBatch(batch, err.message)) + .then(res => confirmSentBatch(t, batch, res)) + .catch(err => setErroredBatch(t, batch, err.message)) }) } function processBatches (settings, lifecycle) { - getBatchesByStatus(['open', 'failed']) - .then(batches => { - _.each(batch => { - const elapsedMS = batch.time_elapsed * 1000 + const transaction = t => { + getBatchesByStatus(t, ['open', 'failed', 'ready']) + .then(batches => { + _.each(batch => { + const elapsedMS = batch.time_elapsed * 1000 - if (elapsedMS >= lifecycle) { - return closeTransactionBatch(batch) - .then(() => submitBatch(settings, batch)) - } - }, batches) - }) + if (elapsedMS >= lifecycle) { + return closeTransactionBatch(t, batch) + .then(() => submitBatch(t, settings, batch)) + } + }, batches) + }) + } + + return db.tx( + new pgp.txMode.TransactionMode({ tiLevel: pgp.txMode.isolationLevel }), + transaction + ) } module.exports = { From 0132227251f0247af5901824991586b1e617c465 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9rgio=20Salgado?= Date: Wed, 24 Nov 2021 18:07:54 +0000 Subject: [PATCH 8/9] fix: batching db transactions --- lib/tx-batching.js | 85 +++++++++++++++++++++------------------------- 1 file changed, 39 insertions(+), 46 deletions(-) diff --git a/lib/tx-batching.js b/lib/tx-batching.js index 0e7ce6ee..a2cc3b42 100644 --- a/lib/tx-batching.js +++ b/lib/tx-batching.js @@ -6,88 +6,81 @@ const BN = require('./bn') const db = require('./db') const wallet = require('./wallet') -function createTransactionBatch (cryptoCode) { - const sql = `INSERT INTO transaction_batches (id, crypto_code) VALUES ($1, $2) RETURNING *` - - return db.one(sql, [uuid.v4(), cryptoCode]) -} - function addTransactionToBatch (tx) { const sql = `SELECT * FROM transaction_batches WHERE crypto_code=$1 AND status='open' ORDER BY created_at DESC LIMIT 1` const sql2 = `UPDATE cash_in_txs SET batch_id=$1 WHERE id=$2` return db.oneOrNone(sql, [tx.cryptoCode]) .then(batch => { - if (_.isNil(batch)) - return createTransactionBatch(tx.cryptoCode) - return Promise.resolve(batch) + if (_.isNil(batch)) { + return db.tx(t => { + const newBatchId = uuid.v4() + const q1 = t.none(`INSERT INTO transaction_batches (id, crypto_code) VALUES ($1, $2)`, [newBatchId, tx.cryptoCode]) + const q2 = t.none(sql2, [newBatchId, tx.id]) + + return t.batch([q1, q2]) + }) + } + return db.none(sql2, [batch.id, tx.id]) }) - .then(batch => db.none(sql2, [batch.id, tx.id])) } -function closeTransactionBatch (t, batch) { +function closeTransactionBatch (batch) { const sql = `UPDATE transaction_batches SET status='ready', closed_at=now() WHERE id=$1` - return t.none(sql, [batch.id]) + return db.none(sql, [batch.id]) } -function confirmSentBatch (t, batch, tx) { - return t.none(`UPDATE transaction_batches SET status='sent', error_message=NULL WHERE id=$1`, [batch.id]) - .then(() => - t.none(`UPDATE cash_in_txs SET tx_hash=$1, fee=$2, send=true, send_confirmed=true, send_time=now(), send_pending=false, error=NULL, error_code=NULL WHERE batch_id=$3`, [tx.txid, tx.fee.toString(), batch.id]) - ) +function confirmSentBatch (batch, tx) { + return db.tx(t => { + const q1 = t.none(`UPDATE transaction_batches SET status='sent', error_message=NULL WHERE id=$1`, [batch.id]) + const q2 = t.none(`UPDATE cash_in_txs SET tx_hash=$1, fee=$2, send=true, send_confirmed=true, send_time=now(), send_pending=false, error=NULL, error_code=NULL WHERE batch_id=$3`, [tx.txid, tx.fee.toString(), batch.id]) + + return t.batch([q1, q2]) + }) } -function setErroredBatch (t, batch, errorMsg) { +function setErroredBatch (batch, errorMsg) { const sql = `UPDATE transaction_batches SET status='failed', error_message=$1 WHERE id=$2` - return t.none(sql, [errorMsg, batch.id]) + return db.none(sql, [errorMsg, batch.id]) } -function getBatchTransactions (t, batch) { +function getBatchTransactions (batch) { const sql = `SELECT * FROM cash_in_txs WHERE batch_id=$1` - return t.manyOrNone(sql, [batch.id]) + return db.manyOrNone(sql, [batch.id]) } -function getBatchesByStatus (t, statuses) { +function getBatchesByStatus (statuses) { const sql = `SELECT *, EXTRACT(EPOCH FROM (now() - created_at)) as time_elapsed FROM transaction_batches WHERE status in ($1^)` - return t.manyOrNone(sql, [_.map(pgp.as.text, statuses).join(',')]) + return db.manyOrNone(sql, [_.map(pgp.as.text, statuses).join(',')]) } -function submitBatch (t, settings, batch) { - getBatchTransactions(t, batch) +function submitBatch (settings, batch) { + getBatchTransactions(batch) .then(txs => { wallet.sendCoinsBatch(settings, txs, batch.crypto_code) - .then(res => confirmSentBatch(t, batch, res)) - .catch(err => setErroredBatch(t, batch, err.message)) + .then(res => confirmSentBatch(batch, res)) + .catch(err => setErroredBatch(batch, err.message)) }) } function processBatches (settings, lifecycle) { - const transaction = t => { - getBatchesByStatus(t, ['open', 'failed', 'ready']) - .then(batches => { - _.each(batch => { - const elapsedMS = batch.time_elapsed * 1000 + return getBatchesByStatus(['open']) + .then(batches => { + _.each(batch => { + const elapsedMS = batch.time_elapsed * 1000 - if (elapsedMS >= lifecycle) { - return closeTransactionBatch(t, batch) - .then(() => submitBatch(t, settings, batch)) - } - }, batches) - }) - } - - return db.tx( - new pgp.txMode.TransactionMode({ tiLevel: pgp.txMode.isolationLevel }), - transaction - ) + if (elapsedMS >= lifecycle) { + return closeTransactionBatch(batch) + .then(() => submitBatch(settings, batch)) + } + }, batches) + }) } module.exports = { - createTransactionBatch, - closeTransactionBatch, addTransactionToBatch, processBatches } From 008dd3e9f61d4dfcb7b03e7ebb15038c7e624395 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9rgio=20Salgado?= Date: Wed, 24 Nov 2021 19:09:15 +0000 Subject: [PATCH 9/9] feat: add crypto reservation for batched transaction --- lib/poller.js | 4 ++-- lib/tx-batching-processing.js | 29 +++++++++++++++++++++++++++++ lib/tx-batching.js | 34 +++++++++++++--------------------- lib/wallet.js | 6 +++++- 4 files changed, 49 insertions(+), 24 deletions(-) create mode 100644 lib/tx-batching-processing.js diff --git a/lib/poller.js b/lib/poller.js index 1c05bef1..25011b17 100644 --- a/lib/poller.js +++ b/lib/poller.js @@ -17,7 +17,7 @@ const NodeCache = require('node-cache') const util = require('util') const db = require('./db') const state = require('./middlewares/state') -const batching = require('./tx-batching') +const processBatches = require('./tx-batching-processing') const INCOMING_TX_INTERVAL = 30 * T.seconds const LIVE_INCOMING_TX_INTERVAL = 5 * T.seconds @@ -216,7 +216,7 @@ function doPolling (schema) { } addToQueue(cashOutTx.monitorUnnotified, UNNOTIFIED_INTERVAL, schema, QUEUE.FAST, settings) addToQueue(cashInTx.monitorPending, PENDING_INTERVAL, schema, QUEUE.FAST, settings) - addToQueue(batching.processBatches, UNNOTIFIED_INTERVAL, schema, QUEUE.FAST, settings, TRANSACTION_BATCH_LIFECYCLE) + addToQueue(processBatches, UNNOTIFIED_INTERVAL, schema, QUEUE.FAST, settings, TRANSACTION_BATCH_LIFECYCLE) addToQueue(pi().sweepHd, SWEEP_HD_INTERVAL, schema, QUEUE.FAST, settings) addToQueue(pi().pong, PONG_INTERVAL, schema, QUEUE.FAST) addToQueue(pi().clearOldLogs, LOGS_CLEAR_INTERVAL, schema, QUEUE.SLOW) diff --git a/lib/tx-batching-processing.js b/lib/tx-batching-processing.js new file mode 100644 index 00000000..1e7045d1 --- /dev/null +++ b/lib/tx-batching-processing.js @@ -0,0 +1,29 @@ +const _ = require('lodash/fp') + +const txBatching = require('./tx-batching') +const wallet = require('./wallet') + +function submitBatch (settings, batch) { + txBatching.getBatchTransactions(batch) + .then(txs => { + wallet.sendCoinsBatch(settings, txs, batch.crypto_code) + .then(res => txBatching.confirmSentBatch(batch, res)) + .catch(err => txBatching.setErroredBatch(batch, err.message)) + }) +} + +function processBatches (settings, lifecycle) { + return txBatching.getBatchesByStatus(['open']) + .then(batches => { + _.each(batch => { + const elapsedMS = batch.time_elapsed * 1000 + + if (elapsedMS >= lifecycle) { + return txBatching.closeTransactionBatch(batch) + .then(() => submitBatch(settings, batch)) + } + }, batches) + }) +} + +module.exports = processBatches diff --git a/lib/tx-batching.js b/lib/tx-batching.js index a2cc3b42..e36c48d7 100644 --- a/lib/tx-batching.js +++ b/lib/tx-batching.js @@ -4,7 +4,6 @@ const uuid = require('uuid') const BN = require('./bn') const db = require('./db') -const wallet = require('./wallet') function addTransactionToBatch (tx) { const sql = `SELECT * FROM transaction_batches WHERE crypto_code=$1 AND status='open' ORDER BY created_at DESC LIMIT 1` @@ -57,30 +56,23 @@ function getBatchesByStatus (statuses) { return db.manyOrNone(sql, [_.map(pgp.as.text, statuses).join(',')]) } -function submitBatch (settings, batch) { - getBatchTransactions(batch) - .then(txs => { - wallet.sendCoinsBatch(settings, txs, batch.crypto_code) - .then(res => confirmSentBatch(batch, res)) - .catch(err => setErroredBatch(batch, err.message)) - }) -} +function getOpenBatchCryptoValue (cryptoCode) { + const sql = `SELECT * FROM transaction_batches WHERE crypto_code=$1 AND status='open' ORDER BY created_at DESC LIMIT 1` -function processBatches (settings, lifecycle) { - return getBatchesByStatus(['open']) - .then(batches => { - _.each(batch => { - const elapsedMS = batch.time_elapsed * 1000 - - if (elapsedMS >= lifecycle) { - return closeTransactionBatch(batch) - .then(() => submitBatch(settings, batch)) - } - }, batches) + return db.oneOrNone(sql, [cryptoCode]) + .then(batch => { + if (_.isNil(batch)) return Promise.resolve([]) + return db.any(`SELECT * FROM cash_in_txs WHERE batch_id=$1`, [batch.id]) }) + .then(txs => _.reduce((acc, tx) => acc.plus(tx.cash_in_fee_crypto).plus(tx.crypto_atoms), BN(0), txs)) } module.exports = { addTransactionToBatch, - processBatches + closeTransactionBatch, + confirmSentBatch, + setErroredBatch, + getBatchTransactions, + getBatchesByStatus, + getOpenBatchCryptoValue } diff --git a/lib/wallet.js b/lib/wallet.js index 6ed54f63..92d1a7b4 100644 --- a/lib/wallet.js +++ b/lib/wallet.js @@ -13,6 +13,8 @@ const ph = require('./plugin-helper') const layer2 = require('./layer2') const httpError = require('./route-helpers').httpError const logger = require('./logger') +const { getOpenBatchCryptoValue } = require('./tx-batching') +const BN = require('./bn') const FETCH_INTERVAL = 5000 const INSUFFICIENT_FUNDS_CODE = 570 @@ -46,7 +48,9 @@ const lastBalance = {} function _balance (settings, cryptoCode) { return fetchWallet(settings, cryptoCode) .then(r => r.wallet.balance(r.account, cryptoCode, settings, r.operatorId)) - .then(balance => ({ balance, timestamp: Date.now() })) + .then(balance => Promise.all([balance, supportsBatching(settings, cryptoCode)])) + .then(([balance, supportsBatching]) => Promise.all([balance, supportsBatching ? getOpenBatchCryptoValue(cryptoCode) : Promise.resolve(BN(0))])) + .then(([balance, reservedBalance]) => ({ balance: balance.minus(reservedBalance), reservedBalance, timestamp: Date.now() })) .then(r => { lastBalance[cryptoCode] = r return r