From ed1fb5252a3616a2bcaac42809a2bebb87912902 Mon Sep 17 00:00:00 2001 From: Rafael Date: Tue, 3 Dec 2024 13:54:28 +0000 Subject: [PATCH 01/10] chore: bump version --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index aef7e6ec..9ec3d0e5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "lamassu-server", - "version": "10.0.6", + "version": "10.1.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 77c3504e..9d4511c6 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "lamassu-server", "description": "bitcoin atm client server protocol module", "keywords": [], - "version": "10.0.6", + "version": "10.1.0", "license": "./LICENSE", "author": "Lamassu (https://lamassu.is)", "dependencies": { From 9592d92bfdb22bbafd6af35493f3e5761a6fc003 Mon Sep 17 00:00:00 2001 From: CrypticaScriptura <7396812+CrypticaScriptura@users.noreply.github.com> Date: Tue, 3 Dec 2024 10:25:09 -0500 Subject: [PATCH 02/10] chore: update wallet node releases --- lib/blockchain/common.js | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/lib/blockchain/common.js b/lib/blockchain/common.js index 8db73a72..5777d2ab 100644 --- a/lib/blockchain/common.js +++ b/lib/blockchain/common.js @@ -30,37 +30,37 @@ const BINARIES = { BTC: { defaultUrl: 'https://bitcoincore.org/bin/bitcoin-core-0.20.1/bitcoin-0.20.1-x86_64-linux-gnu.tar.gz', defaultDir: 'bitcoin-0.20.1/bin', - url: 'https://bitcoincore.org/bin/bitcoin-core-27.1/bitcoin-27.1-x86_64-linux-gnu.tar.gz', - dir: 'bitcoin-27.1/bin' + url: 'https://bitcoincore.org/bin/bitcoin-core-28.0/bitcoin-28.0-x86_64-linux-gnu.tar.gz', + dir: 'bitcoin-28.0/bin' }, ETH: { - url: 'https://gethstore.blob.core.windows.net/builds/geth-linux-amd64-1.14.8-a9523b64.tar.gz', - dir: 'geth-linux-amd64-1.14.8-a9523b64' + url: 'https://gethstore.blob.core.windows.net/builds/geth-linux-amd64-1.14.12-293a300d.tar.gz', + dir: 'geth-linux-amd64-1.14.12-293a300d' }, ZEC: { - url: 'https://github.com/zcash/artifacts/raw/master/v5.9.0/bullseye/zcash-5.9.0-linux64-debian-bullseye.tar.gz', - dir: 'zcash-5.9.0/bin' + url: 'https://download.z.cash/downloads/zcash-6.0.0-linux64-debian-bullseye.tar.gz', + dir: 'zcash-6.0.0/bin' }, DASH: { defaultUrl: 'https://github.com/dashpay/dash/releases/download/v18.1.0/dashcore-18.1.0-x86_64-linux-gnu.tar.gz', defaultDir: 'dashcore-18.1.0/bin', - url: 'https://github.com/dashpay/dash/releases/download/v21.1.0/dashcore-21.1.0-x86_64-linux-gnu.tar.gz', - dir: 'dashcore-21.1.0/bin' + url: 'https://github.com/dashpay/dash/releases/download/v21.1.1/dashcore-21.1.1-x86_64-linux-gnu.tar.gz', + dir: 'dashcore-21.1.1/bin' }, LTC: { defaultUrl: 'https://download.litecoin.org/litecoin-0.18.1/linux/litecoin-0.18.1-x86_64-linux-gnu.tar.gz', defaultDir: 'litecoin-0.18.1/bin', - url: 'https://download.litecoin.org/litecoin-0.21.3/linux/litecoin-0.21.3-x86_64-linux-gnu.tar.gz', - dir: 'litecoin-0.21.3/bin' + url: 'https://download.litecoin.org/litecoin-0.21.4/linux/litecoin-0.21.4-x86_64-linux-gnu.tar.gz', + dir: 'litecoin-0.21.4/bin' }, BCH: { - url: 'https://github.com/bitcoin-cash-node/bitcoin-cash-node/releases/download/v27.1.0/bitcoin-cash-node-27.1.0-x86_64-linux-gnu.tar.gz', - dir: 'bitcoin-cash-node-27.1.0/bin', + url: 'https://github.com/bitcoin-cash-node/bitcoin-cash-node/releases/download/v28.0.0/bitcoin-cash-node-28.0.0-x86_64-linux-gnu.tar.gz', + dir: 'bitcoin-cash-node-28.0.0/bin', files: [['bitcoind', 'bitcoincashd'], ['bitcoin-cli', 'bitcoincash-cli']] }, XMR: { - url: 'https://downloads.getmonero.org/cli/monero-linux-x64-v0.18.3.3.tar.bz2', - dir: 'monero-x86_64-linux-gnu-v0.18.3.3', + url: 'https://downloads.getmonero.org/cli/monero-linux-x64-v0.18.3.4.tar.bz2', + dir: 'monero-x86_64-linux-gnu-v0.18.3.4', files: [['monerod', 'monerod'], ['monero-wallet-rpc', 'monero-wallet-rpc']] } } From 46e34a78a2c3b0d4534d6509307f92a76b03388a Mon Sep 17 00:00:00 2001 From: Rafael Taranto Date: Fri, 26 Apr 2024 10:03:24 +0100 Subject: [PATCH 03/10] feat: add lamassu bumpfee script --- bin/lamassu-btc-bumpfee | 44 +++++++++++++++++++++++++ lib/blockexplorers/mempool.space.js | 24 +++++++++----- lib/plugins/wallet/bitcoind/bitcoind.js | 1 + 3 files changed, 61 insertions(+), 8 deletions(-) create mode 100644 bin/lamassu-btc-bumpfee diff --git a/bin/lamassu-btc-bumpfee b/bin/lamassu-btc-bumpfee new file mode 100644 index 00000000..8bf8cd04 --- /dev/null +++ b/bin/lamassu-btc-bumpfee @@ -0,0 +1,44 @@ +#!/usr/bin/env node + +const inquirer = require('inquirer') + +const bitcoind = require('../lib/plugins/wallet/bitcoind/bitcoind') +const BN = require('../lib/bn') +const mempool = require('../lib/blockexplorers/mempool.space') + +const txId = process.argv[2] +if (!txId) { + console.error('Please provide a BTC transaction hash as input.') + process.exit(1) +} + +const bumpTransactionFee = async (txId) => { + const txData = await bitcoind.fetch('gettransaction', [txId, true, true]) + + const fee = new BN(txData.fee).abs().shiftedBy(8).decimalPlaces(0) + const size = txData.decoded.vsize + const satPerVb = fee.div(size) + + console.log(`Current fee: ${satPerVb.toFixed(2).toString()} sat/vB`) + + const recommendedFees = mempool.getSatBEstimateFees() + + console.log('Recommended fees (sat/vB):', recommendedFees) + + const { selectedFee } = await inquirer.prompt([ + { + type: 'list', + name: 'selectedFee', + message: 'Select a fee higher than the current one:', + choices: Object.entries(recommendedFees) + .filter(([_, value]) => satPerVb.gt(value)) + .map(([key, value]) => ({name: `${key}: ${value} sat/vB`, value})), + }, + ]) + + await bitcoind.fetch('bumpfee', [txId, {fee_rate: selectedFee}]) + + console.log(`Fee bumped to ${selectedFee.toFixed(2)} sat/vB`) +} + +bumpTransactionFee(txId) \ No newline at end of file diff --git a/lib/blockexplorers/mempool.space.js b/lib/blockexplorers/mempool.space.js index 389d6727..383c1c72 100644 --- a/lib/blockexplorers/mempool.space.js +++ b/lib/blockexplorers/mempool.space.js @@ -1,8 +1,16 @@ -const axios = require("axios"); - -const getSatBEstimateFee = () => { - return axios.get('https://mempool.space/api/v1/fees/recommended') - .then(r => r.data.hourFee) -} - -module.exports = { getSatBEstimateFee } \ No newline at end of file +const axios = require("axios"); + +const getSatBEstimateFee = () => { + return axios.get('https://mempool.space/api/v1/fees/recommended') + .then(r => r.data.hourFee) +} + +const getSatBEstimateFees = () => { + return axios.get('https://mempool.space/api/v1/fees/recommended') + .then(r => r.data) +} + +module.exports = { + getSatBEstimateFees, + getSatBEstimateFee +} \ No newline at end of file diff --git a/lib/plugins/wallet/bitcoind/bitcoind.js b/lib/plugins/wallet/bitcoind/bitcoind.js index ad979a5c..ba5d17ae 100644 --- a/lib/plugins/wallet/bitcoind/bitcoind.js +++ b/lib/plugins/wallet/bitcoind/bitcoind.js @@ -219,5 +219,6 @@ module.exports = { sendCoinsBatch, checkBlockchainStatus, getTxHashesByAddress, + fetch, SUPPORTS_BATCHING } From 1e6f762c2c3507e30dfba6dbb7e59729321ff9fe Mon Sep 17 00:00:00 2001 From: Rafael Date: Tue, 3 Dec 2024 17:16:52 +0000 Subject: [PATCH 04/10] fix: comparator, missing await, print new txid --- bin/lamassu-btc-bumpfee | 11 +++++++---- package.json | 1 + 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/bin/lamassu-btc-bumpfee b/bin/lamassu-btc-bumpfee index 8bf8cd04..bd17b3af 100644 --- a/bin/lamassu-btc-bumpfee +++ b/bin/lamassu-btc-bumpfee @@ -21,7 +21,7 @@ const bumpTransactionFee = async (txId) => { console.log(`Current fee: ${satPerVb.toFixed(2).toString()} sat/vB`) - const recommendedFees = mempool.getSatBEstimateFees() + const recommendedFees = await mempool.getSatBEstimateFees() console.log('Recommended fees (sat/vB):', recommendedFees) @@ -31,14 +31,17 @@ const bumpTransactionFee = async (txId) => { name: 'selectedFee', message: 'Select a fee higher than the current one:', choices: Object.entries(recommendedFees) - .filter(([_, value]) => satPerVb.gt(value)) + .filter(([_, value]) => satPerVb.lt(value)) .map(([key, value]) => ({name: `${key}: ${value} sat/vB`, value})), }, ]) - await bitcoind.fetch('bumpfee', [txId, {fee_rate: selectedFee}]) + const { txid } = await bitcoind.fetch('bumpfee', [txId, {fee_rate: selectedFee}]) - console.log(`Fee bumped to ${selectedFee.toFixed(2)} sat/vB`) + console.log(` +Fee bumped to ${selectedFee.toFixed(2)} sat/vB +Transaction ID: ${txid} +`) } bumpTransactionFee(txId) \ No newline at end of file diff --git a/package.json b/package.json index 9d4511c6..47798153 100644 --- a/package.json +++ b/package.json @@ -113,6 +113,7 @@ "lamassu-ofac-update": "./bin/lamassu-ofac-update", "lamassu-send-coins": "./bin/lamassu-send-coins", "lamassu-update-to-mnemonic": "./bin/lamassu-update-to-mnemonic", + "lamassu-btc-bumpfee": "./bin/lamassu-btc-bumpfee", "lamassu-update-wallet-nodes": "./bin/lamassu-update-wallet-nodes", "lamassu-configure-frontcamera": "./bin/lamassu-configure-frontcamera", "lamassu-ofac-update-sources": "./bin/lamassu-ofac-update-sources", From 2ade6c21cc505d00a8fcbb530f0dab6a55ffc786 Mon Sep 17 00:00:00 2001 From: siiky Date: Tue, 17 Dec 2024 13:07:46 +0000 Subject: [PATCH 05/10] fix: serialize ETH `sendCoins()` --- lib/plugins/wallet/geth/base.js | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/lib/plugins/wallet/geth/base.js b/lib/plugins/wallet/geth/base.js index bdf5bd73..8b9fa9dc 100644 --- a/lib/plugins/wallet/geth/base.js +++ b/lib/plugins/wallet/geth/base.js @@ -42,6 +42,10 @@ const SWEEP_QUEUE = new PQueue({ interval: 250, }) +const SEND_QUEUE = new PQueue({ + concurrency: 1, +}) + const infuraCalls = {} const pify = _function => { @@ -78,18 +82,20 @@ function sendCoins (account, tx, settings, operatorId, feeMultiplier) { const { toAddress, cryptoAtoms, cryptoCode } = tx const isErc20Token = coins.utils.isErc20Token(cryptoCode) - return (isErc20Token ? generateErc20Tx : generateTx)(toAddress, defaultWallet(account), cryptoAtoms, false, cryptoCode) - .then(pify(web3.eth.sendSignedTransaction)) - .then(txid => { - return pify(web3.eth.getTransaction)(txid) - .then(tx => { - if (!tx) return { txid } + return SEND_QUEUE.add(() => + (isErc20Token ? generateErc20Tx : generateTx)(toAddress, defaultWallet(account), cryptoAtoms, false, cryptoCode) + .then(pify(web3.eth.sendSignedTransaction)) + .then(txid => { + return pify(web3.eth.getTransaction)(txid) + .then(tx => { + if (!tx) return { txid } - const fee = new BN(tx.gas).times(new BN(tx.gasPrice)).decimalPlaces(0) + const fee = new BN(tx.gas).times(new BN(tx.gasPrice)).decimalPlaces(0) - return { txid, fee } - }) - }) + return { txid, fee } + }) + }) + ) } function checkCryptoCode (cryptoCode) { From 74a026dc1ba2869e7cb5416d673c9e6948607623 Mon Sep 17 00:00:00 2001 From: Rafael Date: Fri, 20 Dec 2024 11:59:35 +0000 Subject: [PATCH 06/10] refactor: don't use env for ofac sources --- .sample.env | 9 -------- bin/lamassu-ofac-update-sources | 11 ---------- docker-compose.yaml | 4 ---- lib/ofac/update.js | 39 +++++++++++++++------------------ package.json | 1 - tools/build-dev-env.js | 3 --- tools/build-prod-env.js | 3 --- 7 files changed, 18 insertions(+), 52 deletions(-) delete mode 100755 bin/lamassu-ofac-update-sources diff --git a/.sample.env b/.sample.env index 09826711..db66efe7 100644 --- a/.sample.env +++ b/.sample.env @@ -31,15 +31,6 @@ OPERATOR_DATA_DIR= COIN_ATM_RADAR_URL= -## OFAC Sources variables - -# These variables map to each other, similar to a zip HOF. Entries are separated by commas -# Example: -# OFAC_SOURCES_NAMES=name1,name2 -# OFAC_SOURCES_URLS=url1,url2 -OFAC_SOURCES_NAMES= -OFAC_SOURCES_URLS= - ## Misc HOSTNAME= diff --git a/bin/lamassu-ofac-update-sources b/bin/lamassu-ofac-update-sources deleted file mode 100755 index a79e6c0c..00000000 --- a/bin/lamassu-ofac-update-sources +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env node - -'use strict' - -require('../lib/environment-helper') -const setEnvVariable = require('../tools/set-env-var') - -if (!process.env.OFAC_SOURCES_NAMES && !process.env.OFAC_SOURCES_URLS) { - setEnvVariable('OFAC_SOURCES_NAMES', 'sdn_advanced,cons_advanced') - setEnvVariable('OFAC_SOURCES_URLS', 'https://www.treasury.gov/ofac/downloads/sanctions/1.0/sdn_advanced.xml,https://www.treasury.gov/ofac/downloads/sanctions/1.0/cons_advanced.xml') -} diff --git a/docker-compose.yaml b/docker-compose.yaml index bb8cb1ce..29ff0a05 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -27,8 +27,6 @@ services: - FRONT_CAMERA_DIR=/lamassu-data/frontcamera - OPERATOR_DATA_DIR=/lamassu-data/operatordata - COIN_ATM_RADAR_URL=https://coinatmradar.info/api/lamassu/ - - OFAC_SOURCES_NAMES=sdn_advanced,cons_advanced - - OFAC_SOURCES_URLS=https://www.treasury.gov/ofac/downloads/sanctions/1.0/sdn_advanced.xml,https://www.treasury.gov/ofac/downloads/sanctions/1.0/cons_advanced.xml - HOSTNAME=localhost - LOG_LEVEL=info @@ -58,8 +56,6 @@ services: - FRONT_CAMERA_DIR=/lamassu-data/frontcamera - OPERATOR_DATA_DIR=/lamassu-data/operatordata - COIN_ATM_RADAR_URL=https://coinatmradar.info/api/lamassu/ - - OFAC_SOURCES_NAMES=sdn_advanced,cons_advanced - - OFAC_SOURCES_URLS=https://www.treasury.gov/ofac/downloads/sanctions/1.0/sdn_advanced.xml,https://www.treasury.gov/ofac/downloads/sanctions/1.0/cons_advanced.xml - HOSTNAME=172.29.0.3 - LOG_LEVEL=info depends_on: diff --git a/lib/ofac/update.js b/lib/ofac/update.js index d1408687..e7eb5b73 100644 --- a/lib/ofac/update.js +++ b/lib/ofac/update.js @@ -11,13 +11,14 @@ const logger = require('../logger') const DOWNLOAD_DIR = path.resolve('/tmp') const OFAC_DATA_DIR = process.env.OFAC_DATA_DIR -const OFAC_SOURCES_NAMES = process.env.OFAC_SOURCES_NAMES.split(',') -const OFAC_SOURCES_URLS = process.env.OFAC_SOURCES_URLS.split(',') -const ofacSources = _.map( - ([name, url]) => ({ name, url }), - _.zip(OFAC_SOURCES_NAMES, OFAC_SOURCES_URLS) -) +const OFAC_SOURCES = [{ + name: 'sdn_advanced', + url: 'https://sanctionslistservice.ofac.treas.gov/api/download/sdn_advanced.xml' +}, { + name: 'cons_advanced', + url: 'https://sanctionslistservice.ofac.treas.gov/api/download/cons_advanced.xml' +}] const mkdir = path => fs.mkdir(path) @@ -92,39 +93,35 @@ function update () { throw new Error('ofacDataDir must be defined in the environment') } - if (!ofacSources) { - logger.error('ofacSources must be defined in the environment') - } - const OFAC_SOURCES_DIR = path.join(OFAC_DATA_DIR, 'sources') const OFAC_ETAGS_FILE = path.join(OFAC_DATA_DIR, 'etags.json') return mkdir(OFAC_DATA_DIR) .then(() => mkdir(OFAC_SOURCES_DIR)) - .then(() => writeFile(OFAC_ETAGS_FILE, '{}', {encoding: 'utf-8', flag: 'wx'})) + .then(() => writeFile(OFAC_ETAGS_FILE, '{}', { encoding: 'utf-8', flag: 'wx' })) .catch(err => { if (err.code === 'EEXIST') return throw err }) .then(() => { - const promiseOldEtags = readFile(OFAC_ETAGS_FILE, {encoding: 'utf-8'}) - .then(json => JSON.parse(json)) - .catch(_ => { - logger.error('Can\'t parse etags.json, getting new data...') - return {} - }) + const promiseOldEtags = readFile(OFAC_ETAGS_FILE, { encoding: 'utf-8' }) + .then(json => JSON.parse(json)) + .catch(_ => { + logger.error('Can\'t parse etags.json, getting new data...') + return {} + }) - const promiseNewEtags = Promise.resolve(ofacSources || []) + const promiseNewEtags = Promise.resolve(OFAC_SOURCES || []) .then(sources => Promise.all(_.map(promiseGetEtag, sources)) .then(etags => _.map( - ([source, etag]) => _.set('etag', etag, source), + ([source, etag]) => _.set('etag', etag, source), _.zip(sources, etags) )) ) return Promise.all([promiseOldEtags, promiseNewEtags]) .then(([oldEtags, newEtags]) => { - const hasNotChanged = ({name, etag}) => oldEtags[name] === etag + const hasNotChanged = ({ name, etag }) => oldEtags[name] === etag const downloads = _.flow( _.reject(hasNotChanged), @@ -155,4 +152,4 @@ function update () { }) } -module.exports = {update} +module.exports = { update } diff --git a/package.json b/package.json index 8f2bfaa7..1fa71916 100644 --- a/package.json +++ b/package.json @@ -114,7 +114,6 @@ "lamassu-update-to-mnemonic": "./bin/lamassu-update-to-mnemonic", "lamassu-update-wallet-nodes": "./bin/lamassu-update-wallet-nodes", "lamassu-configure-frontcamera": "./bin/lamassu-configure-frontcamera", - "lamassu-ofac-update-sources": "./bin/lamassu-ofac-update-sources", "lamassu-devices": "./bin/lamassu-devices", "lamassu-operator": "./bin/lamassu-operator", "lamassu-coinatmradar": "./bin/lamassu-coinatmradar", diff --git a/tools/build-dev-env.js b/tools/build-dev-env.js index eb72ec90..14c160cb 100644 --- a/tools/build-dev-env.js +++ b/tools/build-dev-env.js @@ -26,9 +26,6 @@ setEnvVariable('ID_PHOTO_CARD_DIR', `${process.env.HOME}/.lamassu/idphotocard`) setEnvVariable('FRONT_CAMERA_DIR', `${process.env.HOME}/.lamassu/frontcamera`) setEnvVariable('OPERATOR_DATA_DIR', `${process.env.HOME}/.lamassu/operatordata`) -setEnvVariable('OFAC_SOURCES_NAMES', 'sdn_advanced,cons_advanced') -setEnvVariable('OFAC_SOURCES_URLS', 'https://www.treasury.gov/ofac/downloads/sanctions/1.0/sdn_advanced.xml,https://www.treasury.gov/ofac/downloads/sanctions/1.0/cons_advanced.xml') - setEnvVariable('BTC_NODE_LOCATION', 'remote') setEnvVariable('BTC_WALLET_LOCATION', 'local') diff --git a/tools/build-prod-env.js b/tools/build-prod-env.js index 41e59aae..59a10ffd 100644 --- a/tools/build-prod-env.js +++ b/tools/build-prod-env.js @@ -36,9 +36,6 @@ setEnvVariable('OPERATOR_DATA_DIR', `/opt/lamassu-server/operatordata`) setEnvVariable('COIN_ATM_RADAR_URL', `https://coinatmradar.info/api/lamassu/`) -setEnvVariable('OFAC_SOURCES_NAMES', 'sdn_advanced,cons_advanced') -setEnvVariable('OFAC_SOURCES_URLS', 'https://www.treasury.gov/ofac/downloads/sanctions/1.0/sdn_advanced.xml,https://www.treasury.gov/ofac/downloads/sanctions/1.0/cons_advanced.xml') - setEnvVariable('BTC_NODE_LOCATION', 'local') setEnvVariable('BTC_WALLET_LOCATION', 'local') setEnvVariable('BCH_NODE_LOCATION', 'local') From 908baa4fb02831a93a5ec87184f9be71ff5e735a Mon Sep 17 00:00:00 2001 From: Rafael Date: Fri, 20 Dec 2024 12:21:08 +0000 Subject: [PATCH 07/10] refactor: stop using etags --- lib/ofac/update.js | 74 +++++----------------------------------------- 1 file changed, 8 insertions(+), 66 deletions(-) diff --git a/lib/ofac/update.js b/lib/ofac/update.js index e7eb5b73..5a91819f 100644 --- a/lib/ofac/update.js +++ b/lib/ofac/update.js @@ -1,12 +1,10 @@ const parser = require('./parsing') const https = require('https') -const URL = require('url') const { createWriteStream } = require('fs') const fs = require('fs/promises') -const { readFile, writeFile, rename, unlink } = fs +const { rename } = fs const path = require('path') const _ = require('lodash/fp') -const logger = require('../logger') const DOWNLOAD_DIR = path.resolve('/tmp') @@ -24,25 +22,6 @@ const mkdir = path => fs.mkdir(path) .catch(err => err.code === 'EEXIST' ? Promise.resolve() : Promise.reject(err)) -const promiseGetEtag = ({ url }) => - new Promise((resolve, reject) => { - const parsed = URL.parse(url) - const requestOptions = { - hostname: parsed.hostname, - path: parsed.path, - method: 'HEAD' - } - - const request = https.request(requestOptions, _.flow( - _.get(['headers', 'etag']), - resolve - )) - - request.on('error', reject) - - request.end() - }) - const download = (dstDir, { name, url }) => { const dstFile = path.join(dstDir, name + '.xml') const file = createWriteStream(dstFile) @@ -94,60 +73,23 @@ function update () { } const OFAC_SOURCES_DIR = path.join(OFAC_DATA_DIR, 'sources') - const OFAC_ETAGS_FILE = path.join(OFAC_DATA_DIR, 'etags.json') return mkdir(OFAC_DATA_DIR) .then(() => mkdir(OFAC_SOURCES_DIR)) - .then(() => writeFile(OFAC_ETAGS_FILE, '{}', { encoding: 'utf-8', flag: 'wx' })) .catch(err => { if (err.code === 'EEXIST') return throw err }) .then(() => { - const promiseOldEtags = readFile(OFAC_ETAGS_FILE, { encoding: 'utf-8' }) - .then(json => JSON.parse(json)) - .catch(_ => { - logger.error('Can\'t parse etags.json, getting new data...') - return {} - }) + const downloads = _.flow( + _.map(file => download(DOWNLOAD_DIR, file).then(parseToJson)) + )(OFAC_SOURCES) - const promiseNewEtags = Promise.resolve(OFAC_SOURCES || []) - .then(sources => Promise.all(_.map(promiseGetEtag, sources)) - .then(etags => _.map( - ([source, etag]) => _.set('etag', etag, source), - _.zip(sources, etags) - )) - ) + return Promise.all(downloads) + .then(parsed => { + const moves = _.map(src => moveToSourcesDir(src, OFAC_SOURCES_DIR), parsed) - return Promise.all([promiseOldEtags, promiseNewEtags]) - .then(([oldEtags, newEtags]) => { - const hasNotChanged = ({ name, etag }) => oldEtags[name] === etag - - const downloads = _.flow( - _.reject(hasNotChanged), - _.map(file => download(DOWNLOAD_DIR, file).then(parseToJson)) - )(newEtags) - - const oldFileNames = _.keys(oldEtags) - const newFileNames = _.map(_.get('name'), newEtags) - const missingFileNames = _.difference(oldFileNames, newFileNames) - const resolve = name => path.join(OFAC_SOURCES_DIR, name + '.json') - const missing = _.map(resolve, missingFileNames) - - const etagsJson = _.flow( - _.map(source => [source.name, source.etag]), - _.fromPairs, - obj => JSON.stringify(obj, null, 4) - )(newEtags) - - return Promise.all(downloads) - .then(parsed => { - const moves = _.map(src => moveToSourcesDir(src, OFAC_SOURCES_DIR), parsed) - const deletions = _.map(unlink, missing) - const updateEtags = writeFile(OFAC_ETAGS_FILE, etagsJson) - - return Promise.all([updateEtags, ...moves, ...deletions]) - }) + return Promise.all([...moves]) }) }) } From c4d695934c9065378d84ac3f3f5846823ae6ff15 Mon Sep 17 00:00:00 2001 From: Rafael Date: Fri, 20 Dec 2024 14:02:06 +0000 Subject: [PATCH 08/10] chore: use axios for the download and typo --- lib/compliance.js | 4 ++-- lib/ofac/update.js | 36 ++++++++++++++++++++++++++---------- 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/lib/compliance.js b/lib/compliance.js index 8ecd1770..becf9ee3 100644 --- a/lib/compliance.js +++ b/lib/compliance.js @@ -72,8 +72,8 @@ function validateOfac (deviceId, sanctionsActive, customer) { function validationPatch (deviceId, sanctionsActive, customer) { return validateOfac(deviceId, sanctionsActive, customer) - .then(sactions => - _.isNil(customer.sanctions) || customer.sanctions !== sactions ? + .then(sanctions => + _.isNil(customer.sanctions) || customer.sanctions !== sanctions ? { sanctions } : {} ) diff --git a/lib/ofac/update.js b/lib/ofac/update.js index 5a91819f..16d03d3d 100644 --- a/lib/ofac/update.js +++ b/lib/ofac/update.js @@ -1,5 +1,5 @@ const parser = require('./parsing') -const https = require('https') +const axios = require('axios') const { createWriteStream } = require('fs') const fs = require('fs/promises') const { rename } = fs @@ -22,18 +22,34 @@ const mkdir = path => fs.mkdir(path) .catch(err => err.code === 'EEXIST' ? Promise.resolve() : Promise.reject(err)) +const newDownload = (dstDir, { name, url }) => { + return path.join('/tmp/', name + '.xml') +} + const download = (dstDir, { name, url }) => { const dstFile = path.join(dstDir, name + '.xml') - const file = createWriteStream(dstFile) + const writer = createWriteStream(dstFile) - return new Promise((resolve, reject) => { - const request = https.get(url, response => { - response.pipe(file) - file.on('finish', () => file.close(() => resolve(dstFile))) - }) - - request.on('error', reject) - }) + return axios({ + method: 'get', + url: url, + responseType: 'stream', + }).then(response => { + return new Promise((resolve, reject) => { + response.data.pipe(writer); + let error = null; + writer.on('error', err => { + error = err; + writer.close(); + reject(err); + }); + writer.on('close', () => { + if (!error) { + resolve(dstFile); + } + }); + }); + }); } const parseToJson = srcFile => { From a68aae0965e053384a11f51b3b26df64cbbf3be7 Mon Sep 17 00:00:00 2001 From: Rafael Date: Fri, 20 Dec 2024 14:47:32 +0000 Subject: [PATCH 09/10] feat: add another timer mechanism --- lib/ofac/update.js | 94 ++++++++++++++++++++++++++++------------------ 1 file changed, 58 insertions(+), 36 deletions(-) diff --git a/lib/ofac/update.js b/lib/ofac/update.js index 16d03d3d..8c520a69 100644 --- a/lib/ofac/update.js +++ b/lib/ofac/update.js @@ -1,14 +1,14 @@ const parser = require('./parsing') const axios = require('axios') const { createWriteStream } = require('fs') -const fs = require('fs/promises') -const { rename } = fs +const { rename, writeFile, readFile, mkdir, copyFile, unlink } = require('fs/promises') const path = require('path') const _ = require('lodash/fp') const DOWNLOAD_DIR = path.resolve('/tmp') - const OFAC_DATA_DIR = process.env.OFAC_DATA_DIR +const OFAC_SOURCES_DIR = path.join(OFAC_DATA_DIR, 'sources') +const LAST_UPDATED_FILE = path.resolve(OFAC_DATA_DIR, 'last_updated.dat') const OFAC_SOURCES = [{ name: 'sdn_advanced', @@ -18,38 +18,34 @@ const OFAC_SOURCES = [{ url: 'https://sanctionslistservice.ofac.treas.gov/api/download/cons_advanced.xml' }] -const mkdir = path => - fs.mkdir(path) +const _mkdir = path => + mkdir(path) .catch(err => err.code === 'EEXIST' ? Promise.resolve() : Promise.reject(err)) -const newDownload = (dstDir, { name, url }) => { - return path.join('/tmp/', name + '.xml') -} - const download = (dstDir, { name, url }) => { const dstFile = path.join(dstDir, name + '.xml') const writer = createWriteStream(dstFile) - return axios({ - method: 'get', - url: url, - responseType: 'stream', - }).then(response => { - return new Promise((resolve, reject) => { - response.data.pipe(writer); - let error = null; - writer.on('error', err => { - error = err; - writer.close(); - reject(err); - }); - writer.on('close', () => { - if (!error) { - resolve(dstFile); - } - }); - }); - }); + return axios({ + method: 'get', + url: url, + responseType: 'stream', + }).then(response => { + return new Promise((resolve, reject) => { + response.data.pipe(writer) + let error = null + writer.on('error', err => { + error = err + writer.close() + reject(err) + }) + writer.on('close', () => { + if (!error) { + resolve(dstFile) + } + }) + }) + }) } const parseToJson = srcFile => { @@ -77,10 +73,21 @@ const parseToJson = srcFile => { }) } -const moveToSourcesDir = (srcFile, ofacSourcesDir) => { +const moveToSourcesDir = async (srcFile, ofacSourcesDir) => { const name = path.basename(srcFile) const dstFile = path.join(ofacSourcesDir, name) - return rename(srcFile, dstFile) + try { + await rename(srcFile, dstFile) + } catch (err) { + if (err.code === 'EXDEV') { + // If rename fails due to cross-device link, fallback to copy + delete + await copyFile(srcFile, dstFile) + await unlink(srcFile) + } else { + throw err + } + } + return dstFile } function update () { @@ -88,15 +95,28 @@ function update () { throw new Error('ofacDataDir must be defined in the environment') } - const OFAC_SOURCES_DIR = path.join(OFAC_DATA_DIR, 'sources') - - return mkdir(OFAC_DATA_DIR) - .then(() => mkdir(OFAC_SOURCES_DIR)) + return _mkdir(OFAC_DATA_DIR) + .then(() => _mkdir(OFAC_SOURCES_DIR)) .catch(err => { if (err.code === 'EEXIST') return throw err }) - .then(() => { + .then(() => readFile(LAST_UPDATED_FILE)) + .then(data => { + const lastUpdate = new Date(data.toString()) + const now = new Date() + const hoursSinceUpdate = (now - lastUpdate) / (1000 * 60 * 60) + + return hoursSinceUpdate < 24 + }) + .catch(err => { + // If file doesn't exist, continue with update + if (err.code === 'ENOENT') return false + throw err + }) + .then(skipUpdate => { + if (skipUpdate) return Promise.resolve() + const downloads = _.flow( _.map(file => download(DOWNLOAD_DIR, file).then(parseToJson)) )(OFAC_SOURCES) @@ -104,8 +124,10 @@ function update () { return Promise.all(downloads) .then(parsed => { const moves = _.map(src => moveToSourcesDir(src, OFAC_SOURCES_DIR), parsed) + const timestamp = new Date().toISOString() return Promise.all([...moves]) + .then(() => writeFile(LAST_UPDATED_FILE, timestamp)) }) }) } From e8f98cc6be2d8d32b194a6b79112fc204ff9630a Mon Sep 17 00:00:00 2001 From: Rafael Date: Fri, 20 Dec 2024 15:08:35 +0000 Subject: [PATCH 10/10] chore: version bump --- package-lock.json | 122 +++++++++++++++++++++++----------------------- package.json | 2 +- 2 files changed, 62 insertions(+), 62 deletions(-) diff --git a/package-lock.json b/package-lock.json index c33b7368..8ea90173 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "lamassu-server", - "version": "10.0.6", + "version": "10.0.7", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -990,38 +990,6 @@ "secp256k1": "^4.0.2", "secrets.js-grempe": "^1.1.0", "superagent": "3.8.3" - }, - "dependencies": { - "bech32": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/bech32/-/bech32-1.1.4.tgz", - "integrity": "sha512-s0IrSOzLlbvX7yp4WBfPITzpAU8sqQcpsmwXDiKwrG4r491vwCO/XpejasRNl0piBMe/DvP4Tz0mIS/X1DPJBQ==" - }, - "bitcoinjs-message": { - "version": "npm:@bitgo-forks/bitcoinjs-message@1.0.0-master.2", - "resolved": "https://registry.npmjs.org/@bitgo-forks/bitcoinjs-message/-/bitcoinjs-message-1.0.0-master.2.tgz", - "integrity": "sha512-XSDGM3rA75vcDxeKqHPexika/TgWUFWdfKTv1lV8TZTb5XFHHD6ARckLdMOBiCf29eZSzbJQvF/OIWqNqMl/2A==", - "requires": { - "bech32": "^1.1.3", - "bs58check": "^2.1.2", - "buffer-equals": "^1.0.3", - "create-hash": "^1.1.2", - "secp256k1": "5.0.0", - "varuint-bitcoin": "^1.0.1" - }, - "dependencies": { - "secp256k1": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/secp256k1/-/secp256k1-5.0.0.tgz", - "integrity": "sha512-TKWX8xvoGHrxVdqbYeZM9w+izTF4b9z3NhSaDkdn81btvuh+ivbIMGT/zQvDtTFWhRlThpoz6LEYTr7n8A5GcA==", - "requires": { - "elliptic": "^6.5.4", - "node-addon-api": "^5.0.0", - "node-gyp-build": "^4.2.0" - } - } - } - } } }, "@bitgo/sdk-coin-bch": { @@ -1230,6 +1198,40 @@ "fastpriorityqueue": "^0.7.1", "typeforce": "^1.11.3", "varuint-bitcoin": "^1.1.2" + }, + "dependencies": { + "bip174": { + "version": "npm:@bitgo-forks/bip174@3.1.0-master.4", + "resolved": "https://registry.npmjs.org/@bitgo-forks/bip174/-/bip174-3.1.0-master.4.tgz", + "integrity": "sha512-WDRNzPSdJGDqQNqfN+L5KHNHFDmNOPYnUnT7NkEkfHWn5m1jSOfcf8Swaslt5P0xcSDiERdN2gZxFc6XtOqRYg==" + }, + "bitcoinjs-lib": { + "version": "npm:@bitgo-forks/bitcoinjs-lib@7.1.0-master.7", + "resolved": "https://registry.npmjs.org/@bitgo-forks/bitcoinjs-lib/-/bitcoinjs-lib-7.1.0-master.7.tgz", + "integrity": "sha512-FZle7954KnbbVXFCc5uYGtjq+0PFOnFxVchNwt3Kcv2nVusezTp29aeQwDi2Y+lM1dCoup2gJGXMkkREenY7KQ==", + "requires": { + "bech32": "^2.0.0", + "bip174": "npm:@bitgo-forks/bip174@3.1.0-master.4", + "bs58check": "^2.1.2", + "create-hash": "^1.1.0", + "fastpriorityqueue": "^0.7.1", + "json5": "^2.2.3", + "ripemd160": "^2.0.2", + "typeforce": "^1.11.3", + "varuint-bitcoin": "^1.1.2", + "wif": "^2.0.1" + } + }, + "ecpair": { + "version": "npm:@bitgo/ecpair@2.1.0-rc.0", + "resolved": "https://registry.npmjs.org/@bitgo/ecpair/-/ecpair-2.1.0-rc.0.tgz", + "integrity": "sha512-qPZetcEA1Lzzm9NsqsGF9NGorAGaXrv20eZjopLUjsdwftWcsYTE7lwzE/Xjdf4fcq6G4+vjrCudWAMGNfJqOQ==", + "requires": { + "randombytes": "^2.1.0", + "typeforce": "^1.18.0", + "wif": "^2.0.6" + } + } } }, "@bitgo/utxo-ord": { @@ -4289,11 +4291,6 @@ "safe-buffer": "^5.2.1" } }, - "bip174": { - "version": "npm:@bitgo-forks/bip174@3.1.0-master.4", - "resolved": "https://registry.npmjs.org/@bitgo-forks/bip174/-/bip174-3.1.0-master.4.tgz", - "integrity": "sha512-WDRNzPSdJGDqQNqfN+L5KHNHFDmNOPYnUnT7NkEkfHWn5m1jSOfcf8Swaslt5P0xcSDiERdN2gZxFc6XtOqRYg==" - }, "bip32": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/bip32/-/bip32-3.1.0.tgz", @@ -4332,21 +4329,34 @@ "resolved": "https://registry.npmjs.org/bitcoin-ops/-/bitcoin-ops-1.4.1.tgz", "integrity": "sha512-pef6gxZFztEhaE9RY9HmWVmiIHqCb2OyS4HPKkpc6CIiiOa3Qmuoylxc5P2EkU3w+5eTSifI9SEZC88idAIGow==" }, - "bitcoinjs-lib": { - "version": "npm:@bitgo-forks/bitcoinjs-lib@7.1.0-master.7", - "resolved": "https://registry.npmjs.org/@bitgo-forks/bitcoinjs-lib/-/bitcoinjs-lib-7.1.0-master.7.tgz", - "integrity": "sha512-FZle7954KnbbVXFCc5uYGtjq+0PFOnFxVchNwt3Kcv2nVusezTp29aeQwDi2Y+lM1dCoup2gJGXMkkREenY7KQ==", + "bitcoinjs-message": { + "version": "npm:@bitgo-forks/bitcoinjs-message@1.0.0-master.2", + "resolved": "https://registry.npmjs.org/@bitgo-forks/bitcoinjs-message/-/bitcoinjs-message-1.0.0-master.2.tgz", + "integrity": "sha512-XSDGM3rA75vcDxeKqHPexika/TgWUFWdfKTv1lV8TZTb5XFHHD6ARckLdMOBiCf29eZSzbJQvF/OIWqNqMl/2A==", "requires": { - "bech32": "^2.0.0", - "bip174": "npm:@bitgo-forks/bip174@3.1.0-master.4", + "bech32": "^1.1.3", "bs58check": "^2.1.2", - "create-hash": "^1.1.0", - "fastpriorityqueue": "^0.7.1", - "json5": "^2.2.3", - "ripemd160": "^2.0.2", - "typeforce": "^1.11.3", - "varuint-bitcoin": "^1.1.2", - "wif": "^2.0.1" + "buffer-equals": "^1.0.3", + "create-hash": "^1.1.2", + "secp256k1": "5.0.0", + "varuint-bitcoin": "^1.0.1" + }, + "dependencies": { + "bech32": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/bech32/-/bech32-1.1.4.tgz", + "integrity": "sha512-s0IrSOzLlbvX7yp4WBfPITzpAU8sqQcpsmwXDiKwrG4r491vwCO/XpejasRNl0piBMe/DvP4Tz0mIS/X1DPJBQ==" + }, + "secp256k1": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/secp256k1/-/secp256k1-5.0.0.tgz", + "integrity": "sha512-TKWX8xvoGHrxVdqbYeZM9w+izTF4b9z3NhSaDkdn81btvuh+ivbIMGT/zQvDtTFWhRlThpoz6LEYTr7n8A5GcA==", + "requires": { + "elliptic": "^6.5.4", + "node-addon-api": "^5.0.0", + "node-gyp-build": "^4.2.0" + } + } } }, "bitcore-lib": { @@ -6158,16 +6168,6 @@ "safe-buffer": "^5.0.1" } }, - "ecpair": { - "version": "npm:@bitgo/ecpair@2.1.0-rc.0", - "resolved": "https://registry.npmjs.org/@bitgo/ecpair/-/ecpair-2.1.0-rc.0.tgz", - "integrity": "sha512-qPZetcEA1Lzzm9NsqsGF9NGorAGaXrv20eZjopLUjsdwftWcsYTE7lwzE/Xjdf4fcq6G4+vjrCudWAMGNfJqOQ==", - "requires": { - "randombytes": "^2.1.0", - "typeforce": "^1.18.0", - "wif": "^2.0.6" - } - }, "ecurve": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/ecurve/-/ecurve-1.0.6.tgz", diff --git a/package.json b/package.json index 1fa71916..25a175c4 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "lamassu-server", "description": "bitcoin atm client server protocol module", "keywords": [], - "version": "10.0.6", + "version": "10.0.7", "license": "./LICENSE", "author": "Lamassu (https://lamassu.is)", "dependencies": {