Merge pull request #1782 from lamassu/chore/merge-v10.1-into-dev

Chore/merge v10.1 into dev

.sample.env

@@ -31,15 +31,6 @@ OPERATOR_DATA_DIR=
 
 COIN_ATM_RADAR_URL=
 
-## OFAC Sources variables
-
-# These variables map to each other, similar to a zip HOF. Entries are separated by commas
-# Example:
-# OFAC_SOURCES_NAMES=name1,name2
-# OFAC_SOURCES_URLS=url1,url2
-OFAC_SOURCES_NAMES=
-OFAC_SOURCES_URLS=
-
 ## Misc
 
 HOSTNAME=

bin/lamassu-btc-bumpfee

@@ -0,0 +1,47 @@
+#!/usr/bin/env node
+
+const inquirer = require('inquirer')
+
+const bitcoind = require('../lib/plugins/wallet/bitcoind/bitcoind')
+const BN = require('../lib/bn')
+const mempool = require('../lib/blockexplorers/mempool.space')
+
+const txId = process.argv[2]
+if (!txId) {
+  console.error('Please provide a BTC transaction hash as input.')
+  process.exit(1)
+}
+
+const bumpTransactionFee = async (txId) => {
+  const txData = await bitcoind.fetch('gettransaction', [txId, true, true])
+
+  const fee = new BN(txData.fee).abs().shiftedBy(8).decimalPlaces(0)
+  const size = txData.decoded.vsize
+  const satPerVb = fee.div(size)
+
+  console.log(`Current fee: ${satPerVb.toFixed(2).toString()} sat/vB`)
+
+  const recommendedFees = await mempool.getSatBEstimateFees()
+
+  console.log('Recommended fees (sat/vB):', recommendedFees)
+
+  const { selectedFee } = await inquirer.prompt([
+    {
+      type: 'list',
+      name: 'selectedFee',
+      message: 'Select a fee higher than the current one:',
+      choices: Object.entries(recommendedFees)
+        .filter(([_, value]) => satPerVb.lt(value))
+        .map(([key, value]) => ({name: `${key}: ${value} sat/vB`, value})),
+    },
+  ])
+
+  const { txid } = await bitcoind.fetch('bumpfee', [txId, {fee_rate: selectedFee}])
+
+  console.log(`
+Fee bumped to ${selectedFee.toFixed(2)} sat/vB
+Transaction ID: ${txid}
+`)
+}
+
+bumpTransactionFee(txId)

bin/lamassu-ofac-update-sources

@@ -1,11 +0,0 @@
-#!/usr/bin/env node
-
-'use strict'
-
-require('../lib/environment-helper')
-const setEnvVariable = require('../tools/set-env-var')
-
-if (!process.env.OFAC_SOURCES_NAMES && !process.env.OFAC_SOURCES_URLS) {
-  setEnvVariable('OFAC_SOURCES_NAMES', 'sdn_advanced,cons_advanced')
-  setEnvVariable('OFAC_SOURCES_URLS', 'https://www.treasury.gov/ofac/downloads/sanctions/1.0/sdn_advanced.xml,https://www.treasury.gov/ofac/downloads/sanctions/1.0/cons_advanced.xml')
-}

docker-compose.yaml

@@ -27,8 +27,6 @@ services:
       - FRONT_CAMERA_DIR=/lamassu-data/frontcamera
       - OPERATOR_DATA_DIR=/lamassu-data/operatordata
       - COIN_ATM_RADAR_URL=https://coinatmradar.info/api/lamassu/
-      - OFAC_SOURCES_NAMES=sdn_advanced,cons_advanced
-      - OFAC_SOURCES_URLS=https://www.treasury.gov/ofac/downloads/sanctions/1.0/sdn_advanced.xml,https://www.treasury.gov/ofac/downloads/sanctions/1.0/cons_advanced.xml
       - HOSTNAME=localhost
       - LOG_LEVEL=info
 
@@ -58,8 +56,6 @@ services:
       - FRONT_CAMERA_DIR=/lamassu-data/frontcamera
       - OPERATOR_DATA_DIR=/lamassu-data/operatordata
       - COIN_ATM_RADAR_URL=https://coinatmradar.info/api/lamassu/
-      - OFAC_SOURCES_NAMES=sdn_advanced,cons_advanced
-      - OFAC_SOURCES_URLS=https://www.treasury.gov/ofac/downloads/sanctions/1.0/sdn_advanced.xml,https://www.treasury.gov/ofac/downloads/sanctions/1.0/cons_advanced.xml
       - HOSTNAME=172.29.0.3
       - LOG_LEVEL=info
     depends_on:

lib/blockchain/common.js

@@ -31,47 +31,47 @@ const BINARIES = {
     defaultUrl: 'https://bitcoincore.org/bin/bitcoin-core-0.20.1/bitcoin-0.20.1-x86_64-linux-gnu.tar.gz',
     defaultUrlHash: '376194f06596ecfa40331167c39bc70c355f960280bd2a645fdbf18f66527397',
     defaultDir: 'bitcoin-0.20.1/bin',
-    url: 'https://bitcoincore.org/bin/bitcoin-core-27.1/bitcoin-27.1-x86_64-linux-gnu.tar.gz',
+    url: 'https://bitcoincore.org/bin/bitcoin-core-28.0/bitcoin-28.0-x86_64-linux-gnu.tar.gz',
-    UrlHash: 'c9840607d230d65f6938b81deaec0b98fe9cb14c3a41a5b13b2c05d044a48422',
+    dir: 'bitcoin-28.0/bin',
-    dir: 'bitcoin-27.1/bin'
+    urlHash: '7fe294b02b25b51acb8e8e0a0eb5af6bbafa7cd0c5b0e5fcbb61263104a82fbc',
   },
   ETH: {
-    url: 'https://gethstore.blob.core.windows.net/builds/geth-linux-amd64-1.14.8-a9523b64.tar.gz',
+    url: 'https://gethstore.blob.core.windows.net/builds/geth-linux-amd64-1.14.12-293a300d.tar.gz',
-    urlHash: 'fff507c90c180443456950e4fc0bf224d26ce5ea6896194ff864c3c3754c136b',
+    dir: 'geth-linux-amd64-1.14.12-293a300d',
-    dir: 'geth-linux-amd64-1.14.8-a9523b64'
+    urlHash: 'e56216b9d179a66a8f71d3dee13ad554da5544d3d29dba33f64c9c0eda5a2237',
   },
   ZEC: {
-    url: 'https://github.com/zcash/artifacts/raw/master/v5.9.0/bullseye/zcash-5.9.0-linux64-debian-bullseye.tar.gz',
+    url: 'https://download.z.cash/downloads/zcash-6.0.0-linux64-debian-bullseye.tar.gz',
-    urlHash: 'd385b9fbeeb145f60b0b339d256cabb342713ed3014cd634cf2d68078365abd2',
+    dir: 'zcash-6.0.0/bin',
-    dir: 'zcash-5.9.0/bin'
+    urlHash: '3cb82f490e9c8e88007a0216b5261b33ef0fda962b9258441b2def59cb272a4d',
   },
   DASH: {
     defaultUrl: 'https://github.com/dashpay/dash/releases/download/v18.1.0/dashcore-18.1.0-x86_64-linux-gnu.tar.gz',
     defaultUrlHash: 'd89c2afd78183f3ee815adcccdff02098be0c982633889e7b1e9c9656fbef219',
     defaultDir: 'dashcore-18.1.0/bin',
-    url: 'https://github.com/dashpay/dash/releases/download/v21.1.0/dashcore-21.1.0-x86_64-linux-gnu.tar.gz',
+    url: 'https://github.com/dashpay/dash/releases/download/v21.1.1/dashcore-21.1.1-x86_64-linux-gnu.tar.gz',
-    urlHash: 'a7d0c1b04d53a9b1b3499eb82182c0fa57f4c8768c16163e5d05971bf45d7928',
+    dir: 'dashcore-21.1.1/bin'
-    dir: 'dashcore-21.1.0/bin'
+    urlHash: 'c3157d4a82a3cb7c904a68e827bd1e629854fefcc0dcaf1de4343a810a190bf5',
   },
   LTC: {
     defaultUrl: 'https://download.litecoin.org/litecoin-0.18.1/linux/litecoin-0.18.1-x86_64-linux-gnu.tar.gz',
     defaultUrlHash: 'ca50936299e2c5a66b954c266dcaaeef9e91b2f5307069b9894048acf3eb5751',
     defaultDir: 'litecoin-0.18.1/bin',
-    url: 'https://download.litecoin.org/litecoin-0.21.3/linux/litecoin-0.21.3-x86_64-linux-gnu.tar.gz',
+    url: 'https://download.litecoin.org/litecoin-0.21.4/linux/litecoin-0.21.4-x86_64-linux-gnu.tar.gz',
-    urlHash: 'ea231c630e2a243cb01affd4c2b95a2be71560f80b64b9f4bceaa13d736aa7cb',
+    dir: 'litecoin-0.21.4/bin',
-    dir: 'litecoin-0.21.3/bin'
+    urlHash: '857fc41091f2bae65c3bf0fd4d388fca915fc93a03f16dd2578ac3cc92898390',
   },
   BCH: {
-    url: 'https://github.com/bitcoin-cash-node/bitcoin-cash-node/releases/download/v27.1.0/bitcoin-cash-node-27.1.0-x86_64-linux-gnu.tar.gz',
+    url: 'https://github.com/bitcoin-cash-node/bitcoin-cash-node/releases/download/v28.0.0/bitcoin-cash-node-28.0.0-x86_64-linux-gnu.tar.gz',
-    urlHash: '0dcc387cbaa3a039c97ddc8fb99c1fa7bff5dc6e4bd3a01d3c3095f595ad2dce',
+    dir: 'bitcoin-cash-node-28.0.0/bin',
-    dir: 'bitcoin-cash-node-27.1.0/bin',
+    files: [['bitcoind', 'bitcoincashd'], ['bitcoin-cli', 'bitcoincash-cli']],
-    files: [['bitcoind', 'bitcoincashd'], ['bitcoin-cli', 'bitcoincash-cli']]
+    urlHash: 'ba735cd3b70fab35ac1496e38596cec1f8d34989924376de001d4a86198f7158',
   },
   XMR: {
-    url: 'https://downloads.getmonero.org/cli/monero-linux-x64-v0.18.3.3.tar.bz2',
+    url: 'https://downloads.getmonero.org/cli/monero-linux-x64-v0.18.3.4.tar.bz2',
-    urlHash: '47c7e6b4b88a57205800a2538065a7874174cd087eedc2526bee1ebcce0cc5e3',
+    dir: 'monero-x86_64-linux-gnu-v0.18.3.4',
-    dir: 'monero-x86_64-linux-gnu-v0.18.3.3',
+    files: [['monerod', 'monerod'], ['monero-wallet-rpc', 'monero-wallet-rpc']],
-    files: [['monerod', 'monerod'], ['monero-wallet-rpc', 'monero-wallet-rpc']]
+    urlHash: '51ba03928d189c1c11b5379cab17dd9ae8d2230056dc05c872d0f8dba4a87f1d',
   }
 }
 

lib/blockexplorers/mempool.space.js

@@ -1,8 +1,16 @@
-const axios = require("axios");
+const axios = require("axios");
-
+
-const getSatBEstimateFee = () => {
+const getSatBEstimateFee = () => {
-  return axios.get('https://mempool.space/api/v1/fees/recommended')
+  return axios.get('https://mempool.space/api/v1/fees/recommended')
-    .then(r => r.data.hourFee)
+    .then(r => r.data.hourFee)
-}
+}
-
+
-module.exports = { getSatBEstimateFee }
+const getSatBEstimateFees = () => {
+  return axios.get('https://mempool.space/api/v1/fees/recommended')
+    .then(r => r.data)
+}
+
+module.exports = {
+  getSatBEstimateFees,
+  getSatBEstimateFee
+}

lib/compliance.js

@@ -72,8 +72,8 @@ function validateOfac (deviceId, sanctionsActive, customer) {
 
 function validationPatch (deviceId, sanctionsActive, customer) {
   return validateOfac(deviceId, sanctionsActive, customer)
-    .then(sactions =>
+    .then(sanctions =>
-      _.isNil(customer.sanctions) || customer.sanctions !== sactions ?
+      _.isNil(customer.sanctions) || customer.sanctions !== sanctions ?
         { sanctions } :
         {}
     )

lib/ofac/update.js

@@ -1,58 +1,50 @@
 const parser = require('./parsing')
-const https = require('https')
+const axios = require('axios')
-const URL = require('url')
 const { createWriteStream } = require('fs')
-const fs = require('fs/promises')
+const { rename, writeFile, readFile, mkdir, copyFile, unlink } = require('fs/promises')
-const { readFile, writeFile, rename, unlink } = fs
 const path = require('path')
 const _ = require('lodash/fp')
-const logger = require('../logger')
 
 const DOWNLOAD_DIR = path.resolve('/tmp')
-
 const OFAC_DATA_DIR = process.env.OFAC_DATA_DIR
-const OFAC_SOURCES_NAMES = process.env.OFAC_SOURCES_NAMES.split(',')
+const OFAC_SOURCES_DIR = path.join(OFAC_DATA_DIR, 'sources')
-const OFAC_SOURCES_URLS = process.env.OFAC_SOURCES_URLS.split(',')
+const LAST_UPDATED_FILE = path.resolve(OFAC_DATA_DIR, 'last_updated.dat')
 
-const ofacSources = _.map(
+const OFAC_SOURCES = [{
-  ([name, url]) => ({ name, url }),
+  name: 'sdn_advanced',
-  _.zip(OFAC_SOURCES_NAMES, OFAC_SOURCES_URLS)
+  url: 'https://sanctionslistservice.ofac.treas.gov/api/download/sdn_advanced.xml'
-)
+}, {
+  name: 'cons_advanced',
+  url: 'https://sanctionslistservice.ofac.treas.gov/api/download/cons_advanced.xml'
+}]
 
-const mkdir = path =>
+const _mkdir = path =>
-  fs.mkdir(path)
+  mkdir(path)
     .catch(err => err.code === 'EEXIST' ? Promise.resolve() : Promise.reject(err))
 
-const promiseGetEtag = ({ url }) =>
-  new Promise((resolve, reject) => {
-    const parsed = URL.parse(url)
-    const requestOptions = {
-      hostname: parsed.hostname,
-      path: parsed.path,
-      method: 'HEAD'
-    }
-
-    const request = https.request(requestOptions, _.flow(
-      _.get(['headers', 'etag']),
-      resolve
-    ))
-
-    request.on('error', reject)
-
-    request.end()
-  })
-
 const download = (dstDir, { name, url }) => {
   const dstFile = path.join(dstDir, name + '.xml')
-  const file = createWriteStream(dstFile)
+  const writer = createWriteStream(dstFile)
 
-  return new Promise((resolve, reject) => {
+  return axios({
-    const request = https.get(url, response => {
+    method: 'get',
-      response.pipe(file)
+    url: url,
-      file.on('finish', () => file.close(() => resolve(dstFile)))
+    responseType: 'stream',
+  }).then(response => {
+    return new Promise((resolve, reject) => {
+      response.data.pipe(writer)
+      let error = null
+      writer.on('error', err => {
+        error = err
+        writer.close()
+        reject(err)
+      })
+      writer.on('close', () => {
+        if (!error) {
+          resolve(dstFile)
+        }
+      })
     })
-
-    request.on('error', reject)
   })
 }
 
@@ -81,10 +73,21 @@ const parseToJson = srcFile => {
   })
 }
 
-const moveToSourcesDir = (srcFile, ofacSourcesDir) => {
+const moveToSourcesDir = async (srcFile, ofacSourcesDir) => {
   const name = path.basename(srcFile)
   const dstFile = path.join(ofacSourcesDir, name)
-  return rename(srcFile, dstFile)
+  try {
+    await rename(srcFile, dstFile)
+  } catch (err) {
+    if (err.code === 'EXDEV') {
+      // If rename fails due to cross-device link, fallback to copy + delete
+      await copyFile(srcFile, dstFile)
+      await unlink(srcFile)
+    } else {
+      throw err
+    }
+  }
+  return dstFile
 }
 
 function update () {
@@ -92,67 +95,41 @@ function update () {
     throw new Error('ofacDataDir must be defined in the environment')
   }
 
-  if (!ofacSources) {
+  return _mkdir(OFAC_DATA_DIR)
-    logger.error('ofacSources must be defined in the environment')
+    .then(() => _mkdir(OFAC_SOURCES_DIR))
-  }
-
-  const OFAC_SOURCES_DIR = path.join(OFAC_DATA_DIR, 'sources')
-  const OFAC_ETAGS_FILE = path.join(OFAC_DATA_DIR, 'etags.json')
-
-  return mkdir(OFAC_DATA_DIR)
-    .then(() => mkdir(OFAC_SOURCES_DIR))
-    .then(() => writeFile(OFAC_ETAGS_FILE, '{}', {encoding: 'utf-8', flag: 'wx'}))
     .catch(err => {
       if (err.code === 'EEXIST') return
       throw err
     })
-    .then(() => {
+    .then(() => readFile(LAST_UPDATED_FILE))
-      const promiseOldEtags = readFile(OFAC_ETAGS_FILE, {encoding: 'utf-8'})
+    .then(data => {
-      .then(json => JSON.parse(json))
+      const lastUpdate = new Date(data.toString())
-      .catch(_ => {
+      const now = new Date()
-        logger.error('Can\'t parse etags.json, getting new data...')
+      const hoursSinceUpdate = (now - lastUpdate) / (1000 * 60 * 60)
-        return {}
-      })
 
-      const promiseNewEtags = Promise.resolve(ofacSources || [])
+      return hoursSinceUpdate < 24
-        .then(sources => Promise.all(_.map(promiseGetEtag, sources))
+    })
-          .then(etags => _.map(
+    .catch(err => {
-            ([source, etag]) => _.set('etag', etag,  source),
+      // If file doesn't exist, continue with update
-            _.zip(sources, etags)
+      if (err.code === 'ENOENT') return false
-          ))
+      throw err
-        )
+    })
+    .then(skipUpdate => {
+      if (skipUpdate) return Promise.resolve()
 
-      return Promise.all([promiseOldEtags, promiseNewEtags])
+      const downloads = _.flow(
-        .then(([oldEtags, newEtags]) => {
+        _.map(file => download(DOWNLOAD_DIR, file).then(parseToJson))
-          const hasNotChanged = ({name, etag}) => oldEtags[name] === etag
+      )(OFAC_SOURCES)
 
-          const downloads = _.flow(
+      return Promise.all(downloads)
-            _.reject(hasNotChanged),
+        .then(parsed => {
-            _.map(file => download(DOWNLOAD_DIR, file).then(parseToJson))
+          const moves = _.map(src => moveToSourcesDir(src, OFAC_SOURCES_DIR), parsed)
-          )(newEtags)
+          const timestamp = new Date().toISOString()
 
-          const oldFileNames = _.keys(oldEtags)
+          return Promise.all([...moves])
-          const newFileNames = _.map(_.get('name'), newEtags)
+            .then(() => writeFile(LAST_UPDATED_FILE, timestamp))
-          const missingFileNames = _.difference(oldFileNames, newFileNames)
-          const resolve = name => path.join(OFAC_SOURCES_DIR, name + '.json')
-          const missing = _.map(resolve, missingFileNames)
-
-          const etagsJson = _.flow(
-            _.map(source => [source.name, source.etag]),
-            _.fromPairs,
-            obj => JSON.stringify(obj, null, 4)
-          )(newEtags)
-
-          return Promise.all(downloads)
-            .then(parsed => {
-              const moves = _.map(src => moveToSourcesDir(src, OFAC_SOURCES_DIR), parsed)
-              const deletions = _.map(unlink, missing)
-              const updateEtags = writeFile(OFAC_ETAGS_FILE, etagsJson)
-
-              return Promise.all([updateEtags, ...moves, ...deletions])
-            })
         })
     })
 }
 
-module.exports = {update}
+module.exports = { update }

lib/plugins/wallet/bitcoind/bitcoind.js

@@ -219,5 +219,6 @@ module.exports = {
   sendCoinsBatch,
   checkBlockchainStatus,
   getTxHashesByAddress,
+  fetch,
   SUPPORTS_BATCHING
 }

lib/plugins/wallet/geth/base.js

@@ -42,6 +42,10 @@ const SWEEP_QUEUE = new PQueue({
   interval: 250,
 })
 
+const SEND_QUEUE = new PQueue({
+  concurrency: 1,
+})
+
 const infuraCalls = {}
 
 const pify = _function => {
@@ -78,18 +82,20 @@ function sendCoins (account, tx, settings, operatorId, feeMultiplier) {
   const { toAddress, cryptoAtoms, cryptoCode } = tx
   const isErc20Token = coins.utils.isErc20Token(cryptoCode)
 
-  return (isErc20Token ? generateErc20Tx : generateTx)(toAddress, defaultWallet(account), cryptoAtoms, false, cryptoCode)
+  return SEND_QUEUE.add(() =>
-    .then(pify(web3.eth.sendSignedTransaction))
+    (isErc20Token ? generateErc20Tx : generateTx)(toAddress, defaultWallet(account), cryptoAtoms, false, cryptoCode)
-    .then(txid => {
+      .then(pify(web3.eth.sendSignedTransaction))
-      return pify(web3.eth.getTransaction)(txid)
+      .then(txid => {
-        .then(tx => {
+        return pify(web3.eth.getTransaction)(txid)
-          if (!tx) return { txid }
+          .then(tx => {
+            if (!tx) return { txid }
 
-          const fee = new BN(tx.gas).times(new BN(tx.gasPrice)).decimalPlaces(0)
+            const fee = new BN(tx.gas).times(new BN(tx.gasPrice)).decimalPlaces(0)
 
-          return { txid, fee }
+            return { txid, fee }
-        })
+          })
-    })
+      })
+  )
 }
 
 function checkCryptoCode (cryptoCode) {

package.json

@@ -113,9 +113,9 @@
     "lamassu-ofac-update": "./bin/lamassu-ofac-update",
     "lamassu-send-coins": "./bin/lamassu-send-coins",
     "lamassu-update-to-mnemonic": "./bin/lamassu-update-to-mnemonic",
+    "lamassu-btc-bumpfee": "./bin/lamassu-btc-bumpfee",
     "lamassu-update-wallet-nodes": "./bin/lamassu-update-wallet-nodes",
     "lamassu-configure-frontcamera": "./bin/lamassu-configure-frontcamera",
-    "lamassu-ofac-update-sources": "./bin/lamassu-ofac-update-sources",
     "lamassu-devices": "./bin/lamassu-devices",
     "lamassu-operator": "./bin/lamassu-operator",
     "lamassu-coinatmradar": "./bin/lamassu-coinatmradar",

tools/build-dev-env.js

@@ -26,9 +26,6 @@ setEnvVariable('ID_PHOTO_CARD_DIR', `${process.env.HOME}/.lamassu/idphotocard`)
 setEnvVariable('FRONT_CAMERA_DIR', `${process.env.HOME}/.lamassu/frontcamera`)
 setEnvVariable('OPERATOR_DATA_DIR', `${process.env.HOME}/.lamassu/operatordata`)
 
-setEnvVariable('OFAC_SOURCES_NAMES', 'sdn_advanced,cons_advanced')
-setEnvVariable('OFAC_SOURCES_URLS', 'https://www.treasury.gov/ofac/downloads/sanctions/1.0/sdn_advanced.xml,https://www.treasury.gov/ofac/downloads/sanctions/1.0/cons_advanced.xml')
-
 setEnvVariable('BTC_NODE_LOCATION', 'remote')
 setEnvVariable('BTC_WALLET_LOCATION', 'local')
 

tools/build-prod-env.js

@@ -36,9 +36,6 @@ setEnvVariable('OPERATOR_DATA_DIR', `/opt/lamassu-server/operatordata`)
 
 setEnvVariable('COIN_ATM_RADAR_URL', `https://coinatmradar.info/api/lamassu/`)
 
-setEnvVariable('OFAC_SOURCES_NAMES', 'sdn_advanced,cons_advanced')
-setEnvVariable('OFAC_SOURCES_URLS', 'https://www.treasury.gov/ofac/downloads/sanctions/1.0/sdn_advanced.xml,https://www.treasury.gov/ofac/downloads/sanctions/1.0/cons_advanced.xml')
-
 setEnvVariable('BTC_NODE_LOCATION', 'local')
 setEnvVariable('BTC_WALLET_LOCATION', 'local')
 setEnvVariable('BCH_NODE_LOCATION', 'local')