diff --git a/.sample.env b/.sample.env index 09826711..db66efe7 100644 --- a/.sample.env +++ b/.sample.env @@ -31,15 +31,6 @@ OPERATOR_DATA_DIR= COIN_ATM_RADAR_URL= -## OFAC Sources variables - -# These variables map to each other, similar to a zip HOF. Entries are separated by commas -# Example: -# OFAC_SOURCES_NAMES=name1,name2 -# OFAC_SOURCES_URLS=url1,url2 -OFAC_SOURCES_NAMES= -OFAC_SOURCES_URLS= - ## Misc HOSTNAME= diff --git a/bin/lamassu-ofac-update-sources b/bin/lamassu-ofac-update-sources deleted file mode 100755 index a79e6c0c..00000000 --- a/bin/lamassu-ofac-update-sources +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env node - -'use strict' - -require('../lib/environment-helper') -const setEnvVariable = require('../tools/set-env-var') - -if (!process.env.OFAC_SOURCES_NAMES && !process.env.OFAC_SOURCES_URLS) { - setEnvVariable('OFAC_SOURCES_NAMES', 'sdn_advanced,cons_advanced') - setEnvVariable('OFAC_SOURCES_URLS', 'https://www.treasury.gov/ofac/downloads/sanctions/1.0/sdn_advanced.xml,https://www.treasury.gov/ofac/downloads/sanctions/1.0/cons_advanced.xml') -} diff --git a/docker-compose.yaml b/docker-compose.yaml index bb8cb1ce..29ff0a05 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -27,8 +27,6 @@ services: - FRONT_CAMERA_DIR=/lamassu-data/frontcamera - OPERATOR_DATA_DIR=/lamassu-data/operatordata - COIN_ATM_RADAR_URL=https://coinatmradar.info/api/lamassu/ - - OFAC_SOURCES_NAMES=sdn_advanced,cons_advanced - - OFAC_SOURCES_URLS=https://www.treasury.gov/ofac/downloads/sanctions/1.0/sdn_advanced.xml,https://www.treasury.gov/ofac/downloads/sanctions/1.0/cons_advanced.xml - HOSTNAME=localhost - LOG_LEVEL=info @@ -58,8 +56,6 @@ services: - FRONT_CAMERA_DIR=/lamassu-data/frontcamera - OPERATOR_DATA_DIR=/lamassu-data/operatordata - COIN_ATM_RADAR_URL=https://coinatmradar.info/api/lamassu/ - - OFAC_SOURCES_NAMES=sdn_advanced,cons_advanced - - OFAC_SOURCES_URLS=https://www.treasury.gov/ofac/downloads/sanctions/1.0/sdn_advanced.xml,https://www.treasury.gov/ofac/downloads/sanctions/1.0/cons_advanced.xml - HOSTNAME=172.29.0.3 - LOG_LEVEL=info depends_on: diff --git a/lib/compliance.js b/lib/compliance.js index 8ecd1770..becf9ee3 100644 --- a/lib/compliance.js +++ b/lib/compliance.js @@ -72,8 +72,8 @@ function validateOfac (deviceId, sanctionsActive, customer) { function validationPatch (deviceId, sanctionsActive, customer) { return validateOfac(deviceId, sanctionsActive, customer) - .then(sactions => - _.isNil(customer.sanctions) || customer.sanctions !== sactions ? + .then(sanctions => + _.isNil(customer.sanctions) || customer.sanctions !== sanctions ? { sanctions } : {} ) diff --git a/lib/ofac/update.js b/lib/ofac/update.js index d1408687..8c520a69 100644 --- a/lib/ofac/update.js +++ b/lib/ofac/update.js @@ -1,58 +1,50 @@ const parser = require('./parsing') -const https = require('https') -const URL = require('url') +const axios = require('axios') const { createWriteStream } = require('fs') -const fs = require('fs/promises') -const { readFile, writeFile, rename, unlink } = fs +const { rename, writeFile, readFile, mkdir, copyFile, unlink } = require('fs/promises') const path = require('path') const _ = require('lodash/fp') -const logger = require('../logger') const DOWNLOAD_DIR = path.resolve('/tmp') - const OFAC_DATA_DIR = process.env.OFAC_DATA_DIR -const OFAC_SOURCES_NAMES = process.env.OFAC_SOURCES_NAMES.split(',') -const OFAC_SOURCES_URLS = process.env.OFAC_SOURCES_URLS.split(',') +const OFAC_SOURCES_DIR = path.join(OFAC_DATA_DIR, 'sources') +const LAST_UPDATED_FILE = path.resolve(OFAC_DATA_DIR, 'last_updated.dat') -const ofacSources = _.map( - ([name, url]) => ({ name, url }), - _.zip(OFAC_SOURCES_NAMES, OFAC_SOURCES_URLS) -) +const OFAC_SOURCES = [{ + name: 'sdn_advanced', + url: 'https://sanctionslistservice.ofac.treas.gov/api/download/sdn_advanced.xml' +}, { + name: 'cons_advanced', + url: 'https://sanctionslistservice.ofac.treas.gov/api/download/cons_advanced.xml' +}] -const mkdir = path => - fs.mkdir(path) +const _mkdir = path => + mkdir(path) .catch(err => err.code === 'EEXIST' ? Promise.resolve() : Promise.reject(err)) -const promiseGetEtag = ({ url }) => - new Promise((resolve, reject) => { - const parsed = URL.parse(url) - const requestOptions = { - hostname: parsed.hostname, - path: parsed.path, - method: 'HEAD' - } - - const request = https.request(requestOptions, _.flow( - _.get(['headers', 'etag']), - resolve - )) - - request.on('error', reject) - - request.end() - }) - const download = (dstDir, { name, url }) => { const dstFile = path.join(dstDir, name + '.xml') - const file = createWriteStream(dstFile) + const writer = createWriteStream(dstFile) - return new Promise((resolve, reject) => { - const request = https.get(url, response => { - response.pipe(file) - file.on('finish', () => file.close(() => resolve(dstFile))) + return axios({ + method: 'get', + url: url, + responseType: 'stream', + }).then(response => { + return new Promise((resolve, reject) => { + response.data.pipe(writer) + let error = null + writer.on('error', err => { + error = err + writer.close() + reject(err) + }) + writer.on('close', () => { + if (!error) { + resolve(dstFile) + } + }) }) - - request.on('error', reject) }) } @@ -81,10 +73,21 @@ const parseToJson = srcFile => { }) } -const moveToSourcesDir = (srcFile, ofacSourcesDir) => { +const moveToSourcesDir = async (srcFile, ofacSourcesDir) => { const name = path.basename(srcFile) const dstFile = path.join(ofacSourcesDir, name) - return rename(srcFile, dstFile) + try { + await rename(srcFile, dstFile) + } catch (err) { + if (err.code === 'EXDEV') { + // If rename fails due to cross-device link, fallback to copy + delete + await copyFile(srcFile, dstFile) + await unlink(srcFile) + } else { + throw err + } + } + return dstFile } function update () { @@ -92,67 +95,41 @@ function update () { throw new Error('ofacDataDir must be defined in the environment') } - if (!ofacSources) { - logger.error('ofacSources must be defined in the environment') - } - - const OFAC_SOURCES_DIR = path.join(OFAC_DATA_DIR, 'sources') - const OFAC_ETAGS_FILE = path.join(OFAC_DATA_DIR, 'etags.json') - - return mkdir(OFAC_DATA_DIR) - .then(() => mkdir(OFAC_SOURCES_DIR)) - .then(() => writeFile(OFAC_ETAGS_FILE, '{}', {encoding: 'utf-8', flag: 'wx'})) + return _mkdir(OFAC_DATA_DIR) + .then(() => _mkdir(OFAC_SOURCES_DIR)) .catch(err => { if (err.code === 'EEXIST') return throw err }) - .then(() => { - const promiseOldEtags = readFile(OFAC_ETAGS_FILE, {encoding: 'utf-8'}) - .then(json => JSON.parse(json)) - .catch(_ => { - logger.error('Can\'t parse etags.json, getting new data...') - return {} - }) + .then(() => readFile(LAST_UPDATED_FILE)) + .then(data => { + const lastUpdate = new Date(data.toString()) + const now = new Date() + const hoursSinceUpdate = (now - lastUpdate) / (1000 * 60 * 60) - const promiseNewEtags = Promise.resolve(ofacSources || []) - .then(sources => Promise.all(_.map(promiseGetEtag, sources)) - .then(etags => _.map( - ([source, etag]) => _.set('etag', etag, source), - _.zip(sources, etags) - )) - ) + return hoursSinceUpdate < 24 + }) + .catch(err => { + // If file doesn't exist, continue with update + if (err.code === 'ENOENT') return false + throw err + }) + .then(skipUpdate => { + if (skipUpdate) return Promise.resolve() - return Promise.all([promiseOldEtags, promiseNewEtags]) - .then(([oldEtags, newEtags]) => { - const hasNotChanged = ({name, etag}) => oldEtags[name] === etag + const downloads = _.flow( + _.map(file => download(DOWNLOAD_DIR, file).then(parseToJson)) + )(OFAC_SOURCES) - const downloads = _.flow( - _.reject(hasNotChanged), - _.map(file => download(DOWNLOAD_DIR, file).then(parseToJson)) - )(newEtags) + return Promise.all(downloads) + .then(parsed => { + const moves = _.map(src => moveToSourcesDir(src, OFAC_SOURCES_DIR), parsed) + const timestamp = new Date().toISOString() - const oldFileNames = _.keys(oldEtags) - const newFileNames = _.map(_.get('name'), newEtags) - const missingFileNames = _.difference(oldFileNames, newFileNames) - const resolve = name => path.join(OFAC_SOURCES_DIR, name + '.json') - const missing = _.map(resolve, missingFileNames) - - const etagsJson = _.flow( - _.map(source => [source.name, source.etag]), - _.fromPairs, - obj => JSON.stringify(obj, null, 4) - )(newEtags) - - return Promise.all(downloads) - .then(parsed => { - const moves = _.map(src => moveToSourcesDir(src, OFAC_SOURCES_DIR), parsed) - const deletions = _.map(unlink, missing) - const updateEtags = writeFile(OFAC_ETAGS_FILE, etagsJson) - - return Promise.all([updateEtags, ...moves, ...deletions]) - }) + return Promise.all([...moves]) + .then(() => writeFile(LAST_UPDATED_FILE, timestamp)) }) }) } -module.exports = {update} +module.exports = { update } diff --git a/package.json b/package.json index 47798153..c40fec92 100644 --- a/package.json +++ b/package.json @@ -116,7 +116,6 @@ "lamassu-btc-bumpfee": "./bin/lamassu-btc-bumpfee", "lamassu-update-wallet-nodes": "./bin/lamassu-update-wallet-nodes", "lamassu-configure-frontcamera": "./bin/lamassu-configure-frontcamera", - "lamassu-ofac-update-sources": "./bin/lamassu-ofac-update-sources", "lamassu-devices": "./bin/lamassu-devices", "lamassu-operator": "./bin/lamassu-operator", "lamassu-coinatmradar": "./bin/lamassu-coinatmradar", diff --git a/tools/build-dev-env.js b/tools/build-dev-env.js index eb72ec90..14c160cb 100644 --- a/tools/build-dev-env.js +++ b/tools/build-dev-env.js @@ -26,9 +26,6 @@ setEnvVariable('ID_PHOTO_CARD_DIR', `${process.env.HOME}/.lamassu/idphotocard`) setEnvVariable('FRONT_CAMERA_DIR', `${process.env.HOME}/.lamassu/frontcamera`) setEnvVariable('OPERATOR_DATA_DIR', `${process.env.HOME}/.lamassu/operatordata`) -setEnvVariable('OFAC_SOURCES_NAMES', 'sdn_advanced,cons_advanced') -setEnvVariable('OFAC_SOURCES_URLS', 'https://www.treasury.gov/ofac/downloads/sanctions/1.0/sdn_advanced.xml,https://www.treasury.gov/ofac/downloads/sanctions/1.0/cons_advanced.xml') - setEnvVariable('BTC_NODE_LOCATION', 'remote') setEnvVariable('BTC_WALLET_LOCATION', 'local') diff --git a/tools/build-prod-env.js b/tools/build-prod-env.js index 41e59aae..59a10ffd 100644 --- a/tools/build-prod-env.js +++ b/tools/build-prod-env.js @@ -36,9 +36,6 @@ setEnvVariable('OPERATOR_DATA_DIR', `/opt/lamassu-server/operatordata`) setEnvVariable('COIN_ATM_RADAR_URL', `https://coinatmradar.info/api/lamassu/`) -setEnvVariable('OFAC_SOURCES_NAMES', 'sdn_advanced,cons_advanced') -setEnvVariable('OFAC_SOURCES_URLS', 'https://www.treasury.gov/ofac/downloads/sanctions/1.0/sdn_advanced.xml,https://www.treasury.gov/ofac/downloads/sanctions/1.0/cons_advanced.xml') - setEnvVariable('BTC_NODE_LOCATION', 'local') setEnvVariable('BTC_WALLET_LOCATION', 'local') setEnvVariable('BCH_NODE_LOCATION', 'local')