From 80e851fb59aee989507ad6d650bda035bad42a45 Mon Sep 17 00:00:00 2001 From: Josh Harvey Date: Tue, 1 May 2018 19:35:54 +0300 Subject: [PATCH] WIP --- bin/lamassu-ofac-update | 9 +++ lib/app.js | 4 +- lib/compliance.js | 23 ++++++++ lib/customers.js | 2 +- lib/ofac/index.js | 28 ++++----- lib/ofac/matching.js | 7 +-- lib/ofac/update.js | 128 +++++++++++++++++++++++----------------- lib/routes.js | 3 + package.json | 3 +- 9 files changed, 131 insertions(+), 76 deletions(-) create mode 100755 bin/lamassu-ofac-update create mode 100644 lib/compliance.js diff --git a/bin/lamassu-ofac-update b/bin/lamassu-ofac-update new file mode 100755 index 00000000..0e514aa8 --- /dev/null +++ b/bin/lamassu-ofac-update @@ -0,0 +1,9 @@ +#!/usr/bin/env node + +const ofac = require('../lib/ofac/update') + +console.log('Updating OFAC databases.') + +ofac.update() + .then(() => console.log('Success.')) + .catch(console.log) diff --git a/lib/app.js b/lib/app.js index b4ef4bfa..dcbd6f59 100644 --- a/lib/app.js +++ b/lib/app.js @@ -8,6 +8,7 @@ const logger = require('./logger') const poller = require('./poller') const settingsLoader = require('./settings-loader') const options = require('./options') +const ofac = require('./ofac/index') const devMode = argv.dev || options.http @@ -32,7 +33,8 @@ function run () { } function runOnce () { - return settingsLoader.loadLatest() + return ofac.load() + .then(settingsLoader.loadLatest()) .then(settings => { poller.start(settings) diff --git a/lib/compliance.js b/lib/compliance.js new file mode 100644 index 00000000..4d6938d4 --- /dev/null +++ b/lib/compliance.js @@ -0,0 +1,23 @@ +const _ = require('lodash/fp') + +const ofac = require('./ofac/index') + +function matchOfac (customer) { + const nameParts = _.flatMap(_.split(/\s+/), [customer.firstName, customer.lastName]) + const birthDate = customer.dateOfBirth + + const result = ofac.match(nameParts, birthDate) + console.log('DEBUG200: %s', result) + + if (result > 0.8) throw new Error('Compliance error') +} + +function validateCustomer (config, customer) { + if (config.sanctionsVerificationActive) { + matchOfac(customer) + } + + return customer +} + +module.exports = {validateCustomer} diff --git a/lib/customers.js b/lib/customers.js index 7021ebca..b201ff83 100644 --- a/lib/customers.js +++ b/lib/customers.js @@ -328,4 +328,4 @@ function batch () { }, customers))) } -module.exports = { add, get, batch, getById, update } +module.exports = {add, get, batch, getById, update} diff --git a/lib/ofac/index.js b/lib/ofac/index.js index a71ec62b..f2c1ead6 100644 --- a/lib/ofac/index.js +++ b/lib/ofac/index.js @@ -7,9 +7,7 @@ const nameUtils = require('./name-utils') const options = require('../options') const _ = require('lodash/fp') -const debug_log = require('../pp')(__filename) // KOSTIS TODO: remove - -const OFAC_SOURCES_DIR = path.join(options.ofacDataDir, 'sources') +const debugLog = require('../pp')(__filename) // KOSTIS TODO: remove let structs = null @@ -18,19 +16,21 @@ const readdir = util.promisify(fs.readdir) function load () { // NOTE: Not sure how you push code updates to existing clients. This problem // might pop up if new code is pushed, without re-doing setup. - if (!OFAC_SOURCES_DIR) { + if (!options.ofacDataDir) { const message = 'The ofacDataDir option has not been set in lamassu.json' return Promise.reject(new Error(message)) } - return readdir(OFAC_SOURCES_DIR) - .then(_.flow( - _.map(file => path.join(OFAC_SOURCES_DIR, file)), - loader.load - )) - .then(result => { - return (structs = result) - }) + const ofacSourcesDir = path.join(options.ofacDataDir, 'sources') + + return readdir(ofacSourcesDir) + .then(_.flow( + _.map(file => path.join(ofacSourcesDir, file)), + loader.load + )) + .then(result => { + return (structs = result) + }) } // nameParts should be an object like {firstName: "John", lastName: "Doe", ...} @@ -69,10 +69,10 @@ function match (nameParts, birthDateString, options) { ])(birthDateString) const candidate = {parts, fullName, words, birthDate} - debug && debug_log(candidate) + debug && debugLog(candidate) const result = matcher.match(structs, candidate, options) - debug && debug_log(result) + debug && debugLog(result) return result } diff --git a/lib/ofac/matching.js b/lib/ofac/matching.js index c03139a4..c1178177 100644 --- a/lib/ofac/matching.js +++ b/lib/ofac/matching.js @@ -1,7 +1,7 @@ const jaro = require('talisman/metrics/distance/jaro') const _ = require('lodash/fp') -const debug_log = require('../pp')(__filename) // KOSTIS TODO: remove +const debugLog = require('../pp')(__filename) // KOSTIS TODO: remove const stringSimilarity = _.curry(jaro) @@ -44,7 +44,6 @@ function match (structs, candidate, options) { _.map(_.get('id')) )(aliases) - const phoneticWeight = ratio const stringWeight = 1 - phoneticWeight @@ -90,8 +89,8 @@ function match (structs, candidate, options) { _.map(_.first) )(matches) - debug && debug_log(aliasIdsFromFullName) - debug && debug_log(aliasIdsFromNamePart) + debug && debugLog(aliasIdsFromFullName) + debug && debugLog(aliasIdsFromNamePart) // Get the full record for each matched id const getIndividual = aliasId => { diff --git a/lib/ofac/update.js b/lib/ofac/update.js index 951a9483..b812a4bf 100644 --- a/lib/ofac/update.js +++ b/lib/ofac/update.js @@ -7,23 +7,28 @@ const util = require('util') const options = require('../options') const _ = require('lodash/fp') -const OFAC_DATA_DIR = options.ofacDataDir -const OFAC_SOURCES_DIR = path.join(OFAC_DATA_DIR, 'sources') -const OFAC_ETAGS_FILE = path.join(OFAC_DATA_DIR, 'etags.json') const DOWNLOAD_DIR = path.resolve('/tmp') +function mkdir (path) { + return new Promise((resolve, reject) => { + fs.mkdir(path, err => { + if (!err) return resolve() + if (err.code === 'EEXIST') return resolve() + reject(err) + }) + }) +} -const mkdir = util.promisify(fs.mkdir) const readFile = util.promisify(fs.readFile) const writeFile = util.promisify(fs.writeFile) const rename = util.promisify(fs.rename) const unlink = util.promisify(fs.unlink) + const remove = file => { - console.log("remove", file) + console.log('remove', file) return unlink(file) } - const promiseGetEtag = (source) => { return new Promise((resolve, reject) => { const {url: sourceUrl} = source @@ -46,14 +51,14 @@ const promiseGetEtag = (source) => { } const download = _.curry((dstDir, source) => { - console.log("download", source) + console.log('download', source) const {name, url: sourceUrl} = source const dstFile = path.join(dstDir, name + '.xml') const file = fs.createWriteStream(dstFile) return new Promise((resolve, reject) => { const request = https.get(sourceUrl, response => { - response.pipe(file); + response.pipe(file) file.on('finish', () => file.close(() => resolve(dstFile))) }) @@ -62,14 +67,14 @@ const download = _.curry((dstDir, source) => { }) const parseToJson = srcFile => { - console.log("parseToJson", srcFile) + console.log('parseToJson', srcFile) const dstFile = srcFile.replace(/\.xml$/, '.json') const writeStream = fs.createWriteStream(dstFile) return new Promise((resolve, reject) => { parser.parse(srcFile, (err, profile) => { - console.log("callback", err, profile) + console.log('callback', err, profile) if (err) { reject(err) @@ -90,65 +95,78 @@ const parseToJson = srcFile => { }) } -const moveToSourcesDir = srcFile => { - console.log("moveToSourcesDir", srcFile) +const moveToSourcesDir = (srcFile, ofacSourcesDir) => { + console.log('moveToSourcesDir', srcFile) const name = path.basename(srcFile) - const dstFile = path.join(OFAC_SOURCES_DIR, name) + const dstFile = path.join(ofacSourcesDir, name) return rename(srcFile, dstFile) } +function update () { + const OFAC_DATA_DIR = options.ofacDataDir -const update = () => mkdir(OFAC_DATA_DIR).catch(err => null) - .then(() => mkdir(OFAC_SOURCES_DIR)).catch(err => null) - .then(() => writeFile(OFAC_ETAGS_FILE, '{}', {encoding: 'utf-8', flag: 'wx'})) - .catch(err => null) - .then(() => { - const promiseOldEtags = readFile(OFAC_ETAGS_FILE, {encoding: 'utf-8'}) - .then(json => JSON.parse(json) || {}) + if (!OFAC_DATA_DIR) { + throw new Error('ofacDataDir must be defined in lamassu.json') + } - const promiseNewEtags = Promise.resolve(options.ofacSources || []) - .then(sources => Promise.all(_.map(promiseGetEtag, sources)) - .then(etags => _.map( - ([source, etag]) => ({...source, etag}), - _.zip(sources, etags) - )) - ) + const OFAC_SOURCES_DIR = path.join(OFAC_DATA_DIR, 'sources') + const OFAC_ETAGS_FILE = path.join(OFAC_DATA_DIR, 'etags.json') - return Promise.all([promiseOldEtags, promiseNewEtags]) - .then(([oldEtags, newEtags]) => { - console.log("OLD", JSON.stringify(oldEtags, null, 4)) - console.log("NEW", JSON.stringify(newEtags, null, 4)) + return mkdir(OFAC_DATA_DIR) + .then(() => mkdir(OFAC_SOURCES_DIR)) + .then(() => writeFile(OFAC_ETAGS_FILE, '{}', {encoding: 'utf-8', flag: 'wx'})) + .catch(err => { + if (err.code === 'EEXIST') return + throw err + }) + .then(() => { + const promiseOldEtags = readFile(OFAC_ETAGS_FILE, {encoding: 'utf-8'}) + .then(json => JSON.parse(json) || {}) - const hasNotChanged = ({name, etag}) => oldEtags[name] === etag + const promiseNewEtags = Promise.resolve(options.ofacSources || []) + .then(sources => Promise.all(_.map(promiseGetEtag, sources)) + .then(etags => _.map( + ([source, etag]) => ({...source, etag}), + _.zip(sources, etags) + )) + ) - const downloads = _.flow( - _.reject(hasNotChanged), - _.map(file => download(DOWNLOAD_DIR, file).then(parseToJson)) - )(newEtags) + return Promise.all([promiseOldEtags, promiseNewEtags]) + .then(([oldEtags, newEtags]) => { + console.log('OLD', JSON.stringify(oldEtags, null, 4)) + console.log('NEW', JSON.stringify(newEtags, null, 4)) - const oldFileNames = _.keys(oldEtags) - const newFileNames = _.map(_.get('name'), newEtags) - const missingFileNames = _.difference(oldFileNames, newFileNames) - const resolve = name => path.join(OFAC_SOURCES_DIR, name + '.json') - const missing = _.map(resolve, missingFileNames) + const hasNotChanged = ({name, etag}) => oldEtags[name] === etag - const etagsJson = _.flow( - _.map(source => [source.name, source.etag]), - _.fromPairs, - obj => JSON.stringify(obj, null, 4) - )(newEtags) + const downloads = _.flow( + _.reject(hasNotChanged), + _.map(file => download(DOWNLOAD_DIR, file).then(parseToJson)) + )(newEtags) - return Promise.all(downloads) - .then(parsed => { - console.log("finished", parsed) + const oldFileNames = _.keys(oldEtags) + const newFileNames = _.map(_.get('name'), newEtags) + const missingFileNames = _.difference(oldFileNames, newFileNames) + const resolve = name => path.join(OFAC_SOURCES_DIR, name + '.json') + const missing = _.map(resolve, missingFileNames) - const moves = _.map(moveToSourcesDir, parsed) - const deletions = _.map(remove, missing) - const updateEtags = writeFile(OFAC_ETAGS_FILE, etagsJson) + const etagsJson = _.flow( + _.map(source => [source.name, source.etag]), + _.fromPairs, + obj => JSON.stringify(obj, null, 4) + )(newEtags) - return Promise.all([updateEtags, ...moves, ...deletions]) + return Promise.all(downloads) + .then(parsed => { + console.log('finished', parsed) + + const moves = _.map(src => moveToSourcesDir(src, OFAC_SOURCES_DIR), parsed) + const deletions = _.map(remove, missing) + const updateEtags = writeFile(OFAC_ETAGS_FILE, etagsJson) + + return Promise.all([updateEtags, ...moves, ...deletions]) + }) }) - }) - }) + }) +} module.exports = {update} diff --git a/lib/routes.js b/lib/routes.js index 3cf97f82..bbe7a95b 100644 --- a/lib/routes.js +++ b/lib/routes.js @@ -186,11 +186,14 @@ function getCustomerWithPhoneCode (req, res, next) { function updateCustomer (req, res, next) { const id = req.params.id const patch = req.body + const config = configManager.unscoped(req.settings.config) + customers.getById(id) .then(customer => { if (!customer) { throw httpError('Not Found', 404) } return customers.update(id, patch) }) + .then(customer => customers.validate(config, customer)) .then(customer => respond(req, res, {customer})) .catch(next) } diff --git a/package.json b/package.json index acc76cc5..b4fe5170 100644 --- a/package.json +++ b/package.json @@ -75,7 +75,8 @@ "lamassu-cancel": "./bin/lamassu-cancel", "lamassu-nuke-db": "./bin/lamassu-nuke-db", "lamassu-coins": "./bin/lamassu-coins", - "lamassu-update": "./bin/lamassu-update" + "lamassu-update": "./bin/lamassu-update", + "lamassu-ofac-update": "./bin/lamassu-ofac-update" }, "scripts": { "start": "node bin/lamassu-server",