lamassu/lamassu-server
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

WIP

Browse source
This commit is contained in:
Josh Harvey 2018-05-01 19:35:54 +03:00
parent f7561acf3c
commit 80e851fb59
9 changed files with 131 additions and 76 deletions
Download patch file Download diff file Expand all files Collapse all files

9
bin/lamassu-ofac-update Executable file
View file

@ -0,0 +1,9 @@
#!/usr/bin/env node
const ofac = require('../lib/ofac/update')
console.log('Updating OFAC databases.')
ofac.update()
.then(() => console.log('Success.'))
.catch(console.log)

4
lib/app.js
View file

@ -8,6 +8,7 @@ const logger = require('./logger')
const poller = require('./poller')
const settingsLoader = require('./settings-loader')
const options = require('./options')
const ofac = require('./ofac/index')
const devMode = argv.dev || options.http
@ -32,7 +33,8 @@ function run () {
}
function runOnce () {
return settingsLoader.loadLatest()
return ofac.load()
.then(settingsLoader.loadLatest())
.then(settings => {
poller.start(settings)

23
lib/compliance.js Normal file
View file

@ -0,0 +1,23 @@
const _ = require('lodash/fp')
const ofac = require('./ofac/index')
function matchOfac (customer) {
const nameParts = _.flatMap(_.split(/\s+/), [customer.firstName, customer.lastName])
const birthDate = customer.dateOfBirth
const result = ofac.match(nameParts, birthDate)
console.log('DEBUG200: %s', result)
if (result > 0.8) throw new Error('Compliance error')
}
function validateCustomer (config, customer) {
if (config.sanctionsVerificationActive) {
matchOfac(customer)
}
return customer
}
module.exports = {validateCustomer}

2
lib/customers.js
View file

@ -328,4 +328,4 @@ function batch () {
}, customers)))
}
module.exports = { add, get, batch, getById, update }
module.exports = {add, get, batch, getById, update}

28
lib/ofac/index.js
View file

@ -7,9 +7,7 @@ const nameUtils = require('./name-utils')
const options = require('../options')
const _ = require('lodash/fp')
const debug_log = require('../pp')(__filename) // KOSTIS TODO: remove
const OFAC_SOURCES_DIR = path.join(options.ofacDataDir, 'sources')
const debugLog = require('../pp')(__filename) // KOSTIS TODO: remove
let structs = null
@ -18,19 +16,21 @@ const readdir = util.promisify(fs.readdir)
function load () {
// NOTE: Not sure how you push code updates to existing clients. This problem
// might pop up if new code is pushed, without re-doing setup.
if (!OFAC_SOURCES_DIR) {
if (!options.ofacDataDir) {
const message = 'The ofacDataDir option has not been set in lamassu.json'
return Promise.reject(new Error(message))
}
return readdir(OFAC_SOURCES_DIR)
.then(_.flow(
_.map(file => path.join(OFAC_SOURCES_DIR, file)),
loader.load
))
.then(result => {
return (structs = result)
})
const ofacSourcesDir = path.join(options.ofacDataDir, 'sources')
return readdir(ofacSourcesDir)
.then(_.flow(
_.map(file => path.join(ofacSourcesDir, file)),
loader.load
))
.then(result => {
return (structs = result)
})
}
// nameParts should be an object like {firstName: "John", lastName: "Doe", ...}
@ -69,10 +69,10 @@ function match (nameParts, birthDateString, options) {
])(birthDateString)
const candidate = {parts, fullName, words, birthDate}
debug && debug_log(candidate)
debug && debugLog(candidate)
const result = matcher.match(structs, candidate, options)
debug && debug_log(result)
debug && debugLog(result)
return result
}

7
lib/ofac/matching.js
View file

@ -1,7 +1,7 @@
const jaro = require('talisman/metrics/distance/jaro')
const _ = require('lodash/fp')
const debug_log = require('../pp')(__filename) // KOSTIS TODO: remove
const debugLog = require('../pp')(__filename) // KOSTIS TODO: remove
const stringSimilarity = _.curry(jaro)
@ -44,7 +44,6 @@ function match (structs, candidate, options) {
_.map(_.get('id'))
)(aliases)
const phoneticWeight = ratio
const stringWeight = 1 - phoneticWeight
@ -90,8 +89,8 @@ function match (structs, candidate, options) {
_.map(_.first)
)(matches)
debug && debug_log(aliasIdsFromFullName)
debug && debug_log(aliasIdsFromNamePart)
debug && debugLog(aliasIdsFromFullName)
debug && debugLog(aliasIdsFromNamePart)
// Get the full record for each matched id
const getIndividual = aliasId => {

128
lib/ofac/update.js
View file

@ -7,23 +7,28 @@ const util = require('util')
const options = require('../options')
const _ = require('lodash/fp')
const OFAC_DATA_DIR = options.ofacDataDir
const OFAC_SOURCES_DIR = path.join(OFAC_DATA_DIR, 'sources')
const OFAC_ETAGS_FILE = path.join(OFAC_DATA_DIR, 'etags.json')
const DOWNLOAD_DIR = path.resolve('/tmp')
function mkdir (path) {
return new Promise((resolve, reject) => {
fs.mkdir(path, err => {
if (!err) return resolve()
if (err.code === 'EEXIST') return resolve()
reject(err)
})
})
}
const mkdir = util.promisify(fs.mkdir)
const readFile = util.promisify(fs.readFile)
const writeFile = util.promisify(fs.writeFile)
const rename = util.promisify(fs.rename)
const unlink = util.promisify(fs.unlink)
const remove = file => {
console.log("remove", file)
console.log('remove', file)
return unlink(file)
}
const promiseGetEtag = (source) => {
return new Promise((resolve, reject) => {
const {url: sourceUrl} = source
@ -46,14 +51,14 @@ const promiseGetEtag = (source) => {
}
const download = _.curry((dstDir, source) => {
console.log("download", source)
console.log('download', source)
const {name, url: sourceUrl} = source
const dstFile = path.join(dstDir, name + '.xml')
const file = fs.createWriteStream(dstFile)
return new Promise((resolve, reject) => {
const request = https.get(sourceUrl, response => {
response.pipe(file);
response.pipe(file)
file.on('finish', () => file.close(() => resolve(dstFile)))
})
@ -62,14 +67,14 @@ const download = _.curry((dstDir, source) => {
})
const parseToJson = srcFile => {
console.log("parseToJson", srcFile)
console.log('parseToJson', srcFile)
const dstFile = srcFile.replace(/\.xml$/, '.json')
const writeStream = fs.createWriteStream(dstFile)
return new Promise((resolve, reject) => {
parser.parse(srcFile, (err, profile) => {
console.log("callback", err, profile)
console.log('callback', err, profile)
if (err) {
reject(err)
@ -90,65 +95,78 @@ const parseToJson = srcFile => {
})
}
const moveToSourcesDir = srcFile => {
console.log("moveToSourcesDir", srcFile)
const moveToSourcesDir = (srcFile, ofacSourcesDir) => {
console.log('moveToSourcesDir', srcFile)
const name = path.basename(srcFile)
const dstFile = path.join(OFAC_SOURCES_DIR, name)
const dstFile = path.join(ofacSourcesDir, name)
return rename(srcFile, dstFile)
}
function update () {
const OFAC_DATA_DIR = options.ofacDataDir
const update = () => mkdir(OFAC_DATA_DIR).catch(err => null)
.then(() => mkdir(OFAC_SOURCES_DIR)).catch(err => null)
.then(() => writeFile(OFAC_ETAGS_FILE, '{}', {encoding: 'utf-8', flag: 'wx'}))
.catch(err => null)
.then(() => {
const promiseOldEtags = readFile(OFAC_ETAGS_FILE, {encoding: 'utf-8'})
.then(json => JSON.parse(json) || {})
if (!OFAC_DATA_DIR) {
throw new Error('ofacDataDir must be defined in lamassu.json')
}
const promiseNewEtags = Promise.resolve(options.ofacSources || [])
.then(sources => Promise.all(_.map(promiseGetEtag, sources))
.then(etags => _.map(
([source, etag]) => ({...source, etag}),
_.zip(sources, etags)
))
)
const OFAC_SOURCES_DIR = path.join(OFAC_DATA_DIR, 'sources')
const OFAC_ETAGS_FILE = path.join(OFAC_DATA_DIR, 'etags.json')
return Promise.all([promiseOldEtags, promiseNewEtags])
.then(([oldEtags, newEtags]) => {
console.log("OLD", JSON.stringify(oldEtags, null, 4))
console.log("NEW", JSON.stringify(newEtags, null, 4))
return mkdir(OFAC_DATA_DIR)
.then(() => mkdir(OFAC_SOURCES_DIR))
.then(() => writeFile(OFAC_ETAGS_FILE, '{}', {encoding: 'utf-8', flag: 'wx'}))
.catch(err => {
if (err.code === 'EEXIST') return
throw err
})
.then(() => {
const promiseOldEtags = readFile(OFAC_ETAGS_FILE, {encoding: 'utf-8'})
.then(json => JSON.parse(json) || {})
const hasNotChanged = ({name, etag}) => oldEtags[name] === etag
const promiseNewEtags = Promise.resolve(options.ofacSources || [])
.then(sources => Promise.all(_.map(promiseGetEtag, sources))
.then(etags => _.map(
([source, etag]) => ({...source, etag}),
_.zip(sources, etags)
))
)
const downloads = _.flow(
_.reject(hasNotChanged),
_.map(file => download(DOWNLOAD_DIR, file).then(parseToJson))
)(newEtags)
return Promise.all([promiseOldEtags, promiseNewEtags])
.then(([oldEtags, newEtags]) => {
console.log('OLD', JSON.stringify(oldEtags, null, 4))
console.log('NEW', JSON.stringify(newEtags, null, 4))
const oldFileNames = _.keys(oldEtags)
const newFileNames = _.map(_.get('name'), newEtags)
const missingFileNames = _.difference(oldFileNames, newFileNames)
const resolve = name => path.join(OFAC_SOURCES_DIR, name + '.json')
const missing = _.map(resolve, missingFileNames)
const hasNotChanged = ({name, etag}) => oldEtags[name] === etag
const etagsJson = _.flow(
_.map(source => [source.name, source.etag]),
_.fromPairs,
obj => JSON.stringify(obj, null, 4)
)(newEtags)
const downloads = _.flow(
_.reject(hasNotChanged),
_.map(file => download(DOWNLOAD_DIR, file).then(parseToJson))
)(newEtags)
return Promise.all(downloads)
.then(parsed => {
console.log("finished", parsed)
const oldFileNames = _.keys(oldEtags)
const newFileNames = _.map(_.get('name'), newEtags)
const missingFileNames = _.difference(oldFileNames, newFileNames)
const resolve = name => path.join(OFAC_SOURCES_DIR, name + '.json')
const missing = _.map(resolve, missingFileNames)
const moves = _.map(moveToSourcesDir, parsed)
const deletions = _.map(remove, missing)
const updateEtags = writeFile(OFAC_ETAGS_FILE, etagsJson)
const etagsJson = _.flow(
_.map(source => [source.name, source.etag]),
_.fromPairs,
obj => JSON.stringify(obj, null, 4)
)(newEtags)
return Promise.all([updateEtags, ...moves, ...deletions])
return Promise.all(downloads)
.then(parsed => {
console.log('finished', parsed)
const moves = _.map(src => moveToSourcesDir(src, OFAC_SOURCES_DIR), parsed)
const deletions = _.map(remove, missing)
const updateEtags = writeFile(OFAC_ETAGS_FILE, etagsJson)
return Promise.all([updateEtags, ...moves, ...deletions])
})
})
})
})
})
}
module.exports = {update}

3
lib/routes.js
View file

@ -186,11 +186,14 @@ function getCustomerWithPhoneCode (req, res, next) {
function updateCustomer (req, res, next) {
const id = req.params.id
const patch = req.body
const config = configManager.unscoped(req.settings.config)
customers.getById(id)
.then(customer => {
if (!customer) { throw httpError('Not Found', 404) }
return customers.update(id, patch)
})
.then(customer => customers.validate(config, customer))
.then(customer => respond(req, res, {customer}))
.catch(next)
}

3
package.json
View file

@ -75,7 +75,8 @@
"lamassu-cancel": "./bin/lamassu-cancel",
"lamassu-nuke-db": "./bin/lamassu-nuke-db",
"lamassu-coins": "./bin/lamassu-coins",
"lamassu-update": "./bin/lamassu-update"
"lamassu-update": "./bin/lamassu-update",
"lamassu-ofac-update": "./bin/lamassu-ofac-update"
},
"scripts": {
"start": "node bin/lamassu-server",