diff --git a/lib/ofac/index.js b/lib/ofac/index.js index fff2f70e..8f34027b 100644 --- a/lib/ofac/index.js +++ b/lib/ofac/index.js @@ -1,20 +1,15 @@ -const fs = require('fs') +const { readdir } = require('fs/promises') const path = require('path') -const util = require('util') const loader = require('./loading') const matcher = require('./matching') const nameUtils = require('./name-utils') const _ = require('lodash/fp') const logger = require('../logger') -const debugLog = require('../pp')(__filename) // KOSTIS TODO: remove - const OFAC_DATA_DIR = process.env.OFAC_DATA_DIR let structs = null -const readdir = util.promisify(fs.readdir) - function load () { if (!OFAC_DATA_DIR) { const message = 'The ofacDataDir option has not been set in the environment' @@ -43,8 +38,6 @@ function makeCompatible (nameParts) { } function match (nameParts, birthDateString, options) { - const {debug} = options - if (!structs) { logger.error(new Error('The OFAC data sources have not been loaded yet.')) return false @@ -69,10 +62,7 @@ function match (nameParts, birthDateString, options) { ])(birthDateString) const candidate = {parts, fullName, words, birthDate} - debug && debugLog(candidate) - const result = matcher.match(structs, candidate, options) - debug && debugLog(result) return result } diff --git a/lib/ofac/matching.js b/lib/ofac/matching.js index 3ddb883e..f8c089e8 100644 --- a/lib/ofac/matching.js +++ b/lib/ofac/matching.js @@ -1,7 +1,6 @@ const jaro = require('talisman/metrics/distance/jaro') const _ = require('lodash/fp') -const debugLog = require('../pp')(__filename) // KOSTIS TODO: remove const logger = require('../logger') const stringSimilarity = _.curry(jaro) @@ -30,7 +29,7 @@ const isBornTooLongSince = _.curry((days, dateObject, individual) => { // algorithm function match (structs, candidate, options) { - const {threshold, fullNameThreshold, ratio = 0.5, debug, verboseFor} = options + const {threshold, fullNameThreshold, ratio = 0.5, verboseFor} = options const {fullName, words, birthDate} = candidate // Accept aliases who's full name matches. @@ -90,9 +89,6 @@ function match (structs, candidate, options) { _.map(_.first) )(matches) - debug && debugLog(aliasIdsFromFullName) - debug && debugLog(aliasIdsFromNamePart) - // Get the full record for each matched id const getIndividual = aliasId => { const individualId = structs.aliasToIndividual.get(aliasId) diff --git a/lib/ofac/parsing.js b/lib/ofac/parsing.js index a62eb69f..36b7dcd8 100644 --- a/lib/ofac/parsing.js +++ b/lib/ofac/parsing.js @@ -4,8 +4,6 @@ const nameUtils = require('./name-utils') const logger = require('../logger') const _ = require('lodash/fp') -const debug_log = require('../pp')(__filename) // KOSTIS TODO: remove - // KOSTIS TODO: get these from the document itself const INDIVIDUAL = '4' const NAME = '1403' @@ -132,8 +130,6 @@ function processProfile (profileNode) { const birthDatePeriods = mapCompact(processFeature, profileNode.Feature) const individual = {id, aliases, birthDatePeriods} - // debug_log(individual) - return individual } diff --git a/lib/ofac/update.js b/lib/ofac/update.js index 11fe480d..d1408687 100644 --- a/lib/ofac/update.js +++ b/lib/ofac/update.js @@ -1,9 +1,10 @@ const parser = require('./parsing') const https = require('https') -const url = require('url') -const fs = require('fs') +const URL = require('url') +const { createWriteStream } = require('fs') +const fs = require('fs/promises') +const { readFile, writeFile, rename, unlink } = fs const path = require('path') -const util = require('util') const _ = require('lodash/fp') const logger = require('../logger') @@ -14,36 +15,17 @@ const OFAC_SOURCES_NAMES = process.env.OFAC_SOURCES_NAMES.split(',') const OFAC_SOURCES_URLS = process.env.OFAC_SOURCES_URLS.split(',') const ofacSources = _.map( - it => ({ - name: it[0], - url: it[1] - }), + ([name, url]) => ({ name, url }), _.zip(OFAC_SOURCES_NAMES, OFAC_SOURCES_URLS) ) -function mkdir (path) { - return new Promise((resolve, reject) => { - fs.mkdir(path, err => { - if (!err) return resolve() - if (err.code === 'EEXIST') return resolve() - reject(err) - }) - }) -} +const mkdir = path => + fs.mkdir(path) + .catch(err => err.code === 'EEXIST' ? Promise.resolve() : Promise.reject(err)) -const readFile = util.promisify(fs.readFile) -const writeFile = util.promisify(fs.writeFile) -const rename = util.promisify(fs.rename) -const unlink = util.promisify(fs.unlink) - -const remove = file => { - return unlink(file) -} - -const promiseGetEtag = (source) => { - return new Promise((resolve, reject) => { - const {url: sourceUrl} = source - const parsed = url.parse(sourceUrl) +const promiseGetEtag = ({ url }) => + new Promise((resolve, reject) => { + const parsed = URL.parse(url) const requestOptions = { hostname: parsed.hostname, path: parsed.path, @@ -59,26 +41,24 @@ const promiseGetEtag = (source) => { request.end() }) -} -const download = _.curry((dstDir, source) => { - const {name, url: sourceUrl} = source +const download = (dstDir, { name, url }) => { const dstFile = path.join(dstDir, name + '.xml') - const file = fs.createWriteStream(dstFile) + const file = createWriteStream(dstFile) return new Promise((resolve, reject) => { - const request = https.get(sourceUrl, response => { + const request = https.get(url, response => { response.pipe(file) file.on('finish', () => file.close(() => resolve(dstFile))) }) request.on('error', reject) }) -}) +} const parseToJson = srcFile => { const dstFile = srcFile.replace(/\.xml$/, '.json') - const writeStream = fs.createWriteStream(dstFile) + const writeStream = createWriteStream(dstFile) return new Promise((resolve, reject) => { parser.parse(srcFile, (err, profile) => { @@ -137,7 +117,7 @@ function update () { const promiseNewEtags = Promise.resolve(ofacSources || []) .then(sources => Promise.all(_.map(promiseGetEtag, sources)) .then(etags => _.map( - ([source, etag]) => ({...source, etag}), + ([source, etag]) => _.set('etag', etag, source), _.zip(sources, etags) )) ) @@ -166,7 +146,7 @@ function update () { return Promise.all(downloads) .then(parsed => { const moves = _.map(src => moveToSourcesDir(src, OFAC_SOURCES_DIR), parsed) - const deletions = _.map(remove, missing) + const deletions = _.map(unlink, missing) const updateEtags = writeFile(OFAC_ETAGS_FILE, etagsJson) return Promise.all([updateEtags, ...moves, ...deletions])