From 9a8bbbd87f0284dc5599f7a3395af5deb24940c4 Mon Sep 17 00:00:00 2001 From: siiky Date: Fri, 28 Jun 2024 12:18:25 +0100 Subject: [PATCH 1/9] refactor: use `set` from `lodash/fp` instead of syntax --- lib/ofac/update.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ofac/update.js b/lib/ofac/update.js index 11fe480d..8630cdf6 100644 --- a/lib/ofac/update.js +++ b/lib/ofac/update.js @@ -137,7 +137,7 @@ function update () { const promiseNewEtags = Promise.resolve(ofacSources || []) .then(sources => Promise.all(_.map(promiseGetEtag, sources)) .then(etags => _.map( - ([source, etag]) => ({...source, etag}), + ([source, etag]) => _.set('etag', etag, source), _.zip(sources, etags) )) ) From 704e1673bd4995b3f60739d24ab4a6114730cb4c Mon Sep 17 00:00:00 2001 From: siiky Date: Fri, 28 Jun 2024 12:18:54 +0100 Subject: [PATCH 2/9] refactor: destruct in parameters list instead --- lib/ofac/update.js | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/ofac/update.js b/lib/ofac/update.js index 8630cdf6..66f7dd25 100644 --- a/lib/ofac/update.js +++ b/lib/ofac/update.js @@ -1,6 +1,6 @@ const parser = require('./parsing') const https = require('https') -const url = require('url') +const URL = require('url') const fs = require('fs') const path = require('path') const util = require('util') @@ -40,10 +40,9 @@ const remove = file => { return unlink(file) } -const promiseGetEtag = (source) => { +const promiseGetEtag = ({ url }) => { return new Promise((resolve, reject) => { - const {url: sourceUrl} = source - const parsed = url.parse(sourceUrl) + const parsed = URL.parse(url) const requestOptions = { hostname: parsed.hostname, path: parsed.path, From 3e058c6a98d5dce47f5ca414cb0d6c2923185775 Mon Sep 17 00:00:00 2001 From: siiky Date: Fri, 28 Jun 2024 12:20:09 +0100 Subject: [PATCH 3/9] refactor: destruct in parameters list to build object --- lib/ofac/update.js | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/ofac/update.js b/lib/ofac/update.js index 66f7dd25..4abf30cc 100644 --- a/lib/ofac/update.js +++ b/lib/ofac/update.js @@ -14,10 +14,7 @@ const OFAC_SOURCES_NAMES = process.env.OFAC_SOURCES_NAMES.split(',') const OFAC_SOURCES_URLS = process.env.OFAC_SOURCES_URLS.split(',') const ofacSources = _.map( - it => ({ - name: it[0], - url: it[1] - }), + ([name, url]) => ({ name, url }), _.zip(OFAC_SOURCES_NAMES, OFAC_SOURCES_URLS) ) From 7c5e26968576eda80c132b40eb30e66b78741726 Mon Sep 17 00:00:00 2001 From: siiky Date: Fri, 28 Jun 2024 12:21:42 +0100 Subject: [PATCH 4/9] chore: drop unused import --- lib/ofac/parsing.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/ofac/parsing.js b/lib/ofac/parsing.js index a62eb69f..36b7dcd8 100644 --- a/lib/ofac/parsing.js +++ b/lib/ofac/parsing.js @@ -4,8 +4,6 @@ const nameUtils = require('./name-utils') const logger = require('../logger') const _ = require('lodash/fp') -const debug_log = require('../pp')(__filename) // KOSTIS TODO: remove - // KOSTIS TODO: get these from the document itself const INDIVIDUAL = '4' const NAME = '1403' @@ -132,8 +130,6 @@ function processProfile (profileNode) { const birthDatePeriods = mapCompact(processFeature, profileNode.Feature) const individual = {id, aliases, birthDatePeriods} - // debug_log(individual) - return individual } From 6d9bf1fefcb84f28e4158a03192600e55589da20 Mon Sep 17 00:00:00 2001 From: siiky Date: Fri, 28 Jun 2024 15:42:34 +0100 Subject: [PATCH 5/9] refactor: drop unnecessary curly braces --- lib/ofac/update.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/ofac/update.js b/lib/ofac/update.js index 4abf30cc..83e9b7a4 100644 --- a/lib/ofac/update.js +++ b/lib/ofac/update.js @@ -37,8 +37,8 @@ const remove = file => { return unlink(file) } -const promiseGetEtag = ({ url }) => { - return new Promise((resolve, reject) => { +const promiseGetEtag = ({ url }) => + new Promise((resolve, reject) => { const parsed = URL.parse(url) const requestOptions = { hostname: parsed.hostname, @@ -55,7 +55,6 @@ const promiseGetEtag = ({ url }) => { request.end() }) -} const download = _.curry((dstDir, source) => { const {name, url: sourceUrl} = source From 0d94c2e58911a23ab7a37abb048065238666d82c Mon Sep 17 00:00:00 2001 From: siiky Date: Fri, 28 Jun 2024 15:43:17 +0100 Subject: [PATCH 6/9] refactor: destruct in parameters list --- lib/ofac/update.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/ofac/update.js b/lib/ofac/update.js index 83e9b7a4..39738853 100644 --- a/lib/ofac/update.js +++ b/lib/ofac/update.js @@ -56,13 +56,12 @@ const promiseGetEtag = ({ url }) => request.end() }) -const download = _.curry((dstDir, source) => { - const {name, url: sourceUrl} = source +const download = _.curry((dstDir, { name, url }) => { const dstFile = path.join(dstDir, name + '.xml') const file = fs.createWriteStream(dstFile) return new Promise((resolve, reject) => { - const request = https.get(sourceUrl, response => { + const request = https.get(url, response => { response.pipe(file) file.on('finish', () => file.close(() => resolve(dstFile))) }) From 3f87685865384bd627109486b18a967f1a02a35b Mon Sep 17 00:00:00 2001 From: siiky Date: Fri, 28 Jun 2024 15:43:36 +0100 Subject: [PATCH 7/9] refactor: drop unnecessary use of `_.curry` --- lib/ofac/update.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/ofac/update.js b/lib/ofac/update.js index 39738853..a8a45d5b 100644 --- a/lib/ofac/update.js +++ b/lib/ofac/update.js @@ -56,7 +56,7 @@ const promiseGetEtag = ({ url }) => request.end() }) -const download = _.curry((dstDir, { name, url }) => { +const download = (dstDir, { name, url }) => { const dstFile = path.join(dstDir, name + '.xml') const file = fs.createWriteStream(dstFile) @@ -68,7 +68,7 @@ const download = _.curry((dstDir, { name, url }) => { request.on('error', reject) }) -}) +} const parseToJson = srcFile => { const dstFile = srcFile.replace(/\.xml$/, '.json') From 4ead3dc4bfdb49f380a46ae9049267720260b2db Mon Sep 17 00:00:00 2001 From: siiky Date: Fri, 28 Jun 2024 15:53:22 +0100 Subject: [PATCH 8/9] refactor: replace `fs`+`util.promisify` with `fs/promises` --- lib/ofac/index.js | 5 +---- lib/ofac/update.js | 32 +++++++++----------------------- 2 files changed, 10 insertions(+), 27 deletions(-) diff --git a/lib/ofac/index.js b/lib/ofac/index.js index fff2f70e..f51a3ad8 100644 --- a/lib/ofac/index.js +++ b/lib/ofac/index.js @@ -1,6 +1,5 @@ -const fs = require('fs') +const { readdir } = require('fs/promises') const path = require('path') -const util = require('util') const loader = require('./loading') const matcher = require('./matching') const nameUtils = require('./name-utils') @@ -13,8 +12,6 @@ const OFAC_DATA_DIR = process.env.OFAC_DATA_DIR let structs = null -const readdir = util.promisify(fs.readdir) - function load () { if (!OFAC_DATA_DIR) { const message = 'The ofacDataDir option has not been set in the environment' diff --git a/lib/ofac/update.js b/lib/ofac/update.js index a8a45d5b..d1408687 100644 --- a/lib/ofac/update.js +++ b/lib/ofac/update.js @@ -1,9 +1,10 @@ const parser = require('./parsing') const https = require('https') const URL = require('url') -const fs = require('fs') +const { createWriteStream } = require('fs') +const fs = require('fs/promises') +const { readFile, writeFile, rename, unlink } = fs const path = require('path') -const util = require('util') const _ = require('lodash/fp') const logger = require('../logger') @@ -18,24 +19,9 @@ const ofacSources = _.map( _.zip(OFAC_SOURCES_NAMES, OFAC_SOURCES_URLS) ) -function mkdir (path) { - return new Promise((resolve, reject) => { - fs.mkdir(path, err => { - if (!err) return resolve() - if (err.code === 'EEXIST') return resolve() - reject(err) - }) - }) -} - -const readFile = util.promisify(fs.readFile) -const writeFile = util.promisify(fs.writeFile) -const rename = util.promisify(fs.rename) -const unlink = util.promisify(fs.unlink) - -const remove = file => { - return unlink(file) -} +const mkdir = path => + fs.mkdir(path) + .catch(err => err.code === 'EEXIST' ? Promise.resolve() : Promise.reject(err)) const promiseGetEtag = ({ url }) => new Promise((resolve, reject) => { @@ -58,7 +44,7 @@ const promiseGetEtag = ({ url }) => const download = (dstDir, { name, url }) => { const dstFile = path.join(dstDir, name + '.xml') - const file = fs.createWriteStream(dstFile) + const file = createWriteStream(dstFile) return new Promise((resolve, reject) => { const request = https.get(url, response => { @@ -72,7 +58,7 @@ const download = (dstDir, { name, url }) => { const parseToJson = srcFile => { const dstFile = srcFile.replace(/\.xml$/, '.json') - const writeStream = fs.createWriteStream(dstFile) + const writeStream = createWriteStream(dstFile) return new Promise((resolve, reject) => { parser.parse(srcFile, (err, profile) => { @@ -160,7 +146,7 @@ function update () { return Promise.all(downloads) .then(parsed => { const moves = _.map(src => moveToSourcesDir(src, OFAC_SOURCES_DIR), parsed) - const deletions = _.map(remove, missing) + const deletions = _.map(unlink, missing) const updateEtags = writeFile(OFAC_ETAGS_FILE, etagsJson) return Promise.all([updateEtags, ...moves, ...deletions]) From 8660f00257590397dfedf5a4246f727a68949bbc Mon Sep 17 00:00:00 2001 From: siiky Date: Fri, 28 Jun 2024 15:58:59 +0100 Subject: [PATCH 9/9] refactor: drop unused debug logs --- lib/ofac/index.js | 7 ------- lib/ofac/matching.js | 6 +----- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/lib/ofac/index.js b/lib/ofac/index.js index f51a3ad8..8f34027b 100644 --- a/lib/ofac/index.js +++ b/lib/ofac/index.js @@ -6,8 +6,6 @@ const nameUtils = require('./name-utils') const _ = require('lodash/fp') const logger = require('../logger') -const debugLog = require('../pp')(__filename) // KOSTIS TODO: remove - const OFAC_DATA_DIR = process.env.OFAC_DATA_DIR let structs = null @@ -40,8 +38,6 @@ function makeCompatible (nameParts) { } function match (nameParts, birthDateString, options) { - const {debug} = options - if (!structs) { logger.error(new Error('The OFAC data sources have not been loaded yet.')) return false @@ -66,10 +62,7 @@ function match (nameParts, birthDateString, options) { ])(birthDateString) const candidate = {parts, fullName, words, birthDate} - debug && debugLog(candidate) - const result = matcher.match(structs, candidate, options) - debug && debugLog(result) return result } diff --git a/lib/ofac/matching.js b/lib/ofac/matching.js index 3ddb883e..f8c089e8 100644 --- a/lib/ofac/matching.js +++ b/lib/ofac/matching.js @@ -1,7 +1,6 @@ const jaro = require('talisman/metrics/distance/jaro') const _ = require('lodash/fp') -const debugLog = require('../pp')(__filename) // KOSTIS TODO: remove const logger = require('../logger') const stringSimilarity = _.curry(jaro) @@ -30,7 +29,7 @@ const isBornTooLongSince = _.curry((days, dateObject, individual) => { // algorithm function match (structs, candidate, options) { - const {threshold, fullNameThreshold, ratio = 0.5, debug, verboseFor} = options + const {threshold, fullNameThreshold, ratio = 0.5, verboseFor} = options const {fullName, words, birthDate} = candidate // Accept aliases who's full name matches. @@ -90,9 +89,6 @@ function match (structs, candidate, options) { _.map(_.first) )(matches) - debug && debugLog(aliasIdsFromFullName) - debug && debugLog(aliasIdsFromNamePart) - // Get the full record for each matched id const getIndividual = aliasId => { const individualId = structs.aliasToIndividual.get(aliasId)