Merge pull request #1692 from siiky/refactor/lam-1122

LAM-1122 refactor: simplify OFAC code

lib/ofac/index.js

@@ -1,20 +1,15 @@
-const fs = require('fs')
+const { readdir } = require('fs/promises')
 const path = require('path')
-const util = require('util')
 const loader = require('./loading')
 const matcher = require('./matching')
 const nameUtils = require('./name-utils')
 const _ = require('lodash/fp')
 const logger = require('../logger')
 
-const debugLog = require('../pp')(__filename) // KOSTIS TODO: remove
-
 const OFAC_DATA_DIR = process.env.OFAC_DATA_DIR
 
 let structs = null
 
-const readdir = util.promisify(fs.readdir)
-
 function load () {
   if (!OFAC_DATA_DIR) {
     const message = 'The ofacDataDir option has not been set in the environment'
@@ -43,8 +38,6 @@ function makeCompatible (nameParts) {
 }
 
 function match (nameParts, birthDateString, options) {
-  const {debug} = options
-
   if (!structs) {
     logger.error(new Error('The OFAC data sources have not been loaded yet.'))
     return false
@@ -69,10 +62,7 @@ function match (nameParts, birthDateString, options) {
   ])(birthDateString)
 
   const candidate = {parts, fullName, words, birthDate}
-  debug && debugLog(candidate)
-
   const result = matcher.match(structs, candidate, options)
-  debug && debugLog(result)
   return result
 }
 

lib/ofac/matching.js

@@ -1,7 +1,6 @@
 const jaro = require('talisman/metrics/distance/jaro')
 const _ = require('lodash/fp')
 
-const debugLog = require('../pp')(__filename) // KOSTIS TODO: remove
 const logger = require('../logger')
 
 const stringSimilarity = _.curry(jaro)
@@ -30,7 +29,7 @@ const isBornTooLongSince = _.curry((days, dateObject, individual) => {
 // algorithm
 
 function match (structs, candidate, options) {
-  const {threshold, fullNameThreshold, ratio = 0.5, debug, verboseFor} = options
+  const {threshold, fullNameThreshold, ratio = 0.5, verboseFor} = options
   const {fullName, words, birthDate} = candidate
 
   // Accept aliases who's full name matches.
@@ -90,9 +89,6 @@ function match (structs, candidate, options) {
     _.map(_.first)
   )(matches)
 
-  debug && debugLog(aliasIdsFromFullName)
-  debug && debugLog(aliasIdsFromNamePart)
-
   // Get the full record for each matched id
   const getIndividual = aliasId => {
     const individualId = structs.aliasToIndividual.get(aliasId)

lib/ofac/parsing.js

@@ -4,8 +4,6 @@ const nameUtils = require('./name-utils')
 const logger = require('../logger')
 const _ = require('lodash/fp')
 
-const debug_log = require('../pp')(__filename) // KOSTIS TODO: remove
-
 // KOSTIS TODO: get these from the document itself
 const INDIVIDUAL = '4'
 const NAME = '1403'
@@ -132,8 +130,6 @@ function processProfile (profileNode) {
   const birthDatePeriods = mapCompact(processFeature, profileNode.Feature)
   const individual = {id, aliases, birthDatePeriods}
 
-  // debug_log(individual)
-
   return individual
 }
 

lib/ofac/update.js

@@ -1,9 +1,10 @@
 const parser = require('./parsing')
 const https = require('https')
-const url = require('url')
+const URL = require('url')
-const fs = require('fs')
+const { createWriteStream } = require('fs')
+const fs = require('fs/promises')
+const { readFile, writeFile, rename, unlink } = fs
 const path = require('path')
-const util = require('util')
 const _ = require('lodash/fp')
 const logger = require('../logger')
 
@@ -14,36 +15,17 @@ const OFAC_SOURCES_NAMES = process.env.OFAC_SOURCES_NAMES.split(',')
 const OFAC_SOURCES_URLS = process.env.OFAC_SOURCES_URLS.split(',')
 
 const ofacSources = _.map(
-  it => ({
+  ([name, url]) => ({ name, url }),
-    name: it[0],
-    url: it[1]
-  }),
   _.zip(OFAC_SOURCES_NAMES, OFAC_SOURCES_URLS)
 )
 
-function mkdir (path) {
+const mkdir = path =>
-  return new Promise((resolve, reject) => {
+  fs.mkdir(path)
-    fs.mkdir(path, err => {
+    .catch(err => err.code === 'EEXIST' ? Promise.resolve() : Promise.reject(err))
-      if (!err) return resolve()
-      if (err.code === 'EEXIST') return resolve()
-      reject(err)
-    })
-  })
-}
 
-const readFile = util.promisify(fs.readFile)
+const promiseGetEtag = ({ url }) =>
-const writeFile = util.promisify(fs.writeFile)
+  new Promise((resolve, reject) => {
-const rename = util.promisify(fs.rename)
+    const parsed = URL.parse(url)
-const unlink = util.promisify(fs.unlink)
-
-const remove = file => {
-  return unlink(file)
-}
-
-const promiseGetEtag = (source) => {
-  return new Promise((resolve, reject) => {
-    const {url: sourceUrl} = source
-    const parsed = url.parse(sourceUrl)
     const requestOptions = {
       hostname: parsed.hostname,
       path: parsed.path,
@@ -59,26 +41,24 @@ const promiseGetEtag = (source) => {
 
     request.end()
   })
-}
 
-const download = _.curry((dstDir, source) => {
+const download = (dstDir, { name, url }) => {
-  const {name, url: sourceUrl} = source
   const dstFile = path.join(dstDir, name + '.xml')
-  const file = fs.createWriteStream(dstFile)
+  const file = createWriteStream(dstFile)
 
   return new Promise((resolve, reject) => {
-    const request = https.get(sourceUrl, response => {
+    const request = https.get(url, response => {
       response.pipe(file)
       file.on('finish', () => file.close(() => resolve(dstFile)))
     })
 
     request.on('error', reject)
   })
-})
+}
 
 const parseToJson = srcFile => {
   const dstFile = srcFile.replace(/\.xml$/, '.json')
-  const writeStream = fs.createWriteStream(dstFile)
+  const writeStream = createWriteStream(dstFile)
 
   return new Promise((resolve, reject) => {
     parser.parse(srcFile, (err, profile) => {
@@ -137,7 +117,7 @@ function update () {
       const promiseNewEtags = Promise.resolve(ofacSources || [])
         .then(sources => Promise.all(_.map(promiseGetEtag, sources))
           .then(etags => _.map(
-            ([source, etag]) => ({...source, etag}),
+            ([source, etag]) => _.set('etag', etag,  source),
             _.zip(sources, etags)
           ))
         )
@@ -166,7 +146,7 @@ function update () {
           return Promise.all(downloads)
             .then(parsed => {
               const moves = _.map(src => moveToSourcesDir(src, OFAC_SOURCES_DIR), parsed)
-              const deletions = _.map(remove, missing)
+              const deletions = _.map(unlink, missing)
               const updateEtags = writeFile(OFAC_ETAGS_FILE, etagsJson)
 
               return Promise.all([updateEtags, ...moves, ...deletions])