Corrections and initial algorithm

lib/ofac/index.js

@@ -0,0 +1,266 @@
+const fs = require('fs')
+const path = require('path')
+const XmlStream = require('xml-stream')
+const jaroWinkler = require('talisman/metrics/distance/jaro-winkler')
+const metaphone = require('talisman/phonetics/metaphone')
+const options = require('../options')
+const logger = require('../logger')
+const _ = require('lodash/fp')
+
+// PARSING
+
+const OFAC_DATA_DIR = options.ofacDataDir
+
+// TODO: get these from the document itself
+const INDIVIDUAL = '4'
+const NAME = '1403'
+const BIRTH_DATE = '8'
+
+const LAST_NAME = '1520'
+const FIRST_NAME = '1521'
+const MIDDLE_NAME = '1522'
+const MAIDEN_NAME = '1523'
+const PATRONYMIC = '91708'
+const MATRONYMIC = '91709'
+const NICKNAME = '1528'
+
+const partNames = new Map([
+  [LAST_NAME, 'lastName'],
+  [FIRST_NAME, 'firstName'],
+  [MIDDLE_NAME, 'middleName'],
+  [MAIDEN_NAME, 'maidenName'],
+  [PATRONYMIC, 'patronymic'],
+  [MATRONYMIC, 'matronymic'],
+  [NICKNAME, 'nickname']
+])
+
+// TODO: get this from admin configuration
+const SIMILARITY_THRESHOLD = 0.5
+
+// TODO: remove
+const debug_log = (...args) => console.log(require('util').inspect(args, {depth: null, colors: true}))
+
+let individuals = []
+const individualsById = new Map()
+
+// group-id to type-id
+
+function processMasterNamePartGroup (groupNode) {
+  const namePartGroupNode = groupNode.NamePartGroup
+  const groupId = namePartGroupNode.$.ID
+  const typeId = namePartGroupNode.$.NamePartTypeID
+  return [groupId, typeId]
+}
+
+// name parts
+
+function makeFullNameFromParts (nameParts) {
+  // Combine name-parts in a standared order.
+  const namePartPairs = _.toPairs(nameParts)
+  const sortedPairs = _.sortBy(_.nth(0), namePartPairs)
+  return _.map(_.nth(1), sortedPairs).join(' ')
+}
+
+function makePhonetic (name) {
+  return metaphone(name)
+}
+
+function processDocumentedNamePart (groupTypes) {
+  return function (namePartNode) {
+    const valueNode = namePartNode.NamePartValue
+    const groupId = valueNode.$.NamePartGroupID
+    const typeId = groupTypes.get(groupId)
+    const partName = partNames.get(typeId)
+    const value = valueNode.$text
+    return {[partName]: value}
+  }
+}
+
+function processAlias (groupTypes) {
+  return function (aliasNode) {
+    if (aliasNode.$.AliasTypeID !== NAME) return
+
+    const nameParts = _.map(processDocumentedNamePart(groupTypes), aliasNode.DocumentedName.DocumentedNamePart)
+    const parts = _.assignAll(nameParts)
+    const fullName = makeFullNameFromParts(parts)
+
+    const phoneticParts = _.mapValues(makePhonetic, parts)
+    const phoneticFullName = makePhonetic(fullName)
+
+    return {parts, fullName, phoneticParts, phoneticFullName}
+  }
+}
+
+// birth date
+
+function processDate (dateNode) {
+  const year = parseInt(dateNode.Year)
+  const month = parseInt(dateNode.Month)
+  const day = parseInt(dateNode.Day)
+  const date = new Date(year, month - 1, day)
+
+  return {year, month, day, date}
+}
+
+function processFeature (featureNode) {
+  if (featureNode.$.FeatureTypeID !== BIRTH_DATE) return
+
+  const datePeriodNode = featureNode.FeatureVersion.DatePeriod
+  // Ignore the fact that both Start and end can be a range.
+  // By using Start.From and End.To we use the extremes of the date-period.
+  const period = {
+    start: datePeriodNode.Start.From,
+    end: datePeriodNode.End.To
+  }
+
+  return _.mapValues(processDate, period)
+}
+
+// profile
+
+function processProfile (profileNode) {
+  if (profileNode.$.PartySubTypeID !== INDIVIDUAL) return
+
+  const id = profileNode.$.ID
+
+  const identityNode = profileNode.Identity
+  const groupTypesEntries = _.map(processMasterNamePartGroup, identityNode.NamePartGroups.MasterNamePartGroup)
+  const groupTypes = new Map(groupTypesEntries)
+
+  const aliases = _.compact(_.map(processAlias(groupTypes), identityNode.Alias))
+  const birthDatePeriods = _.compact(_.map(processFeature, profileNode.Feature))
+  const individual = {aliases, birthDatePeriods}
+
+  individualsById.set(id, individual)
+  debug_log(individual)
+}
+
+function promiseParseDocument (source) {
+  return new Promise(resolve => {
+    const fileName = path.join(OFAC_DATA_DIR, source)
+    const stream = fs.createReadStream(fileName)
+    const xml = new XmlStream(stream)
+
+    xml.on('error', error => {
+      logger.error('Error while parsing the OFAC data sources.')
+      logger.error(error)
+      xml.pause()
+      resolve()
+    })
+
+    xml.collect('Alias')
+    xml.collect('DocumentedNamePart')
+    xml.collect('Feature')
+    xml.collect('MasterNamePartGroup')
+
+    xml.on('updateElement: Profile', processProfile)
+
+    xml.on('end', resolve)
+  })
+}
+
+function load () {
+  // NOTE: Not sure how you push code updates to existing clients. This problem
+  // might pop up if new code is pushed, without re-doing setup.
+  if (!OFAC_DATA_DIR) {
+    logger.error('The ofacDataDir option has not been set in lamassu.json')
+    return
+  }
+
+  individualsById.clear()
+
+  const sources = fs.readdirSync(OFAC_DATA_DIR)
+  const promises = _.map(promiseParseDocument, sources)
+
+  return Promise.all(promises)
+    .then(() => {
+      individuals = Array.from(individualsById.values())
+    })
+}
+
+// MATCHING
+
+// birth date
+
+function isDateWithinTwoYearsOfPeriod (targetDate) {
+  return function (period) {
+    const startDate = new Date(period.from.date)
+    const startYear = startDate.getFullYear()
+    startDate.setFullYear(startYear - 2)
+
+    const endDate = new Date(period.to.date)
+    const endYear = endDate.getFullYear()
+    endDate.setFullYear(endYear + 2)
+
+    return (startDate < targetDate && targetDate < endDate)
+  }
+}
+
+function isBornWithinTwoYears (individual, dateObject) {
+  return _.some(isDateWithinTwoYearsOfPeriod(dateObject.date), individual.birthDatePeriods)
+}
+
+// exact match
+
+function calcExactMatchScore (candidateFullName) {
+  return function (alias) {
+    return jaroWinkler(alias.fullName, candidateFullName)
+  }
+}
+
+// phonetic match
+
+function calcPhoneticMatchScore (candidatePhoneticFullName) {
+  return function (alias) {
+    return jaroWinkler(alias.phoneticFullName, candidatePhoneticFullName)
+  }
+}
+
+// algorithm
+
+// NOTE: I'm still not 100% on what matching algorithm is the best choice.
+// I just experiment with a few metrics for now.
+
+function doesMatch (nameParts, fullName, phoneticParts, phoneticFullName, birthDate) {
+  return function (individual) {
+    // Calculate if his birth date is within two years of the given date.
+    // If an individual has multiple birth-date periods, return wether any are
+    // within two years. Reject individuals who don't match this criterion.
+    if (individual.birthDatePeriods.length && !isBornWithinTwoYears(individual, birthDate)) return false
+
+    // Calculate the Jaro-Winkler similarity of the full name.
+    // If an individual has multiple aliases, use the maximum score.
+    const exactMatchScore = _.max(_.map(calcExactMatchScore(fullName), individual.aliases))
+
+    if (exactMatchScore > SIMILARITY_THRESHOLD) return true
+
+    // Calculate the Jaro-Winkler similarity of the phonetic representation of the full name.
+    // This should approximate the phonetic similarity of the two names.
+    // If an individual has multiple aliases, use the maximum score.
+    const phoneticMatchScore = _.max(_.map(calcPhoneticMatchScore(phoneticFullName), individual.aliases))
+
+    if (phoneticMatchScore > SIMILARITY_THRESHOLD) return true
+
+    return false
+  }
+}
+
+function match (nameParts, birthDateString) {
+  // nameParts should be an object like {firstName: "John", lastName: "Doe", ...}
+  const fullName = makeFullNameFromParts(nameParts)
+
+  const phoneticParts = _.mapValues(makePhonetic, nameParts)
+  const phoneticFullName = makePhonetic(fullName)
+
+  // birthDateString is in YYYYMMDD format
+  const year = parseInt(birthDateString.slice(0, 4))
+  const month = parseInt(birthDateString.slice(4, 6))
+  const day = parseInt(birthDateString.slice(6, 8))
+  const date = new Date(year, month - 1, day)
+
+  const birthDate = {year, month, day, date}
+
+  return _.some(doesMatch(nameParts, fullName, phoneticParts, phoneticFullName, birthDate), individuals)
+}
+
+module.exports = {load, match}