diff --git a/lib/ofac/update.js b/lib/ofac/update.js index 16d03d3d..8c520a69 100644 --- a/lib/ofac/update.js +++ b/lib/ofac/update.js @@ -1,14 +1,14 @@ const parser = require('./parsing') const axios = require('axios') const { createWriteStream } = require('fs') -const fs = require('fs/promises') -const { rename } = fs +const { rename, writeFile, readFile, mkdir, copyFile, unlink } = require('fs/promises') const path = require('path') const _ = require('lodash/fp') const DOWNLOAD_DIR = path.resolve('/tmp') - const OFAC_DATA_DIR = process.env.OFAC_DATA_DIR +const OFAC_SOURCES_DIR = path.join(OFAC_DATA_DIR, 'sources') +const LAST_UPDATED_FILE = path.resolve(OFAC_DATA_DIR, 'last_updated.dat') const OFAC_SOURCES = [{ name: 'sdn_advanced', @@ -18,38 +18,34 @@ const OFAC_SOURCES = [{ url: 'https://sanctionslistservice.ofac.treas.gov/api/download/cons_advanced.xml' }] -const mkdir = path => - fs.mkdir(path) +const _mkdir = path => + mkdir(path) .catch(err => err.code === 'EEXIST' ? Promise.resolve() : Promise.reject(err)) -const newDownload = (dstDir, { name, url }) => { - return path.join('/tmp/', name + '.xml') -} - const download = (dstDir, { name, url }) => { const dstFile = path.join(dstDir, name + '.xml') const writer = createWriteStream(dstFile) - return axios({ - method: 'get', - url: url, - responseType: 'stream', - }).then(response => { - return new Promise((resolve, reject) => { - response.data.pipe(writer); - let error = null; - writer.on('error', err => { - error = err; - writer.close(); - reject(err); - }); - writer.on('close', () => { - if (!error) { - resolve(dstFile); - } - }); - }); - }); + return axios({ + method: 'get', + url: url, + responseType: 'stream', + }).then(response => { + return new Promise((resolve, reject) => { + response.data.pipe(writer) + let error = null + writer.on('error', err => { + error = err + writer.close() + reject(err) + }) + writer.on('close', () => { + if (!error) { + resolve(dstFile) + } + }) + }) + }) } const parseToJson = srcFile => { @@ -77,10 +73,21 @@ const parseToJson = srcFile => { }) } -const moveToSourcesDir = (srcFile, ofacSourcesDir) => { +const moveToSourcesDir = async (srcFile, ofacSourcesDir) => { const name = path.basename(srcFile) const dstFile = path.join(ofacSourcesDir, name) - return rename(srcFile, dstFile) + try { + await rename(srcFile, dstFile) + } catch (err) { + if (err.code === 'EXDEV') { + // If rename fails due to cross-device link, fallback to copy + delete + await copyFile(srcFile, dstFile) + await unlink(srcFile) + } else { + throw err + } + } + return dstFile } function update () { @@ -88,15 +95,28 @@ function update () { throw new Error('ofacDataDir must be defined in the environment') } - const OFAC_SOURCES_DIR = path.join(OFAC_DATA_DIR, 'sources') - - return mkdir(OFAC_DATA_DIR) - .then(() => mkdir(OFAC_SOURCES_DIR)) + return _mkdir(OFAC_DATA_DIR) + .then(() => _mkdir(OFAC_SOURCES_DIR)) .catch(err => { if (err.code === 'EEXIST') return throw err }) - .then(() => { + .then(() => readFile(LAST_UPDATED_FILE)) + .then(data => { + const lastUpdate = new Date(data.toString()) + const now = new Date() + const hoursSinceUpdate = (now - lastUpdate) / (1000 * 60 * 60) + + return hoursSinceUpdate < 24 + }) + .catch(err => { + // If file doesn't exist, continue with update + if (err.code === 'ENOENT') return false + throw err + }) + .then(skipUpdate => { + if (skipUpdate) return Promise.resolve() + const downloads = _.flow( _.map(file => download(DOWNLOAD_DIR, file).then(parseToJson)) )(OFAC_SOURCES) @@ -104,8 +124,10 @@ function update () { return Promise.all(downloads) .then(parsed => { const moves = _.map(src => moveToSourcesDir(src, OFAC_SOURCES_DIR), parsed) + const timestamp = new Date().toISOString() return Promise.all([...moves]) + .then(() => writeFile(LAST_UPDATED_FILE, timestamp)) }) }) }