diff --git a/bin/cert-gen.sh b/bin/cert-gen.sh index e0a4e956..ba656ab0 100755 --- a/bin/cert-gen.sh +++ b/bin/cert-gen.sh @@ -80,7 +80,17 @@ cat < $CONFIG_DIR/lamassu.json "lamassuCaPath": "$LAMASSU_CA_PATH", "lamassuServerPath": "$PWD", "migrateStatePath": "$MIGRATE_STATE_PATH", - "ofacDataDir": "$OFAC_DATA_DIR" + "ofacDataDir": "$OFAC_DATA_DIR", + "ofacSources": [ + { + "name": "sdn_advanced", + "url": "https://www.treasury.gov/ofac/downloads/sanctions/1.0/sdn_advanced.xml" + }, + { + "name": "cons_advanced", + "url": "https://www.treasury.gov/ofac/downloads/sanctions/1.0/cons_advanced.xml" + } + ] } EOF diff --git a/lib/ofac/update.js b/lib/ofac/update.js index ad510418..951a9483 100644 --- a/lib/ofac/update.js +++ b/lib/ofac/update.js @@ -9,11 +9,11 @@ const _ = require('lodash/fp') const OFAC_DATA_DIR = options.ofacDataDir const OFAC_SOURCES_DIR = path.join(OFAC_DATA_DIR, 'sources') -const OFAC_SOURCES_FILE = path.join(OFAC_DATA_DIR, 'sources.json') const OFAC_ETAGS_FILE = path.join(OFAC_DATA_DIR, 'etags.json') const DOWNLOAD_DIR = path.resolve('/tmp') +const mkdir = util.promisify(fs.mkdir) const readFile = util.promisify(fs.readFile) const writeFile = util.promisify(fs.writeFile) const rename = util.promisify(fs.rename) @@ -47,9 +47,8 @@ const promiseGetEtag = (source) => { const download = _.curry((dstDir, source) => { console.log("download", source) - const {url: sourceUrl} = source - const fileName = path.basename(sourceUrl) - const dstFile = path.join(dstDir, fileName) + const {name, url: sourceUrl} = source + const dstFile = path.join(dstDir, name + '.xml') const file = fs.createWriteStream(dstFile) return new Promise((resolve, reject) => { @@ -99,57 +98,57 @@ const moveToSourcesDir = srcFile => { } -function update () { - const promiseOldEtags = readFile(OFAC_ETAGS_FILE, {encoding: 'utf-8'}) - .then(json => JSON.parse(json) || {}) +const update = () => mkdir(OFAC_DATA_DIR).catch(err => null) + .then(() => mkdir(OFAC_SOURCES_DIR)).catch(err => null) + .then(() => writeFile(OFAC_ETAGS_FILE, '{}', {encoding: 'utf-8', flag: 'wx'})) + .catch(err => null) + .then(() => { + const promiseOldEtags = readFile(OFAC_ETAGS_FILE, {encoding: 'utf-8'}) + .then(json => JSON.parse(json) || {}) - const promiseNewEtags = readFile(OFAC_SOURCES_FILE, {encoding: 'utf-8'}) - .then(json => { - const obj = JSON.parse(json) - return obj ? obj.sources : [] - }) - .then(sources => Promise.all(_.map(promiseGetEtag, sources)) - .then(etags => _.map( - ([source, etag]) => ({...source, etag}), - _.zip(sources, etags) - )) - ) + const promiseNewEtags = Promise.resolve(options.ofacSources || []) + .then(sources => Promise.all(_.map(promiseGetEtag, sources)) + .then(etags => _.map( + ([source, etag]) => ({...source, etag}), + _.zip(sources, etags) + )) + ) - return Promise.all([promiseOldEtags, promiseNewEtags]) - .then(([oldEtags, newEtags]) => { - console.log("OLD", JSON.stringify(oldEtags, null, 4)) - console.log("NEW", JSON.stringify(newEtags, null, 4)) + return Promise.all([promiseOldEtags, promiseNewEtags]) + .then(([oldEtags, newEtags]) => { + console.log("OLD", JSON.stringify(oldEtags, null, 4)) + console.log("NEW", JSON.stringify(newEtags, null, 4)) - const hasNotChanged = ({name, etag}) => oldEtags[name] === etag + const hasNotChanged = ({name, etag}) => oldEtags[name] === etag - const downloads = _.flow( - _.reject(hasNotChanged), - _.map(file => download(DOWNLOAD_DIR, file).then(parseToJson)) - )(newEtags) + const downloads = _.flow( + _.reject(hasNotChanged), + _.map(file => download(DOWNLOAD_DIR, file).then(parseToJson)) + )(newEtags) - const oldFileNames = _.keys(oldEtags) - const newFileNames = _.map(_.get('name'), newEtags) - const missingFileNames = _.difference(oldFileNames, newFileNames) - const resolve = name => path.join(OFAC_SOURCES_DIR, name + '.json') - const missing = _.map(resolve, missingFileNames) + const oldFileNames = _.keys(oldEtags) + const newFileNames = _.map(_.get('name'), newEtags) + const missingFileNames = _.difference(oldFileNames, newFileNames) + const resolve = name => path.join(OFAC_SOURCES_DIR, name + '.json') + const missing = _.map(resolve, missingFileNames) - const etagsJson = _.flow( - _.map(source => [source.name, source.etag]), - _.fromPairs, - obj => JSON.stringify(obj, null, 4) - )(newEtags) + const etagsJson = _.flow( + _.map(source => [source.name, source.etag]), + _.fromPairs, + obj => JSON.stringify(obj, null, 4) + )(newEtags) - return Promise.all(downloads) - .then(parsed => { - console.log("finished", parsed) + return Promise.all(downloads) + .then(parsed => { + console.log("finished", parsed) - const moves = _.map(moveToSourcesDir, parsed) - const deletions = _.map(remove, missing) - const updateEtags = writeFile(OFAC_ETAGS_FILE, etagsJson) + const moves = _.map(moveToSourcesDir, parsed) + const deletions = _.map(remove, missing) + const updateEtags = writeFile(OFAC_ETAGS_FILE, etagsJson) - return Promise.all([updateEtags, ...moves, ...deletions]) - }) + return Promise.all([updateEtags, ...moves, ...deletions]) + }) + }) }) -} module.exports = {update}