97 lines
2.5 KiB
JavaScript
97 lines
2.5 KiB
JavaScript
const parser = require('./parsing')
|
|
const https = require('https')
|
|
const { createWriteStream } = require('fs')
|
|
const fs = require('fs/promises')
|
|
const { rename } = fs
|
|
const path = require('path')
|
|
const _ = require('lodash/fp')
|
|
|
|
const DOWNLOAD_DIR = path.resolve('/tmp')
|
|
|
|
const OFAC_DATA_DIR = process.env.OFAC_DATA_DIR
|
|
|
|
const OFAC_SOURCES = [{
|
|
name: 'sdn_advanced',
|
|
url: 'https://sanctionslistservice.ofac.treas.gov/api/download/sdn_advanced.xml'
|
|
}, {
|
|
name: 'cons_advanced',
|
|
url: 'https://sanctionslistservice.ofac.treas.gov/api/download/cons_advanced.xml'
|
|
}]
|
|
|
|
const mkdir = path =>
|
|
fs.mkdir(path)
|
|
.catch(err => err.code === 'EEXIST' ? Promise.resolve() : Promise.reject(err))
|
|
|
|
const download = (dstDir, { name, url }) => {
|
|
const dstFile = path.join(dstDir, name + '.xml')
|
|
const file = createWriteStream(dstFile)
|
|
|
|
return new Promise((resolve, reject) => {
|
|
const request = https.get(url, response => {
|
|
response.pipe(file)
|
|
file.on('finish', () => file.close(() => resolve(dstFile)))
|
|
})
|
|
|
|
request.on('error', reject)
|
|
})
|
|
}
|
|
|
|
const parseToJson = srcFile => {
|
|
const dstFile = srcFile.replace(/\.xml$/, '.json')
|
|
const writeStream = createWriteStream(dstFile)
|
|
|
|
return new Promise((resolve, reject) => {
|
|
parser.parse(srcFile, (err, profile) => {
|
|
if (err) {
|
|
reject(err)
|
|
return
|
|
}
|
|
|
|
if (!profile) {
|
|
writeStream.end()
|
|
return
|
|
}
|
|
|
|
const json = JSON.stringify(profile)
|
|
writeStream.write(json + '\n', 'utf-8')
|
|
})
|
|
|
|
writeStream.on('error', reject)
|
|
writeStream.on('finish', () => resolve(dstFile))
|
|
})
|
|
}
|
|
|
|
const moveToSourcesDir = (srcFile, ofacSourcesDir) => {
|
|
const name = path.basename(srcFile)
|
|
const dstFile = path.join(ofacSourcesDir, name)
|
|
return rename(srcFile, dstFile)
|
|
}
|
|
|
|
function update () {
|
|
if (!OFAC_DATA_DIR) {
|
|
throw new Error('ofacDataDir must be defined in the environment')
|
|
}
|
|
|
|
const OFAC_SOURCES_DIR = path.join(OFAC_DATA_DIR, 'sources')
|
|
|
|
return mkdir(OFAC_DATA_DIR)
|
|
.then(() => mkdir(OFAC_SOURCES_DIR))
|
|
.catch(err => {
|
|
if (err.code === 'EEXIST') return
|
|
throw err
|
|
})
|
|
.then(() => {
|
|
const downloads = _.flow(
|
|
_.map(file => download(DOWNLOAD_DIR, file).then(parseToJson))
|
|
)(OFAC_SOURCES)
|
|
|
|
return Promise.all(downloads)
|
|
.then(parsed => {
|
|
const moves = _.map(src => moveToSourcesDir(src, OFAC_SOURCES_DIR), parsed)
|
|
|
|
return Promise.all([...moves])
|
|
})
|
|
})
|
|
}
|
|
|
|
module.exports = { update }
|