lamassu/lamassu-server
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Put sources in lamassu.json

Browse source
This commit is contained in:
Konstantin Mamalakis 2018-03-16 15:35:32 +02:00 • committed by Josh Harvey
parent b72f5549a5
commit 65f0e852d0
2 changed files with 56 additions and 47 deletions
Download patch file Download diff file Expand all files Collapse all files

12
bin/cert-gen.sh
View file

@ -80,7 +80,17 @@ cat <<EOF > $CONFIG_DIR/lamassu.json
"lamassuCaPath": "$LAMASSU_CA_PATH", "lamassuCaPath": "$LAMASSU_CA_PATH",
"lamassuServerPath": "$PWD", "lamassuServerPath": "$PWD",
"migrateStatePath": "$MIGRATE_STATE_PATH", "migrateStatePath": "$MIGRATE_STATE_PATH",
"ofacDataDir": "$OFAC_DATA_DIR" "ofacDataDir": "$OFAC_DATA_DIR",
"ofacSources": [
{
"name": "sdn_advanced",
"url": "https://www.treasury.gov/ofac/downloads/sanctions/1.0/sdn_advanced.xml"
},
{
"name": "cons_advanced",
"url": "https://www.treasury.gov/ofac/downloads/sanctions/1.0/cons_advanced.xml"
}
]
} }
EOF EOF

91
lib/ofac/update.js
View file

@ -9,11 +9,11 @@ const _ = require('lodash/fp')
const OFAC_DATA_DIR = options.ofacDataDir const OFAC_DATA_DIR = options.ofacDataDir
const OFAC_SOURCES_DIR = path.join(OFAC_DATA_DIR, 'sources') const OFAC_SOURCES_DIR = path.join(OFAC_DATA_DIR, 'sources')
const OFAC_SOURCES_FILE = path.join(OFAC_DATA_DIR, 'sources.json')
const OFAC_ETAGS_FILE = path.join(OFAC_DATA_DIR, 'etags.json') const OFAC_ETAGS_FILE = path.join(OFAC_DATA_DIR, 'etags.json')
const DOWNLOAD_DIR = path.resolve('/tmp') const DOWNLOAD_DIR = path.resolve('/tmp')
const mkdir = util.promisify(fs.mkdir)
const readFile = util.promisify(fs.readFile) const readFile = util.promisify(fs.readFile)
const writeFile = util.promisify(fs.writeFile) const writeFile = util.promisify(fs.writeFile)
const rename = util.promisify(fs.rename) const rename = util.promisify(fs.rename)
@ -47,9 +47,8 @@ const promiseGetEtag = (source) => {
const download = _.curry((dstDir, source) => { const download = _.curry((dstDir, source) => {
console.log("download", source) console.log("download", source)
const {url: sourceUrl} = source const {name, url: sourceUrl} = source
const fileName = path.basename(sourceUrl) const dstFile = path.join(dstDir, name + '.xml')
const dstFile = path.join(dstDir, fileName)
const file = fs.createWriteStream(dstFile) const file = fs.createWriteStream(dstFile)
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
@ -99,57 +98,57 @@ const moveToSourcesDir = srcFile => {
} }
function update () { const update = () => mkdir(OFAC_DATA_DIR).catch(err => null)
const promiseOldEtags = readFile(OFAC_ETAGS_FILE, {encoding: 'utf-8'}) .then(() => mkdir(OFAC_SOURCES_DIR)).catch(err => null)
.then(json => JSON.parse(json) || {}) .then(() => writeFile(OFAC_ETAGS_FILE, '{}', {encoding: 'utf-8', flag: 'wx'}))
.catch(err => null)
.then(() => {
const promiseOldEtags = readFile(OFAC_ETAGS_FILE, {encoding: 'utf-8'})
.then(json => JSON.parse(json) || {})
const promiseNewEtags = readFile(OFAC_SOURCES_FILE, {encoding: 'utf-8'}) const promiseNewEtags = Promise.resolve(options.ofacSources || [])
.then(json => { .then(sources => Promise.all(_.map(promiseGetEtag, sources))
const obj = JSON.parse(json) .then(etags => _.map(
return obj ? obj.sources : [] ([source, etag]) => ({...source, etag}),
}) _.zip(sources, etags)
.then(sources => Promise.all(_.map(promiseGetEtag, sources)) ))
.then(etags => _.map( )
([source, etag]) => ({...source, etag}),
_.zip(sources, etags)
))
)
return Promise.all([promiseOldEtags, promiseNewEtags]) return Promise.all([promiseOldEtags, promiseNewEtags])
.then(([oldEtags, newEtags]) => { .then(([oldEtags, newEtags]) => {
console.log("OLD", JSON.stringify(oldEtags, null, 4)) console.log("OLD", JSON.stringify(oldEtags, null, 4))
console.log("NEW", JSON.stringify(newEtags, null, 4)) console.log("NEW", JSON.stringify(newEtags, null, 4))
const hasNotChanged = ({name, etag}) => oldEtags[name] === etag const hasNotChanged = ({name, etag}) => oldEtags[name] === etag
const downloads = _.flow( const downloads = _.flow(
_.reject(hasNotChanged), _.reject(hasNotChanged),
_.map(file => download(DOWNLOAD_DIR, file).then(parseToJson)) _.map(file => download(DOWNLOAD_DIR, file).then(parseToJson))
)(newEtags) )(newEtags)
const oldFileNames = _.keys(oldEtags) const oldFileNames = _.keys(oldEtags)
const newFileNames = _.map(_.get('name'), newEtags) const newFileNames = _.map(_.get('name'), newEtags)
const missingFileNames = _.difference(oldFileNames, newFileNames) const missingFileNames = _.difference(oldFileNames, newFileNames)
const resolve = name => path.join(OFAC_SOURCES_DIR, name + '.json') const resolve = name => path.join(OFAC_SOURCES_DIR, name + '.json')
const missing = _.map(resolve, missingFileNames) const missing = _.map(resolve, missingFileNames)
const etagsJson = _.flow( const etagsJson = _.flow(
_.map(source => [source.name, source.etag]), _.map(source => [source.name, source.etag]),
_.fromPairs, _.fromPairs,
obj => JSON.stringify(obj, null, 4) obj => JSON.stringify(obj, null, 4)
)(newEtags) )(newEtags)
return Promise.all(downloads) return Promise.all(downloads)
.then(parsed => { .then(parsed => {
console.log("finished", parsed) console.log("finished", parsed)
const moves = _.map(moveToSourcesDir, parsed) const moves = _.map(moveToSourcesDir, parsed)
const deletions = _.map(remove, missing) const deletions = _.map(remove, missing)
const updateEtags = writeFile(OFAC_ETAGS_FILE, etagsJson) const updateEtags = writeFile(OFAC_ETAGS_FILE, etagsJson)
return Promise.all([updateEtags, ...moves, ...deletions]) return Promise.all([updateEtags, ...moves, ...deletions])
}) })
})
}) })
}
module.exports = {update} module.exports = {update}