lamassu/lamassu-server
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Put sources in lamassu.json

Browse source
This commit is contained in:
Konstantin Mamalakis 2018-03-16 15:35:32 +02:00 • committed by Josh Harvey
parent b72f5549a5
commit 65f0e852d0
2 changed files with 56 additions and 47 deletions
Download patch file Download diff file Expand all files Collapse all files

91
lib/ofac/update.js
View file

@ -9,11 +9,11 @@ const _ = require('lodash/fp')
const OFAC_DATA_DIR = options.ofacDataDir
const OFAC_SOURCES_DIR = path.join(OFAC_DATA_DIR, 'sources')
const OFAC_SOURCES_FILE = path.join(OFAC_DATA_DIR, 'sources.json')
const OFAC_ETAGS_FILE = path.join(OFAC_DATA_DIR, 'etags.json')
const DOWNLOAD_DIR = path.resolve('/tmp')
const mkdir = util.promisify(fs.mkdir)
const readFile = util.promisify(fs.readFile)
const writeFile = util.promisify(fs.writeFile)
const rename = util.promisify(fs.rename)
@ -47,9 +47,8 @@ const promiseGetEtag = (source) => {
const download = _.curry((dstDir, source) => {
console.log("download", source)
const {url: sourceUrl} = source
const fileName = path.basename(sourceUrl)
const dstFile = path.join(dstDir, fileName)
const {name, url: sourceUrl} = source
const dstFile = path.join(dstDir, name + '.xml')
const file = fs.createWriteStream(dstFile)
return new Promise((resolve, reject) => {
@ -99,57 +98,57 @@ const moveToSourcesDir = srcFile => {
}
function update () {
const promiseOldEtags = readFile(OFAC_ETAGS_FILE, {encoding: 'utf-8'})
.then(json => JSON.parse(json) || {})
const update = () => mkdir(OFAC_DATA_DIR).catch(err => null)
.then(() => mkdir(OFAC_SOURCES_DIR)).catch(err => null)
.then(() => writeFile(OFAC_ETAGS_FILE, '{}', {encoding: 'utf-8', flag: 'wx'}))
.catch(err => null)
.then(() => {
const promiseOldEtags = readFile(OFAC_ETAGS_FILE, {encoding: 'utf-8'})
.then(json => JSON.parse(json) || {})
const promiseNewEtags = readFile(OFAC_SOURCES_FILE, {encoding: 'utf-8'})
.then(json => {
const obj = JSON.parse(json)
return obj ? obj.sources : []
})
.then(sources => Promise.all(_.map(promiseGetEtag, sources))
.then(etags => _.map(
([source, etag]) => ({...source, etag}),
_.zip(sources, etags)
))
)
const promiseNewEtags = Promise.resolve(options.ofacSources || [])
.then(sources => Promise.all(_.map(promiseGetEtag, sources))
.then(etags => _.map(
([source, etag]) => ({...source, etag}),
_.zip(sources, etags)
))
)
return Promise.all([promiseOldEtags, promiseNewEtags])
.then(([oldEtags, newEtags]) => {
console.log("OLD", JSON.stringify(oldEtags, null, 4))
console.log("NEW", JSON.stringify(newEtags, null, 4))
return Promise.all([promiseOldEtags, promiseNewEtags])
.then(([oldEtags, newEtags]) => {
console.log("OLD", JSON.stringify(oldEtags, null, 4))
console.log("NEW", JSON.stringify(newEtags, null, 4))
const hasNotChanged = ({name, etag}) => oldEtags[name] === etag
const hasNotChanged = ({name, etag}) => oldEtags[name] === etag
const downloads = _.flow(
_.reject(hasNotChanged),
_.map(file => download(DOWNLOAD_DIR, file).then(parseToJson))
)(newEtags)
const downloads = _.flow(
_.reject(hasNotChanged),
_.map(file => download(DOWNLOAD_DIR, file).then(parseToJson))
)(newEtags)
const oldFileNames = _.keys(oldEtags)
const newFileNames = _.map(_.get('name'), newEtags)
const missingFileNames = _.difference(oldFileNames, newFileNames)
const resolve = name => path.join(OFAC_SOURCES_DIR, name + '.json')
const missing = _.map(resolve, missingFileNames)
const oldFileNames = _.keys(oldEtags)
const newFileNames = _.map(_.get('name'), newEtags)
const missingFileNames = _.difference(oldFileNames, newFileNames)
const resolve = name => path.join(OFAC_SOURCES_DIR, name + '.json')
const missing = _.map(resolve, missingFileNames)
const etagsJson = _.flow(
_.map(source => [source.name, source.etag]),
_.fromPairs,
obj => JSON.stringify(obj, null, 4)
)(newEtags)
const etagsJson = _.flow(
_.map(source => [source.name, source.etag]),
_.fromPairs,
obj => JSON.stringify(obj, null, 4)
)(newEtags)
return Promise.all(downloads)
.then(parsed => {
console.log("finished", parsed)
return Promise.all(downloads)
.then(parsed => {
console.log("finished", parsed)
const moves = _.map(moveToSourcesDir, parsed)
const deletions = _.map(remove, missing)
const updateEtags = writeFile(OFAC_ETAGS_FILE, etagsJson)
const moves = _.map(moveToSourcesDir, parsed)
const deletions = _.map(remove, missing)
const updateEtags = writeFile(OFAC_ETAGS_FILE, etagsJson)
return Promise.all([updateEtags, ...moves, ...deletions])
})
return Promise.all([updateEtags, ...moves, ...deletions])
})
})
})
}
module.exports = {update}