WIP
This commit is contained in:
Josh Harvey
parent
f7561acf3c
commit
80e851fb59
9 changed files with 131 additions and 76 deletions
|
|
@ -7,23 +7,28 @@ const util = require('util')
|
|||
const options = require('../options')
|
||||
const _ = require('lodash/fp')
|
||||
|
||||
const OFAC_DATA_DIR = options.ofacDataDir
|
||||
const OFAC_SOURCES_DIR = path.join(OFAC_DATA_DIR, 'sources')
|
||||
const OFAC_ETAGS_FILE = path.join(OFAC_DATA_DIR, 'etags.json')
|
||||
const DOWNLOAD_DIR = path.resolve('/tmp')
|
||||
|
||||
function mkdir (path) {
|
||||
return new Promise((resolve, reject) => {
|
||||
fs.mkdir(path, err => {
|
||||
if (!err) return resolve()
|
||||
if (err.code === 'EEXIST') return resolve()
|
||||
reject(err)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
const mkdir = util.promisify(fs.mkdir)
|
||||
const readFile = util.promisify(fs.readFile)
|
||||
const writeFile = util.promisify(fs.writeFile)
|
||||
const rename = util.promisify(fs.rename)
|
||||
const unlink = util.promisify(fs.unlink)
|
||||
|
||||
const remove = file => {
|
||||
console.log("remove", file)
|
||||
console.log('remove', file)
|
||||
return unlink(file)
|
||||
}
|
||||
|
||||
|
||||
const promiseGetEtag = (source) => {
|
||||
return new Promise((resolve, reject) => {
|
||||
const {url: sourceUrl} = source
|
||||
|
|
@ -46,14 +51,14 @@ const promiseGetEtag = (source) => {
|
|||
}
|
||||
|
||||
const download = _.curry((dstDir, source) => {
|
||||
console.log("download", source)
|
||||
console.log('download', source)
|
||||
const {name, url: sourceUrl} = source
|
||||
const dstFile = path.join(dstDir, name + '.xml')
|
||||
const file = fs.createWriteStream(dstFile)
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const request = https.get(sourceUrl, response => {
|
||||
response.pipe(file);
|
||||
response.pipe(file)
|
||||
file.on('finish', () => file.close(() => resolve(dstFile)))
|
||||
})
|
||||
|
||||
|
|
@ -62,14 +67,14 @@ const download = _.curry((dstDir, source) => {
|
|||
})
|
||||
|
||||
const parseToJson = srcFile => {
|
||||
console.log("parseToJson", srcFile)
|
||||
console.log('parseToJson', srcFile)
|
||||
|
||||
const dstFile = srcFile.replace(/\.xml$/, '.json')
|
||||
const writeStream = fs.createWriteStream(dstFile)
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
parser.parse(srcFile, (err, profile) => {
|
||||
console.log("callback", err, profile)
|
||||
console.log('callback', err, profile)
|
||||
|
||||
if (err) {
|
||||
reject(err)
|
||||
|
|
@ -90,65 +95,78 @@ const parseToJson = srcFile => {
|
|||
})
|
||||
}
|
||||
|
||||
const moveToSourcesDir = srcFile => {
|
||||
console.log("moveToSourcesDir", srcFile)
|
||||
const moveToSourcesDir = (srcFile, ofacSourcesDir) => {
|
||||
console.log('moveToSourcesDir', srcFile)
|
||||
const name = path.basename(srcFile)
|
||||
const dstFile = path.join(OFAC_SOURCES_DIR, name)
|
||||
const dstFile = path.join(ofacSourcesDir, name)
|
||||
return rename(srcFile, dstFile)
|
||||
}
|
||||
|
||||
function update () {
|
||||
const OFAC_DATA_DIR = options.ofacDataDir
|
||||
|
||||
const update = () => mkdir(OFAC_DATA_DIR).catch(err => null)
|
||||
.then(() => mkdir(OFAC_SOURCES_DIR)).catch(err => null)
|
||||
.then(() => writeFile(OFAC_ETAGS_FILE, '{}', {encoding: 'utf-8', flag: 'wx'}))
|
||||
.catch(err => null)
|
||||
.then(() => {
|
||||
const promiseOldEtags = readFile(OFAC_ETAGS_FILE, {encoding: 'utf-8'})
|
||||
.then(json => JSON.parse(json) || {})
|
||||
if (!OFAC_DATA_DIR) {
|
||||
throw new Error('ofacDataDir must be defined in lamassu.json')
|
||||
}
|
||||
|
||||
const promiseNewEtags = Promise.resolve(options.ofacSources || [])
|
||||
.then(sources => Promise.all(_.map(promiseGetEtag, sources))
|
||||
.then(etags => _.map(
|
||||
([source, etag]) => ({...source, etag}),
|
||||
_.zip(sources, etags)
|
||||
))
|
||||
)
|
||||
const OFAC_SOURCES_DIR = path.join(OFAC_DATA_DIR, 'sources')
|
||||
const OFAC_ETAGS_FILE = path.join(OFAC_DATA_DIR, 'etags.json')
|
||||
|
||||
return Promise.all([promiseOldEtags, promiseNewEtags])
|
||||
.then(([oldEtags, newEtags]) => {
|
||||
console.log("OLD", JSON.stringify(oldEtags, null, 4))
|
||||
console.log("NEW", JSON.stringify(newEtags, null, 4))
|
||||
return mkdir(OFAC_DATA_DIR)
|
||||
.then(() => mkdir(OFAC_SOURCES_DIR))
|
||||
.then(() => writeFile(OFAC_ETAGS_FILE, '{}', {encoding: 'utf-8', flag: 'wx'}))
|
||||
.catch(err => {
|
||||
if (err.code === 'EEXIST') return
|
||||
throw err
|
||||
})
|
||||
.then(() => {
|
||||
const promiseOldEtags = readFile(OFAC_ETAGS_FILE, {encoding: 'utf-8'})
|
||||
.then(json => JSON.parse(json) || {})
|
||||
|
||||
const hasNotChanged = ({name, etag}) => oldEtags[name] === etag
|
||||
const promiseNewEtags = Promise.resolve(options.ofacSources || [])
|
||||
.then(sources => Promise.all(_.map(promiseGetEtag, sources))
|
||||
.then(etags => _.map(
|
||||
([source, etag]) => ({...source, etag}),
|
||||
_.zip(sources, etags)
|
||||
))
|
||||
)
|
||||
|
||||
const downloads = _.flow(
|
||||
_.reject(hasNotChanged),
|
||||
_.map(file => download(DOWNLOAD_DIR, file).then(parseToJson))
|
||||
)(newEtags)
|
||||
return Promise.all([promiseOldEtags, promiseNewEtags])
|
||||
.then(([oldEtags, newEtags]) => {
|
||||
console.log('OLD', JSON.stringify(oldEtags, null, 4))
|
||||
console.log('NEW', JSON.stringify(newEtags, null, 4))
|
||||
|
||||
const oldFileNames = _.keys(oldEtags)
|
||||
const newFileNames = _.map(_.get('name'), newEtags)
|
||||
const missingFileNames = _.difference(oldFileNames, newFileNames)
|
||||
const resolve = name => path.join(OFAC_SOURCES_DIR, name + '.json')
|
||||
const missing = _.map(resolve, missingFileNames)
|
||||
const hasNotChanged = ({name, etag}) => oldEtags[name] === etag
|
||||
|
||||
const etagsJson = _.flow(
|
||||
_.map(source => [source.name, source.etag]),
|
||||
_.fromPairs,
|
||||
obj => JSON.stringify(obj, null, 4)
|
||||
)(newEtags)
|
||||
const downloads = _.flow(
|
||||
_.reject(hasNotChanged),
|
||||
_.map(file => download(DOWNLOAD_DIR, file).then(parseToJson))
|
||||
)(newEtags)
|
||||
|
||||
return Promise.all(downloads)
|
||||
.then(parsed => {
|
||||
console.log("finished", parsed)
|
||||
const oldFileNames = _.keys(oldEtags)
|
||||
const newFileNames = _.map(_.get('name'), newEtags)
|
||||
const missingFileNames = _.difference(oldFileNames, newFileNames)
|
||||
const resolve = name => path.join(OFAC_SOURCES_DIR, name + '.json')
|
||||
const missing = _.map(resolve, missingFileNames)
|
||||
|
||||
const moves = _.map(moveToSourcesDir, parsed)
|
||||
const deletions = _.map(remove, missing)
|
||||
const updateEtags = writeFile(OFAC_ETAGS_FILE, etagsJson)
|
||||
const etagsJson = _.flow(
|
||||
_.map(source => [source.name, source.etag]),
|
||||
_.fromPairs,
|
||||
obj => JSON.stringify(obj, null, 4)
|
||||
)(newEtags)
|
||||
|
||||
return Promise.all([updateEtags, ...moves, ...deletions])
|
||||
return Promise.all(downloads)
|
||||
.then(parsed => {
|
||||
console.log('finished', parsed)
|
||||
|
||||
const moves = _.map(src => moveToSourcesDir(src, OFAC_SOURCES_DIR), parsed)
|
||||
const deletions = _.map(remove, missing)
|
||||
const updateEtags = writeFile(OFAC_ETAGS_FILE, etagsJson)
|
||||
|
||||
return Promise.all([updateEtags, ...moves, ...deletions])
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
module.exports = {update}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue