lamassu/lamassu-server
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactor: stop using etags

Browse source
This commit is contained in:
Rafael 2024-12-20 12:21:08 +00:00
parent 74a026dc1b
commit 908baa4fb0
1 changed files with 8 additions and 66 deletions
Download patch file Download diff file Expand all files Collapse all files

74
lib/ofac/update.js
View file

@ -1,12 +1,10 @@
const parser = require('./parsing')
const https = require('https')
const URL = require('url')
const { createWriteStream } = require('fs')
const fs = require('fs/promises')
const { readFile, writeFile, rename, unlink } = fs
const { rename } = fs
const path = require('path')
const _ = require('lodash/fp')
const logger = require('../logger')
const DOWNLOAD_DIR = path.resolve('/tmp')
@ -24,25 +22,6 @@ const mkdir = path =>
fs.mkdir(path)
.catch(err => err.code === 'EEXIST' ? Promise.resolve() : Promise.reject(err))
const promiseGetEtag = ({ url }) =>
new Promise((resolve, reject) => {
const parsed = URL.parse(url)
const requestOptions = {
hostname: parsed.hostname,
path: parsed.path,
method: 'HEAD'
}
const request = https.request(requestOptions, _.flow(
_.get(['headers', 'etag']),
resolve
))
request.on('error', reject)
request.end()
})
const download = (dstDir, { name, url }) => {
const dstFile = path.join(dstDir, name + '.xml')
const file = createWriteStream(dstFile)
@ -94,60 +73,23 @@ function update () {
}
const OFAC_SOURCES_DIR = path.join(OFAC_DATA_DIR, 'sources')
const OFAC_ETAGS_FILE = path.join(OFAC_DATA_DIR, 'etags.json')
return mkdir(OFAC_DATA_DIR)
.then(() => mkdir(OFAC_SOURCES_DIR))
.then(() => writeFile(OFAC_ETAGS_FILE, '{}', { encoding: 'utf-8', flag: 'wx' }))
.catch(err => {
if (err.code === 'EEXIST') return
throw err
})
.then(() => {
const promiseOldEtags = readFile(OFAC_ETAGS_FILE, { encoding: 'utf-8' })
.then(json => JSON.parse(json))
.catch(_ => {
logger.error('Can\'t parse etags.json, getting new data...')
return {}
})
const downloads = _.flow(
_.map(file => download(DOWNLOAD_DIR, file).then(parseToJson))
)(OFAC_SOURCES)
const promiseNewEtags = Promise.resolve(OFAC_SOURCES || [])
.then(sources => Promise.all(_.map(promiseGetEtag, sources))
.then(etags => _.map(
([source, etag]) => _.set('etag', etag, source),
_.zip(sources, etags)
))
)
return Promise.all(downloads)
.then(parsed => {
const moves = _.map(src => moveToSourcesDir(src, OFAC_SOURCES_DIR), parsed)
return Promise.all([promiseOldEtags, promiseNewEtags])
.then(([oldEtags, newEtags]) => {
const hasNotChanged = ({ name, etag }) => oldEtags[name] === etag
const downloads = _.flow(
_.reject(hasNotChanged),
_.map(file => download(DOWNLOAD_DIR, file).then(parseToJson))
)(newEtags)
const oldFileNames = _.keys(oldEtags)
const newFileNames = _.map(_.get('name'), newEtags)
const missingFileNames = _.difference(oldFileNames, newFileNames)
const resolve = name => path.join(OFAC_SOURCES_DIR, name + '.json')
const missing = _.map(resolve, missingFileNames)
const etagsJson = _.flow(
_.map(source => [source.name, source.etag]),
_.fromPairs,
obj => JSON.stringify(obj, null, 4)
)(newEtags)
return Promise.all(downloads)
.then(parsed => {
const moves = _.map(src => moveToSourcesDir(src, OFAC_SOURCES_DIR), parsed)
const deletions = _.map(unlink, missing)
const updateEtags = writeFile(OFAC_ETAGS_FILE, etagsJson)
return Promise.all([updateEtags, ...moves, ...deletions])
})
return Promise.all([...moves])
})
})
}