diff --git a/lib/ofac/update.js b/lib/ofac/update.js index e7eb5b73..5a91819f 100644 --- a/lib/ofac/update.js +++ b/lib/ofac/update.js @@ -1,12 +1,10 @@ const parser = require('./parsing') const https = require('https') -const URL = require('url') const { createWriteStream } = require('fs') const fs = require('fs/promises') -const { readFile, writeFile, rename, unlink } = fs +const { rename } = fs const path = require('path') const _ = require('lodash/fp') -const logger = require('../logger') const DOWNLOAD_DIR = path.resolve('/tmp') @@ -24,25 +22,6 @@ const mkdir = path => fs.mkdir(path) .catch(err => err.code === 'EEXIST' ? Promise.resolve() : Promise.reject(err)) -const promiseGetEtag = ({ url }) => - new Promise((resolve, reject) => { - const parsed = URL.parse(url) - const requestOptions = { - hostname: parsed.hostname, - path: parsed.path, - method: 'HEAD' - } - - const request = https.request(requestOptions, _.flow( - _.get(['headers', 'etag']), - resolve - )) - - request.on('error', reject) - - request.end() - }) - const download = (dstDir, { name, url }) => { const dstFile = path.join(dstDir, name + '.xml') const file = createWriteStream(dstFile) @@ -94,60 +73,23 @@ function update () { } const OFAC_SOURCES_DIR = path.join(OFAC_DATA_DIR, 'sources') - const OFAC_ETAGS_FILE = path.join(OFAC_DATA_DIR, 'etags.json') return mkdir(OFAC_DATA_DIR) .then(() => mkdir(OFAC_SOURCES_DIR)) - .then(() => writeFile(OFAC_ETAGS_FILE, '{}', { encoding: 'utf-8', flag: 'wx' })) .catch(err => { if (err.code === 'EEXIST') return throw err }) .then(() => { - const promiseOldEtags = readFile(OFAC_ETAGS_FILE, { encoding: 'utf-8' }) - .then(json => JSON.parse(json)) - .catch(_ => { - logger.error('Can\'t parse etags.json, getting new data...') - return {} - }) + const downloads = _.flow( + _.map(file => download(DOWNLOAD_DIR, file).then(parseToJson)) + )(OFAC_SOURCES) - const promiseNewEtags = Promise.resolve(OFAC_SOURCES || []) - .then(sources => Promise.all(_.map(promiseGetEtag, sources)) - .then(etags => _.map( - ([source, etag]) => _.set('etag', etag, source), - _.zip(sources, etags) - )) - ) + return Promise.all(downloads) + .then(parsed => { + const moves = _.map(src => moveToSourcesDir(src, OFAC_SOURCES_DIR), parsed) - return Promise.all([promiseOldEtags, promiseNewEtags]) - .then(([oldEtags, newEtags]) => { - const hasNotChanged = ({ name, etag }) => oldEtags[name] === etag - - const downloads = _.flow( - _.reject(hasNotChanged), - _.map(file => download(DOWNLOAD_DIR, file).then(parseToJson)) - )(newEtags) - - const oldFileNames = _.keys(oldEtags) - const newFileNames = _.map(_.get('name'), newEtags) - const missingFileNames = _.difference(oldFileNames, newFileNames) - const resolve = name => path.join(OFAC_SOURCES_DIR, name + '.json') - const missing = _.map(resolve, missingFileNames) - - const etagsJson = _.flow( - _.map(source => [source.name, source.etag]), - _.fromPairs, - obj => JSON.stringify(obj, null, 4) - )(newEtags) - - return Promise.all(downloads) - .then(parsed => { - const moves = _.map(src => moveToSourcesDir(src, OFAC_SOURCES_DIR), parsed) - const deletions = _.map(unlink, missing) - const updateEtags = writeFile(OFAC_ETAGS_FILE, etagsJson) - - return Promise.all([updateEtags, ...moves, ...deletions]) - }) + return Promise.all([...moves]) }) }) }