refactor: stop using etags

lib/ofac/update.js

@@ -1,12 +1,10 @@
 const parser = require('./parsing')
 const https = require('https')
-const URL = require('url')
 const { createWriteStream } = require('fs')
 const fs = require('fs/promises')
-const { readFile, writeFile, rename, unlink } = fs
+const { rename } = fs
 const path = require('path')
 const _ = require('lodash/fp')
-const logger = require('../logger')
 
 const DOWNLOAD_DIR = path.resolve('/tmp')
 
@@ -24,25 +22,6 @@ const mkdir = path =>
   fs.mkdir(path)
     .catch(err => err.code === 'EEXIST' ? Promise.resolve() : Promise.reject(err))
 
-const promiseGetEtag = ({ url }) =>
-  new Promise((resolve, reject) => {
-    const parsed = URL.parse(url)
-    const requestOptions = {
-      hostname: parsed.hostname,
-      path: parsed.path,
-      method: 'HEAD'
-    }
-
-    const request = https.request(requestOptions, _.flow(
-      _.get(['headers', 'etag']),
-      resolve
-    ))
-
-    request.on('error', reject)
-
-    request.end()
-  })
-
 const download = (dstDir, { name, url }) => {
   const dstFile = path.join(dstDir, name + '.xml')
   const file = createWriteStream(dstFile)
@@ -94,60 +73,23 @@ function update () {
   }
 
   const OFAC_SOURCES_DIR = path.join(OFAC_DATA_DIR, 'sources')
-  const OFAC_ETAGS_FILE = path.join(OFAC_DATA_DIR, 'etags.json')
 
   return mkdir(OFAC_DATA_DIR)
     .then(() => mkdir(OFAC_SOURCES_DIR))
-    .then(() => writeFile(OFAC_ETAGS_FILE, '{}', { encoding: 'utf-8', flag: 'wx' }))
     .catch(err => {
       if (err.code === 'EEXIST') return
       throw err
     })
     .then(() => {
-      const promiseOldEtags = readFile(OFAC_ETAGS_FILE, { encoding: 'utf-8' })
-        .then(json => JSON.parse(json))
-        .catch(_ => {
-          logger.error('Can\'t parse etags.json, getting new data...')
-          return {}
-        })
-
-      const promiseNewEtags = Promise.resolve(OFAC_SOURCES || [])
-        .then(sources => Promise.all(_.map(promiseGetEtag, sources))
-          .then(etags => _.map(
-            ([source, etag]) => _.set('etag', etag, source),
-            _.zip(sources, etags)
-          ))
-        )
-
-      return Promise.all([promiseOldEtags, promiseNewEtags])
-        .then(([oldEtags, newEtags]) => {
-          const hasNotChanged = ({ name, etag }) => oldEtags[name] === etag
-
       const downloads = _.flow(
-            _.reject(hasNotChanged),
         _.map(file => download(DOWNLOAD_DIR, file).then(parseToJson))
-          )(newEtags)
-
-          const oldFileNames = _.keys(oldEtags)
-          const newFileNames = _.map(_.get('name'), newEtags)
-          const missingFileNames = _.difference(oldFileNames, newFileNames)
-          const resolve = name => path.join(OFAC_SOURCES_DIR, name + '.json')
-          const missing = _.map(resolve, missingFileNames)
-
-          const etagsJson = _.flow(
-            _.map(source => [source.name, source.etag]),
-            _.fromPairs,
-            obj => JSON.stringify(obj, null, 4)
-          )(newEtags)
+      )(OFAC_SOURCES)
 
       return Promise.all(downloads)
         .then(parsed => {
           const moves = _.map(src => moveToSourcesDir(src, OFAC_SOURCES_DIR), parsed)
-              const deletions = _.map(unlink, missing)
-              const updateEtags = writeFile(OFAC_ETAGS_FILE, etagsJson)
 
-              return Promise.all([updateEtags, ...moves, ...deletions])
-            })
+          return Promise.all([...moves])
         })
     })
 }