feat: add another timer mechanism

lib/ofac/update.js

@@ -1,14 +1,14 @@
 const parser = require('./parsing')
 const axios = require('axios')
 const { createWriteStream } = require('fs')
-const fs = require('fs/promises')
+const { rename, writeFile, readFile, mkdir, copyFile, unlink } = require('fs/promises')
-const { rename } = fs
 const path = require('path')
 const _ = require('lodash/fp')
 
 const DOWNLOAD_DIR = path.resolve('/tmp')
-
 const OFAC_DATA_DIR = process.env.OFAC_DATA_DIR
+const OFAC_SOURCES_DIR = path.join(OFAC_DATA_DIR, 'sources')
+const LAST_UPDATED_FILE = path.resolve(OFAC_DATA_DIR, 'last_updated.dat')
 
 const OFAC_SOURCES = [{
   name: 'sdn_advanced',
@@ -18,38 +18,34 @@ const OFAC_SOURCES = [{
   url: 'https://sanctionslistservice.ofac.treas.gov/api/download/cons_advanced.xml'
 }]
 
-const mkdir = path =>
+const _mkdir = path =>
-  fs.mkdir(path)
+  mkdir(path)
     .catch(err => err.code === 'EEXIST' ? Promise.resolve() : Promise.reject(err))
 
-const newDownload = (dstDir, { name, url }) => {
-  return path.join('/tmp/', name + '.xml')
-}
-
 const download = (dstDir, { name, url }) => {
   const dstFile = path.join(dstDir, name + '.xml')
   const writer = createWriteStream(dstFile)
 
-   return axios({
+  return axios({
-     method: 'get',
+    method: 'get',
-     url: url,
+    url: url,
-     responseType: 'stream',
+    responseType: 'stream',
-   }).then(response => {
+  }).then(response => {
-     return new Promise((resolve, reject) => {
+    return new Promise((resolve, reject) => {
-       response.data.pipe(writer);
+      response.data.pipe(writer)
-       let error = null;
+      let error = null
-       writer.on('error', err => {
+      writer.on('error', err => {
-         error = err;
+        error = err
-         writer.close();
+        writer.close()
-         reject(err);
+        reject(err)
-       });
+      })
-       writer.on('close', () => {
+      writer.on('close', () => {
-         if (!error) {
+        if (!error) {
-           resolve(dstFile);
+          resolve(dstFile)
-         }
+        }
-       });
+      })
-     });
+    })
-   });
+  })
 }
 
 const parseToJson = srcFile => {
@@ -77,10 +73,21 @@ const parseToJson = srcFile => {
   })
 }
 
-const moveToSourcesDir = (srcFile, ofacSourcesDir) => {
+const moveToSourcesDir = async (srcFile, ofacSourcesDir) => {
   const name = path.basename(srcFile)
   const dstFile = path.join(ofacSourcesDir, name)
-  return rename(srcFile, dstFile)
+  try {
+    await rename(srcFile, dstFile)
+  } catch (err) {
+    if (err.code === 'EXDEV') {
+      // If rename fails due to cross-device link, fallback to copy + delete
+      await copyFile(srcFile, dstFile)
+      await unlink(srcFile)
+    } else {
+      throw err
+    }
+  }
+  return dstFile
 }
 
 function update () {
@@ -88,15 +95,28 @@ function update () {
     throw new Error('ofacDataDir must be defined in the environment')
   }
 
-  const OFAC_SOURCES_DIR = path.join(OFAC_DATA_DIR, 'sources')
+  return _mkdir(OFAC_DATA_DIR)
-
+    .then(() => _mkdir(OFAC_SOURCES_DIR))
-  return mkdir(OFAC_DATA_DIR)
-    .then(() => mkdir(OFAC_SOURCES_DIR))
     .catch(err => {
       if (err.code === 'EEXIST') return
       throw err
     })
-    .then(() => {
+    .then(() => readFile(LAST_UPDATED_FILE))
+    .then(data => {
+      const lastUpdate = new Date(data.toString())
+      const now = new Date()
+      const hoursSinceUpdate = (now - lastUpdate) / (1000 * 60 * 60)
+
+      return hoursSinceUpdate < 24
+    })
+    .catch(err => {
+      // If file doesn't exist, continue with update
+      if (err.code === 'ENOENT') return false
+      throw err
+    })
+    .then(skipUpdate => {
+      if (skipUpdate) return Promise.resolve()
+
       const downloads = _.flow(
         _.map(file => download(DOWNLOAD_DIR, file).then(parseToJson))
       )(OFAC_SOURCES)
@@ -104,8 +124,10 @@ function update () {
       return Promise.all(downloads)
         .then(parsed => {
           const moves = _.map(src => moveToSourcesDir(src, OFAC_SOURCES_DIR), parsed)
+          const timestamp = new Date().toISOString()
 
           return Promise.all([...moves])
+            .then(() => writeFile(LAST_UPDATED_FILE, timestamp))
         })
     })
 }