From 60b016efb445c4f9a5e03ae7a6b47f644e93e0db Mon Sep 17 00:00:00 2001 From: siiky Date: Mon, 17 Jun 2024 11:51:36 +0100 Subject: [PATCH 01/10] refactor: generalize `updatePhotos` --- lib/machine-loader.js | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/lib/machine-loader.js b/lib/machine-loader.js index fd9375b1..78c9a794 100644 --- a/lib/machine-loader.js +++ b/lib/machine-loader.js @@ -487,7 +487,7 @@ function updateDiagnostics (deviceId, images) { const directory = `${OPERATOR_DATA_DIR}/diagnostics/${deviceId}/` const { scan, front } = images - return updatePhotos(scan, front, directory) + return updatePhotos(directory, [['scan.jpg', scan], ['front.jpg', front]]) .then(() => db.none(sql, [deviceId, !!scan, !!front])) .catch(err => logger.error('while running machine diagnostics: ', err)) } @@ -503,16 +503,13 @@ function createPhoto (name, data, dir) { return fsPromises.writeFile(filename, decodedImageData) } -function updatePhotos (scan, front, dir) { +function updatePhotos (dir, photoPairs) { const dirname = path.join(dir) _.attempt(() => makeDir.sync(dirname)) - - const promises = [ - createPhoto('scan.jpg', scan, dirname), - createPhoto('front.jpg', front, dirname) - ] - - return Promise.all(promises) + return Promise.all(_.map( + ([filename, data]) => createPhoto(filename, data, dirname), + photoPairs + )) } module.exports = { From 79fc7f3c446de450655552605ab3b9130ef393fa Mon Sep 17 00:00:00 2001 From: siiky Date: Fri, 14 Jun 2024 00:28:41 +0100 Subject: [PATCH 02/10] feat: add `failedqrscans` route --- lib/machine-loader.js | 14 ++++++++++---- lib/routes.js | 2 ++ lib/routes/failedQRScans.js | 14 ++++++++++++++ 3 files changed, 26 insertions(+), 4 deletions(-) create mode 100644 lib/routes/failedQRScans.js diff --git a/lib/machine-loader.js b/lib/machine-loader.js index 78c9a794..5d5b0682 100644 --- a/lib/machine-loader.js +++ b/lib/machine-loader.js @@ -492,6 +492,12 @@ function updateDiagnostics (deviceId, images) { .catch(err => logger.error('while running machine diagnostics: ', err)) } +const updateFailedQRScans = (deviceId, frames) => { + const directory = `${OPERATOR_DATA_DIR}/failedQRScans/${deviceId}/` + const filenames = _.map(no => `${no}.jpg`, _.range(0, _.size(frames))) + return updatePhotos(directory, _.zip(filenames, frames)) +} + function createPhoto (name, data, dir) { if (!data) { logger.error(`Diagnostics error: No data to save for ${name} photo`) @@ -506,9 +512,8 @@ function createPhoto (name, data, dir) { function updatePhotos (dir, photoPairs) { const dirname = path.join(dir) _.attempt(() => makeDir.sync(dirname)) - return Promise.all(_.map( - ([filename, data]) => createPhoto(filename, data, dirname), - photoPairs + return Promise.all(photoPairs.map( + ([filename, data]) => createPhoto(filename, data, dirname) )) } @@ -527,5 +532,6 @@ module.exports = { getMachineIds, emptyMachineUnits, refillMachineUnits, - updateDiagnostics + updateDiagnostics, + updateFailedQRScans } diff --git a/lib/routes.js b/lib/routes.js index 4165ea58..e5ac126b 100644 --- a/lib/routes.js +++ b/lib/routes.js @@ -32,6 +32,7 @@ const verifyUserRoutes = require('./routes/verifyUserRoutes') const verifyTxRoutes = require('./routes/verifyTxRoutes') const verifyPromoCodeRoutes = require('./routes/verifyPromoCodeRoutes') const probeRoutes = require('./routes/probeLnRoutes') +const failedQRScansRoutes = require('./routes/failedQRScans') const graphQLServer = require('./graphql/server') @@ -75,6 +76,7 @@ app.use('/cashbox', cashboxRoutes) app.use('/network', performanceRoutes) app.use('/diagnostics', diagnosticsRoutes) +app.use('/failedqrscans', failedQRScansRoutes) app.use('/verify_user', verifyUserRoutes) app.use('/verify_transaction', verifyTxRoutes) diff --git a/lib/routes/failedQRScans.js b/lib/routes/failedQRScans.js new file mode 100644 index 00000000..e484ef36 --- /dev/null +++ b/lib/routes/failedQRScans.js @@ -0,0 +1,14 @@ +const express = require('express') +const router = express.Router() + +const { updateFailedQRScans } = require('../machine-loader') + +function failedQRScans (req, res, next) { + return updateFailedQRScans(req.deviceId, req.body) + .then(() => res.status(200).send({ status: 'OK' })) + .catch(next) +} + +router.post('/', failedQRScans) + +module.exports = router From 7e6783c7b29ea0dd4e5b21f386f74e2e3d00d708 Mon Sep 17 00:00:00 2001 From: siiky Date: Mon, 17 Jun 2024 15:46:13 +0100 Subject: [PATCH 03/10] feat: include timestamp in filename --- lib/machine-loader.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/machine-loader.js b/lib/machine-loader.js index 5d5b0682..d9113eb5 100644 --- a/lib/machine-loader.js +++ b/lib/machine-loader.js @@ -493,8 +493,9 @@ function updateDiagnostics (deviceId, images) { } const updateFailedQRScans = (deviceId, frames) => { + const timestamp = (new Date()).toISOString() const directory = `${OPERATOR_DATA_DIR}/failedQRScans/${deviceId}/` - const filenames = _.map(no => `${no}.jpg`, _.range(0, _.size(frames))) + const filenames = _.map(no => `${timestamp}-${no}.jpg`, _.range(0, _.size(frames))) return updatePhotos(directory, _.zip(filenames, frames)) } From 2d91c55539af18c6cc5fca804c991b7dbf87a431 Mon Sep 17 00:00:00 2001 From: siiky Date: Tue, 18 Jun 2024 17:06:37 +0100 Subject: [PATCH 04/10] feat: remove old failed QR scans --- lib/poller.js | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/lib/poller.js b/lib/poller.js index ca002612..d58deb71 100644 --- a/lib/poller.js +++ b/lib/poller.js @@ -1,3 +1,5 @@ +const fs = require('fs/promises') +const path = require('path') const _ = require('lodash/fp') const Queue = require('queue-promise') const plugins = require('./plugins') @@ -31,6 +33,7 @@ const RADAR_UPDATE_INTERVAL = 5 * T.minutes const PRUNE_MACHINES_HEARTBEAT = 1 * T.day const TRANSACTION_BATCH_LIFECYCLE = 20 * T.minutes const TICKER_RATES_INTERVAL = 59 * T.seconds +const FAILED_SCANS_INTERVAL = 1 * T.day const CHECK_NOTIFICATION_INTERVAL = 20 * T.seconds const PENDING_INTERVAL = 10 * T.seconds @@ -39,6 +42,8 @@ const CACHE_ENTRY_TTL = 3600 // seconds const FAST_QUEUE_WAIT = 1 * T.seconds const SLOW_QUEUE_WAIT = 10 * T.seconds +const OPERATOR_DATA_DIR = process.env.OPERATOR_DATA_DIR + const FAST_QUEUE = new Queue({ concurrent: 600, interval: FAST_QUEUE_WAIT @@ -127,6 +132,28 @@ function updateCoinAtmRadar () { .then(rates => coinAtmRadar.update(rates, settings())) } +const cleanOldFailedScans = () => { + const old = new Date() + old.setDate(old.getDate() - 2) // 2 days ago + const isOld = filepath => { + const then = new Date(path.basename(filepath).replace(/-[0-9]+\.jpg$/, '')) + return then < old + } + const baseDirPath = path.join(OPERATOR_DATA_DIR, 'failedQRScans') + fs.readdir(baseDirPath) + .then(machines => Promise.all( + machines.map( + machine => { + const machineDirPath = path.join(baseDirPath, machine) + return fs.readdir(machineDirPath) + .then(fnames => fnames.map(fname => path.join(machineDirPath, fname))) + } + ) + )) + .then(_.flatten) + .then(filepaths => Promise.all(filepaths.filter(isOld).map(fs.unlink))) +} + function initializeEachSchema (schemas = ['public']) { // for each schema set "thread variables" and do polling return _.forEach(schema => { @@ -206,6 +233,7 @@ function doPolling (schema) { addToQueue(updateAndLoadSanctions, SANCTIONS_UPDATE_INTERVAL, schema, QUEUE.SLOW) addToQueue(updateCoinAtmRadar, RADAR_UPDATE_INTERVAL, schema, QUEUE.SLOW) addToQueue(pi().pruneMachinesHeartbeat, PRUNE_MACHINES_HEARTBEAT, schema, QUEUE.SLOW, settings) + addToQueue(cleanOldFailedScans, FAILED_SCANS_INTERVAL, schema, QUEUE.SLOW, settings) } function setup (schemasToAdd = [], schemasToRemove = []) { From 806a2716f0070fe21fd97d3e286563673ae98906 Mon Sep 17 00:00:00 2001 From: siiky Date: Wed, 3 Jul 2024 11:25:53 +0100 Subject: [PATCH 05/10] refactor: rename failed QR scans' GC function --- lib/poller.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/poller.js b/lib/poller.js index d58deb71..75ffd6fd 100644 --- a/lib/poller.js +++ b/lib/poller.js @@ -132,7 +132,7 @@ function updateCoinAtmRadar () { .then(rates => coinAtmRadar.update(rates, settings())) } -const cleanOldFailedScans = () => { +const cleanOldFailedQRScans = () => { const old = new Date() old.setDate(old.getDate() - 2) // 2 days ago const isOld = filepath => { @@ -233,7 +233,7 @@ function doPolling (schema) { addToQueue(updateAndLoadSanctions, SANCTIONS_UPDATE_INTERVAL, schema, QUEUE.SLOW) addToQueue(updateCoinAtmRadar, RADAR_UPDATE_INTERVAL, schema, QUEUE.SLOW) addToQueue(pi().pruneMachinesHeartbeat, PRUNE_MACHINES_HEARTBEAT, schema, QUEUE.SLOW, settings) - addToQueue(cleanOldFailedScans, FAILED_SCANS_INTERVAL, schema, QUEUE.SLOW, settings) + addToQueue(cleanOldFailedQRScans, FAILED_SCANS_INTERVAL, schema, QUEUE.SLOW, settings) } function setup (schemasToAdd = [], schemasToRemove = []) { From a573419dd3b8eb4d37a468c1d1cab1130ea6b80d Mon Sep 17 00:00:00 2001 From: siiky Date: Wed, 3 Jul 2024 12:23:04 +0100 Subject: [PATCH 06/10] refactor: extract files listing out of `cleanOldFailedQRScans()` --- lib/poller.js | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/lib/poller.js b/lib/poller.js index 75ffd6fd..2178255e 100644 --- a/lib/poller.js +++ b/lib/poller.js @@ -132,6 +132,18 @@ function updateCoinAtmRadar () { .then(rates => coinAtmRadar.update(rates, settings())) } +const readdir = dirpath => + fs.readdir(dirpath, { withFileTypes: true }) + .then(_.map(entry => _.set('path', path.join(dirpath, entry.name), entry))) + +const readdirRec = rootPath => + readdir(rootPath) + .then(entries => Promise.all( + entries.map(entry => entry.isDirectory() ? readdirRec(entry.path) : [entry]) + )) + .then(_.flatten) + +// @see lib/machine-loader.js:updateFailedQRScans() const cleanOldFailedQRScans = () => { const old = new Date() old.setDate(old.getDate() - 2) // 2 days ago @@ -139,19 +151,17 @@ const cleanOldFailedQRScans = () => { const then = new Date(path.basename(filepath).replace(/-[0-9]+\.jpg$/, '')) return then < old } - const baseDirPath = path.join(OPERATOR_DATA_DIR, 'failedQRScans') - fs.readdir(baseDirPath) - .then(machines => Promise.all( - machines.map( - machine => { - const machineDirPath = path.join(baseDirPath, machine) - return fs.readdir(machineDirPath) - .then(fnames => fnames.map(fname => path.join(machineDirPath, fname))) - } - ) + readdirRec(path.join(OPERATOR_DATA_DIR, 'failedQRScans')) + .then(entries => Promise.all( + entries + .filter(entry => entry.isFile()) + .map(entry => entry.path) + .filter(isOld) + .map(fs.unlink) )) - .then(_.flatten) - .then(filepaths => Promise.all(filepaths.filter(isOld).map(fs.unlink))) + .catch(err => { + console.log("Error cleaning up failed QR scans:", err) + }) } function initializeEachSchema (schemas = ['public']) { From 63f51f4f3ed8078406de87159aebb5485e1bedab Mon Sep 17 00:00:00 2001 From: siiky Date: Wed, 3 Jul 2024 12:55:15 +0100 Subject: [PATCH 07/10] feat: add GC for failed PDF417 scans --- lib/poller.js | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/lib/poller.js b/lib/poller.js index 2178255e..02834902 100644 --- a/lib/poller.js +++ b/lib/poller.js @@ -143,14 +143,41 @@ const readdirRec = rootPath => )) .then(_.flatten) +// @see lib/customers.js:updateIdCardData() +const cleanOldFailedPDF417Scans = () => { + let old = new Date() + old.setDate(old.getDate() - 2) // 2 days ago + old = old.getTime() + + /* NOTE: Small caveat to mtime: last time the file was written to. */ + const isOld = filestat => filestat.mtimeMs < old + + readdirRec(path.join(OPERATOR_DATA_DIR, 'id-operator')) + .then(entries => Promise.all( + entries + .filter(entry => entry.isFile()) + .map(entry => fs.stat(entry.path).then(_.set('path', entry.path))) + )) + .then(filestats => Promise.all( + filestats + .filter(isOld) + .map(_.flow(_.get(['path']), fs.unlink)) + )) + .catch(err => { + console.log("Error cleaning up failed PDF417 scans:", err) + }) +} + // @see lib/machine-loader.js:updateFailedQRScans() const cleanOldFailedQRScans = () => { const old = new Date() old.setDate(old.getDate() - 2) // 2 days ago + const isOld = filepath => { const then = new Date(path.basename(filepath).replace(/-[0-9]+\.jpg$/, '')) return then < old } + readdirRec(path.join(OPERATOR_DATA_DIR, 'failedQRScans')) .then(entries => Promise.all( entries @@ -244,6 +271,7 @@ function doPolling (schema) { addToQueue(updateCoinAtmRadar, RADAR_UPDATE_INTERVAL, schema, QUEUE.SLOW) addToQueue(pi().pruneMachinesHeartbeat, PRUNE_MACHINES_HEARTBEAT, schema, QUEUE.SLOW, settings) addToQueue(cleanOldFailedQRScans, FAILED_SCANS_INTERVAL, schema, QUEUE.SLOW, settings) + addToQueue(cleanOldFailedPDF417Scans, FAILED_SCANS_INTERVAL, schema, QUEUE.SLOW, settings) } function setup (schemasToAdd = [], schemasToRemove = []) { From 393e149b91c1f459a30e9f62990f2e8342ef6f0e Mon Sep 17 00:00:00 2001 From: siiky Date: Wed, 3 Jul 2024 13:05:12 +0100 Subject: [PATCH 08/10] refactor: extract `stat()` out of `cleanOldFailedPDF417Scans()` --- lib/poller.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/poller.js b/lib/poller.js index 02834902..b3bba26b 100644 --- a/lib/poller.js +++ b/lib/poller.js @@ -143,6 +143,8 @@ const readdirRec = rootPath => )) .then(_.flatten) +const stat = path => fs.stat(path).then(_.set('path', path)) + // @see lib/customers.js:updateIdCardData() const cleanOldFailedPDF417Scans = () => { let old = new Date() @@ -156,7 +158,7 @@ const cleanOldFailedPDF417Scans = () => { .then(entries => Promise.all( entries .filter(entry => entry.isFile()) - .map(entry => fs.stat(entry.path).then(_.set('path', entry.path))) + .map(entry => stat(entry.path)) )) .then(filestats => Promise.all( filestats From 1451f7821172fb155e836c23e23db2355725d28d Mon Sep 17 00:00:00 2001 From: siiky Date: Wed, 3 Jul 2024 17:22:21 +0100 Subject: [PATCH 09/10] fix: make sure to remove only failed PDF417 scans --- lib/poller.js | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/lib/poller.js b/lib/poller.js index b3bba26b..65418df4 100644 --- a/lib/poller.js +++ b/lib/poller.js @@ -144,20 +144,32 @@ const readdirRec = rootPath => .then(_.flatten) const stat = path => fs.stat(path).then(_.set('path', path)) +const pathComponents = p => path.normalize(p).split(path.sep) // @see lib/customers.js:updateIdCardData() const cleanOldFailedPDF417Scans = () => { + const matcher = (c, pat) => typeof pat === 'function' ? pat(c) : c === pat + const PDF417ScanPathPattern = _.concat( + pathComponents(OPERATOR_DATA_DIR), + ["id-operator", s => /* customerid*/ true, "idcarddata", fname => path.extname(fname) === 'jpg'] + ) + const isPDF417Scan = entry => { + entry = pathComponents(entry.path) + return entry.length === PDF417ScanPathPattern.length + && _.isMatchWith(matcher, PDF417ScanPathPattern, pathComponents(entry.path)) + } + let old = new Date() old.setDate(old.getDate() - 2) // 2 days ago old = old.getTime() - /* NOTE: Small caveat to mtime: last time the file was written to. */ + /* NOTE: Small caveat to mtime: last time the file was written to. */ const isOld = filestat => filestat.mtimeMs < old readdirRec(path.join(OPERATOR_DATA_DIR, 'id-operator')) .then(entries => Promise.all( entries - .filter(entry => entry.isFile()) + .filter(entry => entry.isFile() && isPDF417Scan(entry)) .map(entry => stat(entry.path)) )) .then(filestats => Promise.all( From 7ab44602eed58251e34b9999c0d4e791f8433cba Mon Sep 17 00:00:00 2001 From: siiky Date: Wed, 3 Jul 2024 17:25:13 +0100 Subject: [PATCH 10/10] refactor: reduce number of intermediate arrays --- lib/poller.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/poller.js b/lib/poller.js index 65418df4..210a5cba 100644 --- a/lib/poller.js +++ b/lib/poller.js @@ -195,10 +195,8 @@ const cleanOldFailedQRScans = () => { readdirRec(path.join(OPERATOR_DATA_DIR, 'failedQRScans')) .then(entries => Promise.all( entries - .filter(entry => entry.isFile()) - .map(entry => entry.path) - .filter(isOld) - .map(fs.unlink) + .filter(entry => entry.isFile() && isOld(entry.path)) + .map(entry => fs.unlink(entry.path)) )) .catch(err => { console.log("Error cleaning up failed QR scans:", err)