From dec172576ca02a554ded698c2308c1a6f42d6fac Mon Sep 17 00:00:00 2001 From: siiky Date: Tue, 1 Jul 2025 18:55:06 +0100 Subject: [PATCH 1/5] refactor: clarify machine pings object construction --- packages/server/lib/notifier/index.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/server/lib/notifier/index.js b/packages/server/lib/notifier/index.js index a45ac7d4..f0db3cfe 100644 --- a/packages/server/lib/notifier/index.js +++ b/packages/server/lib/notifier/index.js @@ -107,9 +107,7 @@ function buildAlerts(pings, balances, events, devices) { } function checkPings(devices) { - const deviceIds = _.map('deviceId', devices) - const pings = _.map(utils.checkPing, devices) - return _.zipObject(deviceIds)(pings) + return Object.fromEntries(devices.map(d => [d.deviceId, utils.checkPing(d)])) } function checkStuckScreen(deviceEvents, machine) { From f7c42992d8a6993353f0ba959f868f1e1e9e44ed Mon Sep 17 00:00:00 2001 From: siiky Date: Wed, 2 Jul 2025 12:12:45 +0100 Subject: [PATCH 2/5] feat: don't try to parse already-parsed JSON note --- packages/server/lib/notifier/utils.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/server/lib/notifier/utils.js b/packages/server/lib/notifier/utils.js index f02c0e8a..0e36a34a 100644 --- a/packages/server/lib/notifier/utils.js +++ b/packages/server/lib/notifier/utils.js @@ -24,7 +24,11 @@ const DETAIL_TEMPLATE = { } function parseEventNote(event) { - return _.set('note', JSON.parse(event.note), event) + return _.update( + 'note', + note => (typeof note === 'string' ? JSON.parse(note) : note), + event, + ) } function checkPing(device) { From ff3c51623f70029a5f678640dd29d12cd5d2deee Mon Sep 17 00:00:00 2001 From: siiky Date: Thu, 26 Jun 2025 16:36:30 +0100 Subject: [PATCH 3/5] refactor: prepare machine statuses code for new statuses --- packages/server/lib/machine-loader.js | 31 +++++++++++++++++---------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/packages/server/lib/machine-loader.js b/packages/server/lib/machine-loader.js index 8e822fad..105e45fc 100644 --- a/packages/server/lib/machine-loader.js +++ b/packages/server/lib/machine-loader.js @@ -104,26 +104,35 @@ function getConfig(defaultConfig) { return defaultConfig ? Promise.resolve(defaultConfig) : loadLatestConfig() } -const getStatus = (ping, stuck) => { - if (ping && ping.age) return unresponsiveStatus +const getMachineStatuses = (pings, events, machine) => { + const lastPing = pings[machine.deviceId][0] + if (lastPing?.age) return [unresponsiveStatus] - if (stuck && stuck.age) return stuckStatus + const machineEvents = events + .filter( + ev => + ev.device_id === machine.deviceId && ev.event_type === 'stateChange', + ) + .map(ev => + Object.assign({}, ev, { + age: Math.floor(ev.age), + note: JSON.parse(ev.note), + }), + ) + .sort((e1, e2) => e2.age - e1.age) - return fullyFunctionalStatus + const stuckScreen = checkStuckScreen(machineEvents, machine)[0] + if (stuckScreen?.age) return [stuckStatus] + + return [fullyFunctionalStatus] } function addName(pings, events, config) { return machine => { const cashOutConfig = configManager.getCashOut(machine.deviceId, config) - const cashOut = !!cashOutConfig.active - const statuses = [ - getStatus( - _.first(pings[machine.deviceId]), - _.first(checkStuckScreen(events, machine)), - ), - ] + const statuses = getMachineStatuses(pings, events, machine) return _.assign(machine, { cashOut, statuses }) } From 9ad884be5d5ac3e9639c7981a84f62166e0b94d0 Mon Sep 17 00:00:00 2001 From: siiky Date: Wed, 2 Jul 2025 12:41:42 +0100 Subject: [PATCH 4/5] feat: add _Booting up_ machine status --- packages/server/lib/machine-loader.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/packages/server/lib/machine-loader.js b/packages/server/lib/machine-loader.js index 105e45fc..e025f4f8 100644 --- a/packages/server/lib/machine-loader.js +++ b/packages/server/lib/machine-loader.js @@ -20,6 +20,7 @@ const logger = require('./logger') const fullyFunctionalStatus = { label: 'Fully functional', type: 'success' } const unresponsiveStatus = { label: 'Unresponsive', type: 'error' } const stuckStatus = { label: 'Stuck', type: 'error' } +const bootingUpStatus = { label: 'Booting up', type: 'warning' } const OPERATOR_DATA_DIR = process.env.OPERATOR_DATA_DIR const MACHINE_WITH_CALCULATED_FIELD_SQL = ` @@ -104,6 +105,11 @@ function getConfig(defaultConfig) { return defaultConfig ? Promise.resolve(defaultConfig) : loadLatestConfig() } +const isBootingState = state => ['booting', 'pendingIdle'].includes(state) + +const isBooting = machineEvents => + isBootingState(machineEvents[machineEvents.length - 1]?.note?.state) + const getMachineStatuses = (pings, events, machine) => { const lastPing = pings[machine.deviceId][0] if (lastPing?.age) return [unresponsiveStatus] @@ -124,6 +130,8 @@ const getMachineStatuses = (pings, events, machine) => { const stuckScreen = checkStuckScreen(machineEvents, machine)[0] if (stuckScreen?.age) return [stuckStatus] + if (isBooting(machineEvents)) return [bootingUpStatus] + return [fullyFunctionalStatus] } From 8654ad0b197eb28f546ab624a7924b7e310fd9ab Mon Sep 17 00:00:00 2001 From: siiky Date: Wed, 2 Jul 2025 12:42:10 +0100 Subject: [PATCH 5/5] feat: add _Stuck booting up_ machine status --- packages/server/lib/machine-loader.js | 29 +++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/packages/server/lib/machine-loader.js b/packages/server/lib/machine-loader.js index e025f4f8..d9c4b067 100644 --- a/packages/server/lib/machine-loader.js +++ b/packages/server/lib/machine-loader.js @@ -16,9 +16,11 @@ const notifierQueries = require('./notifier/queries') const { GraphQLError } = require('graphql') const { loadLatestConfig } = require('./new-settings-loader') const logger = require('./logger') +const T = require('./time') const fullyFunctionalStatus = { label: 'Fully functional', type: 'success' } const unresponsiveStatus = { label: 'Unresponsive', type: 'error' } +const stuckOnBootStatus = { label: 'Stuck booting up', type: 'error' } const stuckStatus = { label: 'Stuck', type: 'error' } const bootingUpStatus = { label: 'Booting up', type: 'warning' } const OPERATOR_DATA_DIR = process.env.OPERATOR_DATA_DIR @@ -107,6 +109,31 @@ function getConfig(defaultConfig) { const isBootingState = state => ['booting', 'pendingIdle'].includes(state) +const isStuckOnBoot = machineEvents => { + // Consider the machine stuck on boot if it's been booting for at least 30s + const lowerLimit = 30 * T.seconds + + // Heuristic to ignore older events (possibly from previous boots), obviously + // fallible + const higherLimit = 4 * lowerLimit + + // machineEvents is sorted from oldest to newest + const newest = machineEvents[machineEvents.length - 1] + + // Find the first event that makes a lowerLimit time interval with the + // newest, ignoring older events + const firstOverLimit = machineEvents.findLastIndex(ev => { + const ageDiff = ev.age - newest.age + return ageDiff >= lowerLimit && ageDiff <= higherLimit + }) + if (firstOverLimit < 0) return false + + // Check all the events are for a booting state + return machineEvents + .slice(firstOverLimit) + .every(ev => isBootingState(ev.note.state)) +} + const isBooting = machineEvents => isBootingState(machineEvents[machineEvents.length - 1]?.note?.state) @@ -127,6 +154,8 @@ const getMachineStatuses = (pings, events, machine) => { ) .sort((e1, e2) => e2.age - e1.age) + if (isStuckOnBoot(machineEvents)) return [stuckOnBootStatus] + const stuckScreen = checkStuckScreen(machineEvents, machine)[0] if (stuckScreen?.age) return [stuckStatus]