diff --git a/packages/server/lib/machine-loader.js b/packages/server/lib/machine-loader.js index 8e822fad..d9c4b067 100644 --- a/packages/server/lib/machine-loader.js +++ b/packages/server/lib/machine-loader.js @@ -16,10 +16,13 @@ const notifierQueries = require('./notifier/queries') const { GraphQLError } = require('graphql') const { loadLatestConfig } = require('./new-settings-loader') const logger = require('./logger') +const T = require('./time') const fullyFunctionalStatus = { label: 'Fully functional', type: 'success' } const unresponsiveStatus = { label: 'Unresponsive', type: 'error' } +const stuckOnBootStatus = { label: 'Stuck booting up', type: 'error' } const stuckStatus = { label: 'Stuck', type: 'error' } +const bootingUpStatus = { label: 'Booting up', type: 'warning' } const OPERATOR_DATA_DIR = process.env.OPERATOR_DATA_DIR const MACHINE_WITH_CALCULATED_FIELD_SQL = ` @@ -104,26 +107,69 @@ function getConfig(defaultConfig) { return defaultConfig ? Promise.resolve(defaultConfig) : loadLatestConfig() } -const getStatus = (ping, stuck) => { - if (ping && ping.age) return unresponsiveStatus +const isBootingState = state => ['booting', 'pendingIdle'].includes(state) - if (stuck && stuck.age) return stuckStatus +const isStuckOnBoot = machineEvents => { + // Consider the machine stuck on boot if it's been booting for at least 30s + const lowerLimit = 30 * T.seconds - return fullyFunctionalStatus + // Heuristic to ignore older events (possibly from previous boots), obviously + // fallible + const higherLimit = 4 * lowerLimit + + // machineEvents is sorted from oldest to newest + const newest = machineEvents[machineEvents.length - 1] + + // Find the first event that makes a lowerLimit time interval with the + // newest, ignoring older events + const firstOverLimit = machineEvents.findLastIndex(ev => { + const ageDiff = ev.age - newest.age + return ageDiff >= lowerLimit && ageDiff <= higherLimit + }) + if (firstOverLimit < 0) return false + + // Check all the events are for a booting state + return machineEvents + .slice(firstOverLimit) + .every(ev => isBootingState(ev.note.state)) +} + +const isBooting = machineEvents => + isBootingState(machineEvents[machineEvents.length - 1]?.note?.state) + +const getMachineStatuses = (pings, events, machine) => { + const lastPing = pings[machine.deviceId][0] + if (lastPing?.age) return [unresponsiveStatus] + + const machineEvents = events + .filter( + ev => + ev.device_id === machine.deviceId && ev.event_type === 'stateChange', + ) + .map(ev => + Object.assign({}, ev, { + age: Math.floor(ev.age), + note: JSON.parse(ev.note), + }), + ) + .sort((e1, e2) => e2.age - e1.age) + + if (isStuckOnBoot(machineEvents)) return [stuckOnBootStatus] + + const stuckScreen = checkStuckScreen(machineEvents, machine)[0] + if (stuckScreen?.age) return [stuckStatus] + + if (isBooting(machineEvents)) return [bootingUpStatus] + + return [fullyFunctionalStatus] } function addName(pings, events, config) { return machine => { const cashOutConfig = configManager.getCashOut(machine.deviceId, config) - const cashOut = !!cashOutConfig.active - const statuses = [ - getStatus( - _.first(pings[machine.deviceId]), - _.first(checkStuckScreen(events, machine)), - ), - ] + const statuses = getMachineStatuses(pings, events, machine) return _.assign(machine, { cashOut, statuses }) } diff --git a/packages/server/lib/notifier/index.js b/packages/server/lib/notifier/index.js index a45ac7d4..f0db3cfe 100644 --- a/packages/server/lib/notifier/index.js +++ b/packages/server/lib/notifier/index.js @@ -107,9 +107,7 @@ function buildAlerts(pings, balances, events, devices) { } function checkPings(devices) { - const deviceIds = _.map('deviceId', devices) - const pings = _.map(utils.checkPing, devices) - return _.zipObject(deviceIds)(pings) + return Object.fromEntries(devices.map(d => [d.deviceId, utils.checkPing(d)])) } function checkStuckScreen(deviceEvents, machine) { diff --git a/packages/server/lib/notifier/utils.js b/packages/server/lib/notifier/utils.js index f02c0e8a..0e36a34a 100644 --- a/packages/server/lib/notifier/utils.js +++ b/packages/server/lib/notifier/utils.js @@ -24,7 +24,11 @@ const DETAIL_TEMPLATE = { } function parseEventNote(event) { - return _.set('note', JSON.parse(event.note), event) + return _.update( + 'note', + note => (typeof note === 'string' ? JSON.parse(note) : note), + event, + ) } function checkPing(device) {