Merge pull request #1904 from siiky/refactor/lam-1137/machine-status

LAM-1137 feat: add _Booting up_ and _Stuck booting up_ machine statuses

packages/server/lib/machine-loader.js

@@ -16,10 +16,13 @@ const notifierQueries = require('./notifier/queries')
 const { GraphQLError } = require('graphql')
 const { loadLatestConfig } = require('./new-settings-loader')
 const logger = require('./logger')
+const T = require('./time')
 
 const fullyFunctionalStatus = { label: 'Fully functional', type: 'success' }
 const unresponsiveStatus = { label: 'Unresponsive', type: 'error' }
+const stuckOnBootStatus = { label: 'Stuck booting up', type: 'error' }
 const stuckStatus = { label: 'Stuck', type: 'error' }
+const bootingUpStatus = { label: 'Booting up', type: 'warning' }
 const OPERATOR_DATA_DIR = process.env.OPERATOR_DATA_DIR
 
 const MACHINE_WITH_CALCULATED_FIELD_SQL = `
@@ -104,26 +107,69 @@ function getConfig(defaultConfig) {
   return defaultConfig ? Promise.resolve(defaultConfig) : loadLatestConfig()
 }
 
-const getStatus = (ping, stuck) => {
-  if (ping && ping.age) return unresponsiveStatus
+const isBootingState = state => ['booting', 'pendingIdle'].includes(state)
 
-  if (stuck && stuck.age) return stuckStatus
+const isStuckOnBoot = machineEvents => {
+  // Consider the machine stuck on boot if it's been booting for at least 30s
+  const lowerLimit = 30 * T.seconds
 
-  return fullyFunctionalStatus
+  // Heuristic to ignore older events (possibly from previous boots), obviously
+  // fallible
+  const higherLimit = 4 * lowerLimit
+
+  // machineEvents is sorted from oldest to newest
+  const newest = machineEvents[machineEvents.length - 1]
+
+  // Find the first event that makes a lowerLimit time interval with the
+  // newest, ignoring older events
+  const firstOverLimit = machineEvents.findLastIndex(ev => {
+    const ageDiff = ev.age - newest.age
+    return ageDiff >= lowerLimit && ageDiff <= higherLimit
+  })
+  if (firstOverLimit < 0) return false
+
+  // Check all the events are for a booting state
+  return machineEvents
+    .slice(firstOverLimit)
+    .every(ev => isBootingState(ev.note.state))
+}
+
+const isBooting = machineEvents =>
+  isBootingState(machineEvents[machineEvents.length - 1]?.note?.state)
+
+const getMachineStatuses = (pings, events, machine) => {
+  const lastPing = pings[machine.deviceId][0]
+  if (lastPing?.age) return [unresponsiveStatus]
+
+  const machineEvents = events
+    .filter(
+      ev =>
+        ev.device_id === machine.deviceId && ev.event_type === 'stateChange',
+    )
+    .map(ev =>
+      Object.assign({}, ev, {
+        age: Math.floor(ev.age),
+        note: JSON.parse(ev.note),
+      }),
+    )
+    .sort((e1, e2) => e2.age - e1.age)
+
+  if (isStuckOnBoot(machineEvents)) return [stuckOnBootStatus]
+
+  const stuckScreen = checkStuckScreen(machineEvents, machine)[0]
+  if (stuckScreen?.age) return [stuckStatus]
+
+  if (isBooting(machineEvents)) return [bootingUpStatus]
+
+  return [fullyFunctionalStatus]
 }
 
 function addName(pings, events, config) {
   return machine => {
     const cashOutConfig = configManager.getCashOut(machine.deviceId, config)
-
     const cashOut = !!cashOutConfig.active
 
-    const statuses = [
-      getStatus(
-        _.first(pings[machine.deviceId]),
-        _.first(checkStuckScreen(events, machine)),
-      ),
-    ]
+    const statuses = getMachineStatuses(pings, events, machine)
 
     return _.assign(machine, { cashOut, statuses })
   }

packages/server/lib/notifier/index.js

@@ -107,9 +107,7 @@ function buildAlerts(pings, balances, events, devices) {
 }
 
 function checkPings(devices) {
-  const deviceIds = _.map('deviceId', devices)
-  const pings = _.map(utils.checkPing, devices)
-  return _.zipObject(deviceIds)(pings)
+  return Object.fromEntries(devices.map(d => [d.deviceId, utils.checkPing(d)]))
 }
 
 function checkStuckScreen(deviceEvents, machine) {

packages/server/lib/notifier/utils.js

@@ -24,7 +24,11 @@ const DETAIL_TEMPLATE = {
 }
 
 function parseEventNote(event) {
-  return _.set('note', JSON.parse(event.note), event)
+  return _.update(
+    'note',
+    note => (typeof note === 'string' ? JSON.parse(note) : note),
+    event,
+  )
 }
 
 function checkPing(device) {