Commit e19c7781ae76dc9d91d2f17492bb53a7f16bdc70

Authored by Greg Sandstrom
1 parent b8ce08195f
Exists in main

switch to warn / alarm states for yellow and red lights

Showing 1 changed file with 18 additions and 11 deletions Inline Diff

ssa/alarm/alarmist.py View file @ e19c778
""" 1 1 """
alarmist.py 2 2 alarmist.py
3 3
Periodically checks on SOARS state and processes alarms 4 4 Periodically checks on SOARS state and processes alarms
""" 5 5 """
6 6
import configparser 7 7 import configparser
import collections 8 8 import collections
import influxdb 9 9 import influxdb
import sys 10 10 import sys
import threading 11 11 import threading
import ssa.alarm.lightstack as lightstack 12 12 import ssa.alarm.lightstack as lightstack
13 13
DEBUG = False 14 14 DEBUG = False
15 15
# bearing coolant is measured by gauge A 16 16 # bearing coolant is measured by gauge A
BEARING_COOLANT_QUERY = """ 17 17 BEARING_COOLANT_QUERY = """
SELECT last(flow_rate) as flow_rate 18 18 SELECT last(flow_rate) as flow_rate
FROM coolant_flow_rate 19 19 FROM coolant_flow_rate
WHERE meter_id='A' 20 20 WHERE meter_id='A'
""" 21 21 """
# we have to infer if the paddle is moving based on the motor torque 22 22 # we have to infer if the paddle is moving based on the motor torque
PADDLE_MOVING_QUERY = """ 23 23 PADDLE_MOVING_QUERY = """
SELECT last(value) as value FROM Torque GROUP BY * 24 24 SELECT last(value) as value FROM Torque GROUP BY *
""" 25 25 """
# figure out the hottest bearing 26 26 # figure out the hottest bearing
MAX_BEARING_TEMPERATURE_QUERY = """ 27 27 MAX_BEARING_TEMPERATURE_QUERY = """
SELECT max(temperature) as max_temperature FROM ( 28 28 SELECT max(temperature) as max_temperature FROM (
SELECT last(temperature) as temperature 29 29 SELECT last(temperature) as temperature
FROM paddle_bearings 30 30 FROM paddle_bearings
WHERE location != 'Ambient' 31 31 WHERE location != 'Ambient'
GROUP BY location 32 32 GROUP BY location
) 33 33 )
""" 34 34 """
# grab ambient temperature 35 35 # grab ambient temperature
AMBIENT_TEMPERATURE_QUERY = """ 36 36 AMBIENT_TEMPERATURE_QUERY = """
SELECT last(temperature) as temperature 37 37 SELECT last(temperature) as temperature
FROM paddle_bearings 38 38 FROM paddle_bearings
WHERE location = 'Ambient' 39 39 WHERE location = 'Ambient'
""" 40 40 """
41 41
# annoying, but we need to map the measurement name and value key from influxdb 42 42 # annoying, but we need to map the measurement name and value key from influxdb
# onto a key in the soars state object 43 43 # onto a key in the soars state object
MeasurementInfo = collections.namedtuple("MeasurementInfo", "name value_key state_key") 44 44 MeasurementInfo = collections.namedtuple("MeasurementInfo", "name value_key state_key")
45 45
46 46
def dprint(str): 47 47 def dprint(str):
if DEBUG: 48 48 if DEBUG:
print(str) 49 49 print(str)
50 50
51 51
class Alarmist: 52 52 class Alarmist:
def __init__(self): 53 53 def __init__(self):
self.config = configparser.ConfigParser() 54 54 self.config = configparser.ConfigParser()
self.config.read("alarmist_config.ini") 55 55 self.config.read("alarmist_config.ini")
self.state_update_event = threading.Event() 56 56 self.state_update_event = threading.Event()
self.soars_state = ThreadsafeState(self.state_update_event) 57 57 self.soars_state = ThreadsafeState(self.state_update_event)
self.lights = lightstack.LightStack() 58 58 self.lights = lightstack.LightStack()
self.threads = list() 59 59 self.threads = list()
self.threads.append( 60 60 self.threads.append(
PeriodicInfluxdbPoller( 61 61 PeriodicInfluxdbPoller(
interval=2, 62 62 interval=2,
soars_state=self.soars_state, 63 63 soars_state=self.soars_state,
query=BEARING_COOLANT_QUERY, 64 64 query=BEARING_COOLANT_QUERY,
measurment_info=MeasurementInfo( 65 65 measurment_info=MeasurementInfo(
"coolant_flow_rate", "flow_rate", "coolant_flow_rate" 66 66 "coolant_flow_rate", "flow_rate", "coolant_flow_rate"
), 67 67 ),
db_config=self.config["influxdb"], 68 68 db_config=self.config["influxdb"],
) 69 69 )
) 70 70 )
self.threads.append( 71 71 self.threads.append(
PeriodicInfluxdbPoller( 72 72 PeriodicInfluxdbPoller(
interval=2, 73 73 interval=2,
soars_state=self.soars_state, 74 74 soars_state=self.soars_state,
query=PADDLE_MOVING_QUERY, 75 75 query=PADDLE_MOVING_QUERY,
measurment_info=MeasurementInfo("Torque", "value", "motor_torque"), 76 76 measurment_info=MeasurementInfo("Torque", "value", "motor_torque"),
db_config=self.config["influxdb"], 77 77 db_config=self.config["influxdb"],
) 78 78 )
) 79 79 )
self.threads.append( 80 80 self.threads.append(
PeriodicInfluxdbPoller( 81 81 PeriodicInfluxdbPoller(
interval=5, 82 82 interval=5,
soars_state=self.soars_state, 83 83 soars_state=self.soars_state,
query=MAX_BEARING_TEMPERATURE_QUERY, 84 84 query=MAX_BEARING_TEMPERATURE_QUERY,
measurment_info=MeasurementInfo( 85 85 measurment_info=MeasurementInfo(
"paddle_bearings", "max_temperature", "max_temperature" 86 86 "paddle_bearings", "max_temperature", "max_temperature"
), 87 87 ),
db_config=self.config["influxdb"], 88 88 db_config=self.config["influxdb"],
) 89 89 )
) 90 90 )
self.threads.append( 91 91 self.threads.append(
PeriodicInfluxdbPoller( 92 92 PeriodicInfluxdbPoller(
interval=5, 93 93 interval=5,
soars_state=self.soars_state, 94 94 soars_state=self.soars_state,
query=AMBIENT_TEMPERATURE_QUERY, 95 95 query=AMBIENT_TEMPERATURE_QUERY,
measurment_info=MeasurementInfo( 96 96 measurment_info=MeasurementInfo(
"paddle_bearings", "temperature", "ambient_temperature" 97 97 "paddle_bearings", "temperature", "ambient_temperature"
), 98 98 ),
db_config=self.config["influxdb"], 99 99 db_config=self.config["influxdb"],
) 100 100 )
) 101 101 )
self.event_watcher_thread = threading.Thread(target=self.check_state) 102 102 self.event_watcher_thread = threading.Thread(target=self.check_state)
self.event_watcher_thread_cancel_flag = threading.Event() 103 103 self.event_watcher_thread_cancel_flag = threading.Event()
104 104
def run(self): 105 105 def run(self):
# start polling threads 106 106 # start polling threads
for thread in self.threads: 107 107 for thread in self.threads:
thread.start() 108 108 thread.start()
# start update event watcher 109 109 # start update event watcher
self.event_watcher_thread.start() 110 110 self.event_watcher_thread.start()
111 111
def cleanup(self): 112 112 def cleanup(self):
# cancel polling loops 113 113 # cancel polling loops
for thread in self.threads: 114 114 for thread in self.threads:
thread.cancel() 115 115 thread.cancel()
# cancel the event watcher thread 116 116 # cancel the event watcher thread
self.event_watcher_thread_cancel_flag.set() 117 117 self.event_watcher_thread_cancel_flag.set()
self.state_update_event.set() 118 118 self.state_update_event.set()
# wait for threads to stop 119 119 # wait for threads to stop
self.join(timeout=1.0) 120 120 self.join(timeout=1.0)
# finally, call cleanup on lights 121 121 # finally, call cleanup on lights
self.lights.cleanup() 122 122 self.lights.cleanup()
123 123
def join(self, timeout=None): 124 124 def join(self, timeout=None):
for thread in self.threads: 125 125 for thread in self.threads:
thread.join(timeout=timeout) 126 126 thread.join(timeout=timeout)
self.event_watcher_thread.join(timeout=timeout) 127 127 self.event_watcher_thread.join(timeout=timeout)
128 128
def check_state(self): 129 129 def check_state(self):
# TODO - create an influxdb client here and log alarm states 130 130 # TODO - create an influxdb client here and log alarm states
131 131
# this method gets run in its own thread, so loop indefinitely, 132 132 # this method gets run in its own thread, so loop indefinitely,
# waking up whenever the state_update_event flag is set 133 133 # waking up whenever the state_update_event flag is set
while self.state_update_event.wait(timeout=None): 134 134 while self.state_update_event.wait(timeout=None):
# exit thread loop if the cancel flag is set 135 135 # exit thread loop if the cancel flag is set
if self.event_watcher_thread_cancel_flag.is_set(): 136 136 if self.event_watcher_thread_cancel_flag.is_set():
break 137 137 break
# 138 138 #
# update state values that depend on db values 139 139 # update state values that depend on db values
# 140 140 #
# check if paddle is running 141 141 # check if paddle is running
(torque, _) = self.soars_state.get( 142 142 (torque, _) = self.soars_state.get(
"motor_torque", default=(0.0, None), lookback=0 143 143 "motor_torque", default=(0.0, None), lookback=0
) 144 144 )
(previous_torque, _) = self.soars_state.get( 145 145 (previous_torque, _) = self.soars_state.get(
"motor_torque", default=(0.0, None), lookback=1 146 146 "motor_torque", default=(0.0, None), lookback=1
) 147 147 )
(previouser_torque, _) = self.soars_state.get( 148 148 (previouser_torque, _) = self.soars_state.get(
"motor_torque", default=(0.0, None), lookback=2 149 149 "motor_torque", default=(0.0, None), lookback=2
) 150 150 )
paddle_on = torque > float(self.config["thresholds"]["on_torque"]) 151 151 paddle_on = torque > float(self.config["thresholds"]["on_torque"])
# Check three last points to see if paddle is moving 152 152 # Check three last points to see if paddle is moving
paddle_moving = ( 153 153 paddle_moving = (
torque > float(self.config["thresholds"]["moving_torque"]) 154 154 torque > float(self.config["thresholds"]["moving_torque"])
) or (torque != previous_torque or previous_torque != previouser_torque) 155 155 ) or (torque != previous_torque or previous_torque != previouser_torque)
# check if coolant is flowing 156 156 # check if coolant is flowing
(coolant_flow_rate, _) = self.soars_state.get( 157 157 (coolant_flow_rate, _) = self.soars_state.get(
"coolant_flow_rate", (0.0, None) 158 158 "coolant_flow_rate", (0.0, None)
) 159 159 )
coolant_on = coolant_flow_rate > float(self.config["thresholds"]["coolant"]) 160 160 coolant_on = coolant_flow_rate > float(self.config["thresholds"]["coolant"])
161 161
(ambient, _) = self.soars_state.get("ambient_temperature", (-1, None)) 162 162 (ambient, _) = self.soars_state.get("ambient_temperature", (-1, None))
(max_bearing, _) = self.soars_state.get("max_temperature", (-1, None)) 163 163 (max_bearing, _) = self.soars_state.get("max_temperature", (-1, None))
ambient_delta = float( 164 164 ambient_delta = float(
self.config["thresholds"]["ambient_temperature_delta"] 165 165 self.config["thresholds"]["ambient_temperature_delta"]
) 166 166 )
bearing_temperature_warning = False 167 167 bearing_temperature_warning = False
if ambient > 0 and max_bearing > 0: 168 168 if ambient > 0 and max_bearing > 0:
if max_bearing > ambient + ambient_delta: 169 169 if max_bearing > ambient + ambient_delta:
bearing_temperature_warning = True 170 170 bearing_temperature_warning = True
171 171
dprint( 172 172 dprint(
f"State: paddle_on: {paddle_on}, paddle_moving: {paddle_moving}, coolant_on: {coolant_on}" 173 173 f"State: paddle_on: {paddle_on}, paddle_moving: {paddle_moving}, coolant_on: {coolant_on}"
) 174 174 )
dprint(f"Ambient temperature: {ambient}C Hottest bearing: {max_bearing}C") 175 175 dprint(f"Ambient temperature: {ambient}C Hottest bearing: {max_bearing}C")
176 176
# 177 177 #
# run checks to throw alarms 178 178 # run checks to throw alarms
# 179 179 #
buzzer = False 180 180 warning = False
# is paddle running? (solid white, no buzzer) 181 181 alarm = False
182 # is paddle running? (solid white, no problems)
if paddle_on: 182 183 if paddle_on:
self.lights.on(lightstack.Channel.White) 183 184 self.lights.on(lightstack.Channel.White)
else: 184 185 else:
self.lights.off(lightstack.Channel.White) 185 186 self.lights.off(lightstack.Channel.White)
# is paddle moving? (solid green, no buzzer) 186 187 # is paddle moving? (solid green, no problems)
if paddle_moving: 187 188 if paddle_moving:
self.lights.on(lightstack.Channel.Green) 188 189 self.lights.on(lightstack.Channel.Green)
else: 189 190 else:
self.lights.off(lightstack.Channel.Green) 190 191 self.lights.off(lightstack.Channel.Green)
# is water running? (solid blue, no buzzer) 191 192 # is water running? (solid blue, no problems)
if coolant_on: 192 193 if coolant_on:
self.lights.on(lightstack.Channel.Blue) 193 194 self.lights.on(lightstack.Channel.Blue)
else: 194 195 else:
self.lights.off(lightstack.Channel.Blue) 195 196 self.lights.off(lightstack.Channel.Blue)
# is paddle moving without water? (flash blue w/ buzzer) 196 197 # is paddle moving without water? (alarm state and flash blue)
if paddle_moving and not coolant_on: 197 198 if paddle_moving and not coolant_on:
self.lights.start_blink(0.5, lightstack.Channel.Blue) 198 199 self.lights.start_blink(0.5, lightstack.Channel.Blue)
buzzer = True 199 200 alarm = True
else: 200 201 else:
self.lights.stop_blink(lightstack.Channel.Blue) 201 202 self.lights.stop_blink(lightstack.Channel.Blue)
# is the hottest bearing above ambient? (flash red, no buzzer) 202 203 # is the hottest bearing above ambient? (warning)
if bearing_temperature_warning: 203 204 if bearing_temperature_warning:
self.lights.start_blink(0.5, lightstack.Channel.Red) 204 205 warning = True
else: 205
self.lights.stop_blink(lightstack.Channel.Red) 206
# TODO - piston temps > 100F? (flash yellow w/ buzzer) 207 206 # TODO - piston temps > 100F? (flash yellow w/ buzzer)
208 207
208 # did any warnings get thrown? (turn on yellow flashing)
209 if warning:
210 self.lights.start_blink(0.5, lightstack.Channel.Yellow)
211 else:
212 self.lights.stop_blink(lightstack.Channel.Yellow)
213
# 209 214 #
# did any alarm want to turn on the buzzer? 210 215 # did any alarm want to turn on the buzzer?
# 211 216 #
if buzzer: 212 217 if alarm:
218 self.lights.start_blink(0.3, lightstack.Channel.Red)
self.lights.start_blink(0.3, lightstack.Channel.Buzzer) 213 219 self.lights.start_blink(0.3, lightstack.Channel.Buzzer)
else: 214 220 else:
221 self.lights.stop_blink(lightstack.Channel.Red)
self.lights.stop_blink(lightstack.Channel.Buzzer) 215 222 self.lights.stop_blink(lightstack.Channel.Buzzer)
216 223
# clear event flag 217 224 # clear event flag
self.state_update_event.clear() 218 225 self.state_update_event.clear()
219 226
220 227
class RingBuffer: 221 228 class RingBuffer:
def __init__(self, size): 222 229 def __init__(self, size):
self.size = size 223 230 self.size = size
self.data = [None] * size 224 231 self.data = [None] * size
self.head = -1 225 232 self.head = -1
226 233
def __getitem__(self, index=0): 227 234 def __getitem__(self, index=0):
if index >= self.size: 228 235 if index >= self.size:
raise Exception(f"Index out of bounds (index: {index}, size: {self.size})") 229 236 raise Exception(f"Index out of bounds (index: {index}, size: {self.size})")
internal_index = (self.head - index) % self.size 230 237 internal_index = (self.head - index) % self.size
return self.data[internal_index] 231 238 return self.data[internal_index]
232 239
def get(self, index=0, default=None): 233 240 def get(self, index=0, default=None):
val = self[index] 234 241 val = self[index]
return val if val is not None else default 235 242 return val if val is not None else default
236 243
def push(self, item): 237 244 def push(self, item):
self.head = (self.head + 1) % self.size 238 245 self.head = (self.head + 1) % self.size
self.data[self.head] = item 239 246 self.data[self.head] = item
240 247
241 248
# global state object protected by a lock 242 249 # global state object protected by a lock
# only update values if the new timestamp differs from the most recent item's timestamp 243 250 # only update values if the new timestamp differs from the most recent item's timestamp
# store history of past [lookback] items 244 251 # store history of past [lookback] items
class ThreadsafeState: 245 252 class ThreadsafeState:
def __init__(self, update_event, lookback=5): 246 253 def __init__(self, update_event, lookback=5):
self.lookback = lookback 247 254 self.lookback = lookback
self.state = dict() 248 255 self.state = dict()
self.lock = threading.RLock() 249 256 self.lock = threading.RLock()
self.update_event = update_event 250 257 self.update_event = update_event
251 258
def set(self, key, value, time): 252 259 def set(self, key, value, time):
with self.lock: 253 260 with self.lock:
if self.state.get(key, None) is None: 254 261 if self.state.get(key, None) is None:
self._init_entry(key) 255 262 self._init_entry(key)
history = self.state.get(key) 256 263 history = self.state.get(key)
item = history.get() 257 264 item = history.get()
if item is None: 258 265 if item is None:
history.push((value, time)) 259 266 history.push((value, time))
self.update_event.set() 260 267 self.update_event.set()
else: 261 268 else:
(_, previous_time) = item 262 269 (_, previous_time) = item
if time != previous_time: 263 270 if time != previous_time:
history.push((value, time)) 264 271 history.push((value, time))
self.update_event.set() 265 272 self.update_event.set()
dprint(f"ThreadsafeState: set: {key}: {value} ({time})") 266 273 dprint(f"ThreadsafeState: set: {key}: {value} ({time})")
267 274
def _init_entry(self, key): 268 275 def _init_entry(self, key):
with self.lock: 269 276 with self.lock:
self.state[key] = RingBuffer(self.lookback) 270 277 self.state[key] = RingBuffer(self.lookback)
271 278
def get(self, key, default=None, lookback=0): 272 279 def get(self, key, default=None, lookback=0):
with self.lock: 273 280 with self.lock:
history = self.state.get(key, None) 274 281 history = self.state.get(key, None)
if history is None: 275 282 if history is None:
return default 276 283 return default
else: 277 284 else:
return history.get(index=lookback, default=default) 278 285 return history.get(index=lookback, default=default)
279 286
280 287
# extend the Timer thread object to run repeatedly. Subclass this. 281 288 # extend the Timer thread object to run repeatedly. Subclass this.
class RepeatedTimer(threading.Timer): 282 289 class RepeatedTimer(threading.Timer):
def __init__(self, interval): 283 290 def __init__(self, interval):
# Timer constructer expects a callback function, but we don't care 284 291 # Timer constructer expects a callback function, but we don't care
# about that, so just pass None 285 292 # about that, so just pass None
super(RepeatedTimer, self).__init__(interval, None) 286 293 super(RepeatedTimer, self).__init__(interval, None)
287 294
# override Timer's run method to loop on our injest_data function 288 295 # override Timer's run method to loop on our injest_data function
# wait will block while loop for timeout interval 289 296 # wait will block while loop for timeout interval
# calling timer.cancel() sets the finish flag and exits the loop 290 297 # calling timer.cancel() sets the finish flag and exits the loop
def run(self): 291 298 def run(self):
while not self.finished.wait(timeout=self.interval): 292 299 while not self.finished.wait(timeout=self.interval):
self.do_work() 293 300 self.do_work()
294 301
def do_work(self): 295 302 def do_work(self):
raise Exception( 296 303 raise Exception(
"RepeatedTimer baseclass do_work() called. Override this method in a subclass" 297 304 "RepeatedTimer baseclass do_work() called. Override this method in a subclass"
) 298 305 )
299 306
300 307
# makes a [query] to influxdb every [interval] 301 308 # makes a [query] to influxdb every [interval]
# influxdb returns a resultset object: 302 309 # influxdb returns a resultset object:
# https://influxdb-python.readthedocs.io/en/latest/resultset.html#resultset 303 310 # https://influxdb-python.readthedocs.io/en/latest/resultset.html#resultset
class PeriodicInfluxdbPoller(RepeatedTimer): 304 311 class PeriodicInfluxdbPoller(RepeatedTimer):
def __init__(self, interval, soars_state, query, measurment_info, db_config): 305 312 def __init__(self, interval, soars_state, query, measurment_info, db_config):
super(PeriodicInfluxdbPoller, self).__init__(interval) 306 313 super(PeriodicInfluxdbPoller, self).__init__(interval)
self.soars_state = soars_state 307 314 self.soars_state = soars_state
self.query = query 308 315 self.query = query
self.info = measurment_info 309 316 self.info = measurment_info
self.db_config = db_config 310 317 self.db_config = db_config
self.idb_client = influxdb.InfluxDBClient( 311 318 self.idb_client = influxdb.InfluxDBClient(
self.db_config.get("host", "localhost"), 312 319 self.db_config.get("host", "localhost"),
self.db_config.get("port", "8086"), 313 320 self.db_config.get("port", "8086"),
self.db_config.get("username", None), 314 321 self.db_config.get("username", None),
self.db_config.get("password", None), 315 322 self.db_config.get("password", None),