Greg Sandstrom / SOARS Status and Alerting

Download as
Browse Code ยป

Commit 7b0ebee2544f88612402a9a156c16e35ae1d293b

Authored by Greg Sandstrom
1 parent 6cd96adada
Exists in main

try to fix issue where buzzer stayed on after stop_blink called by explicitely t… 

…urning off buzzer and red light when alarm is cleared

Showing 1 changed file with 2 additions and 0 deletions Inline Diff

ssa/alarm/alarmist.py View file @ 7b0ebee
""" 1 1 """
alarmist.py 2 2 alarmist.py
3 3
Periodically checks on SOARS state and processes alarms 4 4 Periodically checks on SOARS state and processes alarms
""" 5 5 """
6 6
import configparser 7 7 import configparser
import collections 8 8 import collections
import influxdb 9 9 import influxdb
import sys 10 10 import sys
import threading 11 11 import threading
import ssa.alarm.lightstack as lightstack 12 12 import ssa.alarm.lightstack as lightstack
13 13
DEBUG = False 14 14 DEBUG = False
15 15
# bearing coolant is measured by gauge A 16 16 # bearing coolant is measured by gauge A
BEARING_COOLANT_QUERY = """ 17 17 BEARING_COOLANT_QUERY = """
SELECT last(flow_rate) as flow_rate 18 18 SELECT last(flow_rate) as flow_rate
FROM coolant_flow_rate 19 19 FROM coolant_flow_rate
WHERE meter_id='A' 20 20 WHERE meter_id='A'
""" 21 21 """
# we have to infer if the paddle is moving based on the motor torque 22 22 # we have to infer if the paddle is moving based on the motor torque
PADDLE_MOVING_QUERY = """ 23 23 PADDLE_MOVING_QUERY = """
SELECT last(value) as value FROM Torque GROUP BY * 24 24 SELECT last(value) as value FROM Torque GROUP BY *
""" 25 25 """
# figure out the hottest bearing 26 26 # figure out the hottest bearing
MAX_BEARING_TEMPERATURE_QUERY = """ 27 27 MAX_BEARING_TEMPERATURE_QUERY = """
SELECT max(temperature) as max_temperature FROM ( 28 28 SELECT max(temperature) as max_temperature FROM (
SELECT last(temperature) as temperature 29 29 SELECT last(temperature) as temperature
FROM paddle_bearings 30 30 FROM paddle_bearings
WHERE location != 'Ambient' 31 31 WHERE location != 'Ambient'
GROUP BY location 32 32 GROUP BY location
) 33 33 )
""" 34 34 """
# grab ambient temperature 35 35 # grab ambient temperature
AMBIENT_TEMPERATURE_QUERY = """ 36 36 AMBIENT_TEMPERATURE_QUERY = """
SELECT last(temperature) as temperature 37 37 SELECT last(temperature) as temperature
FROM paddle_bearings 38 38 FROM paddle_bearings
WHERE location = 'Ambient' 39 39 WHERE location = 'Ambient'
""" 40 40 """
41 41
# annoying, but we need to map the measurement name and value key from influxdb 42 42 # annoying, but we need to map the measurement name and value key from influxdb
# onto a key in the soars state object 43 43 # onto a key in the soars state object
MeasurementInfo = collections.namedtuple("MeasurementInfo", "name value_key state_key") 44 44 MeasurementInfo = collections.namedtuple("MeasurementInfo", "name value_key state_key")
45 45
46 46
def dprint(str): 47 47 def dprint(str):
if DEBUG: 48 48 if DEBUG:
print(str) 49 49 print(str)
50 50
51 51
class Alarmist: 52 52 class Alarmist:
def __init__(self): 53 53 def __init__(self):
self.config = configparser.ConfigParser() 54 54 self.config = configparser.ConfigParser()
self.config.read("alarmist_config.ini") 55 55 self.config.read("alarmist_config.ini")
self.state_update_event = threading.Event() 56 56 self.state_update_event = threading.Event()
self.soars_state = ThreadsafeState(self.state_update_event) 57 57 self.soars_state = ThreadsafeState(self.state_update_event)
self.lights = lightstack.LightStack() 58 58 self.lights = lightstack.LightStack()
self.threads = list() 59 59 self.threads = list()
self.threads.append( 60 60 self.threads.append(
PeriodicInfluxdbPoller( 61 61 PeriodicInfluxdbPoller(
interval=2, 62 62 interval=2,
soars_state=self.soars_state, 63 63 soars_state=self.soars_state,
query=BEARING_COOLANT_QUERY, 64 64 query=BEARING_COOLANT_QUERY,
measurment_info=MeasurementInfo( 65 65 measurment_info=MeasurementInfo(
"coolant_flow_rate", "flow_rate", "coolant_flow_rate" 66 66 "coolant_flow_rate", "flow_rate", "coolant_flow_rate"
), 67 67 ),
db_config=self.config["influxdb"], 68 68 db_config=self.config["influxdb"],
) 69 69 )
) 70 70 )
self.threads.append( 71 71 self.threads.append(
PeriodicInfluxdbPoller( 72 72 PeriodicInfluxdbPoller(
interval=2, 73 73 interval=2,
soars_state=self.soars_state, 74 74 soars_state=self.soars_state,
query=PADDLE_MOVING_QUERY, 75 75 query=PADDLE_MOVING_QUERY,
measurment_info=MeasurementInfo("Torque", "value", "motor_torque"), 76 76 measurment_info=MeasurementInfo("Torque", "value", "motor_torque"),
db_config=self.config["influxdb"], 77 77 db_config=self.config["influxdb"],
) 78 78 )
) 79 79 )
self.threads.append( 80 80 self.threads.append(
PeriodicInfluxdbPoller( 81 81 PeriodicInfluxdbPoller(
interval=5, 82 82 interval=5,
soars_state=self.soars_state, 83 83 soars_state=self.soars_state,
query=MAX_BEARING_TEMPERATURE_QUERY, 84 84 query=MAX_BEARING_TEMPERATURE_QUERY,
measurment_info=MeasurementInfo( 85 85 measurment_info=MeasurementInfo(
"paddle_bearings", "max_temperature", "max_temperature" 86 86 "paddle_bearings", "max_temperature", "max_temperature"
), 87 87 ),
db_config=self.config["influxdb"], 88 88 db_config=self.config["influxdb"],
) 89 89 )
) 90 90 )
self.threads.append( 91 91 self.threads.append(
PeriodicInfluxdbPoller( 92 92 PeriodicInfluxdbPoller(
interval=5, 93 93 interval=5,
soars_state=self.soars_state, 94 94 soars_state=self.soars_state,
query=AMBIENT_TEMPERATURE_QUERY, 95 95 query=AMBIENT_TEMPERATURE_QUERY,
measurment_info=MeasurementInfo( 96 96 measurment_info=MeasurementInfo(
"paddle_bearings", "temperature", "ambient_temperature" 97 97 "paddle_bearings", "temperature", "ambient_temperature"
), 98 98 ),
db_config=self.config["influxdb"], 99 99 db_config=self.config["influxdb"],
) 100 100 )
) 101 101 )
self.event_watcher_thread = threading.Thread(target=self.check_state) 102 102 self.event_watcher_thread = threading.Thread(target=self.check_state)
self.event_watcher_thread_cancel_flag = threading.Event() 103 103 self.event_watcher_thread_cancel_flag = threading.Event()
104 104
def run(self): 105 105 def run(self):
# start polling threads 106 106 # start polling threads
for thread in self.threads: 107 107 for thread in self.threads:
thread.start() 108 108 thread.start()
# start update event watcher 109 109 # start update event watcher
self.event_watcher_thread.start() 110 110 self.event_watcher_thread.start()
111 111
def cleanup(self): 112 112 def cleanup(self):
# cancel polling loops 113 113 # cancel polling loops
for thread in self.threads: 114 114 for thread in self.threads:
thread.cancel() 115 115 thread.cancel()
# cancel the event watcher thread 116 116 # cancel the event watcher thread
self.event_watcher_thread_cancel_flag.set() 117 117 self.event_watcher_thread_cancel_flag.set()
self.state_update_event.set() 118 118 self.state_update_event.set()
# wait for threads to stop 119 119 # wait for threads to stop
self.join(timeout=1.0) 120 120 self.join(timeout=1.0)
# finally, call cleanup on lights 121 121 # finally, call cleanup on lights
self.lights.cleanup() 122 122 self.lights.cleanup()
123 123
def join(self, timeout=None): 124 124 def join(self, timeout=None):
for thread in self.threads: 125 125 for thread in self.threads:
thread.join(timeout=timeout) 126 126 thread.join(timeout=timeout)
self.event_watcher_thread.join(timeout=timeout) 127 127 self.event_watcher_thread.join(timeout=timeout)
128 128
def check_state(self): 129 129 def check_state(self):
# TODO - create an influxdb client here and log alarm states 130 130 # TODO - create an influxdb client here and log alarm states
131 131
# this method gets run in its own thread, so loop indefinitely, 132 132 # this method gets run in its own thread, so loop indefinitely,
# waking up whenever the state_update_event flag is set 133 133 # waking up whenever the state_update_event flag is set
while self.state_update_event.wait(timeout=None): 134 134 while self.state_update_event.wait(timeout=None):
# exit thread loop if the cancel flag is set 135 135 # exit thread loop if the cancel flag is set
if self.event_watcher_thread_cancel_flag.is_set(): 136 136 if self.event_watcher_thread_cancel_flag.is_set():
break 137 137 break
# 138 138 #
# update state values that depend on db values 139 139 # update state values that depend on db values
# 140 140 #
# check if paddle is running 141 141 # check if paddle is running
(torque, _) = self.soars_state.get( 142 142 (torque, _) = self.soars_state.get(
"motor_torque", default=(0.0, None), lookback=0 143 143 "motor_torque", default=(0.0, None), lookback=0
) 144 144 )
(previous_torque, _) = self.soars_state.get( 145 145 (previous_torque, _) = self.soars_state.get(
"motor_torque", default=(0.0, None), lookback=1 146 146 "motor_torque", default=(0.0, None), lookback=1
) 147 147 )
(previouser_torque, _) = self.soars_state.get( 148 148 (previouser_torque, _) = self.soars_state.get(
"motor_torque", default=(0.0, None), lookback=2 149 149 "motor_torque", default=(0.0, None), lookback=2
) 150 150 )
paddle_on = torque > float(self.config["thresholds"]["on_torque"]) 151 151 paddle_on = torque > float(self.config["thresholds"]["on_torque"])
# Check three last points to see if paddle is moving 152 152 # Check three last points to see if paddle is moving
paddle_moving = paddle_on and ( 153 153 paddle_moving = paddle_on and (
(torque > float(self.config["thresholds"]["moving_torque"])) 154 154 (torque > float(self.config["thresholds"]["moving_torque"]))
or (torque != previous_torque or previous_torque != previouser_torque) 155 155 or (torque != previous_torque or previous_torque != previouser_torque)
) 156 156 )
# check if coolant is flowing 157 157 # check if coolant is flowing
(coolant_flow_rate, _) = self.soars_state.get( 158 158 (coolant_flow_rate, _) = self.soars_state.get(
"coolant_flow_rate", (0.0, None) 159 159 "coolant_flow_rate", (0.0, None)
) 160 160 )
coolant_on = coolant_flow_rate > float(self.config["thresholds"]["coolant"]) 161 161 coolant_on = coolant_flow_rate > float(self.config["thresholds"]["coolant"])
162 162
(ambient, _) = self.soars_state.get("ambient_temperature", (-1, None)) 163 163 (ambient, _) = self.soars_state.get("ambient_temperature", (-1, None))
(max_bearing, _) = self.soars_state.get("max_temperature", (-1, None)) 164 164 (max_bearing, _) = self.soars_state.get("max_temperature", (-1, None))
ambient_delta = float( 165 165 ambient_delta = float(
self.config["thresholds"]["ambient_temperature_delta"] 166 166 self.config["thresholds"]["ambient_temperature_delta"]
) 167 167 )
bearing_temperature_warning = False 168 168 bearing_temperature_warning = False
if ambient > 0 and max_bearing > 0: 169 169 if ambient > 0 and max_bearing > 0:
if max_bearing > ambient + ambient_delta: 170 170 if max_bearing > ambient + ambient_delta:
bearing_temperature_warning = True 171 171 bearing_temperature_warning = True
172 172
dprint( 173 173 dprint(
f"State: paddle_on: {paddle_on}, paddle_moving: {paddle_moving}, coolant_on: {coolant_on}" 174 174 f"State: paddle_on: {paddle_on}, paddle_moving: {paddle_moving}, coolant_on: {coolant_on}"
) 175 175 )
dprint(f"Ambient temperature: {ambient}C Hottest bearing: {max_bearing}C") 176 176 dprint(f"Ambient temperature: {ambient}C Hottest bearing: {max_bearing}C")
177 177
# 178 178 #
# run checks to throw alarms 179 179 # run checks to throw alarms
# 180 180 #
warning = False 181 181 warning = False
alarm = False 182 182 alarm = False
# is paddle running? (solid white, no problems) 183 183 # is paddle running? (solid white, no problems)
if paddle_on: 184 184 if paddle_on:
self.lights.on(lightstack.Channel.White) 185 185 self.lights.on(lightstack.Channel.White)
else: 186 186 else:
self.lights.off(lightstack.Channel.White) 187 187 self.lights.off(lightstack.Channel.White)
# is paddle moving? (solid green, no problems) 188 188 # is paddle moving? (solid green, no problems)
if paddle_moving: 189 189 if paddle_moving:
self.lights.on(lightstack.Channel.Green) 190 190 self.lights.on(lightstack.Channel.Green)
else: 191 191 else:
self.lights.off(lightstack.Channel.Green) 192 192 self.lights.off(lightstack.Channel.Green)
# is water running? (solid blue, no problems) 193 193 # is water running? (solid blue, no problems)
if coolant_on: 194 194 if coolant_on:
self.lights.on(lightstack.Channel.Blue) 195 195 self.lights.on(lightstack.Channel.Blue)
else: 196 196 else:
self.lights.off(lightstack.Channel.Blue) 197 197 self.lights.off(lightstack.Channel.Blue)
# is paddle moving without water? (alarm state and flash blue) 198 198 # is paddle moving without water? (alarm state and flash blue)
if paddle_moving and not coolant_on: 199 199 if paddle_moving and not coolant_on:
self.lights.start_blink(0.5, lightstack.Channel.Blue) 200 200 self.lights.start_blink(0.5, lightstack.Channel.Blue)
alarm = True 201 201 alarm = True
else: 202 202 else:
self.lights.stop_blink(lightstack.Channel.Blue) 203 203 self.lights.stop_blink(lightstack.Channel.Blue)
# is the hottest bearing above ambient? (warning) 204 204 # is the hottest bearing above ambient? (warning)
if bearing_temperature_warning: 205 205 if bearing_temperature_warning:
warning = True 206 206 warning = True
# TODO - piston temps > 100F? (flash yellow w/ buzzer) 207 207 # TODO - piston temps > 100F? (flash yellow w/ buzzer)
208 208
# did any warnings get thrown? (turn on yellow flashing) 209 209 # did any warnings get thrown? (turn on yellow flashing)
if warning: 210 210 if warning:
self.lights.start_blink(0.5, lightstack.Channel.Yellow) 211 211 self.lights.start_blink(0.5, lightstack.Channel.Yellow)
else: 212 212 else:
self.lights.stop_blink(lightstack.Channel.Yellow) 213 213 self.lights.stop_blink(lightstack.Channel.Yellow)
214 214
# 215 215 #
# did any alarm want to turn on the buzzer? 216 216 # did any alarm want to turn on the buzzer?
# 217 217 #
if alarm: 218 218 if alarm:
self.lights.start_blink(0.3, lightstack.Channel.Red) 219 219 self.lights.start_blink(0.3, lightstack.Channel.Red)
self.lights.start_blink(0.3, lightstack.Channel.Buzzer) 220 220 self.lights.start_blink(0.3, lightstack.Channel.Buzzer)
else: 221 221 else:
self.lights.stop_blink(lightstack.Channel.Red) 222 222 self.lights.stop_blink(lightstack.Channel.Red)
223 self.lights.off(lightstack.Channel.Red)
self.lights.stop_blink(lightstack.Channel.Buzzer) 223 224 self.lights.stop_blink(lightstack.Channel.Buzzer)
225 self.lights.off(lightstack.Channel.Buzzer)
224 226
# clear event flag 225 227 # clear event flag
self.state_update_event.clear() 226 228 self.state_update_event.clear()
227 229
228 230
class RingBuffer: 229 231 class RingBuffer:
def __init__(self, size): 230 232 def __init__(self, size):
self.size = size 231 233 self.size = size
self.data = [None] * size 232 234 self.data = [None] * size
self.head = -1 233 235 self.head = -1
234 236
def __getitem__(self, index=0): 235 237 def __getitem__(self, index=0):
if index >= self.size: 236 238 if index >= self.size:
raise Exception(f"Index out of bounds (index: {index}, size: {self.size})") 237 239 raise Exception(f"Index out of bounds (index: {index}, size: {self.size})")
internal_index = (self.head - index) % self.size 238 240 internal_index = (self.head - index) % self.size
return self.data[internal_index] 239 241 return self.data[internal_index]
240 242
def get(self, index=0, default=None): 241 243 def get(self, index=0, default=None):
val = self[index] 242 244 val = self[index]
return val if val is not None else default 243 245 return val if val is not None else default
244 246
def push(self, item): 245 247 def push(self, item):
self.head = (self.head + 1) % self.size 246 248 self.head = (self.head + 1) % self.size
self.data[self.head] = item 247 249 self.data[self.head] = item
248 250
249 251
# global state object protected by a lock 250 252 # global state object protected by a lock
# only update values if the new timestamp differs from the most recent item's timestamp 251 253 # only update values if the new timestamp differs from the most recent item's timestamp
# store history of past [lookback] items 252 254 # store history of past [lookback] items
class ThreadsafeState: 253 255 class ThreadsafeState:
def __init__(self, update_event, lookback=5): 254 256 def __init__(self, update_event, lookback=5):
self.lookback = lookback 255 257 self.lookback = lookback
self.state = dict() 256 258 self.state = dict()
self.lock = threading.RLock() 257 259 self.lock = threading.RLock()
self.update_event = update_event 258 260 self.update_event = update_event
259 261
def set(self, key, value, time): 260 262 def set(self, key, value, time):
with self.lock: 261 263 with self.lock:
if self.state.get(key, None) is None: 262 264 if self.state.get(key, None) is None:
self._init_entry(key) 263 265 self._init_entry(key)
history = self.state.get(key) 264 266 history = self.state.get(key)
item = history.get() 265 267 item = history.get()
if item is None: 266 268 if item is None:
history.push((value, time)) 267 269 history.push((value, time))
self.update_event.set() 268 270 self.update_event.set()
else: 269 271 else:
(_, previous_time) = item 270 272 (_, previous_time) = item
if time != previous_time: 271 273 if time != previous_time:
history.push((value, time)) 272 274 history.push((value, time))
self.update_event.set() 273 275 self.update_event.set()
dprint(f"ThreadsafeState: set: {key}: {value} ({time})") 274 276 dprint(f"ThreadsafeState: set: {key}: {value} ({time})")
275 277
def _init_entry(self, key): 276 278 def _init_entry(self, key):
with self.lock: 277 279 with self.lock:
self.state[key] = RingBuffer(self.lookback) 278 280 self.state[key] = RingBuffer(self.lookback)
279 281
def get(self, key, default=None, lookback=0): 280 282 def get(self, key, default=None, lookback=0):
with self.lock: 281 283 with self.lock:
history = self.state.get(key, None) 282 284 history = self.state.get(key, None)
if history is None: 283 285 if history is None:
return default 284 286 return default
else: 285 287 else:
return history.get(index=lookback, default=default) 286 288 return history.get(index=lookback, default=default)
287 289
288 290
# extend the Timer thread object to run repeatedly. Subclass this. 289 291 # extend the Timer thread object to run repeatedly. Subclass this.
class RepeatedTimer(threading.Timer): 290 292 class RepeatedTimer(threading.Timer):
def __init__(self, interval): 291 293 def __init__(self, interval):
# Timer constructer expects a callback function, but we don't care 292 294 # Timer constructer expects a callback function, but we don't care
# about that, so just pass None 293 295 # about that, so just pass None
super(RepeatedTimer, self).__init__(interval, None) 294 296 super(RepeatedTimer, self).__init__(interval, None)
295 297
# override Timer's run method to loop on our injest_data function 296 298 # override Timer's run method to loop on our injest_data function
# wait will block while loop for timeout interval 297 299 # wait will block while loop for timeout interval
# calling timer.cancel() sets the finish flag and exits the loop 298 300 # calling timer.cancel() sets the finish flag and exits the loop
def run(self): 299 301 def run(self):
while not self.finished.wait(timeout=self.interval): 300 302 while not self.finished.wait(timeout=self.interval):
self.do_work() 301 303 self.do_work()
302 304
def do_work(self): 303 305 def do_work(self):
raise Exception( 304 306 raise Exception(
"RepeatedTimer baseclass do_work() called. Override this method in a subclass" 305 307 "RepeatedTimer baseclass do_work() called. Override this method in a subclass"
) 306 308 )
307 309
308 310
# makes a [query] to influxdb every [interval] 309 311 # makes a [query] to influxdb every [interval]
# influxdb returns a resultset object: 310 312 # influxdb returns a resultset object:
# https://influxdb-python.readthedocs.io/en/latest/resultset.html#resultset 311 313 # https://influxdb-python.readthedocs.io/en/latest/resultset.html#resultset
class PeriodicInfluxdbPoller(RepeatedTimer): 312 314 class PeriodicInfluxdbPoller(RepeatedTimer):
def __init__(self, interval, soars_state, query, measurment_info, db_config): 313 315 def __init__(self, interval, soars_state, query, measurment_info, db_config):