Enhance Safety Monitor with Stale Data Failsafe and Reporting

alpaca/__init__.py

@@ -2,6 +2,7 @@
 Alpaca Package - ASCOM Alpaca SafetyMonitor
 """
 import os
+from dataclasses import fields
 from flask import Flask
 from flask_cors import CORS
 from .config import AlpacaConfig
@@ -22,7 +23,8 @@ def create_app():
         port=int(os.getenv('ALPACA_PORT', '11111')),
         device_number=int(os.getenv('ALPACA_DEVICE_NUMBER', '0')),
         detection_interval=int(os.getenv('DETECT_INTERVAL', '30')),
-        update_interval=int(os.getenv('ALPACA_UPDATE_INTERVAL', '30'))
+        update_interval=int(os.getenv('ALPACA_UPDATE_INTERVAL', '30')),
+        max_image_age_sec=int(os.getenv('MAX_IMAGE_AGE_SEC', '600'))
     )
 
     # 2. Load from file and override environment settings if file exists
@@ -33,7 +35,11 @@ def create_app():
         # Update our base config with ONLY the values explicitly in the file
         for key, value in file_settings.items():
             setattr(alpaca_cfg, key, value)
-
+    # A hand-edited JSON file may hold "300" for an int field; coerce once here
+    for f in fields(alpaca_cfg):
+        if f.type is int:
+            setattr(alpaca_cfg, f.name, int(getattr(alpaca_cfg, f.name)))
+
     # 3. Save the final configuration to ensure the file exists and is current
     alpaca_cfg.save_to_file()
 

alpaca/config.py

@@ -80,6 +80,9 @@ class AlpacaConfig:
     debounce_to_safe_sec: int = 60  # Wait time before switching from Unsafe → Safe
     debounce_to_unsafe_sec: int = 0  # Wait time before switching from Safe → Unsafe (immediate)
 
+    # Stale image fail-safe: force unsafe if the image is unchanged this long (0 = disabled)
+    max_image_age_sec: int = 600
+
     # NTP and timezone settings
     ntp_server: str = field(default_factory=lambda: os.environ.get('NTP_SERVER', 'pool.ntp.org'))
     timezone: str = field(default_factory=lambda: os.environ.get('TZ', 'UTC'))

alpaca/device.py

@@ -50,14 +50,22 @@ def __init__(self, alpaca_config: AlpacaConfig, detect_config: DetectConfig):
             'Detection Time (Seconds)': 0.0,
             'timestamp': None
         }
-        self._cached_is_safe = False
         self._unsafe_conditions_set = set(alpaca_config.unsafe_conditions)
 
         # Debounce state tracking
         self._stable_safe_state = False
         self._pending_safe_state: Optional[bool] = None
         self._state_change_start_time: Optional[datetime] = None
 
+        # Stale image tracking: hash of the last image and when it last changed.
+        # changed_at stays None until a second distinct hash is seen after start.
+        self._last_image_hash: Optional[str] = None
+        self._image_hash_changed_at: Optional[datetime] = None
+        self._warned_no_hash = False
+        self._hash_missing = False
+        # Set by _demote() under the lock, read by _run_single_detection() after release
+        self._demoted = False
+
         # Safety state history (last 100 transitions)
         self._safety_history = deque(maxlen=100)
 
@@ -80,7 +88,7 @@ def __init__(self, alpaca_config: AlpacaConfig, detect_config: DetectConfig):
 
         # Setup HA Discovery if enabled
         if self.mqtt_client and self.detect_config.mqtt_discovery_mode == 'homeassistant':
-            self.ha_discovery = HADiscoveryManager(self.detect_config, self.mqtt_client)
+            self.ha_discovery = HADiscoveryManager(self.detect_config, self.mqtt_client, safe_sensor=True)
             self.ha_discovery.publish_discovery_configs()
 
         # Blocking initial detection to ensure readiness
@@ -102,6 +110,11 @@ def _create_thumbnail_bytes(self, img: Image.Image) -> Optional[bytes]:
 
     def _update_cached_safety(self, detection: Dict[str, Any]):
         """Update cached safety status with debouncing logic (assumes lock is held)"""
+        # latest_detection still holds the previous result here, so this catches
+        # a gap left by failed cycles or a hung detection thread
+        if self.latest_detection.get('timestamp') is not None:
+            self._demote_if_stale()
+
         # Step A: Determine Instantaneous Safety
         class_name = detection.get('class_name', '')
         confidence = detection.get('confidence_score', 0.0)
@@ -119,12 +132,40 @@ def _update_cached_safety(self, detection: Dict[str, Any]):
             class_name not in self._unsafe_conditions_set
         )
 
+        # A source image that has stopped changing is never safe: commit UNSAFE
+        # at once and skip the debounce below, so no pending-safe timer runs
+        # while the image is stale. Recovery still needs debounce_to_safe_sec.
+        max_age = self.alpaca_config.max_image_age_sec
+        image_hash = detection.get('image_hash')
+        self._hash_missing = image_hash is None
+        if image_hash is None:
+            if max_age > 0 and not self._warned_no_hash:
+                self._warned_no_hash = True
+                logger.warning("image_hash missing from detector result: stale-image check disabled")
+        else:
+            # Hash history is kept even with the check disabled, so enabling it
+            # at runtime does not restart the "no change since start" rule
+            now = get_current_time(self.alpaca_config.timezone)
+            if self._last_image_hash is not None and image_hash != self._last_image_hash:
+                self._image_hash_changed_at = now
+            self._last_image_hash = image_hash
+        if max_age > 0 and image_hash is not None:
+            if self._image_hash_changed_at is None:
+                stale = "no image change seen since start"
+            else:
+                unchanged_sec = (now - self._image_hash_changed_at).total_seconds()
+                stale = f"image unchanged for {unchanged_sec:.0f}s, max {max_age}s" if unchanged_sec > max_age else None
+            if stale:
+                if is_safe_now:
+                    logger.info(f"Stale Image: {stale}; holding UNSAFE")
+                self._demote('Stale Image', stale)
+                return
+
         # Step B: Apply Debouncing
         if is_safe_now == self._stable_safe_state:
             # State matches - reset any pending changes
             self._pending_safe_state = None
             self._state_change_start_time = None
-            self._cached_is_safe = self._stable_safe_state
         else:
             # State differs from stable state
             if self._pending_safe_state != is_safe_now:
@@ -147,7 +188,6 @@ def _update_cached_safety(self, detection: Dict[str, Any]):
                     if elapsed_time >= required_duration:
                         # Debounce period complete - commit state change
                         self._stable_safe_state = is_safe_now
-                        self._cached_is_safe = is_safe_now
                         self._pending_safe_state = None
                         self._state_change_start_time = None
 
@@ -158,7 +198,7 @@ def _update_cached_safety(self, detection: Dict[str, Any]):
                             'condition': class_name,
                             'confidence': confidence
                         })
-                        
+
                         logger.warning(f"SAFETY STATE CHANGED: {'SAFE' if is_safe_now else 'UNSAFE'} "
                                      f"(class={class_name}, confidence={confidence:.1f}%, "
                                      f"threshold={threshold:.1f}%, debounce={elapsed_time:.1f}s)")
@@ -246,6 +286,7 @@ def on_connect(client, userdata, flags, rc, properties=None):
 
     def _run_single_detection(self, initial: bool = False):
         """Run a single detection cycle with optimized image handling"""
+        self._demoted = False
         try:
             # Return image so we can process thumbnail
             result = self.cloud_detector.detect(return_image=True)
@@ -266,9 +307,11 @@ def _run_single_detection(self, initial: bool = False):
                 del result['image']
 
             with self.detection_lock:
+                self._update_cached_safety(result)
+                # Hash is only needed for the stale check; keep it out of API and MQTT output
+                result.pop('image_hash', None)
                 self.latest_detection = result
                 self.latest_image_bytes = image_bytes
-                self._update_cached_safety(result)
 
                 if initial:
                     self._safety_history.append({
@@ -278,22 +321,28 @@ def _run_single_detection(self, initial: bool = False):
                         'confidence': result.get('confidence_score', 0.0)
                     })
 
-            # MQTT Publish
+            # MQTT Publish (outside the lock: is_safe() takes it)
             if self.mqtt_client:
                 try:
                     mqtt_result = result.copy()
                     if 'timestamp' in mqtt_result and isinstance(mqtt_result['timestamp'], datetime):
                         mqtt_result['timestamp'] = mqtt_result['timestamp'].isoformat()
-                    
+
                     if self.detect_config.mqtt_discovery_mode == 'homeassistant':
                         self.ha_discovery.publish_states(mqtt_result)
                     else:
+                        mqtt_result['is_safe'] = self.is_safe()
                         self.mqtt_client.publish(self.detect_config.topic, json.dumps(mqtt_result))
                 except Exception as e:
                     logger.error(f"MQTT publish failed: {e}")
+            self._publish_safety(data_online=not self._demoted)
 
         except Exception as e:
             logger.error(f"Detection cycle failed: {e}")
+            with self.detection_lock:
+                self._demote_if_stale()
+            if self._demoted:
+                self._publish_safety(data_online=False)
 
     def _detection_loop(self):
         """Background thread for continuous cloud detection"""
@@ -443,13 +492,78 @@ def disconnect(self, client_ip: str = None, client_id: int = None):
     def is_safe(self) -> bool:
         """Debounced safety state of the sky, independent of ASCOM client connections.
 
-        The Alpaca routes apply the per-client Connected gate themselves
-        (see routes/api.py client_is_safe). Everything else (external REST
-        API, dashboard, MQTT) reports this value directly.
+        Served as-is by the Alpaca IsSafe route, the external REST API, the
+        dashboard and MQTT. The ASCOM spec never requires False when no client
+        is connected (ConformU dropped that check on 2025-03-16).
         """
         with self.detection_lock:
-            return self._stable_safe_state
+            return self._stable_safe_state and self._detection_is_fresh()
+
+    def _demote(self, condition: str, detail: str):
+        """Commit UNSAFE now and drop any pending debounce (assumes lock is held).
+
+        Recovery then has to pass debounce_to_safe_sec like any other transition.
+        """
+        self._demoted = True
+        self._pending_safe_state = None
+        self._state_change_start_time = None
+        if self._stable_safe_state:
+            self._stable_safe_state = False
+            self._safety_history.append({
+                'timestamp': get_current_time(self.alpaca_config.timezone),
+                'is_safe': False,
+                'condition': condition,
+                'confidence': 0.0
+            })
+            logger.warning(f"SAFETY STATE CHANGED: UNSAFE (class={condition}, {detail})")
+
+    def _demote_if_stale(self):
+        """Demote once the last successful detection is too old (assumes lock is held)"""
+        if not self._detection_is_fresh():
+            self._demote('Detection Stale', f"last success {self.latest_detection.get('timestamp')}")
+
+    def _publish_safety(self, data_online: bool):
+        """Publish HA data availability and the IsSafe binary sensor. Never call under detection_lock."""
+        if not self.ha_discovery:
+            return
+        try:
+            self.ha_discovery.publish_data_availability(data_online)
+            self.ha_discovery.publish_safe(self.is_safe())
+        except Exception as e:
+            logger.error(f"MQTT publish failed: {e}")
+
+    def _detection_is_fresh(self) -> bool:
+        """True if the last successful detection is recent enough to trust (assumes lock is held).
+
+        Fails safe when detect() keeps raising: latest_detection stops updating,
+        so its timestamp ages past the limit.
+        """
+        timestamp = self.latest_detection.get('timestamp') if self.latest_detection else None
+        if timestamp is None:
+            return False
+        max_age = max(180, 3 * self.alpaca_config.update_interval)
+        return (get_current_time(self.alpaca_config.timezone) - timestamp).total_seconds() <= max_age
 
+    def get_image_status(self) -> Dict[str, Any]:
+        """Stale-image check state for the dashboard, from the same fields the demote logic uses"""
+        with self.detection_lock:
+            now = get_current_time(self.alpaca_config.timezone)
+            limit = self.alpaca_config.max_image_age_sec
+            changed = self._image_hash_changed_at is not None
+            unchanged_sec = (now - self._image_hash_changed_at).total_seconds() if changed else None
+            last_ts = self.latest_detection.get('timestamp')
+            return {
+                'check_enabled': limit > 0,
+                'limit_sec': limit,
+                'unchanged_sec': unchanged_sec,
+                'changed_since_start': changed,
+                'hash_missing': self._hash_missing,
+                # A missing hash disables the check (see _update_cached_safety), so never report stale then
+                'stale': limit > 0 and not self._hash_missing and (not changed or unchanged_sec > limit),
+                'detection_fresh': self._detection_is_fresh(),
+                'detection_age_sec': (now - last_ts).total_seconds() if last_ts else None,
+            }
+
     def get_pending_status(self) -> Dict[str, Any]:
         """Get information about any pending state changes"""
         with self.detection_lock:

alpaca/routes/api.py

@@ -53,19 +53,6 @@ def endpoint(device_number: int):
         ))
     return endpoint
 
-def client_is_safe() -> bool:
-    """IsSafe as reported to the calling Alpaca client.
-
-    Alpaca tracks Connected per client (IP + ClientID). A client that has not
-    connected is told False, matching the pre-connection behaviour ASCOM
-    clients have always seen from this driver. The detector itself keeps
-    running; monitor.is_safe() carries the real state for non-Alpaca consumers.
-    """
-    client_id, _ = monitor.get_client_params()
-    if not monitor.is_client_connected(request.remote_addr, client_id):
-        return False
-    return monitor.is_safe()
-
 @api_bp.route('/v1/safetymonitor/<int:device_number>/issafe', methods=['GET'])
 def get_issafe(device_number: int):
     """Get safety status"""
@@ -74,7 +61,7 @@ def get_issafe(device_number: int):
         return error_response
     _, client_tx_id = monitor.get_client_params()
     return jsonify(monitor.create_response(
-        value=client_is_safe(),
+        value=monitor.is_safe(),
         client_transaction_id=client_tx_id
     ))
 
@@ -200,7 +187,7 @@ def get_description(device_number: int):
 @api_bp.route('/v1/safetymonitor/<int:device_number>/devicestate', methods=['GET'])
 def get_devicestate(device_number: int):
     return create_simple_get_endpoint(
-        lambda: [{"Name": "IsSafe", "Value": client_is_safe()}]
+        lambda: [{"Name": "IsSafe", "Value": monitor.is_safe()}]
     )(device_number)
 
 @api_bp.route('/v1/safetymonitor/<int:device_number>/driverinfo', methods=['GET'])

alpaca/routes/external_api.py

@@ -51,7 +51,8 @@ def get_status():
     return jsonify({
         "is_safe": is_safe,
         "safety_status": "Safe" if is_safe else "Unsafe",
-        "detection": detection
+        "detection": detection,
+        "image": monitor.get_image_status()
     })
 
 @external_api_bp.route('/api/ext/v1/config', methods=['GET'])