Reshape boiler data

solar_relay/cli.py

@@ -42,7 +42,7 @@ def next_poll_at(now, interval):
 def _run(config, store):
     stop = threading.Event()
     previous = {}
-    uploader = Uploader(config.upload, store) if config.upload.url else None
+    uploader = Uploader(config.upload, store, boiler_threshold_w=config.inverter.boiler_threshold_w) if config.upload.url else None
     recovery = HistoryRecovery(config.history, config.collector.source_id, store,
                                timezone_name=config.collector.timezone) if config.history.enabled else None
     worker = None
@@ -150,7 +150,7 @@ def main(argv=None):
             print(json.dumps({"requeued": store.replay(start, until)}))
         elif args.command == "upload":
             store.prune()
-            uploader = Uploader(config.upload, store)
+            uploader = Uploader(config.upload, store, boiler_threshold_w=config.inverter.boiler_threshold_w)
             count = 0
             while uploaded := uploader.upload_once():
                 count += uploaded

solar_relay/config.py

@@ -51,6 +51,7 @@ class InverterConfig:
     boiler_pin: str = "pin1"
     request_timeout_seconds: float = 5
     min_request_interval_seconds: float = 4
+    boiler_threshold_w: float = 2000
 
 
 @dataclass(frozen=True)
@@ -163,6 +164,9 @@ def load_config(path: Path) -> Config:
         raise ValueError("inverter.boiler_pin must be pin1, pin2, pin3 or pin4")
     _number(inverter.request_timeout_seconds, "inverter.request_timeout_seconds", 0.1)
     _number(inverter.min_request_interval_seconds, "inverter.min_request_interval_seconds", 4)
+    _number(inverter.boiler_threshold_w, "inverter.boiler_threshold_w", 0.01)
+    if inverter.boiler_threshold_w > 99_999_999.99:
+        raise ValueError("inverter.boiler_threshold_w exceeds the stored watts range")
     upload = UploadConfig(**data.get("upload", {}))
     if not isinstance(upload.url, str):
         raise ValueError("upload.url must be a string")

solar_relay/fronius.py

@@ -94,14 +94,31 @@ def boiler_readings(response, pin):
     return {"boiler_enabled": state["state"], "boiler_reason": reason}
 
 
+def boiler_threshold(response, pin):
+    """GEN24's UI reads rules.<pin>.thresholdOn from /api/config/emrs, in watts.
+
+    In pgrid mode, negative thresholds denote surplus exported to the grid.
+    The UI displays their absolute magnitude as the switch-on threshold.
+    """
+    rule = response.get("rules", {}).get(pin)
+    if not isinstance(rule, dict):
+        raise ValueError("Boiler threshold rule is missing")
+    value = _number(rule.get("thresholdOn"))
+    if value is not None and rule.get("mode") == "pgrid":
+        value = abs(value)
+    if value is None or not 0 < value <= 99_999_999.99:
+        raise ValueError("Boiler switch-on threshold is missing or invalid")
+    return value
+
+
 class FroniusClient:
     def __init__(self, config: InverterConfig, *, http=None):
         self.config = config
         self.http = http or JsonHttp(config.request_timeout_seconds, local=True)
         self.last_request_started = None
 
-    def _read(self, path):
-        for attempt in range(REQUEST_RETRIES + 1):
+    def _read(self, path, retries=REQUEST_RETRIES):
+        for attempt in range(retries + 1):
             if self.last_request_started is not None:
                 wait = self.config.min_request_interval_seconds - (time.monotonic() - self.last_request_started)
                 if wait > 0:
@@ -110,7 +127,7 @@ class FroniusClient:
             try:
                 return self.http.request(self.config.base_url.rstrip("/") + path)
             except TransportError as error:
-                if attempt == REQUEST_RETRIES:
+                if attempt == retries:
                     raise
                 LOG.warning("Fronius GET %s failed (%s); retry %d/%d", path, error, attempt + 1, REQUEST_RETRIES)
 
@@ -151,4 +168,18 @@ class FroniusClient:
                 sample["source_timestamps"][name] = timestamp
             except (TransportError, ValueError, KeyError, TypeError, AttributeError) as error:
                 sample["errors"][name] = str(error)
+        # These settings may require inverter authentication (HTTP 401). A failed
+        # settings read must not turn a valid boiler state into a missing sample.
+        sample["readings"]["boiler_threshold_w"] = self.config.boiler_threshold_w
+        sample["readings"]["boiler_threshold_source"] = "config"
+        try:
+            response = self._read("/api/config/emrs", retries=0)
+            threshold = boiler_threshold(response, self.config.boiler_pin)
+            sample["readings"]["boiler_threshold_w"] = threshold
+            sample["readings"]["boiler_threshold_source"] = "api"
+            # Keep only the relevant load-management settings, never an authentication response.
+            rule = response["rules"][self.config.boiler_pin]
+            sample["raw"]["boiler_config"] = {"thresholdOn": rule["thresholdOn"], "mode": rule.get("mode")}
+        except (TransportError, ValueError, KeyError, TypeError, AttributeError):
+            pass
         return sample

solar_relay/storage.py

@@ -25,6 +25,7 @@ NUMBERS = (
     "production_w", "production_ac_w", "consumption_w", "grid_w",
     "grid_import_w", "grid_export_w", "production_ac_total_wh",
     "grid_import_total_wh", "grid_export_total_wh",
+    "boiler_threshold_w",
 )
 SCHEMA = pa.schema([
     ("id", pa.string()), ("source_id", pa.string()), ("observed_at", pa.string()),
@@ -32,12 +33,14 @@ SCHEMA = pa.schema([
     ("uploaded_at", pa.string()), ("payload", pa.string()),
     *((field, pa.float64()) for field in NUMBERS),
     ("boiler_enabled", pa.bool_()), ("boiler_reason", pa.string()),
+    ("boiler_threshold_source", pa.string()),
     ("record_kind", pa.string()),
     ("power_ms", pa.int64()), ("power_ok", pa.bool_()),
     ("meter_ms", pa.int64()), ("meter_ok", pa.bool_()),
     ("boiler_ms", pa.int64()), ("boiler_ok", pa.bool_()),
 ], metadata={b"solar_relay_schema": b"1"})
 COVERAGE_COLUMNS = ["observed_ms", "power_ms", "power_ok", "meter_ms", "meter_ok", "boiler_ms", "boiler_ok"]
+OPTIONAL_COLUMNS = {"boiler_threshold_w", "boiler_threshold_source"}  # Absent in archives written before threshold recording.
 
 
 class StoreError(OSError):
@@ -83,10 +86,16 @@ class Store:
         schema = parquet.schema_arrow
         # Validate the fields this version uses; read only those columns.
         if schema.metadata != SCHEMA.metadata or any(
-            schema.get_field_index(field.name) < 0 or schema.field(field.name) != field for field in SCHEMA
+            (schema.get_field_index(field.name) < 0 and field.name not in OPTIONAL_COLUMNS) or
+            (schema.get_field_index(field.name) >= 0 and schema.field(field.name) != field) for field in SCHEMA
         ):
             raise StoreError("Unsupported Parquet archive schema")
-        return parquet.read(columns=SCHEMA.names if columns is None else columns).to_pylist()
+        wanted = SCHEMA.names if columns is None else columns
+        missing = set(wanted) - set(schema.names)
+        rows = parquet.read(columns=[name for name in wanted if name not in missing]).to_pylist()
+        for row in rows:
+            row.update({name: None for name in missing})
+        return rows
 
     @staticmethod
     def _sync_directory(path):
@@ -127,6 +136,7 @@ class Store:
             "payload": json.dumps(sample, allow_nan=False, separators=(",", ":")),
             **{name: readings.get(name) for name in NUMBERS},
             "boiler_enabled": readings.get("boiler_enabled"), "boiler_reason": readings.get("boiler_reason"),
+            "boiler_threshold_source": readings.get("boiler_threshold_source"),
             "record_kind": "history" if "solarweb_history" in sample.get("raw", {}) else "live",
             **{name: bool(value) if name.endswith("_ok") else value
                for name, value in zip(COVERAGE_COLUMNS[1:], coverage[3:])},

solar_relay/upload.py

@@ -1,9 +1,10 @@
 """Retryable delivery with explicit per-sample acknowledgement."""
 
 import logging
+import math
 import os
 
-from .config import UploadConfig
+from .config import InverterConfig, UploadConfig
 from .http import JsonHttp, TransportError
 from .storage import Store
 
@@ -15,7 +16,7 @@ class UploadError(Exception):
 
 
 class Uploader:
-    def __init__(self, config: UploadConfig, store: Store, *, token=None, http=None):
+    def __init__(self, config: UploadConfig, store: Store, *, token=None, http=None, boiler_threshold_w=None):
         if not config.url:
             raise ValueError("Set upload.url to the website's ingestion endpoint first")
         self.config = config
@@ -24,17 +25,32 @@ class Uploader:
         if not isinstance(self.token, str) or not self.token or any(ord(c) < 33 or ord(c) > 126 for c in self.token):
             raise ValueError(f"Set upload.token in config.toml or {config.token_env} to a nonempty bearer token without whitespace")
         self.http = http or JsonHttp(config.request_timeout_seconds)
+        self.boiler_threshold_w = InverterConfig().boiler_threshold_w if boiler_threshold_w is None else boiler_threshold_w
+        if (isinstance(self.boiler_threshold_w, bool) or not isinstance(self.boiler_threshold_w, (int, float))
+                or not math.isfinite(self.boiler_threshold_w) or not 0 < self.boiler_threshold_w <= 99_999_999.99):
+            raise ValueError("Invalid fallback boiler threshold")
 
     def upload_once(self) -> int:
         batch = self.store.pending(self.config.batch_size)
         if not batch:
             return 0
+        # Older retained samples contain the switch state only. Supply the configured
+        # fallback on delivery; never replace thresholds already recorded with a sample.
+        for sample in batch:
+            readings = sample.get("readings", {})
+            if isinstance(readings.get("boiler_enabled"), bool) and "boiler_threshold_w" not in readings:
+                readings["boiler_threshold_w"] = self.boiler_threshold_w
+                readings["boiler_threshold_source"] = "config"
         try:
             response = self.http.request(
                 self.config.url, payload={"schema_version": 1, "samples": batch}, token=self.token,
             )
         except TransportError as error:
             raise UploadError(str(error)) from None
+        if "result" in response:
+            if response["result"] != "success" or not isinstance(response.get("data"), dict):
+                raise UploadError("Receiver returned an unsuccessful or malformed result")
+            response = response["data"]
         accepted = response.get("accepted_ids")
         if not isinstance(accepted, list) or not all(isinstance(value, str) for value in accepted):
             raise UploadError("Receiver must return an accepted_ids array")