Reshape boiler data

tests/test_parquet.py

@@ -27,12 +27,16 @@ class ParquetTests(unittest.TestCase):
     def test_real_parquet_has_typed_readings_and_original_payload(self):
         value = observed(self.now)
         value["readings"]["boiler_enabled"] = False
+        value["readings"]["boiler_threshold_w"] = 2600
+        value["readings"]["boiler_threshold_source"] = "config"
         self.store.add(value)
         path = next(self.path.glob("date=*/samples.parquet"))
         self.assertEqual(path.read_bytes()[:4], b"PAR1")
         row = pq.ParquetFile(path).read().to_pylist()[0]
         self.assertEqual(row["production_ac_w"], 2000)
         self.assertIs(row["boiler_enabled"], False)
+        self.assertEqual(row["boiler_threshold_w"], 2600)
+        self.assertEqual(row["boiler_threshold_source"], "config")
         self.assertIsNone(row["uploaded_at"])
         self.assertEqual(json.loads(row["payload"]), value)
         self.assertEqual(self.store.status()["format"], "parquet")
@@ -77,6 +81,18 @@ class ParquetTests(unittest.TestCase):
                 with self.assertRaises(StoreError):
                     self.store.pending(100)
 
+    def test_pre_threshold_archive_is_readable_and_upgraded_on_write(self):
+        value = observed(self.now)
+        self.store.add(value)
+        path = next(self.path.glob("date=*/samples.parquet"))
+        legacy = pq.ParquetFile(path).read().drop_columns(["boiler_threshold_w", "boiler_threshold_source"])
+        pq.write_table(legacy, path)
+        self.assertEqual(self.store.pending(100), [value])
+        self.assertEqual(list(self.store.records()), [value])
+        self.store.acknowledge([value["id"]], now=self.now)
+        self.assertTrue(pq.ParquetFile(path).schema_arrow.equals(SCHEMA, check_metadata=True))
+        self.assertIsNone(pq.ParquetFile(path).read().to_pylist()[0]["boiler_threshold_w"])
+
     def test_retention_boundary_is_local_midnight_regardless_of_execution_time(self):
         zone = ZoneInfo("Europe/Zurich")
         for hour in (0, 12, 23):

tests/test_relay.py

@@ -14,7 +14,7 @@ from unittest.mock import Mock, patch
 from urllib.parse import urlsplit
 
 from solar_relay.config import InverterConfig, UploadConfig, date_bound, load_config
-from solar_relay.fronius import FroniusClient, powerflow_readings
+from solar_relay.fronius import FroniusClient, boiler_threshold, powerflow_readings
 from solar_relay.http import JsonHttp, TransportError
 from solar_relay.storage import Store
 from solar_relay.upload import Uploader, UploadError
@@ -36,10 +36,12 @@ METER = api({"0": {
     "EnergyReal_WAC_Sum_Consumed": 50000, "EnergyReal_WAC_Sum_Produced": 80000,
 }})
 BOILER = {"pin1": {"state": True, "reason": "PSTR_EMRS_REASON_3"}}
+BOILER_CONFIG = {"rules": {"pin1": {"mode": "pgrid", "thresholdOn": -2000, "thresholdOff": -500}}}
 RESPONSES = {
     "/solar_api/v1/GetPowerFlowRealtimeData.fcgi": POWER,
     "/solar_api/v1/GetMeterRealtimeData.cgi": METER,
     "/api/status/emrs": BOILER,
+    "/api/config/emrs": BOILER_CONFIG,
 }
 
 
@@ -138,6 +140,8 @@ class NormalizationTests(unittest.TestCase):
         self.assertEqual(readings["grid_export_w"], 0)
         self.assertEqual(readings["grid_import_total_wh"], 50000)
         self.assertTrue(readings["boiler_enabled"])
+        self.assertEqual(readings["boiler_threshold_w"], 2000)
+        self.assertEqual(readings["boiler_threshold_source"], "api")
         self.assertEqual(result["errors"], {})
         self.assertEqual(result["raw"]["powerflow"], POWER)
         self.assertEqual(result["source_timestamps"]["powerflow"], "2026-11-01T12:00:00.000Z")
@@ -167,6 +171,27 @@ class NormalizationTests(unittest.TestCase):
         self.assertIs(result["readings"]["boiler_enabled"], False)
         self.assertNotIn("boiler", result["errors"])
 
+    def test_configured_threshold_is_used_when_settings_require_authentication(self):
+        responses = deepcopy(RESPONSES)
+        responses["/api/config/emrs"] = TransportError("HTTP 401")
+        result = FroniusClient(InverterConfig(min_request_interval_seconds=0, boiler_threshold_w=2500), http=FakeHttp(responses)).collect("test")
+        self.assertTrue(result["readings"]["boiler_enabled"])
+        self.assertEqual(result["readings"]["boiler_threshold_w"], 2500)
+        self.assertEqual(result["readings"]["boiler_threshold_source"], "config")
+        self.assertEqual(result["errors"], {})
+
+    def test_settings_are_matched_to_the_configured_pin_and_bad_values_use_fallback(self):
+        for value in (None, 0, True, "2000", float("nan"), 100_000_000):
+            with self.subTest(value=value):
+                responses = deepcopy(RESPONSES)
+                responses["/api/config/emrs"]["rules"]["pin1"]["thresholdOn"] = value
+                result = FroniusClient(InverterConfig(min_request_interval_seconds=0), http=FakeHttp(responses)).collect("test")
+                self.assertEqual(result["readings"]["boiler_threshold_w"], 2000)
+                self.assertEqual(result["readings"]["boiler_threshold_source"], "config")
+        settings = {"rules": {"pin1": {"mode": "pgrid", "thresholdOn": -1500},
+                              "pin2": {"mode": "ppv", "thresholdOn": 3200}}}
+        self.assertEqual(boiler_threshold(settings, "pin2"), 3200)
+
     def test_solar_api_application_error_does_not_accept_old_values(self):
         responses = deepcopy(RESPONSES)
         responses["/solar_api/v1/GetPowerFlowRealtimeData.fcgi"]["Head"]["Status"]["Code"] = 12
@@ -265,6 +290,53 @@ class DeliveryTests(unittest.TestCase):
                 uploader.upload_once()
             self.assertEqual(self.store.status()["pending"], 1)
 
+    def test_solar_json_result_envelope_acknowledges_the_samples(self):
+        with server() as (httpd, base):
+            reading = sample()
+            self.store.add(reading)
+            httpd.ack_override = {"result": "success", "data": {"accepted_ids": [reading["id"]]},
+                                  "desc_lang_id": "", "desc_lang_params": []}
+            uploader = Uploader(UploadConfig(url=base, token="test-token"), self.store)
+            self.assertEqual(uploader.upload_once(), 1)
+            self.assertEqual(self.store.status()["pending"], 0)
+
+    def test_failed_or_malformed_solar_result_never_clears_the_queue(self):
+        with server() as (httpd, base):
+            reading = sample()
+            self.store.add(reading)
+            uploader = Uploader(UploadConfig(url=base, token="test-token"), self.store)
+            accepted = {"accepted_ids": [reading["id"]]}
+            for response in [
+                {"result": "error", "data": accepted},
+                {"result": "error", **accepted},
+                {"result": True, "data": accepted},
+                {"result": "success", "data": None},
+                {"result": "success", "data": []},
+                {"result": "success", "data": {}, **accepted},
+            ]:
+                with self.subTest(response=response):
+                    httpd.ack_override = response
+                    with self.assertRaises(UploadError):
+                        uploader.upload_once()
+                    self.assertEqual(self.store.status()["pending"], 1)
+
+    def test_legacy_samples_receive_config_fallback_without_replacing_recorded_thresholds(self):
+        legacy = sample()
+        del legacy["readings"]["boiler_threshold_w"]
+        del legacy["readings"]["boiler_threshold_source"]
+        current = sample()
+        current["readings"]["boiler_threshold_w"] = 3100
+        self.store.add(legacy)
+        self.store.add(current)
+        with server() as (httpd, base):
+            uploader = Uploader(UploadConfig(url=base, token="test-token"), self.store, boiler_threshold_w=2500)
+            self.assertEqual(uploader.upload_once(), 2)
+            delivered = {s["id"]: s for s in httpd.requests[0]["samples"]}
+            self.assertEqual(delivered[legacy["id"]]["readings"]["boiler_threshold_w"], 2500)
+            self.assertEqual(delivered[legacy["id"]]["readings"]["boiler_threshold_source"], "config")
+            self.assertEqual(delivered[current["id"]]["readings"]["boiler_threshold_w"], 3100)
+        self.assertEqual(list(self.store.records()), [legacy, current])
+
     def test_partial_ack_only_marks_the_accepted_sample(self):
         with server() as (httpd, base):
             first, second = sample(), sample()
@@ -376,6 +448,14 @@ class ConfigTests(unittest.TestCase):
         self.assertEqual(config.collector.poll_interval_seconds, 300)
         self.assertEqual(config.collector.retention_days, 30)
         self.assertEqual(config.upload.url, "")
+        self.assertEqual(config.inverter.boiler_threshold_w, 2000)
+
+    def test_boiler_threshold_fallback_is_configurable_and_must_be_positive_watts(self):
+        config, _ = self.load('\n[inverter]\nboiler_threshold_w=2750\n')
+        self.assertEqual(config.inverter.boiler_threshold_w, 2750)
+        for value in ('0', '-100', 'true', '"2000"', 'nan', 'inf', '100000000'):
+            with self.subTest(value=value), self.assertRaises(ValueError):
+                self.load(f'\n[inverter]\nboiler_threshold_w={value}\n')
 
     def test_remote_upload_requires_https(self):
         with self.assertRaisesRegex(ValueError, "HTTPS"):

tests/test_retries.py

@@ -17,6 +17,7 @@ from tests.test_schedule import Clock
 POWER_PATH = "/solar_api/v1/GetPowerFlowRealtimeData.fcgi"
 METER_PATH = "/solar_api/v1/GetMeterRealtimeData.cgi"
 BOILER_PATH = "/api/status/emrs"
+BOILER_CONFIG_PATH = "/api/config/emrs"
 
 
 class ScriptedHttp:
@@ -47,7 +48,7 @@ class RetryTests(unittest.TestCase):
                     sample = client.collect("test-house")
                 self.assertEqual(sample["errors"], {})
                 self.assertEqual(sample["readings"]["production_ac_w"], 2000)
-                self.assertEqual(httpd.get_counts, {POWER_PATH: retries + 1, METER_PATH: 1, BOILER_PATH: 1})
+                self.assertEqual(httpd.get_counts, {POWER_PATH: retries + 1, METER_PATH: 1, BOILER_PATH: 1, BOILER_CONFIG_PATH: 1})
                 self.assertEqual(len(messages.output), retries)
 
     def test_giving_up_after_three_retries_preserves_other_sources(self):
@@ -57,7 +58,7 @@ class RetryTests(unittest.TestCase):
                 client = FroniusClient(InverterConfig(base_url=base, min_request_interval_seconds=0))
                 with self.assertLogs("solar_relay.fronius", level="WARNING"):
                     sample = client.collect("test-house")
-                self.assertEqual(httpd.get_counts, {POWER_PATH: 1, METER_PATH: 1, BOILER_PATH: 4})
+                self.assertEqual(httpd.get_counts, {POWER_PATH: 1, METER_PATH: 1, BOILER_PATH: 4, BOILER_CONFIG_PATH: 1})
                 self.assertEqual(len(httpd.get_failures[BOILER_PATH]), 1)
                 self.assertEqual(sample["errors"], {"boiler": f"HTTP {failures[3]}"})
                 self.assertIsNone(sample["readings"]["boiler_enabled"])
@@ -73,7 +74,18 @@ class RetryTests(unittest.TestCase):
                 self.assertLogs("solar_relay.fronius", level="WARNING"):
             result = client.collect("test-house")
         self.assertEqual(result["errors"], {})
-        self.assertEqual([stamp for stamp, _ in http.requests], [0, 4, 8, 12, 16, 20])
+        self.assertEqual([stamp for stamp, _ in http.requests], [0, 4, 8, 12, 16, 20, 24])
+
+    def test_protected_threshold_settings_use_fallback_without_retries_or_losing_state(self):
+        with server() as (httpd, base):
+            httpd.get_failures[BOILER_CONFIG_PATH] = [401, 401]
+            client = FroniusClient(InverterConfig(base_url=base, min_request_interval_seconds=0, boiler_threshold_w=2400))
+            result = client.collect("test")
+            self.assertEqual(httpd.get_counts[BOILER_CONFIG_PATH], 1)
+            self.assertEqual(result["errors"], {})
+            self.assertTrue(result["readings"]["boiler_enabled"])
+            self.assertEqual(result["readings"]["boiler_threshold_w"], 2400)
+            self.assertEqual(result["readings"]["boiler_threshold_source"], "config")
 
     def test_retries_preserve_clock_boundaries_and_one_saved_observation_per_poll(self):
         origin = datetime(2026, 11, 1, 12, tzinfo=timezone.utc).timestamp()