franzz/solar-relay
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove legacy code for format conversion
Deploy Solar Relay / deploy (push) Successful in 1m1s
Details

Browse Source
This commit is contained in:
franzz committed 2026-11-02 23:59:33 +01:00
1 parent e1d22a26ed
commit 9e0230731b
10 files changed
+118 -223
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/test_history.py
+33 -60
View File
@@ -8,7 +8,7 @@ import pyarrow.parquet as pq
from zoneinfo import ZoneInfo
from solar_relay.config import HistoryConfig, utc_text
from solar_relay.history import CHANNELS, HistoryRecovery, chart_entries, history_sample
from solar_relay.history import HistoryRecovery, history_sample
from solar_relay.http import TransportError
from solar_relay.solarweb import ChartDataError
from solar_relay.storage import Store
@@ -17,12 +17,6 @@ from tests.test_status import observed
SYSTEM = "73733052-6f10-47a5-9746-7f7e76ebcb8c"
def entry(when):
values = {"production": 100, "consumption": 120, "self_consumption": 80, "bought": 40, "sold": 20}
return {"logDateTime": utc_text(when), "logDuration": 300,
"channels": [{"channelName": channel, "channelType": "Energy", "unit": "Wh", "value": values[field]}
for field, channel in CHANNELS.items()]}
class FakeCharts:
def __init__(self, when):
@@ -55,9 +49,9 @@ class HistoryTests(unittest.TestCase):
self.store.add(observed(self.when + timedelta(minutes=offset)))
self.assertEqual(self.recover(), 1)
records = list(self.store.records())
recovered = [s for s in records if "history_interval" in s]
recovered = [s for s in records if "solarweb_history" in s["raw"]]
self.assertEqual(len(recovered), 1)
self.assertEqual(recovered[0]["history_interval"]["energy_wh"]["production"], 100)
self.assertEqual(recovered[0]["readings"]["production_ac_w"], 1200)
self.assertIsNone(recovered[0]["readings"]["boiler_enabled"])
self.assertNotIn("boiler", recovered[0]["source_timestamps"])
self.assertIn("boiler", recovered[0]["errors"])
@@ -78,41 +72,20 @@ class HistoryTests(unittest.TestCase):
self.assertEqual(day["complete"], 0)
row = pq.ParquetFile(next(self.store.path.glob("date=*/samples.parquet"))).read().to_pylist()[0]
self.assertEqual(row["record_kind"], "history")
self.assertEqual(row["interval_energy_wh"]["production"], 100)
self.assertEqual(row["production_ac_w"], 1200)
self.assertEqual(row["consumption_w"], 1440)
self.assertEqual(row["grid_import_w"], 240)
self.assertEqual(row["grid_export_w"], 0)
self.assertEqual(day["slots"][120:123], [1, 1, 1])
def test_old_archive_normalization_preserves_delivery_state_and_live_records(self):
old = history_sample(entry(self.when), self.source, SYSTEM)
old["readings"] = dict.fromkeys(old["readings"])
old["source_timestamps"] = {}
old.pop("poll_interval_seconds")
live = observed(self.when + timedelta(minutes=5))
self.store.add(old)
self.store.add(live)
self.store.acknowledge([old["id"]], now=self.when + timedelta(hours=1))
path = next(self.store.path.glob("date=*/samples.parquet"))
before = {row["id"]: row for row in pq.ParquetFile(path).read().to_pylist()}
self.assertEqual(self.store.normalize_history(), 1)
self.assertEqual(self.store.normalize_history(), 0)
after = {row["id"]: row for row in pq.ParquetFile(path).read().to_pylist()}
self.assertEqual(after[live["id"]], before[live["id"]])
row = after[old["id"]]
for field in ("id", "observed_at", "recorded_ns", "uploaded_at"):
self.assertEqual(row[field], before[old["id"]][field])
self.assertEqual(row["production_ac_w"], 1200)
self.assertEqual(row["consumption_w"], 1440)
self.assertIsNone(row["production_w"])
self.assertIsNone(row["boiler_enabled"])
self.assertEqual(self.store.pending(100), [live])
self.assertEqual(list(self.store.records())[0]["source_timestamps"]["powerflow"], utc_text(self.when))
# Replaying the original legacy payload still identifies the same record.
self.store.add(old)
self.assertEqual(self.store.status()["samples"], 2)
def test_recovered_power_uses_the_same_completeness_checks_as_live_power(self):
sample = history_sample(int(self.when.timestamp() * 1000), 1200, 1440, self.source, SYSTEM)
sample["readings"]["consumption_w"] = None
self.store.add(sample)
day = self.store.coverage(self.source, "UTC", 1, self.when + timedelta(days=1))[1]
self.assertEqual(day["energy_complete"], 0)
self.assertEqual(day["boiler_complete"], 0)
self.assertEqual(day["slots"][120], 0)
def test_partial_cloud_records_remain_missing_for_retry(self):
stamp = int((self.when + timedelta(minutes=5)).timestamp() * 1000)
@@ -123,20 +96,23 @@ class HistoryTests(unittest.TestCase):
self.assertEqual(self.recover(), 1)
def test_recovered_identifiers_are_stable_for_duplicate_delivery(self):
first = history_sample(entry(self.when), self.source, SYSTEM)
second = history_sample(entry(self.when), self.source, SYSTEM)
stamp = int(self.when.timestamp() * 1000)
first = history_sample(stamp, 1200, 1440, self.source, SYSTEM)
second = history_sample(stamp, 1200, 1440, self.source, SYSTEM)
self.assertEqual(first, second)
self.store.add(first)
self.store.add(second)
self.assertEqual(self.store.status()["samples"], 1)
def test_non_five_minute_and_bad_units_are_rejected(self):
for change in [{"logDuration": 599}, {"logDateTime": utc_text(self.when + timedelta(seconds=1))}]:
value = entry(self.when)
value.update(change)
with self.assertRaises(ValueError):
history_sample(value, self.source, SYSTEM)
value = entry(self.when)
value["channels"][0]["unit"] = "W"
with self.assertRaises(ValueError):
history_sample(value, self.source, SYSTEM)
def test_unaligned_timestamps_and_invalid_power_are_rejected(self):
stamp = int(self.when.timestamp() * 1000)
for bad_stamp in (stamp + 1000, float(stamp), True):
with self.subTest(stamp=bad_stamp), self.assertRaises(ValueError):
history_sample(bad_stamp, 1200, 1440, self.source, SYSTEM)
for invalid in (None, True, "1200", -1, float("nan"), float("inf"), 1e19):
for generated, consumed in ((invalid, 1440), (1200, invalid)):
with self.subTest(generated=generated, consumed=consumed), self.assertRaises(ValueError):
history_sample(stamp, generated, consumed, self.source, SYSTEM)
def test_missing_private_credentials_fail_before_requests(self):
with self.assertRaisesRegex(ValueError, "SOLARWEB_USERNAME and SOLARWEB_PASSWORD"):
@@ -190,7 +166,7 @@ class HistoryTests(unittest.TestCase):
self.client.points = {"production": points, "consumption": points}
self.store.add(observed(now.replace(minute=0, second=0)))
self.assertEqual(self.recovery.recover(now=now), 3)
recovered = [record for record in self.store.records() if "history_interval" in record]
recovered = [record for record in self.store.records() if "solarweb_history" in record["raw"]]
self.assertEqual([record["observed_at"] for record in recovered], [utc_text(when) for when in recoverable])
self.assertEqual(min(day for _, _, day in self.client.requests), cutoff.astimezone(self.recovery.zone).date())
@@ -266,22 +242,19 @@ class HistoryTests(unittest.TestCase):
self.recovery.recover(now=self.when + timedelta(days=2))
self.assertEqual(client.call_count, 1)
def test_chart_conversion_matches_solar_and_retains_provenance(self):
def test_chart_watts_are_stored_directly_and_retain_provenance(self):
for generated, consumed in [(1200, 1440), (2400, 1200), (0, 0), (1234.56789, 3456.78901)]:
with self.subTest(generated=generated, consumed=consumed):
stamp = int(self.when.timestamp() * 1000)
record = next(chart_entries({stamp: generated}, {stamp: consumed}))
sample = history_sample(record, self.source, SYSTEM)
values = sample["history_interval"]["energy_wh"]
self.assertEqual(values, {"production": generated / 12, "consumption": consumed / 12,
"self_consumption": min(generated, consumed) / 12,
"bought": max(0, consumed - generated) / 12,
"sold": max(0, generated - consumed) / 12})
self.assertEqual(record["energy_method"], "constant_power_over_five_minutes")
sample = history_sample(stamp, generated, consumed, self.source, SYSTEM)
self.assertEqual(sample["readings"]["production_ac_w"], generated)
self.assertEqual(sample["readings"]["consumption_w"], consumed)
self.assertEqual(sample["readings"]["grid_w"], consumed - generated)
self.assertEqual(sample["readings"]["grid_import_w"], max(0, consumed - generated))
self.assertEqual(sample["readings"]["grid_export_w"], max(0, generated - consumed))
self.assertIsNone(sample["readings"]["production_w"])
self.assertEqual(sample["raw"]["solarweb_history"]["chart_power_w"],
{"production": generated, "consumption": consumed})
def test_dst_days_recover_every_actual_five_minute_slot(self):
for day, expected in [("2026-03-29", 276), ("2026-10-25", 300)]:
tests/test_parquet.py
+29 -1
View File
@@ -7,11 +7,12 @@ import unittest
from unittest.mock import patch
from zoneinfo import ZoneInfo
import pyarrow as pa
import pyarrow.parquet as pq
from solar_relay.cli import next_poll_at
from solar_relay.config import retention_cutoff
from solar_relay.storage import Store, StoreError
from solar_relay.storage import SCHEMA, Store, StoreError
from tests.test_status import observed
@@ -49,6 +50,33 @@ class ParquetTests(unittest.TestCase):
self.assertEqual(self.store.prune(now=self.now + timedelta(days=31)), 2)
self.assertEqual(list(self.store.records()), [items[3]])
def test_additional_archive_columns_do_not_change_records_or_acknowledgements(self):
value = observed(self.now)
self.store.add(value)
path = next(self.path.glob("date=*/samples.parquet"))
table = pq.ParquetFile(path).read()
table = table.append_column("unused_estimate", pa.array([100.0]))
pq.write_table(table, path)
self.assertEqual(self.store.pending(100), [value])
self.assertEqual(list(self.store.records()), [value])
self.store.acknowledge([value["id"]], now=self.now)
self.assertEqual(self.store.pending(100), [])
self.assertEqual(list(self.store.records()), [value])
self.assertTrue(pq.ParquetFile(path).schema_arrow.equals(SCHEMA, check_metadata=True))
def test_missing_or_mistyped_archive_columns_are_rejected(self):
value = observed(self.now)
self.store.add(value)
path = next(self.path.glob("date=*/samples.parquet"))
table = pq.ParquetFile(path).read()
index = table.schema.get_field_index("production_ac_w")
for invalid in (table.drop_columns(["production_ac_w"]),
table.set_column(index, "production_ac_w", pa.array(["invalid"]))):
with self.subTest(schema=invalid.schema):
pq.write_table(invalid, path)
with self.assertRaises(StoreError):
self.store.pending(100)
def test_retention_boundary_is_local_midnight_regardless_of_execution_time(self):
zone = ZoneInfo("Europe/Zurich")
for hour in (0, 12, 23):
tests/test_relay.py
+8 -7
View File
@@ -402,27 +402,28 @@ class ConfigTests(unittest.TestCase):
with self.assertRaises(ValueError):
self.load('\n[http]\nbase_path="/../config"\n')
def test_history_account_settings_and_migration_from_query_api_keys(self):
def test_history_account_environment_settings(self):
config, _ = self.load('\n[history]\nusername_env="BACKUP_USER"\npassword_env="BACKUP_PASSWORD"\n')
self.assertEqual(config.history.username_env, "BACKUP_USER")
self.assertEqual(config.history.password_env, "BACKUP_PASSWORD")
self.assertFalse(config.history.enabled)
config, _ = self.load('\n[history]\naccess_key_id_env="OLD_ID"\naccess_key_value_env="OLD_KEY"\n')
self.assertEqual(config.history.username_env, "SOLARWEB_USERNAME")
self.assertEqual(config.history.password_env, "SOLARWEB_PASSWORD")
for name in ("username_env", "password_env"):
with self.subTest(name=name), self.assertRaisesRegex(ValueError, "environment variable"):
self.load(f'\n[history]\n{name}="invalid name"\n')
def test_history_uses_retention_instead_of_legacy_lookback_setting(self):
def test_history_uses_collector_retention(self):
with tempfile.TemporaryDirectory() as directory:
path = Path(directory) / "synthetic-test.toml"
path.write_text('[collector]\nretention_days=2\n[history]\nenabled=true\n'
'pv_system_id="73733052-6f10-47a5-9746-7f7e76ebcb8c"\nlookback_days=7\n')
'pv_system_id="73733052-6f10-47a5-9746-7f7e76ebcb8c"\n')
config = load_config(path)
self.assertEqual(config.collector.retention_days, 2)
self.assertTrue(config.history.enabled)
self.assertFalse(hasattr(config.history, "lookback_days"))
def test_obsolete_history_options_are_rejected(self):
for name in ("access_key_id_env", "access_key_value_env", "lookback_days"):
with self.subTest(name=name), self.assertRaisesRegex(ValueError, "Unknown history settings"):
self.load(f'\n[history]\n{name}=7\n')
def test_private_config_accepts_credentials_and_redacts_them_from_repr(self):
config, _ = self.load('\n[upload]\ntoken="synthetic-shared-token"\n'