259 lines
15 KiB
Python
259 lines
15 KiB
Python
from dataclasses import replace
|
|
from datetime import datetime, timedelta, timezone
|
|
from pathlib import Path
|
|
import tempfile
|
|
import unittest
|
|
from unittest.mock import Mock, patch
|
|
import pyarrow.parquet as pq
|
|
from zoneinfo import ZoneInfo
|
|
|
|
from solar_relay.config import HistoryConfig, utc_text
|
|
from solar_relay.history import CHANNELS, HistoryRecovery, chart_entries, history_sample
|
|
from solar_relay.http import TransportError
|
|
from solar_relay.solarweb import ChartDataError
|
|
from solar_relay.storage import Store
|
|
from tests.test_status import observed
|
|
|
|
SYSTEM = "73733052-6f10-47a5-9746-7f7e76ebcb8c"
|
|
|
|
|
|
def entry(when):
|
|
values = {"production": 100, "consumption": 120, "self_consumption": 80, "bought": 40, "sold": 20}
|
|
return {"logDateTime": utc_text(when), "logDuration": 300,
|
|
"channels": [{"channelName": channel, "channelType": "Energy", "unit": "Wh", "value": values[field]}
|
|
for field, channel in CHANNELS.items()]}
|
|
|
|
|
|
class FakeCharts:
|
|
def __init__(self, when):
|
|
self.points = {dataset: {int((when + timedelta(minutes=offset)).timestamp() * 1000): watts
|
|
for offset in [0, 5, 10]}
|
|
for dataset, watts in [("production", 1200), ("consumption", 1440)]}
|
|
self.requests = []
|
|
|
|
def get_chart(self, system_id, dataset, date):
|
|
self.requests.append((system_id, dataset, date))
|
|
return dict(self.points[dataset])
|
|
|
|
|
|
class HistoryTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.directory = tempfile.TemporaryDirectory()
|
|
self.addCleanup(self.directory.cleanup)
|
|
self.store = Store(Path(self.directory.name) / "archive")
|
|
self.when = datetime(2026, 10, 30, 10, tzinfo=timezone.utc)
|
|
self.source = "test-house"
|
|
self.client = FakeCharts(self.when)
|
|
self.config = HistoryConfig(enabled=True, pv_system_id=SYSTEM)
|
|
self.recovery = HistoryRecovery(self.config, self.source, self.store, client=self.client)
|
|
|
|
def recover(self):
|
|
return self.recovery.recover(utc_text(self.when), utc_text(self.when + timedelta(minutes=15)), now=self.when + timedelta(hours=4))
|
|
|
|
def test_wifi_gap_recovers_only_missing_energy_without_inventing_boiler(self):
|
|
for offset in [0, 10]:
|
|
self.store.add(observed(self.when + timedelta(minutes=offset)))
|
|
self.assertEqual(self.recover(), 1)
|
|
records = list(self.store.records())
|
|
recovered = [s for s in records if "history_interval" in s]
|
|
self.assertEqual(len(recovered), 1)
|
|
self.assertEqual(recovered[0]["history_interval"]["energy_wh"]["production"], 100)
|
|
self.assertIsNone(recovered[0]["readings"]["boiler_enabled"])
|
|
self.assertNotIn("boiler", recovered[0]["source_timestamps"])
|
|
self.assertIn("boiler", recovered[0]["errors"])
|
|
self.assertEqual(self.client.requests, [(SYSTEM, dataset, self.when.date())
|
|
for dataset in ("production", "consumption")])
|
|
self.assertEqual(recovered[0]["raw"]["solarweb_history"]["chart_power_w"],
|
|
{"production": 1200, "consumption": 1440})
|
|
self.assertEqual(self.recover(), 0)
|
|
self.assertEqual(len(self.client.requests), 2)
|
|
|
|
def test_cloud_energy_does_not_mark_boiler_history_complete(self):
|
|
self.assertEqual(self.recover(), 3)
|
|
rows = self.store.coverage(self.source, "UTC", 1, self.when + timedelta(days=1))
|
|
day = rows[1]
|
|
self.assertEqual(day["energy_complete"], 3)
|
|
self.assertEqual(day["boiler_complete"], 0)
|
|
self.assertNotIn("meter_complete", day)
|
|
self.assertEqual(day["complete"], 0)
|
|
row = pq.ParquetFile(next(self.store.path.glob("date=*/samples.parquet"))).read().to_pylist()[0]
|
|
self.assertEqual(row["record_kind"], "history")
|
|
self.assertEqual(row["interval_energy_wh"]["production"], 100)
|
|
|
|
def test_partial_cloud_records_remain_missing_for_retry(self):
|
|
stamp = int((self.when + timedelta(minutes=5)).timestamp() * 1000)
|
|
del self.client.points["production"][stamp]
|
|
self.assertEqual(self.recover(), 2)
|
|
self.assertEqual(self.recover(), 0)
|
|
self.client.points["production"][stamp] = 1200
|
|
self.assertEqual(self.recover(), 1)
|
|
|
|
def test_recovered_identifiers_are_stable_for_duplicate_delivery(self):
|
|
first = history_sample(entry(self.when), self.source, SYSTEM)
|
|
second = history_sample(entry(self.when), self.source, SYSTEM)
|
|
self.assertEqual(first, second)
|
|
|
|
def test_non_five_minute_and_bad_units_are_rejected(self):
|
|
for change in [{"logDuration": 599}, {"logDateTime": utc_text(self.when + timedelta(seconds=1))}]:
|
|
value = entry(self.when)
|
|
value.update(change)
|
|
with self.assertRaises(ValueError):
|
|
history_sample(value, self.source, SYSTEM)
|
|
value = entry(self.when)
|
|
value["channels"][0]["unit"] = "W"
|
|
with self.assertRaises(ValueError):
|
|
history_sample(value, self.source, SYSTEM)
|
|
|
|
def test_missing_private_credentials_fail_before_requests(self):
|
|
with self.assertRaisesRegex(ValueError, "SOLARWEB_USERNAME and SOLARWEB_PASSWORD"):
|
|
HistoryRecovery(self.config, self.source, self.store, credentials=("", ""))
|
|
|
|
def test_config_credentials_take_precedence_over_environment(self):
|
|
config = replace(self.config, username="private@example.test", password="synthetic password é &")
|
|
with patch("solar_relay.history.os.environ", {"SOLARWEB_USERNAME": "env@example.test", "SOLARWEB_PASSWORD": "env-password"}), \
|
|
patch("solar_relay.history.SolarWebClient", return_value=self.client) as client:
|
|
recovery = HistoryRecovery(config, self.source, self.store)
|
|
self.assertEqual(recovery.recover(utc_text(self.when), utc_text(self.when + timedelta(minutes=15)),
|
|
now=self.when + timedelta(hours=4)), 3)
|
|
client.assert_called_once_with("private@example.test", "synthetic password é &", timeout=config.request_timeout_seconds)
|
|
|
|
def test_empty_config_credentials_use_environment_fallback(self):
|
|
config = replace(self.config, username_env="BACKUP_USER", password_env="BACKUP_PASSWORD")
|
|
with patch("solar_relay.history.os.environ", {"BACKUP_USER": "env@example.test", "BACKUP_PASSWORD": "env-password"}), \
|
|
patch("solar_relay.history.SolarWebClient", return_value=self.client) as client:
|
|
HistoryRecovery(config, self.source, self.store)
|
|
client.assert_called_once_with("env@example.test", "env-password", timeout=config.request_timeout_seconds)
|
|
|
|
def test_partial_config_credentials_never_mix_with_environment(self):
|
|
for changes in [{"username": "private@example.test"}, {"password": "private-password"}]:
|
|
with self.subTest(changes=changes), \
|
|
patch("solar_relay.history.os.environ", {"SOLARWEB_USERNAME": "env@example.test", "SOLARWEB_PASSWORD": "env-password"}), \
|
|
patch("solar_relay.history.SolarWebClient") as client:
|
|
with self.assertRaisesRegex(ValueError, "history.username and history.password"):
|
|
HistoryRecovery(replace(self.config, **changes), self.source, self.store)
|
|
client.assert_not_called()
|
|
|
|
def test_requests_use_local_dates_across_dst(self):
|
|
self.recovery.recover("2026-10-24T22:00:00Z", "2026-10-26T00:00:00Z", now=datetime(2026, 10, 27, tzinfo=timezone.utc))
|
|
self.assertEqual([date.isoformat() for _, dataset, date in self.client.requests if dataset == "production"],
|
|
["2026-10-25", "2026-10-26"])
|
|
|
|
def test_recent_intervals_wait_for_the_hourly_cloud_upload(self):
|
|
for points in self.client.points.values():
|
|
points.clear()
|
|
for offset in [0, 115, 120, 235]:
|
|
points[int((self.when + timedelta(minutes=offset)).timestamp() * 1000)] = 1200
|
|
self.assertEqual(self.recovery.recover(utc_text(self.when), utc_text(self.when + timedelta(hours=4)),
|
|
now=self.when + timedelta(hours=4)), 2)
|
|
|
|
def test_automatic_recovery_checks_retention_before_first_observation_and_respects_cutoff(self):
|
|
now = datetime(2026, 11, 2, 12, 2, 17, tzinfo=timezone.utc)
|
|
cutoff = datetime(2026, 10, 3, tzinfo=ZoneInfo("Europe/Zurich")).astimezone(timezone.utc)
|
|
latest = datetime(2026, 11, 2, 10, tzinfo=timezone.utc)
|
|
recoverable = [cutoff, now.replace(minute=0, second=0) - timedelta(days=25), latest - timedelta(minutes=5)]
|
|
points = {int(when.timestamp() * 1000): 1200 for when in
|
|
[cutoff - timedelta(minutes=5), *recoverable, latest]}
|
|
self.client.points = {"production": points, "consumption": points}
|
|
self.store.add(observed(now.replace(minute=0, second=0)))
|
|
self.assertEqual(self.recovery.recover(now=now), 3)
|
|
recovered = [record for record in self.store.records() if "history_interval" in record]
|
|
self.assertEqual([record["observed_at"] for record in recovered], [utc_text(when) for when in recoverable])
|
|
self.assertEqual(min(day for _, _, day in self.client.requests), cutoff.astimezone(self.recovery.zone).date())
|
|
|
|
def test_oldest_retained_day_recovers_all_288_intervals_including_midnight(self):
|
|
zone = ZoneInfo("Europe/Zurich")
|
|
start = datetime(2026, 10, 3, tzinfo=zone)
|
|
end = start + timedelta(days=1)
|
|
points = {stamp: 1200 for stamp in range(int(start.timestamp() * 1000), int(end.timestamp() * 1000), 300_000)}
|
|
self.client.points = {"production": points, "consumption": points}
|
|
self.assertEqual(self.recovery.recover(now=datetime(2026, 11, 2, 22, 30, tzinfo=zone)), 288)
|
|
records = list(self.store.records())
|
|
self.assertEqual(records[0]["observed_at"], utc_text(start))
|
|
self.assertEqual(records[-1]["observed_at"], utc_text(end - timedelta(minutes=5)))
|
|
|
|
def test_retention_uses_the_configured_timezone_instead_of_utc(self):
|
|
now = datetime(2026, 11, 2, 1, 30, tzinfo=timezone.utc)
|
|
recovery = HistoryRecovery(self.config, self.source, self.store, timezone_name="America/New_York", client=self.client)
|
|
recovery.recover(now=now)
|
|
self.assertEqual(min(day for _, _, day in self.client.requests).isoformat(), "2026-10-02")
|
|
|
|
def test_thirty_calendar_day_manual_range_can_cross_a_25_hour_dst_day(self):
|
|
zone = ZoneInfo("Europe/Zurich")
|
|
self.client.points = {"production": {}, "consumption": {}}
|
|
self.assertEqual(self.recovery.recover(utc_text(datetime(2026, 10, 3, tzinfo=zone)),
|
|
utc_text(datetime(2026, 11, 2, tzinfo=zone)),
|
|
now=datetime(2026, 11, 3, tzinfo=zone)), 0)
|
|
|
|
def test_automatic_recovery_checks_entirely_empty_days_and_archives(self):
|
|
self.store.retention_days = 2
|
|
self.assertEqual(self.recovery.recover(now=self.when + timedelta(days=2)), 3)
|
|
self.assertEqual({day for _, _, day in self.client.requests},
|
|
{(self.when + timedelta(days=offset)).date() for offset in range(3)})
|
|
|
|
def test_automatic_recovery_supports_retention_longer_than_manual_limit(self):
|
|
self.store.retention_days = 45
|
|
now = self.when + timedelta(days=40)
|
|
self.assertEqual(self.recovery.recover(now=now), 3)
|
|
self.assertEqual(min(day for _, _, day in self.client.requests), (now - timedelta(days=45)).date())
|
|
with self.assertRaisesRegex(ValueError, "at most 30 days"):
|
|
self.recovery.recover(utc_text(now - timedelta(days=45)), utc_text(now), now=now)
|
|
|
|
def test_complete_retention_window_does_not_request_solarweb(self):
|
|
now = self.when + timedelta(days=1)
|
|
lower = int(datetime(2026, 10, 30, tzinfo=ZoneInfo("Europe/Zurich")).timestamp())
|
|
upper = int((now - timedelta(seconds=self.config.lag_seconds)).timestamp())
|
|
store = Mock(retention_days=1)
|
|
store.energy_slots.return_value = set(range(lower, upper, 300))
|
|
recovery = HistoryRecovery(self.config, self.source, store, client=self.client)
|
|
self.assertEqual(recovery.recover(now=now), 0)
|
|
self.assertEqual(self.client.requests, [])
|
|
store.add.assert_not_called()
|
|
|
|
def test_unavailable_cloud_day_does_not_block_newer_missing_days(self):
|
|
self.store.retention_days = 2
|
|
available = self.when + timedelta(days=1)
|
|
self.client.points = {dataset: {int(available.timestamp() * 1000): 1200}
|
|
for dataset in ("production", "consumption")}
|
|
get_chart = self.client.get_chart
|
|
|
|
def charts(system, dataset, day):
|
|
if day == self.when.date():
|
|
raise ChartDataError("synthetic secret response must not be logged")
|
|
return get_chart(system, dataset, day)
|
|
|
|
with patch.object(self.client, "get_chart", side_effect=charts), self.assertLogs("solar_relay.history", level="WARNING") as logs:
|
|
self.assertEqual(self.recovery.recover(now=self.when + timedelta(days=2)), 1)
|
|
self.assertNotIn("synthetic secret response", " ".join(logs.output))
|
|
self.assertEqual(next(self.store.records())["observed_at"], utc_text(available))
|
|
|
|
def test_login_or_transport_failure_stops_the_scan_without_repeated_attempts(self):
|
|
with patch.object(self.client, "get_chart", side_effect=TransportError("Login failed")) as client:
|
|
with self.assertRaisesRegex(TransportError, "Login failed"):
|
|
self.recovery.recover(now=self.when + timedelta(days=2))
|
|
self.assertEqual(client.call_count, 1)
|
|
|
|
def test_chart_conversion_matches_solar_and_retains_provenance(self):
|
|
for generated, consumed in [(1200, 1440), (2400, 1200), (0, 0)]:
|
|
with self.subTest(generated=generated, consumed=consumed):
|
|
stamp = int(self.when.timestamp() * 1000)
|
|
record = next(chart_entries({stamp: generated}, {stamp: consumed}))
|
|
values = history_sample(record, self.source, SYSTEM)["history_interval"]["energy_wh"]
|
|
self.assertEqual(values, {"production": generated / 12, "consumption": consumed / 12,
|
|
"self_consumption": min(generated, consumed) / 12,
|
|
"bought": max(0, consumed - generated) / 12,
|
|
"sold": max(0, generated - consumed) / 12})
|
|
self.assertEqual(record["energy_method"], "constant_power_over_five_minutes")
|
|
|
|
def test_dst_days_recover_every_actual_five_minute_slot(self):
|
|
for day, expected in [("2026-03-29", 276), ("2026-10-25", 300)]:
|
|
with self.subTest(day=day):
|
|
start = datetime.fromisoformat(day).replace(tzinfo=ZoneInfo("Europe/Zurich"))
|
|
end = start + timedelta(days=1)
|
|
points = {stamp: 1200 for stamp in range(int(start.timestamp() * 1000), int(end.timestamp() * 1000), 300_000)}
|
|
self.client.points = {"production": points, "consumption": points}
|
|
self.assertEqual(self.recovery.recover(utc_text(start), utc_text(end), now=end + timedelta(days=1)), expected)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|