181 lines
10 KiB
Python
181 lines
10 KiB
Python
"""Optional signed-in Solar.web chart recovery of missing five-minute energy.
|
|
|
|
GEN24's local Solar API exposes no archive. Use the same private account login,
|
|
chart series and five-minute W-to-Wh conversion as the sibling solar website.
|
|
"""
|
|
|
|
from datetime import datetime, timedelta, timezone
|
|
import logging
|
|
import math
|
|
import os
|
|
from uuid import NAMESPACE_URL, uuid5
|
|
from zoneinfo import ZoneInfo
|
|
|
|
from .config import retention_cutoff, utc_text
|
|
from .coverage import milliseconds
|
|
from .http import TransportError
|
|
from .solarweb import ChartDataError, SolarWebClient
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
CHANNELS = {
|
|
"production": "EnergyProductionTotal", "consumption": "EnergyConsumptionTotal",
|
|
"self_consumption": "EnergySelfConsumptionTotal", "bought": "EnergyPurchased", "sold": "EnergyFeedIn",
|
|
}
|
|
|
|
|
|
def normalize_history_sample(sample):
|
|
"""Expose recovered power in the same fields as live samples, including old archives."""
|
|
if "history_interval" not in sample:
|
|
return sample
|
|
interval = sample["history_interval"]
|
|
if interval["duration_seconds"] != 300:
|
|
raise ValueError("Historical record is not a five-minute interval")
|
|
energy = interval["energy_wh"]
|
|
if any(isinstance(energy[field], bool) or not isinstance(energy[field], (int, float))
|
|
or not math.isfinite(energy[field]) or not 0 <= energy[field] <= 1e18 for field in CHANNELS):
|
|
raise ValueError("Historical energy is missing or invalid")
|
|
power = {field: energy[field] * 12 for field in CHANNELS}
|
|
chart = sample.get("raw", {}).get("solarweb_history", {}).get("chart_power_w")
|
|
if chart is not None:
|
|
# Retain the chart's original watts instead of a W -> Wh -> W round trip.
|
|
generated, consumed = chart["production"], chart["consumption"]
|
|
if any(isinstance(value, bool) or not isinstance(value, (int, float))
|
|
or not math.isfinite(value) or value < 0 for value in (generated, consumed)):
|
|
raise ValueError("Historical chart power is missing or invalid")
|
|
if not all(math.isclose(value, power[field]) for field, value in
|
|
(("production", generated), ("consumption", consumed))):
|
|
raise ValueError("Historical chart power disagrees with interval energy")
|
|
power = {"production": generated, "consumption": consumed,
|
|
"bought": max(0, consumed - generated), "sold": max(0, generated - consumed)}
|
|
return {
|
|
**sample, "poll_interval_seconds": 300,
|
|
"readings": {**sample["readings"],
|
|
"production_ac_w": power["production"], "consumption_w": power["consumption"],
|
|
"grid_w": power["bought"] - power["sold"],
|
|
"grid_import_w": power["bought"], "grid_export_w": power["sold"]},
|
|
"source_timestamps": {**sample.get("source_timestamps", {}), "powerflow": sample["observed_at"]},
|
|
}
|
|
|
|
|
|
def history_sample(entry, source_id, system_id):
|
|
stamp = milliseconds(entry["logDateTime"])
|
|
if stamp % 300_000 or entry.get("logDuration") != 300:
|
|
raise ValueError("Historical record is not an aligned five-minute interval")
|
|
channels = {}
|
|
for channel in entry["channels"]:
|
|
name, value = channel.get("channelName"), channel.get("value")
|
|
if name not in CHANNELS.values():
|
|
continue
|
|
if name in channels or channel.get("unit") != "Wh" or channel.get("channelType") != "Energy":
|
|
raise ValueError("Unexpected historical channel metadata")
|
|
if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value) or not 0 <= value <= 1e18:
|
|
raise ValueError("Historical energy is missing or invalid")
|
|
channels[name] = value
|
|
energy = {field: channels[name] for field, name in CHANNELS.items()}
|
|
when = utc_text(datetime.fromtimestamp(stamp / 1000, timezone.utc))
|
|
return normalize_history_sample({
|
|
"schema_version": 1,
|
|
# Preserve IDs of intervals already recovered with the former Query API.
|
|
"id": str(uuid5(NAMESPACE_URL, f"https://swqapi.solarweb.com/{system_id}/{source_id}/{when}")),
|
|
"source_id": source_id, "observed_at": when,
|
|
"readings": dict.fromkeys([
|
|
"production_w", "production_ac_w", "consumption_w", "grid_w", "grid_import_w", "grid_export_w",
|
|
"production_ac_total_wh", "grid_import_total_wh", "grid_export_total_wh", "boiler_enabled", "boiler_reason",
|
|
]),
|
|
"source_timestamps": {},
|
|
"errors": {"meter": "Cumulative meter readings are unavailable in recovered energy intervals",
|
|
"boiler": "Historical boiler switch state is unavailable"},
|
|
"raw": {"solarweb_history": entry},
|
|
"history_interval": {"provider": "fronius-solarweb", "start": when, "duration_seconds": 300, "energy_wh": energy},
|
|
})
|
|
|
|
|
|
def chart_entries(production, consumption):
|
|
"""Pair real chart samples; assume constant W over five minutes, as solar does."""
|
|
for stamp in sorted(production.keys() & consumption.keys()):
|
|
generated, consumed = production[stamp], consumption[stamp]
|
|
values = {"production": generated / 12, "consumption": consumed / 12,
|
|
"self_consumption": min(generated, consumed) / 12,
|
|
"bought": max(0, consumed - generated) / 12, "sold": max(0, generated - consumed) / 12}
|
|
yield {"logDateTime": utc_text(datetime.fromtimestamp(stamp / 1000, timezone.utc)), "logDuration": 300,
|
|
"channels": [{"channelName": name, "channelType": "Energy", "unit": "Wh", "value": values[field]}
|
|
for field, name in CHANNELS.items()],
|
|
"chart_power_w": {"production": generated, "consumption": consumed},
|
|
"energy_method": "constant_power_over_five_minutes"}
|
|
|
|
|
|
class HistoryRecovery:
|
|
def __init__(self, config, source_id, store, *, timezone_name="Europe/Zurich", client=None, credentials=None):
|
|
self.config, self.source_id, self.store = config, source_id, store
|
|
self.zone = ZoneInfo(timezone_name)
|
|
if client is None:
|
|
names = (config.username_env, config.password_env)
|
|
if credentials is not None:
|
|
values = credentials
|
|
elif config.username or config.password:
|
|
values = (config.username, config.password)
|
|
else:
|
|
values = [os.environ.get(name, "") for name in names]
|
|
if (len(values) != 2 or any(not isinstance(value, str) or not value for value in values)
|
|
or not values[0].strip()):
|
|
raise ValueError("Set history.username and history.password in config.toml, "
|
|
f"or {names[0]} and {names[1]} before enabling Solar.web recovery")
|
|
client = SolarWebClient(*values, timeout=config.request_timeout_seconds)
|
|
self.client = client
|
|
|
|
def recover(self, start=None, until=None, *, now=None, stop=None):
|
|
now = now or datetime.now(timezone.utc)
|
|
latest = int((now - timedelta(seconds=self.config.lag_seconds)).timestamp()) // 300 * 300
|
|
# Always include the entire oldest local day, even across a DST change.
|
|
earliest = int(retention_cutoff(now, self.store.retention_days, self.zone.key).timestamp())
|
|
lower = milliseconds(start) // 300_000 * 300 if start else earliest
|
|
upper = min(milliseconds(until) // 300_000 * 300, latest) if until else latest
|
|
if lower >= upper:
|
|
return 0
|
|
manual_limit = datetime.fromtimestamp(lower, self.zone) + timedelta(days=30)
|
|
if (start is not None or until is not None) and upper > manual_limit.timestamp():
|
|
raise ValueError("Recover at most 30 days at once")
|
|
stamp = lambda epoch: utc_text(datetime.fromtimestamp(epoch, timezone.utc))
|
|
present = self.store.energy_slots(self.source_id, stamp(lower), stamp(upper))
|
|
missing = set(range(lower, upper, 300)) - present
|
|
LOG.info("Checking %d historical energy intervals; %d are missing", (upper - lower) // 300, len(missing))
|
|
# Chart dates use the installation's local timezone, including 23/25-hour DST days.
|
|
days = {}
|
|
for epoch in sorted(missing):
|
|
local = datetime.fromtimestamp(epoch, self.zone)
|
|
days.setdefault(local.date(), []).append(local.strftime("%H:%M:%S"))
|
|
recovered = 0
|
|
for day in sorted(days):
|
|
if stop and stop.is_set():
|
|
break
|
|
LOG.info("Recovering Solar.web energy history for %s %s", day, ", ".join(days[day]))
|
|
try:
|
|
production = self.client.get_chart(self.config.pv_system_id, "production", day)
|
|
if stop and stop.is_set():
|
|
break
|
|
consumption = self.client.get_chart(self.config.pv_system_id, "consumption", day)
|
|
except ChartDataError:
|
|
# A day with no usable cloud data must not block recovery of newer gaps.
|
|
LOG.warning("Solar.web chart data unavailable for %s; leaving its gaps for the next check", day)
|
|
continue
|
|
for entry in chart_entries(production, consumption):
|
|
epoch = milliseconds(entry["logDateTime"]) // 1000
|
|
if epoch not in missing or datetime.fromtimestamp(epoch, self.zone).date() != day:
|
|
continue
|
|
sample = history_sample(entry, self.source_id, self.config.pv_system_id)
|
|
self.store.add(sample)
|
|
missing.remove(epoch)
|
|
recovered += 1
|
|
LOG.info("History check finished: %d intervals recovered, %d still missing", recovered, len(missing))
|
|
return recovered
|
|
|
|
def run(self, stop):
|
|
while not stop.is_set():
|
|
try:
|
|
recovered = self.recover(stop=stop)
|
|
if recovered:
|
|
LOG.info("Recovered %d historical energy intervals from Solar.web", recovered)
|
|
except (TransportError, OSError, ValueError) as error:
|
|
LOG.warning("Historical recovery failed (%s); retrying on the next scheduled check", type(error).__name__)
|
|
stop.wait(self.config.check_interval_seconds)
|