Swap to private solarweb access

solar_relay/history.py

@@ -1,19 +1,20 @@
-"""Optional private Solar.web Query API recovery of missing five-minute energy.
+"""Optional signed-in Solar.web chart recovery of missing five-minute energy.
 
-GEN24's local Solar API exposes no archive. This reads the independent cloud API
-after the inverter has uploaded its internal backlog; it never requests a guest page.
+GEN24's local Solar API exposes no archive. Use the same private account login,
+chart series and five-minute W-to-Wh conversion as the sibling solar website.
 """
 
 from datetime import datetime, timedelta, timezone
 import logging
 import math
 import os
-from urllib.parse import urlencode
 from uuid import NAMESPACE_URL, uuid5
+from zoneinfo import ZoneInfo
 
 from .config import utc_text
 from .coverage import milliseconds
-from .http import JsonHttp, TransportError
+from .http import TransportError
+from .solarweb import ChartDataError, SolarWebClient
 
 LOG = logging.getLogger(__name__)
 CHANNELS = {
@@ -40,6 +41,7 @@ def history_sample(entry, source_id, system_id):
     when = utc_text(datetime.fromtimestamp(stamp / 1000, timezone.utc))
     return {
         "schema_version": 1,
+        # Preserve IDs of intervals already recovered with the former Query API.
         "id": str(uuid5(NAMESPACE_URL, f"https://swqapi.solarweb.com/{system_id}/{source_id}/{when}")),
         "source_id": source_id, "observed_at": when,
         "readings": dict.fromkeys([
@@ -54,68 +56,80 @@ def history_sample(entry, source_id, system_id):
     }
 
 
+def chart_entries(production, consumption):
+    """Pair real chart samples; assume constant W over five minutes, as solar does."""
+    for stamp in sorted(production.keys() & consumption.keys()):
+        generated, consumed = production[stamp], consumption[stamp]
+        values = {"production": generated / 12, "consumption": consumed / 12,
+                  "self_consumption": min(generated, consumed) / 12,
+                  "bought": max(0, consumed - generated) / 12, "sold": max(0, generated - consumed) / 12}
+        yield {"logDateTime": utc_text(datetime.fromtimestamp(stamp / 1000, timezone.utc)), "logDuration": 300,
+               "channels": [{"channelName": name, "channelType": "Energy", "unit": "Wh", "value": values[field]}
+                            for field, name in CHANNELS.items()],
+               "chart_power_w": {"production": generated, "consumption": consumed},
+               "energy_method": "constant_power_over_five_minutes"}
+
+
 class HistoryRecovery:
-    def __init__(self, config, source_id, store, *, http=None, credentials=None):
+    def __init__(self, config, source_id, store, *, timezone_name="Europe/Zurich", client=None, credentials=None):
         self.config, self.source_id, self.store = config, source_id, store
-        names = (config.access_key_id_env, config.access_key_value_env)
-        values = credentials if credentials is not None else [os.environ.get(name, "") for name in names]
-        if len(values) != 2 or any(not isinstance(value, str) or not value or any(ord(c) < 33 or ord(c) > 126 for c in value) for value in values):
-            raise ValueError(f"Set {names[0]} and {names[1]} before enabling Solar.web recovery")
-        self.headers = dict(zip(("AccessKeyId", "AccessKeyValue"), values))
-        self.http = http or JsonHttp(config.request_timeout_seconds)
+        self.zone = ZoneInfo(timezone_name)
+        if client is None:
+            names = (config.username_env, config.password_env)
+            if credentials is not None:
+                values = credentials
+            elif config.username or config.password:
+                values = (config.username, config.password)
+            else:
+                values = [os.environ.get(name, "") for name in names]
+            if (len(values) != 2 or any(not isinstance(value, str) or not value for value in values)
+                    or not values[0].strip()):
+                raise ValueError("Set history.username and history.password in config.toml, "
+                                 f"or {names[0]} and {names[1]} before enabling Solar.web recovery")
+            client = SolarWebClient(*values, timeout=config.request_timeout_seconds)
+        self.client = client
 
     def recover(self, start=None, until=None, *, now=None, stop=None):
         now = now or datetime.now(timezone.utc)
         latest = int((now - timedelta(seconds=self.config.lag_seconds)).timestamp()) // 300 * 300
-        first = self.store.status()["first_observation"]
-        earliest = max(int((now - timedelta(days=self.config.lookback_days)).timestamp()) // 300 * 300,
-                       milliseconds(first) // 300_000 * 300 if first else latest)
+        # Check the whole retained window, including days before any live observation.
+        # Round up so recovery never requests an interval older than the cutoff.
+        earliest = math.ceil((now - timedelta(days=self.store.retention_days)).timestamp() / 300) * 300
         lower = milliseconds(start) // 300_000 * 300 if start else earliest
         upper = min(milliseconds(until) // 300_000 * 300, latest) if until else latest
         if lower >= upper:
             return 0
-        if upper - lower > 30 * 86400:
+        if (start is not None or until is not None) and upper - lower > 30 * 86400:
             raise ValueError("Recover at most 30 days at once")
         stamp = lambda epoch: utc_text(datetime.fromtimestamp(epoch, timezone.utc))
         present = self.store.energy_slots(self.source_id, stamp(lower), stamp(upper))
         missing = set(range(lower, upper, 300)) - present
-        # Query only UTC days containing gaps. No request spans the API's 24-hour limit.
-        days = sorted({epoch // 86400 * 86400 for epoch in missing})
+        LOG.info("Checking %d historical energy intervals; %d are missing", (upper - lower) // 300, len(missing))
+        # Chart dates use the installation's local timezone, including 23/25-hour DST days.
+        days = sorted({datetime.fromtimestamp(epoch, self.zone).date() for epoch in missing})
         recovered = 0
         for day in days:
             if stop and stop.is_set():
                 break
-            begin, end = max(day, lower), min(day + 86400, upper)
-            page = 0
-            while True:
+            LOG.info("Recovering Solar.web energy history for %s", day)
+            try:
+                production = self.client.get_chart(self.config.pv_system_id, "production", day)
                 if stop and stop.is_set():
                     break
-                query = urlencode({"from": stamp(begin), "to": stamp(end), "timezone": "zulu",
-                                   "channel": ",".join(CHANNELS.values()), "offset": page * 300, "limit": 300})
-                url = f"https://swqapi.solarweb.com/pvsystems/{self.config.pv_system_id}/histdata?{query}"
-                response = self.http.request(url, extra_headers=self.headers)
-                system_id = response.get("pvSystemId")
-                if not isinstance(system_id, str) or system_id.lower() != self.config.pv_system_id.lower() or not isinstance(response.get("data"), list):
-                    raise ValueError("Unexpected Solar.web history response")
-                entries = response["data"]
-                if len(entries) > 300:
-                    raise ValueError("Solar.web history response exceeds the requested page size")
-                for entry in entries:
-                    try:
-                        sample = history_sample(entry, self.source_id, self.config.pv_system_id)
-                        epoch = milliseconds(sample["observed_at"]) // 1000
-                    except (KeyError, TypeError, ValueError, AttributeError):
-                        # Partial/unavailable cloud records remain missing and are tried again later.
-                        continue
-                    if begin <= epoch < end and epoch in missing:
-                        self.store.add(sample)
-                        missing.remove(epoch)
-                        recovered += 1
-                if len(entries) < 300:
-                    break
-                page += 1
-                if page > 2:
-                    raise ValueError("Solar.web history pagination did not finish")
+                consumption = self.client.get_chart(self.config.pv_system_id, "consumption", day)
+            except ChartDataError:
+                # A day with no usable cloud data must not block recovery of newer gaps.
+                LOG.warning("Solar.web chart data unavailable for %s; leaving its gaps for the next check", day)
+                continue
+            for entry in chart_entries(production, consumption):
+                epoch = milliseconds(entry["logDateTime"]) // 1000
+                if epoch not in missing or datetime.fromtimestamp(epoch, self.zone).date() != day:
+                    continue
+                sample = history_sample(entry, self.source_id, self.config.pv_system_id)
+                self.store.add(sample)
+                missing.remove(epoch)
+                recovered += 1
+        LOG.info("History check finished: %d intervals recovered, %d still missing", recovered, len(missing))
         return recovered
 
     def run(self, stop):