65 lines
2.5 KiB
Python
65 lines
2.5 KiB
Python
"""Retryable delivery with explicit per-sample acknowledgement."""
|
|
|
|
import logging
|
|
import os
|
|
|
|
from .config import UploadConfig
|
|
from .http import JsonHttp, TransportError
|
|
from .storage import Store
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
|
|
|
|
class UploadError(Exception):
|
|
pass
|
|
|
|
|
|
class Uploader:
|
|
def __init__(self, config: UploadConfig, store: Store, *, token=None, http=None):
|
|
if not config.url:
|
|
raise ValueError("Set upload.url to the website's ingestion endpoint first")
|
|
self.config = config
|
|
self.store = store
|
|
self.token = token if token is not None else os.environ.get(config.token_env, "")
|
|
if not self.token or any(ord(c) < 33 or ord(c) > 126 for c in self.token):
|
|
raise ValueError(f"Set {config.token_env} to a nonempty bearer token without whitespace")
|
|
self.http = http or JsonHttp(config.request_timeout_seconds)
|
|
|
|
def upload_once(self) -> int:
|
|
batch = self.store.pending(self.config.batch_size)
|
|
if not batch:
|
|
return 0
|
|
try:
|
|
response = self.http.request(
|
|
self.config.url, payload={"schema_version": 1, "samples": batch}, token=self.token,
|
|
)
|
|
except TransportError as error:
|
|
raise UploadError(str(error)) from None
|
|
accepted = response.get("accepted_ids")
|
|
if not isinstance(accepted, list) or not all(isinstance(value, str) for value in accepted):
|
|
raise UploadError("Receiver must return an accepted_ids array")
|
|
requested = {sample["id"] for sample in batch}
|
|
if len(set(accepted)) != len(accepted) or not set(accepted).issubset(requested):
|
|
raise UploadError("Receiver acknowledged duplicate or unknown sample IDs")
|
|
self.store.acknowledge(accepted)
|
|
if set(accepted) != requested:
|
|
raise UploadError(f"Receiver accepted {len(accepted)}/{len(batch)} samples; others remain queued")
|
|
return len(accepted)
|
|
|
|
def run(self, stop):
|
|
delay = self.config.retry_initial_seconds
|
|
while not stop.is_set():
|
|
try:
|
|
count = self.upload_once()
|
|
except (UploadError, OSError) as error:
|
|
LOG.warning("Upload failed: %s; retrying in %.0fs", error, delay)
|
|
stop.wait(delay)
|
|
delay = min(delay * 2, self.config.retry_max_seconds)
|
|
else:
|
|
delay = self.config.retry_initial_seconds
|
|
if count:
|
|
LOG.info("Uploaded and acknowledged %d samples", count)
|
|
else:
|
|
stop.wait(5)
|
|
|