Implement dashboard parsing and delivery sensors
Reads delivery day, next arrival date, both upcoming carts, box contents, add-ons and order totals from a single /p/dashboard/details fetch. Adds five sensors and parser tests over a synthetic fixture mirroring the live markup. Records two portal quirks: every /p/ path returns 200 so signed-in state is detected by a Sign Out control, and cart-contents-skipped marks the locked cart rather than a skipped order.
This commit is contained in:
flan
@@ -1,7 +1,12 @@
|
||||
"""HTTP client for the freshharvest.com customer portal.
|
||||
"""HTTP client and HTML parser for the freshharvest.com customer portal.
|
||||
|
||||
The site is server-rendered (no JSON API), so this client drives the same form
|
||||
flow a browser does and parses HTML out the other side.
|
||||
The site is server-rendered with no JSON API, so this client drives the same
|
||||
form flow a browser does and parses the dashboard markup.
|
||||
|
||||
Everything the integration needs lives on a single page,
|
||||
``/p/dashboard/details``: the account's delivery day and next arrival date, the
|
||||
upcoming carts with their contents, and the order totals. Keeping this to one
|
||||
request per refresh is deliberate — the portal is a small business's website.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -12,6 +17,7 @@ from dataclasses import dataclass, field
|
||||
from datetime import date
|
||||
|
||||
import aiohttp
|
||||
from bs4 import BeautifulSoup
|
||||
from yarl import URL
|
||||
|
||||
_LOGGER = logging.getLogger(__name__)
|
||||
@@ -19,18 +25,26 @@ _LOGGER = logging.getLogger(__name__)
|
||||
BASE = URL("https://freshharvest.com")
|
||||
LOGIN_FORM = "/s/popup/login"
|
||||
LOGIN_SUBMIT = "/s/submit/login"
|
||||
DASHBOARD = "/p/dashboard/details"
|
||||
|
||||
# Browser UA: the portal serves a reduced/blocked page to obvious scripts.
|
||||
# The portal serves a reduced page to obviously-scripted clients.
|
||||
USER_AGENT = (
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/126.0 Safari/537.36"
|
||||
)
|
||||
|
||||
# Hidden anti-replay fields, minted fresh on every GET of the login form and
|
||||
# only valid for the session cookie they were issued against.
|
||||
# Hidden anti-replay fields, minted per session on each GET of the login form.
|
||||
_HIDDEN_RE = re.compile(
|
||||
r"name='(?P<name>LoginSecurity|SubmitToken)'[^>]*value='(?P<value>[^']*)'"
|
||||
)
|
||||
_NEXT_ARRIVING_RE = re.compile(
|
||||
r"Your deliveries are\s*(?P<day>[A-Za-z]+)\.\s*"
|
||||
r"Next Arriving:\s*(?P<date>[A-Za-z]+\s+\d+(?:st|nd|rd|th)?,\s*\d{4})",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_TAB_DATE_RE = re.compile(r"(?P<month>\d{1,2})/(?P<day>\d{1,2})")
|
||||
_ORDINAL_RE = re.compile(r"(\d+)(?:st|nd|rd|th)", re.IGNORECASE)
|
||||
_MONEY_RE = re.compile(r"-?\$\s*([\d,]+\.\d{2})")
|
||||
|
||||
|
||||
class FreshHarvestError(Exception):
|
||||
@@ -41,16 +55,210 @@ class FreshHarvestAuthError(FreshHarvestError):
|
||||
"""Credentials rejected, or the session expired and could not be renewed."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class Delivery:
|
||||
"""A scheduled delivery."""
|
||||
def _money(text: str | None) -> float | None:
|
||||
"""Pull a dollar amount out of a label. Non-amounts (e.g. 'Add Tip') -> None."""
|
||||
if not text:
|
||||
return None
|
||||
match = _MONEY_RE.search(text)
|
||||
if not match:
|
||||
return None
|
||||
return float(match.group(1).replace(",", ""))
|
||||
|
||||
|
||||
def _text(node, selector: str) -> str | None:
|
||||
found = node.select_one(selector)
|
||||
if found is None:
|
||||
return None
|
||||
value = found.get_text(" ", strip=True)
|
||||
return value or None
|
||||
|
||||
|
||||
@dataclass
|
||||
class OrderItem:
|
||||
"""One line in a cart: a produce-box component or a paid add-on."""
|
||||
|
||||
name: str
|
||||
quantity: int | None = None
|
||||
unit: str | None = None
|
||||
price: float | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class DeliveryOrder:
|
||||
"""A single upcoming delivery."""
|
||||
|
||||
delivery_id: str
|
||||
delivery_date: date | None = None
|
||||
window: str | None = None
|
||||
status: str | None = None
|
||||
box_name: str | None = None
|
||||
box_price: float | None = None
|
||||
items: list[OrderItem] = field(default_factory=list)
|
||||
addons: list[OrderItem] = field(default_factory=list)
|
||||
subtotal: float | None = None
|
||||
tax: float | None = None
|
||||
delivery_fee: float | None = None
|
||||
total: float | None = None
|
||||
items: list[str] = field(default_factory=list)
|
||||
cutoff: str | None = None
|
||||
# Non-empty only while the order can still be changed, e.g. "Shop tomorrow"
|
||||
# or "Shop thru Sunday 8/9". Empty once the order is locked for packing.
|
||||
shop_window: str | None = None
|
||||
|
||||
@property
|
||||
def is_open(self) -> bool:
|
||||
"""Whether the order can still be customized."""
|
||||
return bool(self.shop_window)
|
||||
|
||||
@property
|
||||
def all_items(self) -> list[OrderItem]:
|
||||
return [*self.items, *self.addons]
|
||||
|
||||
|
||||
@dataclass
|
||||
class AccountSnapshot:
|
||||
"""Everything read from one dashboard fetch."""
|
||||
|
||||
next_delivery: date | None = None
|
||||
delivery_day: str | None = None
|
||||
orders: list[DeliveryOrder] = field(default_factory=list)
|
||||
|
||||
@property
|
||||
def next_order(self) -> DeliveryOrder | None:
|
||||
"""The cart for the next arriving delivery.
|
||||
|
||||
Matched against the account's stated next-arrival date. Note that this
|
||||
order is usually already locked: the *open* cart is the one after it.
|
||||
"""
|
||||
if self.next_delivery is not None:
|
||||
for order in self.orders:
|
||||
if order.delivery_date == self.next_delivery:
|
||||
return order
|
||||
dated = [o for o in self.orders if o.delivery_date]
|
||||
return min(dated, key=lambda o: o.delivery_date) if dated else None
|
||||
|
||||
@property
|
||||
def open_order(self) -> DeliveryOrder | None:
|
||||
"""The earliest cart that can still be customized."""
|
||||
candidates = [o for o in self.orders if o.is_open and o.delivery_date]
|
||||
return min(candidates, key=lambda o: o.delivery_date) if candidates else None
|
||||
|
||||
|
||||
def _parse_full_date(raw: str) -> date | None:
|
||||
"""Parse 'August 4th, 2026'."""
|
||||
from datetime import datetime
|
||||
|
||||
cleaned = _ORDINAL_RE.sub(r"\1", raw).replace(",", " ")
|
||||
cleaned = re.sub(r"\s+", " ", cleaned).strip()
|
||||
try:
|
||||
return datetime.strptime(cleaned, "%B %d %Y").date()
|
||||
except ValueError:
|
||||
_LOGGER.debug("could not parse date %r", raw)
|
||||
return None
|
||||
|
||||
|
||||
def _parse_tab_date(raw: str, anchor: date | None) -> date | None:
|
||||
"""Parse a cart tab label like 'Tue 8/11', which carries no year.
|
||||
|
||||
The year is taken from ``anchor`` (the account's next-arrival date), rolling
|
||||
forward when the tab falls far enough behind it to be a December/January
|
||||
boundary rather than a genuinely earlier delivery.
|
||||
"""
|
||||
match = _TAB_DATE_RE.search(raw or "")
|
||||
if not match:
|
||||
return None
|
||||
month, day = int(match["month"]), int(match["day"])
|
||||
year = anchor.year if anchor else date.today().year
|
||||
try:
|
||||
candidate = date(year, month, day)
|
||||
except ValueError:
|
||||
return None
|
||||
if anchor and (anchor - candidate).days > 180:
|
||||
try:
|
||||
candidate = date(year + 1, month, day)
|
||||
except ValueError:
|
||||
return None
|
||||
return candidate
|
||||
|
||||
|
||||
def parse_dashboard(html: str) -> AccountSnapshot:
|
||||
"""Parse ``/p/dashboard/details`` into a snapshot."""
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
snapshot = AccountSnapshot()
|
||||
|
||||
account = soup.select_one(".account")
|
||||
if account is not None:
|
||||
match = _NEXT_ARRIVING_RE.search(account.get_text(" ", strip=True))
|
||||
if match:
|
||||
snapshot.delivery_day = match["day"]
|
||||
snapshot.next_delivery = _parse_full_date(match["date"])
|
||||
|
||||
# Tab labels carry the delivery dates; the cart bodies carry everything else.
|
||||
tab_dates = {
|
||||
node.get("data-cart-select"): node.get_text(strip=True)
|
||||
for node in soup.select(".cart-selector-options-wrapper > div[data-cart-select]")
|
||||
}
|
||||
|
||||
for cart in soup.select("div.cart-contents[data-cart-select]"):
|
||||
delivery_id = cart.get("data-cart-select")
|
||||
order = DeliveryOrder(
|
||||
delivery_id=delivery_id,
|
||||
delivery_date=_parse_tab_date(
|
||||
tab_dates.get(delivery_id, ""), snapshot.next_delivery
|
||||
),
|
||||
box_name=_text(cart, ".cart-basket-columns h6"),
|
||||
box_price=_money(_text(cart, ".cart-basket-columns .item-total")),
|
||||
shop_window=_text(cart, ".cart-customize-wrapper"),
|
||||
)
|
||||
|
||||
for row in cart.select(".basket-item"):
|
||||
name = _text(row, ".basket-item-name")
|
||||
if not name:
|
||||
continue
|
||||
quantity = _text(row, ".basket-item-quantity")
|
||||
order.items.append(
|
||||
OrderItem(
|
||||
name=name,
|
||||
quantity=int(quantity) if (quantity or "").isdigit() else None,
|
||||
unit=_text(row, ".basket-item-uom"),
|
||||
)
|
||||
)
|
||||
|
||||
for row in cart.select(".cart-item"):
|
||||
classes = row.get("class") or []
|
||||
if "basket-item" in classes:
|
||||
continue
|
||||
name = _text(row, ".item-name")
|
||||
if not name:
|
||||
continue
|
||||
quantity = _text(row, ".item-order-quantity")
|
||||
order.addons.append(
|
||||
OrderItem(
|
||||
name=name,
|
||||
# .item-total is the extended price: 4 muffins -> $17.96.
|
||||
price=_money(_text(row, ".item-total")),
|
||||
quantity=int(quantity) if (quantity or "").isdigit() else None,
|
||||
unit=_text(row, ".item-uom"),
|
||||
)
|
||||
)
|
||||
|
||||
totals = soup.select_one(f"#OrderTotals-{delivery_id}")
|
||||
if totals is not None:
|
||||
labels = totals.select(".summary-item.label")
|
||||
values = totals.select(".summary-item.value")
|
||||
for label, value in zip(labels, values):
|
||||
key = label.get_text(" ", strip=True).lower()
|
||||
amount = _money(value.get_text(" ", strip=True))
|
||||
if key.startswith("order total"):
|
||||
order.total = amount
|
||||
elif key.startswith("subtotal"):
|
||||
order.subtotal = amount
|
||||
elif key.startswith("tax"):
|
||||
order.tax = amount
|
||||
elif key.startswith("delivery"):
|
||||
order.delivery_fee = amount
|
||||
|
||||
snapshot.orders.append(order)
|
||||
|
||||
if not snapshot.orders and snapshot.next_delivery is None:
|
||||
raise FreshHarvestError("dashboard markup not recognised")
|
||||
return snapshot
|
||||
|
||||
|
||||
class FreshHarvestClient:
|
||||
@@ -75,7 +283,7 @@ class FreshHarvestClient:
|
||||
"""Run the two-step handshake: fetch tokens, then post credentials.
|
||||
|
||||
The tokens are bound to the session cookie issued by the same GET, so
|
||||
the fetch and the post cannot be split across sessions or cached.
|
||||
the fetch and the post must share a cookie jar.
|
||||
"""
|
||||
form = await self._get(LOGIN_FORM)
|
||||
hidden = {m["name"]: m["value"] for m in _HIDDEN_RE.finditer(form)}
|
||||
@@ -92,53 +300,44 @@ class FreshHarvestClient:
|
||||
"SubmitToken": hidden["SubmitToken"],
|
||||
"Redirect": "",
|
||||
}
|
||||
async with self._session.post(
|
||||
BASE.join(URL(LOGIN_SUBMIT)),
|
||||
data=payload,
|
||||
headers={"User-Agent": USER_AGENT},
|
||||
) as resp:
|
||||
resp.raise_for_status()
|
||||
body = await resp.text()
|
||||
try:
|
||||
async with self._session.post(
|
||||
BASE.join(URL(LOGIN_SUBMIT)),
|
||||
data=payload,
|
||||
headers={"User-Agent": USER_AGENT},
|
||||
) as resp:
|
||||
resp.raise_for_status()
|
||||
body = await resp.text()
|
||||
except aiohttp.ClientError as err:
|
||||
raise FreshHarvestError(f"login request failed: {err}") from err
|
||||
|
||||
if not self._looks_authenticated(body):
|
||||
if not self._signed_in(body):
|
||||
raise FreshHarvestAuthError("login rejected")
|
||||
self._authenticated = True
|
||||
|
||||
@staticmethod
|
||||
def _looks_authenticated(body: str) -> bool:
|
||||
"""Distinguish a good login from a rejected one.
|
||||
def _signed_in(body: str) -> bool:
|
||||
"""A signed-in page carries a Sign Out control; a signed-out one does not.
|
||||
|
||||
NOTE: provisional. Every /p/* path returns 200 even for nonsense URLs,
|
||||
so an HTTP status is not a signal here. Needs confirming against a real
|
||||
authenticated response before this integration can be trusted.
|
||||
HTTP status is useless here — the portal answers 200 for every /p/ path,
|
||||
including invented ones.
|
||||
"""
|
||||
lowered = body.lower()
|
||||
if "invalid" in lowered or "incorrect" in lowered:
|
||||
return False
|
||||
return "sign out" in lowered or "log out" in lowered
|
||||
return "sign out" in body.lower()
|
||||
|
||||
async def async_get_next_delivery(self) -> Delivery:
|
||||
"""Return the account's next scheduled delivery.
|
||||
|
||||
UNIMPLEMENTED. The portal serves a catch-all 200 for every /p/* path
|
||||
when signed out, so the real account pages could not be located or
|
||||
parsed without an authenticated session. Fill this in against a live
|
||||
session rather than guessing at selectors.
|
||||
"""
|
||||
raise FreshHarvestError(
|
||||
"delivery parsing is not implemented yet: the portal HTML has not "
|
||||
"been mapped against a signed-in session"
|
||||
)
|
||||
|
||||
async def async_fetch(self, path: str) -> str:
|
||||
"""Fetch a portal page, re-authenticating once if the session lapsed."""
|
||||
async def async_get_snapshot(self) -> AccountSnapshot:
|
||||
"""Fetch and parse the dashboard, re-authenticating once if needed."""
|
||||
if not self._authenticated:
|
||||
await self.async_login()
|
||||
body = await self._get(path)
|
||||
if not self._looks_authenticated(body):
|
||||
self._authenticated = False
|
||||
await self.async_login()
|
||||
body = await self._get(path)
|
||||
if not self._looks_authenticated(body):
|
||||
raise FreshHarvestAuthError(f"could not hold a session for {path}")
|
||||
return body
|
||||
|
||||
try:
|
||||
body = await self._get(DASHBOARD)
|
||||
if not self._signed_in(body):
|
||||
self._authenticated = False
|
||||
await self.async_login()
|
||||
body = await self._get(DASHBOARD)
|
||||
if not self._signed_in(body):
|
||||
raise FreshHarvestAuthError("could not hold a signed-in session")
|
||||
except aiohttp.ClientError as err:
|
||||
raise FreshHarvestError(f"dashboard request failed: {err}") from err
|
||||
|
||||
return parse_dashboard(body)
|
||||
|
||||
Reference in New Issue
Block a user