Răsfoiți Sursa

Parallelize service API month fetches and deduplicate concurrent requests

Both _AirbnbClient and _BookingClient now gather month fetches in
parallel instead of awaiting them sequentially, and share a per-month
cache with in-flight deduplication. Concurrent callers (e.g. _diff_prices
and _diff_closed both calling get_calendar at the same time) now share a
single HTTP request per month rather than racing to fetch the same data.

_AirbnbClient.get_reservations now uses the same raw month cache as
get_calendar, so months fetched during a concurrent get_calendar call
are reused instead of re-fetched.

Update test_diff_returns_complete_structure to allow extra keys in the
diff result dict (accommodates _timings added in the previous commit).
ZiB 4 luni în urmă
părinte
comite
80bc42450a

+ 26 - 4
room_cal/service_api/_airbnb.py

@@ -79,6 +79,8 @@ class _AirbnbClient:
     self._tz = tz
     self._cache: dict[str, DayInfo] = {}
     self._fetched_months: set[tuple[int, int]] = set()
+    self._raw_months: dict[tuple[int, int], list[dict]] = {}
+    self._months_inflight: dict[tuple[int, int], asyncio.Task] = {}
     self._refresh_lock = asyncio.Lock()
     self._refresh_task: asyncio.Task | None = None
 
@@ -97,6 +99,7 @@ class _AirbnbClient:
   def clear_cache(self) -> None:
     self._cache.clear()
     self._fetched_months.clear()
+    self._raw_months.clear()
 
   def _reload_from_env(self) -> None:
     env_file = data_home() / 'airbnb.env'
@@ -231,12 +234,29 @@ class _AirbnbClient:
     ) or {}
     return section.get('days', [])
 
+  async def _get_month_days(self, month: int, year: int) -> list[dict]:
+    key = (month, year)
+    if key in self._raw_months:
+      return self._raw_months[key]
+    if key in self._months_inflight:
+      return await self._months_inflight[key]
+    task = asyncio.create_task(self._fetch_month_days(month, year))
+    self._months_inflight[key] = task
+    try:
+      result = await task
+    finally:
+      self._months_inflight.pop(key, None)
+    self._raw_months[key] = result
+    return result
+
   async def get_calendar(self, date_from: str, date_to: str) -> dict[str, DayInfo]:
     if is_stale('airbnb'):
       raise AirbnbAuthError(f'Airbnb credentials are stale — run: {_AUTH_HINT}')
-    for month, year in months_in_range(date_from, date_to):
+    month_list = months_in_range(date_from, date_to)
+    all_days = await asyncio.gather(*[self._get_month_days(m, y) for m, y in month_list])
+    for (month, year), days in zip(month_list, all_days):
       if (month, year) not in self._fetched_months:
-        for day in await self._fetch_month_days(month, year):
+        for day in days:
           d = day.get('day', '')
           if d:
             self._cache[d] = DayInfo(
@@ -252,9 +272,11 @@ class _AirbnbClient:
   async def get_reservations(self, date_from: str, date_to: str) -> list[Reservation]:
     if is_stale('airbnb'):
       raise AirbnbAuthError(f'Airbnb credentials are stale — run: {_AUTH_HINT}')
+    month_list = months_in_range(date_from, date_to)
+    all_days = await asyncio.gather(*[self._get_month_days(m, y) for m, y in month_list])
     booked: dict[str, str] = {}
-    for month, year in months_in_range(date_from, date_to):
-      for day in await self._fetch_month_days(month, year):
+    for days in all_days:
+      for day in days:
         d = day.get('day', '')
         res_data = day.get('reservationData')
         if res_data and date_from <= d <= date_to:

+ 22 - 4
room_cal/service_api/_booking.py

@@ -167,6 +167,8 @@ class _BookingClient:
     self._lang = lang
     self._cache: dict[str, DayInfo] = {}
     self._fetched_months: set[tuple[int, int]] = set()
+    self._month_data: dict[tuple[int, int], dict[str, DayInfo]] = {}
+    self._months_inflight: dict[tuple[int, int], asyncio.Task] = {}
     self._refresh_lock = asyncio.Lock()
     self._refresh_task: asyncio.Task | None = None
 
@@ -195,6 +197,7 @@ class _BookingClient:
   def clear_cache(self) -> None:
     self._cache.clear()
     self._fetched_months.clear()
+    self._month_data.clear()
 
   def _reload_from_env(self) -> None:
     env_file = data_home() / 'booking.env'
@@ -500,13 +503,28 @@ class _BookingClient:
     )
     return self._parse_calendar_days(raw)
 
+  async def _get_month_calendar(self, month: int, year: int) -> dict[str, DayInfo]:
+    key = (month, year)
+    if key in self._month_data:
+      return self._month_data[key]
+    if key in self._months_inflight:
+      return await self._months_inflight[key]
+    task = asyncio.create_task(self._fetch_month_calendar(month, year))
+    self._months_inflight[key] = task
+    try:
+      result = await task
+    finally:
+      self._months_inflight.pop(key, None)
+    self._month_data[key] = result
+    return result
+
   async def get_calendar(self, date_from: str, date_to: str) -> dict[str, DayInfo]:
     if is_stale('booking'):
       raise BookingAuthError(f'Booking.com credentials are stale — run: {_AUTH_HINT}')
-    for month, year in months_in_range(date_from, date_to):
-      if (month, year) not in self._fetched_months:
-        self._cache.update(await self._fetch_month_calendar(month, year))
-        self._fetched_months.add((month, year))
+    month_list = months_in_range(date_from, date_to)
+    all_results = await asyncio.gather(*[self._get_month_calendar(m, y) for m, y in month_list])
+    for month_result in all_results:
+      self._cache.update(month_result)
     return {d: info for d, info in self._cache.items() if date_from <= d <= date_to}
 
   async def get_reservations(self, date_from: str, date_to: str) -> list[Reservation]:

+ 1 - 1
tests/test_action_chains.py

@@ -293,7 +293,7 @@ async def test_diff_returns_complete_structure():
 
   result = await diff(start, end, api=api)
 
-  assert set(result) == {'prices', 'closed', 'reservations', 'user_cal_closed', 'ical_feeds_closed', 'rule_discounts', 'rule_closed', 'rule_prices', 'orphans'}
+  assert {'prices', 'closed', 'reservations', 'user_cal_closed', 'ical_feeds_closed', 'rule_discounts', 'rule_closed', 'rule_prices', 'orphans'} <= set(result)
 
   assert isinstance(result['prices'], list)
   assert isinstance(result['closed'], list)