Switch FX rates to Frankfurter

agenda/fx.py

@@ -11,6 +11,7 @@ import httpx
 
 DEFAULT_FX_CACHE_TTL_HOURS = 24
 DEFAULT_FX_FAILURE_RETRY_HOURS = 24
+FRANKFURTER_CACHE_PREFIX = "frankfurter"
 
 
 async def get_gbpusd(config: flask.config.Config) -> Decimal:
@@ -57,13 +58,47 @@ def read_cached_rates(
     with open(filename) as file:
         data = json.load(file, parse_float=Decimal)
 
-    quotes = data.get("quotes")
-    if not isinstance(quotes, dict):
+    frankfurter_rates = _frankfurter_rates(data, currencies)
+    if frankfurter_rates:
+        return frankfurter_rates
+
+    if isinstance(data, list):
         return {}
 
-    return {
-        cur: Decimal(quotes[f"GBP{cur}"]) for cur in currencies if f"GBP{cur}" in quotes
-    }
+    if not isinstance(data, dict):
+        return {}
+
+    rates = data.get("rates")
+    if isinstance(rates, dict):
+        return {cur: Decimal(rates[cur]) for cur in currencies if cur in rates}
+
+    quotes = data.get("quotes")
+    if isinstance(quotes, dict):
+        return {
+            cur: Decimal(quotes[f"GBP{cur}"])
+            for cur in currencies
+            if f"GBP{cur}" in quotes
+        }
+
+    return {}
+
+
+def _frankfurter_rates(data: typing.Any, currencies: list[str]) -> dict[str, Decimal]:
+    """Extract rates from Frankfurter's response format."""
+    if not isinstance(data, list):
+        return {}
+
+    rates: dict[str, Decimal] = {}
+    for item in data:
+        if not isinstance(item, dict):
+            continue
+
+        quote = item.get("quote")
+        rate = item.get("rate")
+        if isinstance(quote, str) and quote in currencies and rate is not None:
+            rates[quote] = Decimal(rate)
+
+    return rates
 
 
 def _fx_cache_datetime(filename: str) -> datetime:
@@ -93,8 +128,12 @@ def _latest_file(
     return max(matching_files) if matching_files else None
 
 
-def get_rates(config: flask.config.Config) -> dict[str, Decimal]:
-    """Get current values of exchange rates for a list of currencies against GBP."""
+def get_rates_exchangerate_host(config: flask.config.Config) -> dict[str, Decimal]:
+    """Get exchange rates from exchangerate.host.
+
+    Kept as a fallback implementation in case we decide to switch back from
+    Frankfurter.
+    """
     currencies = config["CURRENCIES"]
     access_key = config["EXCHANGERATE_ACCESS_KEY"]
     data_dir = config["DATA_DIR"]
@@ -160,3 +199,82 @@ def get_rates(config: flask.config.Config) -> dict[str, Decimal]:
         for cur in currencies
         if f"GBP{cur}" in data["quotes"]
     }
+
+
+def get_rates(config: flask.config.Config) -> dict[str, Decimal]:
+    """Get current values of exchange rates for a list of currencies against GBP."""
+    currencies = config["CURRENCIES"]
+    data_dir = config["DATA_DIR"]
+    cache_ttl_hours = int(config.get("FX_CACHE_TTL_HOURS", DEFAULT_FX_CACHE_TTL_HOURS))
+    failure_retry_hours = int(
+        config.get("FX_FAILURE_RETRY_HOURS", DEFAULT_FX_FAILURE_RETRY_HOURS)
+    )
+
+    now = datetime.now()
+    now_str = now.strftime("%Y-%m-%d_%H:%M")
+    fx_dir = os.path.join(data_dir, "fx")
+    os.makedirs(fx_dir, exist_ok=True)  # Ensure the directory exists
+
+    currency_string = ",".join(sorted(currencies))
+    legacy_file_suffix = f"{currency_string}_to_GBP.json"
+    frankfurter_file_suffix = f"{FRANKFURTER_CACHE_PREFIX}_{legacy_file_suffix}"
+    existing_data = os.listdir(fx_dir)
+    valid_cache_files = [
+        f
+        for f in existing_data
+        if f.endswith(legacy_file_suffix) or f.endswith(frankfurter_file_suffix)
+    ]
+    attempt_files = [f for f in existing_data if f.endswith(frankfurter_file_suffix)]
+
+    latest_attempt = _latest_file(fx_dir, attempt_files, currencies, valid_only=False)
+    latest_source_valid = _latest_file(
+        fx_dir, attempt_files, currencies, valid_only=True
+    )
+    latest_valid = _latest_file(fx_dir, valid_cache_files, currencies, valid_only=True)
+    latest_valid_path = (
+        os.path.join(fx_dir, latest_valid) if latest_valid is not None else None
+    )
+
+    if config["OFFLINE_MODE"]:
+        return read_cached_rates(latest_valid_path, currencies)
+
+    if latest_source_valid is not None:
+        recent = _fx_cache_datetime(latest_source_valid)
+        delta = now - recent
+
+        if delta < timedelta(hours=cache_ttl_hours):
+            return read_cached_rates(
+                os.path.join(fx_dir, latest_source_valid), currencies
+            )
+
+    if latest_attempt is not None:
+        recent_attempt = _fx_cache_datetime(latest_attempt)
+        attempt_delta = now - recent_attempt
+        if attempt_delta < timedelta(hours=failure_retry_hours):
+            return read_cached_rates(latest_valid_path, currencies)
+
+    url = "https://api.frankfurter.dev/v2/rates"
+    params = {"base": "GBP", "quotes": currency_string}
+
+    filename = f"{now_str}_{frankfurter_file_suffix}"
+    try:
+        with httpx.Client() as client:
+            response = client.get(url, params=params, timeout=10)
+    except (httpx.ConnectError, httpx.ReadTimeout):
+        return read_cached_rates(latest_valid_path, currencies)
+
+    try:
+        data = json.loads(response.text, parse_float=Decimal)
+    except json.decoder.JSONDecodeError:
+        return read_cached_rates(latest_valid_path, currencies)
+
+    frankfurter_rates = _frankfurter_rates(data, currencies)
+    if not frankfurter_rates:
+        with open(os.path.join(fx_dir, filename), "w") as file:
+            file.write(response.text)
+        return read_cached_rates(latest_valid_path, currencies)
+
+    with open(os.path.join(fx_dir, filename), "w") as file:
+        file.write(response.text)
+
+    return frankfurter_rates