From c115eee74a13441185e6fc4f396bbab5c668dbe1 Mon Sep 17 00:00:00 2001
From: Nolan Kovacs <nolankovacs@gmail.com>
Date: Thu, 26 Mar 2026 14:28:38 -0400
Subject: [PATCH] Fix cache builder rate limiting: slower pass, sparse response
 guard, retry passes

---
 cache_builder.py | 94 +++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 89 insertions(+), 5 deletions(-)

diff --git a/cache_builder.py b/cache_builder.py
index 117b6ca..8cd231a 100644
--- a/cache_builder.py
+++ b/cache_builder.py
@@ -89,9 +89,9 @@ _TICKER_RE    = re.compile(r'^[A-Z]{1,5}(-[A-Z]{1,2})?$')
 _HTTP_HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)"}
 
 INFO_BATCH   = 20
-INFO_PAUSE   = 4.0
-MAX_WORKERS  = 5
-NEWS_WORKERS = 20
+INFO_PAUSE   = 15.0   # slower main pass — reduces Yahoo throttling
+MAX_WORKERS  = 2      # fewer concurrent workers
+NEWS_WORKERS = 10     # reduced from 20
 UPSERT_EVERY = 500
 
 # ---------------------------------------------------------------------------
@@ -257,7 +257,10 @@ def _fetch_analyst(ticker: str) -> dict | None:
     time.sleep(random.uniform(0.0, 1.0))
     try:
         info = yf.Ticker(ticker).info
-        if not info or not isinstance(info, dict) or len(info) < 3:
+        # A throttled Yahoo response returns a sparse dict (< ~30 keys).
+        # Reject it so the ticker is queued for the retry pass instead of
+        # being stored with all-NULL analyst fields.
+        if not info or not isinstance(info, dict) or len(info) < 30:
             return None
 
         try:
@@ -333,7 +336,53 @@ def run_analyst_phase(tickers: list[str]) -> None:
         print()
         _upsert(pending)
 
-    print(f"\n  Phase 1 complete in {time.time() - start:.0f}s\n")
+    print(f"\n  Phase 1 main pass complete in {time.time() - start:.0f}s\n")
+    _retry_analyst_pass(tickers)
+
+
+def _retry_analyst_pass(tickers: list[str]) -> None:
+    """
+    Single-threaded retry for tickers whose Phase 1 fetch returned a sparse
+    (throttled) response.  Queries DB for rows with NULL analyst_count and
+    re-fetches them one at a time with a 30-second delay between requests.
+    """
+    try:
+        conn = _get_conn()
+        with conn.cursor() as cur:
+            cur.execute(
+                f'SELECT ticker FROM "{_TABLE_PREFIX}analyst_cache" '
+                f'WHERE analyst_count IS NULL'
+            )
+            nulls = {r[0] for r in cur.fetchall()}
+        conn.close()
+    except Exception as e:
+        print(f"  [WARN] Retry query failed: {e}\n")
+        return
+
+    retry = [t for t in tickers if t in nulls]
+    if not retry:
+        print("  Retry pass: all tickers have analyst data.\n")
+        return
+
+    print(f"  Retry pass: {len(retry)} tickers need re-fetch (30s delay each) ...")
+    pending: list[dict] = []
+    recovered = 0
+    for i, ticker in enumerate(retry, 1):
+        print(f"\r    {i}/{len(retry)}  {ticker}   ", end="", flush=True)
+        result = _fetch_analyst(ticker)
+        if result and result.get("analyst_count") is not None:
+            pending.append(result)
+            recovered += 1
+        if len(pending) >= UPSERT_EVERY:
+            print()
+            _upsert(pending)
+            pending.clear()
+        time.sleep(30.0)
+
+    if pending:
+        print()
+        _upsert(pending)
+    print(f"\n  Retry pass complete — recovered {recovered}/{len(retry)} tickers\n")
 
 
 # ---------------------------------------------------------------------------
@@ -643,6 +692,7 @@ def run_fundamentals_phase(all_tickers: list[str]) -> dict[str, dict]:
     foreign_tickers = set(US_GAAP_ANNUAL_FILERS) | set(IFRS_ANNUAL_FILERS)
 
     result: dict[str, dict] = {}
+    price_failed: list[str] = []
     for ticker in all_tickers:
         if ticker in foreign_tickers:
             continue
@@ -657,6 +707,7 @@ def run_fundamentals_phase(all_tickers: list[str]) -> dict[str, dict]:
         except Exception:
             price = None
         if price is None:
+            price_failed.append(ticker)
             continue
 
         metrics = _compute_fundamentals(
@@ -676,6 +727,39 @@ def run_fundamentals_phase(all_tickers: list[str]) -> dict[str, dict]:
         result[ticker] = metrics
 
     print(f"  Domestic filers matched: {len(result)}\n")
+
+    # Retry tickers whose price fetch was empty (likely throttled).
+    if price_failed:
+        print(f"  Price retry: {len(price_failed)} tickers (10s delay each) ...")
+        retried = 0
+        for i, ticker in enumerate(price_failed, 1):
+            print(f"\r    {i}/{len(price_failed)}  {ticker}   ", end="", flush=True)
+            cik = ticker_to_cik.get(ticker)
+            try:
+                raw = yf.download(ticker, period="5d", auto_adjust=True, progress=False)
+                price = float(raw["Close"].dropna().iloc[-1]) if not raw.empty else None
+            except Exception:
+                price = None
+            if price is not None and cik is not None:
+                metrics = _compute_fundamentals(
+                    price,
+                    revenue.get(cik),    rev_prev.get(cik),
+                    net_income.get(cik), ni_prev.get(cik),
+                    op_income.get(cik),  dna.get(cik),
+                    op_cf.get(cik),      capex.get(cik),
+                    assets.get(cik),     equity.get(cik),
+                    cur_assets.get(cik), cur_liab.get(cik),
+                    cash.get(cik),       lt_debt.get(cik),
+                    shares.get(cik),
+                )
+                metrics["ticker"]     = ticker
+                metrics["filer_type"] = "us-gaap-quarterly"
+                metrics["updated_at"] = datetime.now(timezone.utc).isoformat()
+                result[ticker] = metrics
+                retried += 1
+            time.sleep(10.0)
+        print(f"\n  Price retry done — recovered {retried}/{len(price_failed)} tickers\n")
+
     return result