Fix hedge fund 13F HTML wrapper, field names; fix cache builder rate limiting

cache_builder.py

@@ -89,9 +89,9 @@ _TICKER_RE    = re.compile(r'^[A-Z]{1,5}(-[A-Z]{1,2})?$')
 _HTTP_HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)"}
 
 INFO_BATCH   = 20
-INFO_PAUSE   = 4.0
+INFO_PAUSE   = 15.0   # slower main pass — reduces Yahoo throttling
-MAX_WORKERS  = 5
+MAX_WORKERS  = 2      # fewer concurrent workers
-NEWS_WORKERS = 20
+NEWS_WORKERS = 10     # reduced from 20
 UPSERT_EVERY = 500
 
 # ---------------------------------------------------------------------------
@@ -257,7 +257,10 @@ def _fetch_analyst(ticker: str) -> dict | None:
     time.sleep(random.uniform(0.0, 1.0))
     try:
         info = yf.Ticker(ticker).info
-        if not info or not isinstance(info, dict) or len(info) < 3:
+        # A throttled Yahoo response returns a sparse dict (< ~30 keys).
+        # Reject it so the ticker is queued for the retry pass instead of
+        # being stored with all-NULL analyst fields.
+        if not info or not isinstance(info, dict) or len(info) < 30:
             return None
 
         try:
@@ -333,7 +336,53 @@ def run_analyst_phase(tickers: list[str]) -> None:
         print()
         _upsert(pending)
 
-    print(f"\n  Phase 1 complete in {time.time() - start:.0f}s\n")
+    print(f"\n  Phase 1 main pass complete in {time.time() - start:.0f}s\n")
+    _retry_analyst_pass(tickers)
+
+
+def _retry_analyst_pass(tickers: list[str]) -> None:
+    """
+    Single-threaded retry for tickers whose Phase 1 fetch returned a sparse
+    (throttled) response.  Queries DB for rows with NULL analyst_count and
+    re-fetches them one at a time with a 30-second delay between requests.
+    """
+    try:
+        conn = _get_conn()
+        with conn.cursor() as cur:
+            cur.execute(
+                f'SELECT ticker FROM "{_TABLE_PREFIX}analyst_cache" '
+                f'WHERE analyst_count IS NULL'
+            )
+            nulls = {r[0] for r in cur.fetchall()}
+        conn.close()
+    except Exception as e:
+        print(f"  [WARN] Retry query failed: {e}\n")
+        return
+
+    retry = [t for t in tickers if t in nulls]
+    if not retry:
+        print("  Retry pass: all tickers have analyst data.\n")
+        return
+
+    print(f"  Retry pass: {len(retry)} tickers need re-fetch (30s delay each) ...")
+    pending: list[dict] = []
+    recovered = 0
+    for i, ticker in enumerate(retry, 1):
+        print(f"\r    {i}/{len(retry)}  {ticker}   ", end="", flush=True)
+        result = _fetch_analyst(ticker)
+        if result and result.get("analyst_count") is not None:
+            pending.append(result)
+            recovered += 1
+        if len(pending) >= UPSERT_EVERY:
+            print()
+            _upsert(pending)
+            pending.clear()
+        time.sleep(30.0)
+
+    if pending:
+        print()
+        _upsert(pending)
+    print(f"\n  Retry pass complete — recovered {recovered}/{len(retry)} tickers\n")
 
 
 # ---------------------------------------------------------------------------
@@ -643,6 +692,7 @@ def run_fundamentals_phase(all_tickers: list[str]) -> dict[str, dict]:
     foreign_tickers = set(US_GAAP_ANNUAL_FILERS) | set(IFRS_ANNUAL_FILERS)
 
     result: dict[str, dict] = {}
+    price_failed: list[str] = []
     for ticker in all_tickers:
         if ticker in foreign_tickers:
             continue
@@ -657,6 +707,7 @@ def run_fundamentals_phase(all_tickers: list[str]) -> dict[str, dict]:
         except Exception:
             price = None
         if price is None:
+            price_failed.append(ticker)
             continue
 
         metrics = _compute_fundamentals(
@@ -676,6 +727,39 @@ def run_fundamentals_phase(all_tickers: list[str]) -> dict[str, dict]:
         result[ticker] = metrics
 
     print(f"  Domestic filers matched: {len(result)}\n")
+
+    # Retry tickers whose price fetch was empty (likely throttled).
+    if price_failed:
+        print(f"  Price retry: {len(price_failed)} tickers (10s delay each) ...")
+        retried = 0
+        for i, ticker in enumerate(price_failed, 1):
+            print(f"\r    {i}/{len(price_failed)}  {ticker}   ", end="", flush=True)
+            cik = ticker_to_cik.get(ticker)
+            try:
+                raw = yf.download(ticker, period="5d", auto_adjust=True, progress=False)
+                price = float(raw["Close"].dropna().iloc[-1]) if not raw.empty else None
+            except Exception:
+                price = None
+            if price is not None and cik is not None:
+                metrics = _compute_fundamentals(
+                    price,
+                    revenue.get(cik),    rev_prev.get(cik),
+                    net_income.get(cik), ni_prev.get(cik),
+                    op_income.get(cik),  dna.get(cik),
+                    op_cf.get(cik),      capex.get(cik),
+                    assets.get(cik),     equity.get(cik),
+                    cur_assets.get(cik), cur_liab.get(cik),
+                    cash.get(cik),       lt_debt.get(cik),
+                    shares.get(cik),
+                )
+                metrics["ticker"]     = ticker
+                metrics["filer_type"] = "us-gaap-quarterly"
+                metrics["updated_at"] = datetime.now(timezone.utc).isoformat()
+                result[ticker] = metrics
+                retried += 1
+            time.sleep(10.0)
+        print(f"\n  Price retry done — recovered {retried}/{len(price_failed)} tickers\n")
+
     return result
 
 

dist/stock_screener.py

@@ -2465,6 +2465,14 @@ def _parse_13f_xml(xml_bytes: bytes) -> list[dict]:
       company, class_, cusip, value (USD), shares, shr_type, put_call
     """
     import xml.etree.ElementTree as ET
+    import re as _re
+    # SEC serves 13F XML wrapped in an HTML document (Content-Type: text/html).
+    # Strip the HTML wrapper and extract just the <informationTable> block.
+    if xml_bytes[:500].upper().find(b'<!DOCTYPE HTML') != -1:
+        m = _re.search(rb'(<informationTable[\s\S]*?</informationTable>)',
+                       xml_bytes, _re.IGNORECASE)
+        if m:
+            xml_bytes = m.group(1)
     try:
         root = ET.fromstring(xml_bytes)
     except ET.ParseError:
@@ -2607,9 +2615,10 @@ def fetch_hedge_fund_filings(
             src       = hit.get("_source", {})
             acc_no    = src.get("adsh", "")
             ciks      = src.get("ciks", [])
-            fund_name = src.get("entity_name", "Unknown Fund")
+            display    = src.get("display_names") or src.get("entity_name") or []
-            filed     = (src.get("file_date") or "")[:10]
+            fund_name  = display[0].split("  (CIK")[0].strip() if display else "Unknown Fund"
-            period    = (src.get("period_of_report") or "")[:10]
+            filed      = (src.get("file_date") or "")[:10]
+            period     = (src.get("period_ending") or src.get("period_of_report") or "")[:10]
 
             if not acc_no or not ciks:
                 return []