cache_builder.py

@@ -89,9 +89,9 @@ _TICKER_RE    = re.compile(r'^[A-Z]{1,5}(-[A-Z]{1,2})?$')
 _HTTP_HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)"}
 
 INFO_BATCH   = 20
-INFO_PAUSE   = 15.0   # slower main pass — reduces Yahoo throttling
+INFO_PAUSE   = 4.0
-MAX_WORKERS  = 2      # fewer concurrent workers
+MAX_WORKERS  = 5
-NEWS_WORKERS = 10     # reduced from 20
+NEWS_WORKERS = 20
 UPSERT_EVERY = 500
 
 # ---------------------------------------------------------------------------
@@ -257,10 +257,7 @@ def _fetch_analyst(ticker: str) -> dict | None:
     time.sleep(random.uniform(0.0, 1.0))
     try:
         info = yf.Ticker(ticker).info
-        # A throttled Yahoo response returns a sparse dict (< ~30 keys).
+        if not info or not isinstance(info, dict) or len(info) < 3:
-        # Reject it so the ticker is queued for the retry pass instead of
-        # being stored with all-NULL analyst fields.
-        if not info or not isinstance(info, dict) or len(info) < 30:
             return None
 
         try:
@@ -336,53 +333,7 @@ def run_analyst_phase(tickers: list[str]) -> None:
         print()
         _upsert(pending)
 
-    print(f"\n  Phase 1 main pass complete in {time.time() - start:.0f}s\n")
+    print(f"\n  Phase 1 complete in {time.time() - start:.0f}s\n")
-    _retry_analyst_pass(tickers)
-
-
-def _retry_analyst_pass(tickers: list[str]) -> None:
-    """
-    Single-threaded retry for tickers whose Phase 1 fetch returned a sparse
-    (throttled) response.  Queries DB for rows with NULL analyst_count and
-    re-fetches them one at a time with a 30-second delay between requests.
-    """
-    try:
-        conn = _get_conn()
-        with conn.cursor() as cur:
-            cur.execute(
-                f'SELECT ticker FROM "{_TABLE_PREFIX}analyst_cache" '
-                f'WHERE analyst_count IS NULL'
-            )
-            nulls = {r[0] for r in cur.fetchall()}
-        conn.close()
-    except Exception as e:
-        print(f"  [WARN] Retry query failed: {e}\n")
-        return
-
-    retry = [t for t in tickers if t in nulls]
-    if not retry:
-        print("  Retry pass: all tickers have analyst data.\n")
-        return
-
-    print(f"  Retry pass: {len(retry)} tickers need re-fetch (30s delay each) ...")
-    pending: list[dict] = []
-    recovered = 0
-    for i, ticker in enumerate(retry, 1):
-        print(f"\r    {i}/{len(retry)}  {ticker}   ", end="", flush=True)
-        result = _fetch_analyst(ticker)
-        if result and result.get("analyst_count") is not None:
-            pending.append(result)
-            recovered += 1
-        if len(pending) >= UPSERT_EVERY:
-            print()
-            _upsert(pending)
-            pending.clear()
-        time.sleep(30.0)
-
-    if pending:
-        print()
-        _upsert(pending)
-    print(f"\n  Retry pass complete — recovered {recovered}/{len(retry)} tickers\n")
 
 
 # ---------------------------------------------------------------------------
@@ -692,7 +643,6 @@ def run_fundamentals_phase(all_tickers: list[str]) -> dict[str, dict]:
     foreign_tickers = set(US_GAAP_ANNUAL_FILERS) | set(IFRS_ANNUAL_FILERS)
 
     result: dict[str, dict] = {}
-    price_failed: list[str] = []
     for ticker in all_tickers:
         if ticker in foreign_tickers:
             continue
@@ -707,7 +657,6 @@ def run_fundamentals_phase(all_tickers: list[str]) -> dict[str, dict]:
         except Exception:
             price = None
         if price is None:
-            price_failed.append(ticker)
             continue
 
         metrics = _compute_fundamentals(
@@ -727,39 +676,6 @@ def run_fundamentals_phase(all_tickers: list[str]) -> dict[str, dict]:
         result[ticker] = metrics
 
     print(f"  Domestic filers matched: {len(result)}\n")
-
-    # Retry tickers whose price fetch was empty (likely throttled).
-    if price_failed:
-        print(f"  Price retry: {len(price_failed)} tickers (10s delay each) ...")
-        retried = 0
-        for i, ticker in enumerate(price_failed, 1):
-            print(f"\r    {i}/{len(price_failed)}  {ticker}   ", end="", flush=True)
-            cik = ticker_to_cik.get(ticker)
-            try:
-                raw = yf.download(ticker, period="5d", auto_adjust=True, progress=False)
-                price = float(raw["Close"].dropna().iloc[-1]) if not raw.empty else None
-            except Exception:
-                price = None
-            if price is not None and cik is not None:
-                metrics = _compute_fundamentals(
-                    price,
-                    revenue.get(cik),    rev_prev.get(cik),
-                    net_income.get(cik), ni_prev.get(cik),
-                    op_income.get(cik),  dna.get(cik),
-                    op_cf.get(cik),      capex.get(cik),
-                    assets.get(cik),     equity.get(cik),
-                    cur_assets.get(cik), cur_liab.get(cik),
-                    cash.get(cik),       lt_debt.get(cik),
-                    shares.get(cik),
-                )
-                metrics["ticker"]     = ticker
-                metrics["filer_type"] = "us-gaap-quarterly"
-                metrics["updated_at"] = datetime.now(timezone.utc).isoformat()
-                result[ticker] = metrics
-                retried += 1
-            time.sleep(10.0)
-        print(f"\n  Price retry done — recovered {retried}/{len(price_failed)} tickers\n")
-
     return result
 
 

dist/stock_screener.py

@@ -2465,14 +2465,6 @@ def _parse_13f_xml(xml_bytes: bytes) -> list[dict]:
       company, class_, cusip, value (USD), shares, shr_type, put_call
     """
     import xml.etree.ElementTree as ET
-    import re as _re
-    # SEC serves 13F XML wrapped in an HTML document (Content-Type: text/html).
-    # Strip the HTML wrapper and extract just the <informationTable> block.
-    if xml_bytes[:500].upper().find(b'<!DOCTYPE HTML') != -1:
-        m = _re.search(rb'(<informationTable[\s\S]*?</informationTable>)',
-                       xml_bytes, _re.IGNORECASE)
-        if m:
-            xml_bytes = m.group(1)
     try:
         root = ET.fromstring(xml_bytes)
     except ET.ParseError:
@@ -2615,10 +2607,9 @@ def fetch_hedge_fund_filings(
             src       = hit.get("_source", {})
             acc_no    = src.get("adsh", "")
             ciks      = src.get("ciks", [])
-            display    = src.get("display_names") or src.get("entity_name") or []
+            fund_name = src.get("entity_name", "Unknown Fund")
-            fund_name  = display[0].split("  (CIK")[0].strip() if display else "Unknown Fund"
+            filed     = (src.get("file_date") or "")[:10]
-            filed      = (src.get("file_date") or "")[:10]
+            period    = (src.get("period_of_report") or "")[:10]
-            period     = (src.get("period_ending") or src.get("period_of_report") or "")[:10]
 
             if not acc_no or not ciks:
                 return []