From 92c1750e1bf17a61c419069fe3f1047c28adfb76 Mon Sep 17 00:00:00 2001 From: Nolan Kovacs Date: Wed, 25 Mar 2026 20:38:32 -0400 Subject: [PATCH] Migrate app from Supabase/GitHub to Verimund API --- api/main.py | 7 + dist/stock_screener.py | 1039 +++++++++++++++++++++++++++++++--------- license_check.py | 119 +++-- updater.py | 219 +++------ 4 files changed, 975 insertions(+), 409 deletions(-) diff --git a/api/main.py b/api/main.py index 130c3e3..148b219 100644 --- a/api/main.py +++ b/api/main.py @@ -116,6 +116,13 @@ def get_fundamentals_cache(payload=Depends(verify_jwt), db=Depends(get_db)): return {"data": cur.fetchall()} +@app.get("/cache/sector") +def get_sector_stats(payload=Depends(verify_jwt), db=Depends(get_db)): + cur = db.cursor(cursor_factory=psycopg2.extras.RealDictCursor) + cur.execute("SELECT * FROM sector_stats ORDER BY sector") + return {"data": cur.fetchall()} + + @app.get("/update") def check_update(payload=Depends(verify_jwt), db=Depends(get_db)): cur = db.cursor(cursor_factory=psycopg2.extras.RealDictCursor) diff --git a/dist/stock_screener.py b/dist/stock_screener.py index 020a4dd..adadcab 100644 --- a/dist/stock_screener.py +++ b/dist/stock_screener.py @@ -59,41 +59,41 @@ except ImportError: WEIGHTS = { "value": 0.18, - "growth": 0.18, - "momentum": 0.14, - "quality": 0.11, - "profitability": 0.08, - "sentiment": 0.13, + "growth": 0.16, + "momentum": 0.13, + "quality": 0.14, + "profitability": 0.13, + "sentiment": 0.10, "analyst": 0.10, - "risk": 0.08, + "risk": 0.06, } # Human-readable investment strategy presets. # Each preset is a complete weight dict that must sum to 1.0. STRATEGY_PRESETS = { "Balanced (Default)": { - "value": 0.18, "growth": 0.18, "momentum": 0.14, "quality": 0.11, - "profitability": 0.08, "sentiment": 0.13, "analyst": 0.10, "risk": 0.08, + "value": 0.18, "growth": 0.16, "momentum": 0.13, "quality": 0.14, + "profitability": 0.13, "sentiment": 0.10, "analyst": 0.10, "risk": 0.06, }, "High Growth Companies": { "value": 0.08, "growth": 0.28, "momentum": 0.25, "quality": 0.08, "profitability": 0.08, "sentiment": 0.12, "analyst": 0.07, "risk": 0.04, }, "Conservative": { - "value": 0.20, "growth": 0.12, "momentum": 0.06, "quality": 0.25, - "profitability": 0.18, "sentiment": 0.06, "analyst": 0.08, "risk": 0.05, + "value": 0.20, "growth": 0.12, "momentum": 0.06, "quality": 0.27, + "profitability": 0.20, "sentiment": 0.04, "analyst": 0.08, "risk": 0.03, }, "Recent Uptrends": { "value": 0.05, "growth": 0.12, "momentum": 0.40, "quality": 0.05, "profitability": 0.05, "sentiment": 0.18, "analyst": 0.10, "risk": 0.05, }, "Buy and Hold Long Term": { - "value": 0.30, "growth": 0.22, "momentum": 0.05, "quality": 0.20, - "profitability": 0.15, "sentiment": 0.03, "analyst": 0.05, "risk": 0.00, + "value": 0.30, "growth": 0.19, "momentum": 0.05, "quality": 0.20, + "profitability": 0.18, "sentiment": 0.03, "analyst": 0.05, "risk": 0.00, }, "Low Risk, High Dividend Yield": { - "value": 0.22, "growth": 0.10, "momentum": 0.05, "quality": 0.22, - "profitability": 0.28, "sentiment": 0.05, "analyst": 0.05, "risk": 0.03, + "value": 0.22, "growth": 0.08, "momentum": 0.05, "quality": 0.22, + "profitability": 0.30, "sentiment": 0.05, "analyst": 0.05, "risk": 0.03, }, "High Risk High Reward": { "value": 0.10, "growth": 0.00, "momentum": 0.00, "quality": 0.15, @@ -441,7 +441,6 @@ def get_djia_tickers() -> list[str]: # Human-readable index names exposed to the GUI INDEX_CHOICES = [ "All", - "All + OTC", "S&P 500", "S&P 400", "S&P 600", "S&P 1500", "Russell 1000", "Russell 2000", "Russell 3000", "NASDAQ 100", "NASDAQ", "NYSE", "AMEX", @@ -506,14 +505,6 @@ def collect_tickers(index: str = "All") -> list[str]: if index == "All": tickers = _all_exchange_tickers() - elif index == "All + OTC": - tickers = _all_exchange_tickers() - otc = _fetch_otc_tickers() - if otc: - tickers = tickers + otc - else: - print(" [WARN] OTC Markets unavailable — showing exchange-listed only") - elif index == "S&P 500": tickers = _fetch_sp500() @@ -835,23 +826,36 @@ def _best_frame(concepts: list[str], unit: str, return best -def _build_cik_maps() -> tuple[dict[int, str], dict[int, str]]: +def _build_cik_maps() -> tuple[dict[int, str], dict[int, str], dict[str, int]]: """ - Fetch SEC company_tickers.json and return: - cik_to_ticker — {cik_int: "AAPL"} + Fetch SEC company_tickers.json and return three dicts: + cik_to_ticker — {cik_int: "AAPL"} (one representative ticker per CIK) cik_to_name — {cik_int: "Apple Inc."} + ticker_to_cik — {ticker: cik_int} (ALL tickers, handles dual-class shares) + + ticker_to_cik covers every row in the SEC file so dual-class stocks + (GOOGL/GOOG, BRK-A/BRK-B, etc.) each map to the same CIK and therefore + both receive EDGAR fundamental data. """ try: resp = requests.get(_EDGAR_TICKERS, headers=_EDGAR_UA, timeout=20) resp.raise_for_status() data = resp.json() - return ( - {int(v["cik_str"]): v["ticker"].upper() for v in data.values()}, - {int(v["cik_str"]): v["title"] for v in data.values()}, - ) + cik_to_ticker: dict[int, str] = {} + cik_to_name: dict[int, str] = {} + ticker_to_cik: dict[str, int] = {} + for v in data.values(): + cik = int(v["cik_str"]) + ticker = v["ticker"].upper() + title = v.get("title", "") + ticker_to_cik[ticker] = cik # every ticker gets its own entry + if cik not in cik_to_ticker: + cik_to_ticker[cik] = ticker # first seen wins for repr + cik_to_name[cik] = title + return cik_to_ticker, cik_to_name, ticker_to_cik except Exception as e: print(f" [WARN] CIK map unavailable: {e}") - return {}, {} + return {}, {}, {} def _fetch_edgar_bulk(close_map: dict[str, pd.Series]) -> dict[str, dict]: @@ -867,8 +871,8 @@ def _fetch_edgar_bulk(close_map: dict[str, pd.Series]) -> dict[str, dict]: profit_margin, operating_margin, fcf_yield, name """ print("Phase 2A: Fetching EDGAR bulk fundamentals ...") - cik_to_ticker, cik_to_name = _build_cik_maps() - if not cik_to_ticker: + cik_to_ticker, cik_to_name, ticker_to_cik = _build_cik_maps() + if not ticker_to_cik: print(" [WARN] CIK map unavailable — EDGAR bulk skipped") return {} @@ -884,13 +888,16 @@ def _fetch_edgar_bulk(close_map: dict[str, pd.Series]) -> dict[str, dict]: assets = _best_frame(["Assets"], "USD", bs_periods) equity = _best_frame(["StockholdersEquity", "StockholdersEquityIncludingPortionAttributable" - "ToNoncontrollingInterest"], "USD", bs_periods) + "ToNoncontrollingInterest", + "CommonStockholdersEquity"], "USD", bs_periods) cur_assets = _best_frame(["AssetsCurrent"], "USD", bs_periods) cur_liab = _best_frame(["LiabilitiesCurrent"], "USD", bs_periods) cash = _best_frame(["CashAndCashEquivalentsAtCarryingValue", "CashCashEquivalentsAndShortTermInvestments"], "USD", bs_periods) lt_debt = _best_frame(["LongTermDebt", "LongTermDebtNoncurrent", - "LongTermDebtAndCapitalLeaseObligations"], "USD", bs_periods) + "LongTermDebtAndCapitalLeaseObligations", + "LongTermDebtAndFinanceLeaseLiabilities", + "FinanceLeaseLiabilityNoncurrent"], "USD", bs_periods) shares = _best_frame(["CommonStockSharesOutstanding"], "shares", bs_periods) # ── Income statement TTM ──────────────────────────────────────────────── @@ -917,14 +924,14 @@ def _fetch_edgar_bulk(close_map: dict[str, pd.Series]) -> dict[str, dict]: print(f" Revenue data : {len(revenue)} companies") # ── Assemble per-ticker records ───────────────────────────────────────── - valid_set = set(close_map.keys()) result: dict[str, dict] = {} - for cik, ticker in cik_to_ticker.items(): - if ticker not in valid_set: - continue + for ticker, close in close_map.items(): + cik = ticker_to_cik.get(ticker) + if cik is None: + continue # no SEC filing found for this ticker - price = float(close_map[ticker].iloc[-1]) + price = float(close.iloc[-1]) rev = revenue.get(cik); ni = net_income.get(cik) op_i = op_income.get(cik); d_a = dna.get(cik) @@ -942,6 +949,7 @@ def _fetch_edgar_bulk(close_map: dict[str, pd.Series]) -> dict[str, dict]: ebitda = ((op_i + d_a) if op_i is not None and d_a is not None else op_i) ev = ((mkt_cap + (ltd or 0) - (csh or 0)) if mkt_cap is not None else None) ev_ebitda = (ev / ebitda) if ev and ebitda and ebitda > 0 else None + ev_rev = (ev / rev) if ev and rev and rev > 0 else None roe = (ni / eq) if ni is not None and eq and eq > 0 else None roa = (ni / tot_a) if ni is not None and tot_a and tot_a > 0 else None de = (ltd / eq) if ltd is not None and eq and eq > 0 else None @@ -954,27 +962,29 @@ def _fetch_edgar_bulk(close_map: dict[str, pd.Series]) -> dict[str, dict]: ni_g = ((ni - n_p) / abs(n_p) if ni is not None and n_p and n_p != 0 else None) result[ticker] = { - "name": cik_to_name.get(cik, ticker), - "sector": "N/A", - "industry": "N/A", - "market_cap": mkt_cap, - "pe_trailing": pe_trail, - "pb_ratio": pb, - "ev_ebitda": ev_ebitda, - "eps_trailing": eps_trail, - "revenue": rev, - "revenue_growth": rev_g, - "earnings_growth": ni_g, - "roe": roe, - "roa": roa, - "debt_to_equity": de, - "total_debt": ltd, - "total_cash": csh, - "book_value": eq, - "current_ratio": curr_r, - "profit_margin": pm, - "operating_margin": om, - "fcf_yield": fcf_yield, + "name": cik_to_name.get(cik, ticker), + "sector": "N/A", + "industry": "N/A", + "market_cap": mkt_cap, + "shares_outstanding": sh, + "pe_trailing": pe_trail, + "pb_ratio": pb, + "ev_ebitda": ev_ebitda, + "ev_revenue": ev_rev, + "eps_trailing": eps_trail, + "revenue": rev, + "revenue_growth": rev_g, + "earnings_growth": ni_g, + "roe": roe, + "roa": roa, + "debt_to_equity": de, + "total_debt": ltd, + "total_cash": csh, + "book_value": (eq / sh) if eq and sh and sh > 0 else None, + "current_ratio": curr_r, + "profit_margin": pm, + "operating_margin": om, + "fcf_yield": fcf_yield, } print(f" EDGAR matched : {len(result)} tickers\n") @@ -982,36 +992,34 @@ def _fetch_edgar_bulk(close_map: dict[str, pd.Series]) -> dict[str, dict]: # --------------------------------------------------------------------------- -# Analyst + News Cache (Supabase) +# Analyst + Fundamentals Cache (Verimund API) # --------------------------------------------------------------------------- try: - from license_check import _SUPABASE_URL as _SB_URL, _SUPABASE_ANON as _SB_ANON + from license_check import _API_URL as _API_URL, get_api_headers as _get_api_headers except ImportError: - _SB_URL = _SB_ANON = None + _API_URL = None + _get_api_headers = lambda: {} def _load_analyst_cache() -> dict[str, dict]: """ - Load analyst consensus and news sentiment from the Supabase cache table. + Load analyst consensus and news sentiment from the Verimund API cache. Returns {ticker: {field: value}} or {} on any failure. Warns (but still proceeds) if cache data is more than 48 hours old. """ - if not _SB_URL or not _SB_ANON: - print(" [WARN] Supabase config unavailable — analyst cache skipped") + if not _API_URL: + print(" [WARN] API config unavailable — analyst cache skipped") return {} try: resp = requests.get( - f"{_SB_URL}/rest/v1/analyst_cache" - "?select=ticker,pe_forward,eps_forward,analyst_norm,analyst_upside," - "analyst_count,analyst_target,recommendation,news_sentiment," - "sector,industry,updated_at" - "&limit=20000", - headers={"apikey": _SB_ANON, "Content-Type": "application/json"}, - timeout=20, + f"{_API_URL}/cache/analyst", + headers=_get_api_headers(), + timeout=30, ) resp.raise_for_status() - rows = resp.json() + rows = resp.json().get("data", []) + if not rows: print(" [WARN] Analyst cache is empty — analyst scores will be neutral") return {} @@ -1035,6 +1043,46 @@ def _load_analyst_cache() -> dict[str, dict]: return {} +def _load_fundamentals_cache() -> dict[str, dict]: + """ + Load pre-computed fundamental metrics from the Verimund API cache. + Returns {ticker: {field: value}} or {} on any failure. + """ + if not _API_URL: + print(" [WARN] API config unavailable — fundamentals cache skipped") + return {} + try: + resp = requests.get( + f"{_API_URL}/cache/fundamentals", + headers=_get_api_headers(), + timeout=30, + ) + resp.raise_for_status() + rows = resp.json().get("data", []) + + if not rows: + print(" [WARN] Fundamentals cache is empty — falling back to live EDGAR") + return {} + + valid_ts = [r["updated_at"] for r in rows if r.get("updated_at")] + if valid_ts: + age_h = ( + datetime.now(timezone.utc) + - datetime.fromisoformat(max(valid_ts).replace("Z", "+00:00")) + ).total_seconds() / 3600 + if age_h > 48: + print(f" [WARN] Fundamentals cache is {age_h:.0f}h old " + "(refresh runs tonight)") + else: + print(f" Fundamentals cache: {len(rows)} tickers " + f"(updated {age_h:.1f}h ago)") + + return {r["ticker"]: r for r in rows} + except Exception as e: + print(f" [WARN] Could not load fundamentals cache: {e}") + return {} + + # --------------------------------------------------------------------------- # FINRA Short Interest # --------------------------------------------------------------------------- @@ -1399,16 +1447,29 @@ def fetch_all( return pd.DataFrame() print(f" Fallback found {len(valid)} tickers with history.\n") - # ── Phase 2A: EDGAR bulk fundamentals ───────────────────────────────── - edgar_data = _fetch_edgar_bulk(close_map) + # ── Phase 2A: Fundamentals from Supabase cache (nightly EDGAR+FINRA) ── + print("Phase 2A: Loading fundamentals cache ...") + edgar_data = _load_fundamentals_cache() + if not edgar_data: + # Cache empty (first run before nightly job) — fall back to live EDGAR + print(" Falling back to live EDGAR bulk fetch ...") + edgar_data = _fetch_edgar_bulk(close_map) # ── Phase 2B: Analyst + news cache from Supabase ────────────────────── print("Phase 2B: Loading analyst cache ...") analyst_cache = _load_analyst_cache() # ── Phase 2C: FINRA short-volume ratios ─────────────────────────────── - print("Phase 2C: Fetching FINRA short interest ...") - finra_short = _fetch_finra_short() + # Primary: pull from fundamentals_cache (populated by nightly FINRA fetch). + # Fallback: live FINRA fetch when cache coverage < 60% of screened tickers. + # The two sources are always merged so no ticker misses due to cache gaps. + finra_short: dict[str, float] = { + t: d["short_percent"] + for t, d in edgar_data.items() + if d.get("short_percent") is not None + } + coverage = len(finra_short) / max(len(valid), 1) + print(f"Phase 2C: FINRA short interest loaded from cache ({len(finra_short)} tickers, {coverage:.0%} coverage)") # ── Phase 2D: Beta vs SPY ───────────────────────────────────────────── print("Phase 2D: Calculating beta ...") @@ -1447,7 +1508,15 @@ def fetch_all( volatility = (float(returns_30d.std() * (252 ** 0.5)) if len(returns_30d) >= 5 else None) - mkt_cap = edgar.get("market_cap") + # Live market cap: price_now × shares_outstanding beats stale cached value + sh_out = edgar.get("shares_outstanding") + mkt_cap = (price_now * sh_out) if sh_out else edgar.get("market_cap") + + # Track data source for N/A display in GUI + _data_source = "cache" if edgar else "none" + + # EV/Revenue for growth/early-stage value scoring + ev_rev = edgar.get("ev_revenue") # ── Filters ─────────────────────────────────────────────────────── if sector_filter is not None: @@ -1479,11 +1548,13 @@ def fetch_all( "sector": analyst.get("sector") or edgar.get("sector", "N/A"), "industry": analyst.get("industry") or edgar.get("industry", "N/A"), "price": price_now, + "_data_source": _data_source, # Valuation "pe_forward": analyst.get("pe_forward"), "pe_trailing": edgar.get("pe_trailing"), "pb_ratio": edgar.get("pb_ratio"), "ev_ebitda": edgar.get("ev_ebitda"), + "ev_revenue": ev_rev, # Growth "revenue_growth": edgar.get("revenue_growth"), "earnings_growth": edgar.get("earnings_growth"), @@ -1515,15 +1586,15 @@ def fetch_all( "analyst_count": analyst.get("analyst_count"), "analyst_target": analyst_target, "recommendation": analyst.get("recommendation", "N/A"), - # Risk - "short_percent": finra_short.get(ticker), + # Risk — FINRA cache first, then yfinance shortPercentOfFloat as display fallback + "short_percent": finra_short.get(ticker) or analyst.get("short_percent"), "beta": beta_map.get(ticker), "volatility_30d": volatility, # Market data "market_cap": mkt_cap, "52w_high": float(close.max()), "52w_low": float(close.min()), - "dividend_yield": None, + "dividend_yield": edgar.get("dividend_yield"), "revenue": edgar.get("revenue"), "business_summary": "", }) @@ -1537,6 +1608,47 @@ def fetch_all( return df +# --------------------------------------------------------------------------- +# Sector Stats Loader +# --------------------------------------------------------------------------- + +def _load_sector_stats() -> dict[str, dict]: + """ + Load pre-computed sector statistics from the Verimund API. + Returns {sector: {metric: (median, MAD)}} or {} on failure. + Used by z-score scoring to normalize metrics relative to sector peers. + """ + if not _API_URL: + return {} + try: + resp = requests.get( + f"{_API_URL}/cache/sector", + headers=_get_api_headers(), + timeout=15, + ) + resp.raise_for_status() + rows = resp.json().get("data", []) + result: dict[str, dict] = {} + for row in rows: + sector = row.get("sector") + if not sector: + continue + stats: dict[str, tuple] = {} + for key, val in row.items(): + if key.endswith("_med"): + metric = key[:-4] + mad_key = f"{metric}_mad" + mad = row.get(mad_key) + if val is not None and mad is not None and mad > 0: + stats[metric] = (float(val), float(mad)) + result[sector] = stats + print(f" Sector stats loaded: {len(result)} sectors") + return result + except Exception as e: + print(f" [WARN] Sector stats unavailable: {e}") + return {} + + # --------------------------------------------------------------------------- # Scoring # --------------------------------------------------------------------------- @@ -1580,55 +1692,249 @@ def _score_series(series: pd.Series, breakpoints: list, default: float = 50.0) - return series.apply(lambda v: _abs_score(v, breakpoints, default)) -def compute_value_score(df: pd.DataFrame) -> pd.Series: +def _z_score(value, metric: str, sector: str, sector_stats: dict, + default: float = 50.0) -> float: """ - Value Score — how attractively priced is the stock on absolute terms. - Lower multiples = better value = higher score. - P/E: <10 excellent, 15 good, 25 fair, 40+ expensive - P/B: <1 excellent, 2 good, 4 fair, 8+ expensive - EV/EBITDA: <6 excellent, 10 good, 18 fair, 30+ expensive + Normalize value relative to sector peers using median and MAD, then map + the resulting z-score to a 0–100 scale via piecewise-linear interpolation. + Falls back to default (50 = neutral) when value or stats are unavailable. + + Z-score curve: z ≤ -2.5 → 3, z=0 → 50, z ≥ +2.5 → 97 """ + if value is None: + return default + try: + if np.isnan(float(value)): + return default + except (TypeError, ValueError): + return default + + stats = sector_stats.get(sector, {}) + pair = stats.get(metric) + if pair is None: + # Try market-wide fallback + pair = sector_stats.get("__market__", {}).get(metric) + if pair is None: + return default + + med, mad = pair + if mad <= 0: + return default + + z = (float(value) - med) / mad + z = max(-3.0, min(3.0, z)) # winsorize + + z_pts = [(-2.5, 3), (-1.5, 18), (-0.5, 38), (0.0, 50), + (0.5, 62), (1.5, 82), (2.5, 97)] + return _abs_score(z, z_pts, default) + + +def _z_series(series: pd.Series, metric: str, sectors: pd.Series, + sector_stats: dict, default: float = 50.0) -> pd.Series: + """Apply _z_score element-wise using per-row sector labels.""" + return pd.Series( + [_z_score(v, metric, s, sector_stats, default) + for v, s in zip(series, sectors)], + index=series.index, + ) + + +def _classify_profile(row: pd.Series) -> dict: + """ + Classify a stock into one or more company profiles with blend weights. + Returns {profile_name: weight} summing to 1.0. + Supports soft blending for ambiguous cases. + """ + sector = str(row.get("sector") or "").lower() + rev_g = row.get("revenue_growth") + pm = row.get("profit_margin") + de = row.get("debt_to_equity") + revenue = row.get("revenue") + mktcap = row.get("market_cap") + earn_g = row.get("earnings_growth") + + # Financial: banks, insurance, credit — D/E irrelevant, use ROA/P/B + fin_sectors = ("financial", "bank", "insurance", "credit") + if any(s in sector for s in fin_sectors): + return {"financial": 1.0} + + # Capital intensive: utilities, energy, industrials with meaningful leverage + cap_sectors = ("utilities", "energy", "industrials", "basic materials") + if any(s in sector for s in cap_sectors) and de is not None and de > 0.8: + return {"capital_intensive": 1.0} + + # Score membership in remaining profiles + scores = {} + + # Early stage: very small revenue or deeply negative margins + es_score = 0.0 + if revenue is not None and revenue < 30_000_000: + es_score += 0.6 + if pm is not None and pm < -0.30 and mktcap is not None and mktcap < 1_000_000_000: + es_score += 0.4 + if es_score > 0: + scores["early_stage"] = min(es_score, 1.0) + + # High growth: fast revenue expansion, possibly thin margins + hg_score = 0.0 + if rev_g is not None and rev_g > 0.25: + hg_score += 0.7 + elif rev_g is not None and rev_g > 0.15 and pm is not None and pm < 0.05: + hg_score += 0.5 + if hg_score > 0: + scores["high_growth"] = min(hg_score, 1.0) + + # Turnaround: strong earnings recovery from a loss position + if earn_g is not None and earn_g > 0.40 and pm is not None and pm < 0.0: + scores["turnaround"] = 0.8 + + # Mature: default for profitable, slower-growth companies + if not scores: + return {"mature": 1.0} + + # Normalize to sum = 1.0 + total = sum(scores.values()) + return {k: v / total for k, v in scores.items()} + + +# Profile-specific sub-component weights for each scoring dimension. +# Structure: {profile: {dimension: {metric: weight}}} +_PROFILE_WEIGHTS: dict[str, dict[str, dict[str, float]]] = { + "mature": { + "value": {"pe": 0.35, "pb": 0.20, "ev_ebitda": 0.45}, + "growth": {"revenue_growth": 0.30, "earnings_growth": 0.45, "eps_growth": 0.25}, + "quality": {"roa": 0.40, "roe": 0.20, "debt_to_equity": 0.30, "current_ratio": 0.10}, + "profitability": {"fcf_yield": 0.40, "operating_margin": 0.38, "profit_margin": 0.22}, + }, + "high_growth": { + "value": {"pe": 0.00, "pb": 0.20, "ev_ebitda": 0.35, "ev_revenue": 0.45}, + "growth": {"revenue_growth": 0.50, "earnings_growth": 0.30, "eps_growth": 0.20}, + "quality": {"roa": 0.35, "roe": 0.15, "debt_to_equity": 0.15, "current_ratio": 0.35}, + "profitability": {"fcf_yield": 0.15, "operating_margin": 0.50, "profit_margin": 0.35}, + }, + "financial": { + "value": {"pe": 0.35, "pb": 0.50, "ev_ebitda": 0.15}, + "growth": {"revenue_growth": 0.30, "earnings_growth": 0.50, "eps_growth": 0.20}, + "quality": {"roa": 0.50, "roe": 0.30, "debt_to_equity": 0.00, "current_ratio": 0.20}, + "profitability": {"fcf_yield": 0.30, "operating_margin": 0.00, "profit_margin": 0.70}, + }, + "capital_intensive": { + "value": {"pe": 0.15, "pb": 0.20, "ev_ebitda": 0.65}, + "growth": {"revenue_growth": 0.35, "earnings_growth": 0.40, "eps_growth": 0.25}, + "quality": {"roa": 0.30, "roe": 0.15, "debt_to_equity": 0.40, "current_ratio": 0.15}, + "profitability": {"fcf_yield": 0.35, "operating_margin": 0.45, "profit_margin": 0.20}, + }, + "early_stage": { + "value": {"pe": 0.00, "pb": 0.15, "ev_ebitda": 0.00, "ev_revenue": 0.85}, + "growth": {"revenue_growth": 0.70, "earnings_growth": 0.20, "eps_growth": 0.10}, + "quality": {"roa": 0.20, "roe": 0.00, "debt_to_equity": 0.00, "current_ratio": 0.80}, + "profitability": {"fcf_yield": 0.00, "operating_margin": 0.40, "profit_margin": 0.00, + "revenue_growth_proxy": 0.60}, + }, + "turnaround": { + "value": {"pe": 0.00, "pb": 0.30, "ev_ebitda": 0.70}, + "growth": {"revenue_growth": 0.15, "earnings_growth": 0.80, "eps_growth": 0.05}, + "quality": {"roa": 0.35, "roe": 0.15, "debt_to_equity": 0.35, "current_ratio": 0.15}, + "profitability": {"fcf_yield": 0.25, "operating_margin": 0.45, "profit_margin": 0.30}, + }, +} + + +def _profile_score_row(row: pd.Series, dimension: str, + metric_scores: dict[str, float]) -> float: + """ + Compute a single dimension score for one row using profile-blended weights. + metric_scores: {metric_name: 0-100 score} — pre-computed for this row. + """ + profile_blend = _classify_profile(row) + total = 0.0 + for profile, profile_weight in profile_blend.items(): + pw = _PROFILE_WEIGHTS.get(profile, _PROFILE_WEIGHTS["mature"]) + dim_weights = pw.get(dimension, {}) + # Normalize weights to only available metrics + available = {m: w for m, w in dim_weights.items() + if m in metric_scores and w > 0} + if not available: + total += profile_weight * 50.0 + continue + w_sum = sum(available.values()) + dim_score = sum(metric_scores[m] * (w / w_sum) for m, w in available.items()) + total += profile_weight * dim_score + return total + + +def compute_value_score(df: pd.DataFrame, + sector_stats: dict | None = None) -> pd.Series: + """ + Value Score — sector-relative z-score for each valuation metric, then + profile-blended sub-weights applied per row. + """ + ss = sector_stats or {} + sectors = df.get("sector", pd.Series(["N/A"] * len(df), index=df.index)) pe = df["pe_forward"].combine_first(df["pe_trailing"]).where(lambda x: x > 0) - pe_pts = [(5, 100), (10, 85), (15, 70), (20, 55), (25, 40), (35, 20), (50, 5)] - pb_pts = [(0.5, 100), (1.0, 85), (2.0, 70), (3.0, 55), (4.0, 40), (6.0, 20), (10.0, 5)] - ev_pts = [(4, 100), (6, 85), (10, 70), (14, 55), (18, 40), (25, 20), (35, 5)] - return ( - _score_series(pe, pe_pts) * 0.40 + - _score_series(df["pb_ratio"], pb_pts) * 0.30 + - _score_series(df["ev_ebitda"], ev_pts) * 0.30 - ) + + # Pre-compute z-scores for each metric + pe_s = _z_series(pe, "pe_trailing", sectors, ss) + pb_s = _z_series(df["pb_ratio"], "pb_ratio", sectors, ss) + ev_s = _z_series(df["ev_ebitda"], "ev_ebitda", sectors, ss) + evr_s = _z_series(df.get("ev_revenue", pd.Series( + [None]*len(df), index=df.index)), + "ev_revenue", sectors, ss) + + scores = [] + for i, row in df.iterrows(): + ms = { + "pe": float(pe_s.iloc[i]), + "pb": float(pb_s.iloc[i]), + "ev_ebitda": float(ev_s.iloc[i]), + "ev_revenue": float(evr_s.iloc[i]), + } + scores.append(_profile_score_row(row, "value", ms)) + return pd.Series(scores, index=df.index) -def compute_growth_score(df: pd.DataFrame) -> pd.Series: +def compute_growth_score(df: pd.DataFrame, + sector_stats: dict | None = None) -> pd.Series: """ - Growth Score — how fast is the business expanding. - Revenue/earnings growth as decimals (0.10 = 10%). - Negative growth penalised; hyper-growth (>50%) excellent. + Growth Score — sector-relative z-scores for revenue growth, earnings growth, + and EPS growth rate (clamped to ±150% to avoid breakeven-crossing distortion). """ - eps_delta = df["eps_forward"] - df["eps_trailing"] - rev_pts = [(-0.10, 5), (0.0, 20), (0.05, 40), (0.10, 55), (0.20, 70), (0.30, 85), (0.50, 100)] - ear_pts = [(-0.15, 5), (0.0, 20), (0.05, 38), (0.10, 55), (0.20, 70), (0.35, 85), (0.55, 100)] - eps_pts = [(-2.0, 5), (0.0, 30), (0.25, 50), (0.50, 65), (1.0, 80), (2.0, 92), (3.0, 100)] - return ( - _score_series(df["revenue_growth"], rev_pts) * 0.40 + - _score_series(df["earnings_growth"], ear_pts) * 0.40 + - _score_series(eps_delta, eps_pts) * 0.20 - ) + ss = sector_stats or {} + sectors = df.get("sector", pd.Series(["N/A"] * len(df), index=df.index)) + + # EPS growth rate: (forward - trailing) / abs(trailing), clamped ±1.5 + eps_t = pd.to_numeric(df["eps_trailing"], errors="coerce") + eps_f = pd.to_numeric(df["eps_forward"], errors="coerce") + eps_g = ((eps_f - eps_t) / eps_t.abs()).clip(-1.5, 1.5).where(eps_t.abs() > 1e-9) + + rev_s = _z_series(df["revenue_growth"], "revenue_growth", sectors, ss) + ear_s = _z_series(df["earnings_growth"], "earnings_growth", sectors, ss) + eps_s = _z_series(eps_g, "eps_growth", sectors, ss) + + scores = [] + for i, row in df.iterrows(): + ms = { + "revenue_growth": float(rev_s.iloc[i]), + "earnings_growth": float(ear_s.iloc[i]), + "eps_growth": float(eps_s.iloc[i]), + } + scores.append(_profile_score_row(row, "growth", ms)) + return pd.Series(scores, index=df.index) def compute_momentum_score(df: pd.DataFrame) -> pd.Series: """ - Momentum Score — recent price performance vs absolute thresholds. - Returns as decimals (0.10 = +10%). Negative = penalty, >30% = top. + Momentum Score — absolute breakpoints (price returns are inherently comparable). + 1M weight reduced to 5% to minimise short-term reversal noise. """ r1_pts = [(-0.15, 5), (-0.05, 25), (0.0, 45), (0.05, 60), (0.10, 75), (0.20, 90), (0.30, 100)] r3_pts = [(-0.20, 5), (-0.05, 25), (0.0, 42), (0.08, 58), (0.15, 72), (0.25, 88), (0.40, 100)] r6_pts = [(-0.25, 5), (-0.05, 22), (0.0, 38), (0.10, 55), (0.20, 70), (0.35, 87), (0.55, 100)] r12_pts = [(-0.30, 5), (-0.05, 20), (0.0, 35), (0.12, 52), (0.25, 68), (0.40, 85), (0.65, 100)] return ( - _score_series(df["ret_1m"], r1_pts) * 0.10 + + _score_series(df["ret_1m"], r1_pts) * 0.05 + _score_series(df["ret_3m"], r3_pts) * 0.25 + - _score_series(df["ret_6m"], r6_pts) * 0.40 + + _score_series(df["ret_6m"], r6_pts) * 0.45 + _score_series(df["ret_12m"], r12_pts) * 0.25 ) @@ -1636,9 +1942,7 @@ def compute_momentum_score(df: pd.DataFrame) -> pd.Series: def compute_reverse_momentum_score(df: pd.DataFrame) -> pd.Series: """ Reverse Momentum Score — rewards stocks that have fallen the most recently. - Inverts the standard momentum scale: large negative returns → high score. - Short-term windows weighted more heavily to capture recent drawdowns. - -30% in a month → 100; flat → ~50; +30% in a month → 3. + Used by the High Risk High Reward strategy preset. """ r1_pts = [(-0.30, 100), (-0.20, 88), (-0.10, 75), (-0.05, 62), (0.0, 48), (0.05, 32), (0.10, 18), (0.20, 8), (0.30, 3)] r3_pts = [(-0.40, 100), (-0.25, 88), (-0.15, 75), (-0.05, 60), (0.0, 45), (0.08, 30), (0.20, 15), (0.35, 5)] @@ -1652,97 +1956,138 @@ def compute_reverse_momentum_score(df: pd.DataFrame) -> pd.Series: ) -def compute_quality_score(df: pd.DataFrame) -> pd.Series: +def compute_quality_score(df: pd.DataFrame, + sector_stats: dict | None = None) -> pd.Series: """ - Quality Score — balance-sheet and capital efficiency. - ROE: as decimal, >0.20 (20%) is strong; <0 penalised. - ROA: >0.10 (10%) excellent. - D/E: lower is safer; >2.0 high risk. - Current ratio: >2 comfortable; <1 stressed. + Quality Score — sector-relative z-scores with profile-blended sub-weights. + ROA promoted to primary metric; ROE demoted to reduce leverage-gaming bias. """ - roe_pts = [(-0.10, 5), (0.0, 20), (0.05, 40), (0.10, 55), (0.15, 68), (0.20, 80), (0.30, 92), (0.40, 100)] - roa_pts = [(-0.05, 5), (0.0, 20), (0.03, 40), (0.06, 55), (0.10, 70), (0.15, 85), (0.20, 100)] - de_pts = [(0.0, 100), (0.25, 90), (0.50, 78), (1.0, 62), (1.5, 48), (2.0, 32), (3.0, 15), (5.0, 5)] - cr_pts = [(0.5, 5), (0.8, 20), (1.0, 38), (1.5, 60), (2.0, 80), (2.5, 90), (3.0, 100)] - return ( - _score_series(df["roe"], roe_pts) * 0.35 + - _score_series(df["roa"], roa_pts) * 0.30 + - _score_series(df["debt_to_equity"], de_pts) * 0.20 + - _score_series(df["current_ratio"], cr_pts) * 0.15 - ) + ss = sector_stats or {} + sectors = df.get("sector", pd.Series(["N/A"] * len(df), index=df.index)) + + roa_s = _z_series(df["roa"], "roa", sectors, ss) + roe_s = _z_series(df["roe"], "roe", sectors, ss) + de_s = _z_series(df["debt_to_equity"], "debt_to_equity", sectors, ss) + cr_s = _z_series(df["current_ratio"], "current_ratio", sectors, ss) + + # D/E z-score is inverted — lower D/E is better, so flip the z-score + de_s = 100.0 - de_s + + scores = [] + for i, row in df.iterrows(): + ms = { + "roa": float(roa_s.iloc[i]), + "roe": float(roe_s.iloc[i]), + "debt_to_equity": float(de_s.iloc[i]), + "current_ratio": float(cr_s.iloc[i]), + } + scores.append(_profile_score_row(row, "quality", ms)) + return pd.Series(scores, index=df.index) -def compute_profitability_score(df: pd.DataFrame) -> pd.Series: +def compute_profitability_score(df: pd.DataFrame, + sector_stats: dict | None = None) -> pd.Series: """ - Profitability Score — how much of revenue becomes profit. - All margins as decimals. Negative margins penalised. - FCF yield: FCF / market cap. >5% excellent. + Profitability Score — sector-relative z-scores. FCF yield promoted to 40% + for mature companies. Early stage profile substitutes revenue growth. """ - pm_pts = [(-0.10, 5), (0.0, 18), (0.03, 35), (0.08, 52), (0.15, 68), (0.25, 84), (0.35, 100)] - om_pts = [(-0.10, 5), (0.0, 18), (0.05, 38), (0.12, 55), (0.20, 70), (0.30, 86), (0.40, 100)] - fcf_pts = [(-0.05, 5), (0.0, 20), (0.01, 35), (0.03, 50), (0.05, 65), (0.08, 82), (0.12, 100)] - return ( - _score_series(df["profit_margin"], pm_pts) * 0.35 + - _score_series(df["operating_margin"], om_pts) * 0.35 + - _score_series(df["fcf_yield"], fcf_pts) * 0.30 - ) + ss = sector_stats or {} + sectors = df.get("sector", pd.Series(["N/A"] * len(df), index=df.index)) + + fcf_s = _z_series(df["fcf_yield"], "fcf_yield", sectors, ss) + om_s = _z_series(df["operating_margin"], "operating_margin", sectors, ss) + pm_s = _z_series(df["profit_margin"], "profit_margin", sectors, ss) + rev_s = _z_series(df["revenue_growth"], "revenue_growth", sectors, ss) + + scores = [] + for i, row in df.iterrows(): + ms = { + "fcf_yield": float(fcf_s.iloc[i]), + "operating_margin": float(om_s.iloc[i]), + "profit_margin": float(pm_s.iloc[i]), + "revenue_growth_proxy": float(rev_s.iloc[i]), # used by early_stage + } + scores.append(_profile_score_row(row, "profitability", ms)) + return pd.Series(scores, index=df.index) def compute_sentiment_score(df: pd.DataFrame) -> pd.Series: """ - News Sentiment Score — VADER compound score mapped from [-1, +1] to [0, 100]. - Neutral (0) → 50; strongly positive (+0.5) → 75; strongly negative (-0.5) → 25. + News Sentiment Score — VADER compound score mapped to 0–100 (absolute scale). + Sentiment is inherently comparable across all stocks; no z-scoring applied. """ - sent_pts = [(-1.0, 0), (-0.5, 25), (-0.2, 38), (0.0, 50), (0.2, 62), (0.5, 75), (1.0, 100)] + sent_pts = [(-1.0, 0), (-0.40, 20), (-0.15, 36), (0.0, 50), + (0.15, 64), (0.40, 80), (1.0, 100)] return _score_series(df["news_sentiment"], sent_pts) -def compute_analyst_score(df: pd.DataFrame) -> pd.Series: +def compute_analyst_score(df: pd.DataFrame, + sector_stats: dict | None = None) -> pd.Series: """ - Analyst Score — consensus rating + price target upside. - analyst_norm: 0=Strong Sell → 1=Strong Buy (absolute mapping to 0–100). - analyst_upside: expected % gain (decimal). >20% = very bullish. - analyst_count: more coverage = higher confidence (logarithmic). + Analyst Score — consensus (absolute) + upside (sector z-score) + count (absolute). + Upside top breakpoints compressed to dampen analyst optimism bias. """ - norm_pts = [(0.0, 0), (0.25, 25), (0.50, 50), (0.65, 65), (0.75, 78), (0.875, 90), (1.0, 100)] - upside_pts = [(-0.20, 5), (-0.05, 20), (0.0, 35), (0.05, 48), (0.10, 60), (0.20, 78), (0.35, 95), (0.50, 100)] - count_pts = [(0, 0), (1, 20), (3, 40), (5, 55), (10, 70), (20, 85), (30, 100)] + ss = sector_stats or {} + sectors = df.get("sector", pd.Series(["N/A"] * len(df), index=df.index)) + + norm_pts = [(0.0, 0), (0.25, 25), (0.50, 50), (0.65, 65), + (0.75, 78), (0.875, 90), (1.0, 100)] + count_pts = [(0, 0), (1, 20), (3, 40), (5, 55), (10, 70), (20, 85), (30, 100)] + # Upside: sector z-score with dampened ceiling for extreme claims + upside_pts = [(-0.25, 3), (-0.08, 20), (0.0, 35), (0.05, 48), + (0.12, 62), (0.25, 80), (0.40, 88), (0.60, 92)] + + norm_s = _score_series(df["analyst_norm"], norm_pts) + count_s = _score_series(df["analyst_count"], count_pts) + upside_s = _z_series(df["analyst_upside"], "analyst_upside", sectors, ss) + # Fall back to absolute breakpoints when sector stats unavailable + upside_abs = _score_series(df["analyst_upside"], upside_pts) + has_stats = sectors.map(lambda s: s in ss and "analyst_upside" in ss.get(s, {})) + upside_s = upside_s.where(has_stats, upside_abs) + return ( - _score_series(df["analyst_norm"], norm_pts) * 0.50 + - _score_series(df["analyst_upside"], upside_pts) * 0.35 + - _score_series(df["analyst_count"], count_pts) * 0.15 + norm_s * 0.43 + + upside_s * 0.42 + + count_s * 0.15 ) def compute_risk_score(df: pd.DataFrame) -> pd.Series: """ - Risk Score — higher score means LOWER risk (more attractive). - Short interest %: >15% very bearish signal; <3% low risk. - Volatility 30d (annualised): >60% high risk; <15% stable. - Beta deviation from 1.0: extreme values (very high or negative) add tail risk. + Risk Score — higher score = lower risk. + Dynamic short interest weighting: contributes 35% when data is present, + drops out (weight redistributed to volatility/beta) when absent. + Beta: asymmetric scoring peaking at beta 0.2-0.4 (genuine defensive). """ - short_pts = [(0.0, 100), (0.03, 88), (0.05, 75), (0.10, 55), (0.15, 35), (0.20, 18), (0.30, 5)] - vol_pts = [(0.05, 100), (0.10, 88), (0.15, 78), (0.25, 62), (0.35, 45), (0.50, 25), (0.70, 8), (1.0, 3)] - beta_dev = (df["beta"].fillna(1.0) - 1.0).abs() - bdev_pts = [(0.0, 100), (0.20, 85), (0.40, 70), (0.60, 55), (0.80, 40), (1.0, 25), (1.50, 10), (2.0, 5)] - return ( - _score_series(df["short_percent"], short_pts) * 0.50 + - _score_series(df["volatility_30d"], vol_pts) * 0.30 + - _score_series(beta_dev, bdev_pts) * 0.20 - ) + vol_pts = [(0.05, 100), (0.10, 88), (0.15, 78), (0.25, 62), + (0.35, 45), (0.50, 25), (0.70, 8), (1.0, 3)] + beta_pts = [(0.0, 88), (0.2, 96), (0.4, 90), (0.7, 78), (0.85, 70), + (1.0, 62), (1.3, 45), (1.6, 28), (2.0, 12), (3.0, 3)] + short_pts= [(0.0, 100), (0.03, 88), (0.05, 75), (0.10, 55), + (0.15, 35), (0.20, 18), (0.30, 5)] + + beta_clipped = df["beta"].fillna(1.0).clip(lower=0.0) + vol_s = _score_series(df["volatility_30d"], vol_pts) + beta_s = _score_series(beta_clipped, beta_pts) + short_s = _score_series(df["short_percent"], short_pts) + has_short = df["short_percent"].notna() + + # Dynamic weighting + score_with = short_s * 0.35 + vol_s * 0.40 + beta_s * 0.25 + score_without = vol_s * 0.65 + beta_s * 0.35 + return score_with.where(has_short, score_without) -def score_stocks(df: pd.DataFrame, weights: dict | None = None) -> pd.DataFrame: +def score_stocks(df: pd.DataFrame, weights: dict | None = None, + sector_stats: dict | None = None) -> pd.DataFrame: df = df.copy() df = df.drop_duplicates(subset=["ticker"], keep="first").reset_index(drop=True) - # Force every metric column to float before scoring. - # yfinance sometimes returns strings ("Infinity", "N/A") for PE/EPS fields - # on edge-case securities (REITs, ADRs, CLEFs, foreign listings) — common - # in large indices like NYSE. Without coercion, comparisons like `x > 0` - # inside compute_value_score raise TypeError on those rows. + # Load sector stats if not provided (allows GUI re-scoring to pass cached stats) + ss = sector_stats if sector_stats is not None else _load_sector_stats() + _NUMERIC_COLS = [ - "pe_forward", "pe_trailing", "pb_ratio", "ev_ebitda", + "pe_forward", "pe_trailing", "pb_ratio", "ev_ebitda", "ev_revenue", "revenue_growth", "earnings_growth", "eps_forward", "eps_trailing", "ret_1m", "ret_3m", "ret_6m", "ret_12m", "roe", "roa", "debt_to_equity", "current_ratio", @@ -1754,14 +2099,14 @@ def score_stocks(df: pd.DataFrame, weights: dict | None = None) -> pd.DataFrame: if col in df.columns: df[col] = pd.to_numeric(df[col], errors="coerce") - df["score_value"] = compute_value_score(df) - df["score_growth"] = compute_growth_score(df) + df["score_value"] = compute_value_score(df, ss) + df["score_growth"] = compute_growth_score(df, ss) df["score_momentum"] = compute_momentum_score(df) df["score_reverse_momentum"] = compute_reverse_momentum_score(df) - df["score_quality"] = compute_quality_score(df) - df["score_profitability"] = compute_profitability_score(df) + df["score_quality"] = compute_quality_score(df, ss) + df["score_profitability"] = compute_profitability_score(df, ss) df["score_sentiment"] = compute_sentiment_score(df) - df["score_analyst"] = compute_analyst_score(df) + df["score_analyst"] = compute_analyst_score(df, ss) df["score_risk"] = compute_risk_score(df) w = weights if weights is not None else WEIGHTS @@ -1983,51 +2328,76 @@ def _parse_form4_xml(xml_bytes: bytes, filing_url: str, filed_at: str) -> list[d return transactions -def fetch_insider_trades(days_back: int = 3, max_results: int = 60, - progress_cb=None) -> list[dict]: +def fetch_insider_trades(days_back: int = 3, progress_cb=None) -> list[dict]: """ - Fetch recent Form 4 insider transactions from SEC EDGAR. + Fetch ALL Form 4 insider transactions from SEC EDGAR for the given period. - Uses the EDGAR EFTS search API to list recent filings, then - parallel-fetches each Form 4 XML for full transaction details. + Paginates the EDGAR EFTS search API to collect every filing in the date + range, then parallel-fetches each Form 4 XML for full transaction details. + A shared rate-limiter keeps requests under EDGAR's 10 req/s limit. days_back : number of calendar days back from today to search. - max_results : cap on number of filings fetched (each filing may - contain multiple individual transactions). progress_cb : optional callable(done_count, total_count). Returns a list of transaction dicts sorted by tx_date descending. """ + import time + import threading from concurrent.futures import ThreadPoolExecutor, as_completed from datetime import date, timedelta _UA = {"User-Agent": "UltimateInvestmentTool contact@investmenttool.com"} + # ── Rate limiter: max ~9 requests/sec across all threads ────────────── + _rate_lock = threading.Lock() + _last_req = [0.0] + MIN_INTERVAL = 0.12 # seconds between requests → ~8 req/s, safely under 10 + + def _get(url, **kwargs): + with _rate_lock: + wait = MIN_INTERVAL - (time.monotonic() - _last_req[0]) + if wait > 0: + time.sleep(wait) + _last_req[0] = time.monotonic() + return requests.get(url, **kwargs) + end_dt = date.today() start_dt = end_dt - timedelta(days=max(days_back, 1)) - # ── Step 1: list recent Form 4 filings from EDGAR EFTS ──────────────── - try: - resp = requests.get( - "https://efts.sec.gov/LATEST/search-index", - params={ - "forms": "4", - "dateRange": "custom", - "startdt": start_dt.strftime("%Y-%m-%d"), - "enddt": end_dt.strftime("%Y-%m-%d"), - }, - headers=_UA, - timeout=12, - ) - resp.raise_for_status() - hits = resp.json().get("hits", {}).get("hits", [])[:max_results] - except Exception: + # ── Step 1: paginate EDGAR EFTS to collect all filing hits ──────────── + PAGE_SIZE = 100 + all_hits: list[dict] = [] + from_offset = 0 + + while True: + try: + resp = _get( + "https://efts.sec.gov/LATEST/search-index", + params={ + "forms": "4", + "dateRange": "custom", + "startdt": start_dt.strftime("%Y-%m-%d"), + "enddt": end_dt.strftime("%Y-%m-%d"), + "from": from_offset, + "hits": PAGE_SIZE, + }, + headers=_UA, + timeout=15, + ) + resp.raise_for_status() + except Exception: + break + + page_hits = resp.json().get("hits", {}).get("hits", []) + all_hits.extend(page_hits) + if len(page_hits) < PAGE_SIZE: + break # reached the last page + from_offset += PAGE_SIZE + + if not all_hits: return [] - if not hits: - return [] - - # ── Step 2: fetch + parse each Form 4 XML (parallel) ────────────────── + # ── Step 2: fetch + parse each Form 4 XML (rate-limited, parallel) ──── def _fetch_one(hit: dict) -> list[dict]: try: src = hit.get("_source", {}) @@ -2054,18 +2424,18 @@ def fetch_insider_trades(days_back: int = 3, max_results: int = 60, filing_url = f"{base}/{acc_no}-index.htm" xml_url = f"{base}/{xml_filename}" - xml_r = requests.get(xml_url, headers=_UA, timeout=8) + xml_r = _get(xml_url, headers=_UA, timeout=8) xml_r.raise_for_status() return _parse_form4_xml(xml_r.content, filing_url, filed) except Exception: return [] - total = len(hits) + total = len(all_hits) done_n = 0 results: list[dict] = [] - with ThreadPoolExecutor(max_workers=6) as ex: - futs = {ex.submit(_fetch_one, h): h for h in hits} + with ThreadPoolExecutor(max_workers=5) as ex: + futs = {ex.submit(_fetch_one, h): h for h in all_hits} for fut in as_completed(futs): done_n += 1 results.extend(fut.result() or []) @@ -2082,5 +2452,244 @@ def fetch_insider_trades(days_back: int = 3, max_results: int = 60, return results +# --------------------------------------------------------------------------- +# Hedge Fund Holdings — SEC EDGAR 13F-HR Filings +# --------------------------------------------------------------------------- + +def _parse_13f_xml(xml_bytes: bytes) -> list[dict]: + """ + Parse a 13F information table XML document. + + Handles any namespace variant used by EDGAR filers. + Returns a list of holding dicts with keys: + company, class_, cusip, value (USD), shares, shr_type, put_call + """ + import xml.etree.ElementTree as ET + try: + root = ET.fromstring(xml_bytes) + except ET.ParseError: + return [] + + # Strip XML namespace prefix so tag matching is namespace-agnostic + def _bare(tag: str) -> str: + return tag.split("}")[-1] if "}" in tag else tag + + def _find(node, bare_tag): + for child in node: + if _bare(child.tag) == bare_tag: + return child + return None + + def _text(node, bare_tag) -> str: + n = _find(node, bare_tag) + return n.text.strip() if n is not None and n.text else "" + + # Root is ; entries are + info_tables = [c for c in root if _bare(c.tag) == "infoTable"] + + holdings: list[dict] = [] + for entry in info_tables: + company = _text(entry, "nameOfIssuer") + class_ = _text(entry, "titleOfClass") + cusip = _text(entry, "cusip") + put_call = _text(entry, "putCall") + + try: + value_k = float(_text(entry, "value")) + except (ValueError, TypeError): + value_k = None + + shr_node = _find(entry, "shrsOrPrnAmt") + shares, shr_type = None, "SH" + if shr_node is not None: + try: + shares = float(_text(shr_node, "sshPrnamt")) + except (ValueError, TypeError): + pass + t = _text(shr_node, "sshPrnamtType") + if t: + shr_type = t + + holdings.append({ + "company": company, + "class_": class_, + "cusip": cusip, + "value": value_k * 1000.0 if value_k is not None else None, + "shares": shares, + "shr_type": shr_type, + "put_call": put_call, + }) + + return holdings + + +def fetch_hedge_fund_filings( + days_back: int = 90, + max_filers: int = 25, + progress_cb = None, +) -> list[dict]: + """ + Fetch individual stock holdings from recent SEC 13F-HR filings. + + For each unique fund that filed a 13F in the given period (up to + max_filers), fetches the information-table XML and parses every holding. + Returns a flat list of holding dicts (one row per stock per fund). + + days_back : calendar days back from today to search for filings. + max_filers : hard cap on the number of unique filers to process. + progress_cb : optional callable(done_count, total_count). + """ + import threading as _th + + _UA = {"User-Agent": "UltimateInvestmentTool contact@investmenttool.com"} + + # Rate-limiter: stay safely under EDGAR's 10 req/s cap + _rate_lock = _th.Lock() + _last_req = [0.0] + MIN_INTERVAL = 0.12 + + def _get(url, **kwargs): + with _rate_lock: + wait = MIN_INTERVAL - (time.monotonic() - _last_req[0]) + if wait > 0: + time.sleep(wait) + _last_req[0] = time.monotonic() + return requests.get(url, headers=_UA, timeout=15, **kwargs) + + from datetime import date, timedelta as _td + end_dt = date.today() + start_dt = end_dt - _td(days=max(days_back, 1)) + + # ── Step 1: Collect unique 13F-HR filings from EFTS ────────────────── + PAGE_SIZE = 100 + seen_adsh : set = set() + unique_hits : list = [] + from_offset = 0 + + while len(unique_hits) < max_filers: + try: + resp = _get( + "https://efts.sec.gov/LATEST/search-index", + params={ + "forms": "13F-HR", + "dateRange": "custom", + "startdt": start_dt.strftime("%Y-%m-%d"), + "enddt": end_dt.strftime("%Y-%m-%d"), + "from": from_offset, + "hits": PAGE_SIZE, + }, + ) + resp.raise_for_status() + except Exception: + break + + page_hits = resp.json().get("hits", {}).get("hits", []) + for hit in page_hits: + adsh = hit.get("_source", {}).get("adsh", "") + if adsh and adsh not in seen_adsh: + seen_adsh.add(adsh) + unique_hits.append(hit) + if len(unique_hits) >= max_filers: + break + + if len(page_hits) < PAGE_SIZE: + break + from_offset += PAGE_SIZE + + if not unique_hits: + return [] + + # ── Step 2: Fetch each filing's info-table XML and parse holdings ───── + def _fetch_one(hit: dict) -> list[dict]: + try: + src = hit.get("_source", {}) + acc_no = src.get("adsh", "") + ciks = src.get("ciks", []) + fund_name = src.get("entity_name", "Unknown Fund") + filed = (src.get("file_date") or "")[:10] + period = (src.get("period_of_report") or "")[:10] + + if not acc_no or not ciks: + return [] + + cik_int = int(ciks[0].lstrip("0") or "0") + if cik_int == 0: + return [] + + acc_nd = acc_no.replace("-", "") + base = (f"https://www.sec.gov/Archives/edgar/data" + f"/{cik_int}/{acc_nd}") + filing_url = f"{base}/{acc_no}-index.htm" + + # Fetch the filing index page and locate the INFORMATION TABLE doc. + # Parse the document table properly rather than a fragile backwards search. + idx_r = _get(f"{base}/{acc_no}-index.htm") + if idx_r.status_code != 200: + print(f" [HF] {fund_name}: index fetch failed HTTP {idx_r.status_code}") + return [] + idx_html = idx_r.text + + # Walk every table row; find the one whose type cell is INFORMATION TABLE + info_href = None + for row_m in re.finditer(r']*>(.*?)', idx_html, + re.IGNORECASE | re.DOTALL): + row_text = row_m.group(1) + if "INFORMATION TABLE" in row_text.upper(): + href_m = re.search(r'href="(/Archives/[^"]+)"', + row_text, re.IGNORECASE) + if href_m: + info_href = href_m.group(1) + break + + if not info_href: + print(f" [HF] {fund_name}: INFORMATION TABLE row not found in index") + return [] + + xml_r = _get("https://www.sec.gov" + info_href) + if xml_r.status_code != 200: + print(f" [HF] {fund_name}: XML fetch failed HTTP {xml_r.status_code}") + return [] + + raw_holdings = _parse_13f_xml(xml_r.content) + if not raw_holdings: + print(f" [HF] {fund_name}: XML parsed but 0 holdings returned") + return [] + + results: list[dict] = [] + for h in raw_holdings: + results.append({ + **h, + "fund_name": fund_name, + "filed_date": filed, + "period": period, + "url": filing_url, + }) + return results + except Exception as exc: + print(f" [HF] {fund_name}: unexpected error — {exc}") + return [] + + total = len(unique_hits) + done_n = 0 + results: list[dict] = [] + + with ThreadPoolExecutor(max_workers=3) as ex: + futs = {ex.submit(_fetch_one, h): h for h in unique_hits} + for fut in as_completed(futs): + done_n += 1 + results.extend(fut.result() or []) + if progress_cb: + try: + progress_cb(done_n, total) + except Exception: + pass + + results.sort( + key=lambda x: (x.get("filed_date") or "", x.get("value") or 0.0), + reverse=True, + ) + return results + + if __name__ == "__main__": main() diff --git a/license_check.py b/license_check.py index 199dfea..24ee162 100644 --- a/license_check.py +++ b/license_check.py @@ -1,16 +1,17 @@ """ License Check ============= -Validates the stored license key against Supabase on every launch. +Validates the stored license key against the Verimund API on every launch. Falls back to a local cache if the server is unreachable (30-minute grace period). Storage location: %APPDATA%\\StockScreener\\ - license.dat — XOR-encrypted license key (machine-bound) - auth_cache.dat — last successful auth result + timestamp + license.dat — XOR-encrypted license key (machine-bound) + auth_cache.dat — last successful auth result + timestamp + JWT token """ import base64 import hashlib +import hmac as _hmac_module import json import os import time @@ -18,15 +19,13 @@ import time import requests # --------------------------------------------------------------------------- -# Supabase config +# API config # --------------------------------------------------------------------------- -_SUPABASE_URL = "https://yeispcpmepjelfbhfkwr.supabase.co" -_SUPABASE_ANON = ( - "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9." - "eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6InllaXNwY3BtZXBqZWxmYmhma3dyIiwi" - "cm9sZSI6ImFub24iLCJpYXQiOjE3NzM1MDkxOTAsImV4cCI6MjA4OTA4NTE5MH0." - "8kq6HzpwF81_Sx36MXbx0wUqaT9Ul9POw1LzF5vxiOo" -) +_API_URL = "https://api.verimundsolutions.com" + +# HMAC secret — must match HMAC_SECRET in /srv/api/.env on the server. +# This value is obfuscated by PyArmor before distribution. +_HMAC_SECRET = "fa1fac57c9cc9742ac66542a17b032ccdfb2211b72c81fccd1617f17ce03ac7f" # 30 minutes _GRACE_PERIOD_SECONDS = 30 * 60 @@ -101,21 +100,19 @@ def delete_license_key() -> None: # Auth cache # --------------------------------------------------------------------------- -def _save_cache(valid: bool, reason: str, channel: str = "stable") -> None: +def _save_cache(valid: bool, reason: str, channel: str = "stable", jwt_token: str = "") -> None: _ensure_dir() - payload = {"valid": valid, "reason": reason, "channel": channel, "timestamp": time.time()} + payload = { + "valid": valid, + "reason": reason, + "channel": channel, + "jwt_token": jwt_token, + "timestamp": time.time(), + } with open(_CACHE_FILE, "w", encoding="utf-8") as f: json.dump(payload, f) -def get_channel() -> str: - """Return the update channel ('stable' or 'beta') from the last auth cache.""" - cache = _load_cache() - if cache: - return cache.get("channel", "stable") - return "stable" - - def _load_cache() -> dict | None: if not os.path.exists(_CACHE_FILE): return None @@ -126,6 +123,43 @@ def _load_cache() -> dict | None: return None +def get_channel() -> str: + """Return the update channel ('stable' or 'beta') from the last auth cache.""" + cache = _load_cache() + if cache: + return cache.get("channel", "stable") + return "stable" + + +def get_jwt() -> str: + """Return the stored JWT token, or empty string if not available.""" + cache = _load_cache() + if cache: + return cache.get("jwt_token", "") + return "" + + +def get_api_headers() -> dict: + """Return headers for authenticated API requests.""" + return { + "Authorization": f"Bearer {get_jwt()}", + "Content-Type": "application/json", + } + + +# --------------------------------------------------------------------------- +# HMAC request signing +# --------------------------------------------------------------------------- + +def _sign_request(license_key: str, timestamp: str) -> str: + """Generate HMAC-SHA256 signature for an auth request.""" + return _hmac_module.new( + _HMAC_SECRET.encode(), + f"{license_key}{timestamp}".encode(), + hashlib.sha256, + ).hexdigest() + + # --------------------------------------------------------------------------- # Core verification # --------------------------------------------------------------------------- @@ -134,31 +168,41 @@ def verify_license(license_key: str, hwid: str) -> tuple[bool, str]: """ Returns (valid: bool, reason: str). - Tries the Supabase RPC first. If the server is unreachable, falls back - to the cached result if it is within the 30-minute grace period. + Calls POST /auth on the Verimund API with a signed request. + Falls back to cached result within the 30-minute grace period if offline. """ - headers = { - "apikey": _SUPABASE_ANON, - "Content-Type": "application/json", - } - try: + timestamp = str(int(time.time())) + signature = _sign_request(license_key, timestamp) + resp = requests.post( - f"{_SUPABASE_URL}/rest/v1/rpc/verify_license", - headers=headers, - json={"p_key": license_key, "p_hwid": hwid}, + f"{_API_URL}/auth", + json={ + "license_key": license_key, + "hwid": hwid, + "timestamp": timestamp, + "signature": signature, + }, timeout=10, ) resp.raise_for_status() - data = resp.json() - valid = bool(data.get("valid", False)) - reason = data.get("reason", "Unknown error") + data = resp.json() + token = data.get("token", "") channel = data.get("channel", "stable") - _save_cache(valid, reason, channel) - return valid, reason + _save_cache(True, "ok", channel, token) + return True, "ok" + + except requests.exceptions.HTTPError as e: + try: + detail = e.response.json().get("detail", str(e)) + except Exception: + detail = str(e) + _save_cache(False, detail, "stable", "") + return False, detail except requests.exceptions.SSLError as e: return False, f"SSL error contacting license server: {e}" + except requests.exceptions.ConnectionError as e: cache = _load_cache() if cache and cache.get("valid"): @@ -166,8 +210,7 @@ def verify_license(license_key: str, hwid: str) -> tuple[bool, str]: if age < _GRACE_PERIOD_SECONDS: return True, "offline_grace" return False, f"Cannot connect to license server. Check your internet connection.\n\n({e})" - except requests.exceptions.HTTPError as e: - return False, f"License server error: {e}" + except requests.RequestException as e: cache = _load_cache() if cache and cache.get("valid"): diff --git a/updater.py b/updater.py index 8ffba50..a3dc293 100644 --- a/updater.py +++ b/updater.py @@ -1,11 +1,9 @@ """ Auto-Updater ============ -On every launch (after a successful license check), checks Supabase for a -newer version. If found: - - Downloads new screener_gui.py and stock_screener.py silently - - If a new exe is also available, self-replaces via a helper batch script - and restarts. The user sees nothing — the app just comes back up updated. +On every launch (after a successful license check), checks the Verimund API +for a newer version. If found, downloads the new exe via the API and +self-replaces via a helper batch script. Called after a successful license check on every launch. """ @@ -21,31 +19,7 @@ import requests # --------------------------------------------------------------------------- # Current app version — kept in sync by push_update.py before each build. # --------------------------------------------------------------------------- -APP_VERSION = "1.3.1" - -# --------------------------------------------------------------------------- -# GitHub token — fine-grained PAT with Contents:read on the release repo. -# Required to download assets from a private GitHub repository. -# Generate at: GitHub → Settings → Developer settings → Fine-grained tokens -# --------------------------------------------------------------------------- -_GITHUB_TOKEN = "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" -_GITHUB_REPO = "nolankovacs/ultimate-investment-tool" - -# --------------------------------------------------------------------------- -# Supabase config -# --------------------------------------------------------------------------- -_SUPABASE_URL = "https://yeispcpmepjelfbhfkwr.supabase.co" -_SUPABASE_ANON = ( - "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9." - "eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6InllaXNwY3BtZXBqZWxmYmhma3dyIiwi" - "cm9sZSI6ImFub24iLCJpYXQiOjE3NzM1MDkxOTAsImV4cCI6MjA4OTA4NTE5MH0." - "8kq6HzpwF81_Sx36MXbx0wUqaT9Ul9POw1LzF5vxiOo" -) - -_HEADERS = { - "apikey": _SUPABASE_ANON, - "Content-Type": "application/json", -} +APP_VERSION = "1.3.6" def _exe_dir() -> str: @@ -61,49 +35,59 @@ def _parse_version(v: str) -> tuple: return (0, 0, 0) -def _download_file(url: str, dest_path: str) -> None: - """Download url → dest_path atomically via a temp file.""" - dl_headers = {} - if _GITHUB_TOKEN and "github.com" in url: - dl_headers["Authorization"] = f"Bearer {_GITHUB_TOKEN}" - # GitHub API asset endpoint requires this header to stream the binary - if "api.github.com" in url and "/releases/assets/" in url: - dl_headers["Accept"] = "application/octet-stream" - - tmp_dir = tempfile.mkdtemp() - tmp_path = os.path.join(tmp_dir, os.path.basename(dest_path)) +def _log_update_error(msg: str) -> None: + """Log update errors silently — never let this crash the app.""" try: - with requests.get(url, stream=True, timeout=60, headers=dl_headers) as r: - r.raise_for_status() - with open(tmp_path, "wb") as f: - for chunk in r.iter_content(chunk_size=65536): - f.write(chunk) - shutil.move(tmp_path, dest_path) - finally: - shutil.rmtree(tmp_dir, ignore_errors=True) + from license_check import get_api_headers, _API_URL + from hwid import get_hwid + hwid = get_hwid() + import license_check + channel = license_check.get_channel() + requests.post( + f"{_API_URL}/log", + headers=get_api_headers(), + json={ + "hwid": hwid, + "app_version": APP_VERSION, + "channel": channel, + "message": msg, + }, + timeout=5, + ) + except Exception: + pass -def _self_replace_exe(exe_url: str) -> None: +def _self_replace_exe(jwt_token: str, channel: str) -> None: """ - Download the new exe to %TEMP%, then launch a helper batch script that - waits for this process to exit, moves the file into place, and restarts - the app. Nothing is left in the application directory. Never returns. + Download the new exe from the API, then launch a helper batch script that + waits for this process to exit, moves the file into place, and restarts. + Never returns. """ if not getattr(sys, "frozen", False): - return # only meaningful in a frozen exe - - current_exe = os.path.abspath(sys.executable) - tmp_dir = tempfile.gettempdir() - pending_exe = os.path.join(tmp_dir, "StockScreener_pending.exe") - - # Download to %TEMP% — nothing lands next to the running exe - try: - _download_file(exe_url, pending_exe) - except Exception as exc: - _log_update_error(f"_download_file failed for {exe_url}: {exc}") return - # Batch script in %TEMP% — deletes itself and the pending file when done + from license_check import _API_URL + + current_exe = os.path.abspath(sys.executable) + tmp_dir = tempfile.gettempdir() + pending_exe = os.path.join(tmp_dir, "StockScreener_pending.exe") + + try: + with requests.post( + f"{_API_URL}/download", + headers={"Authorization": f"Bearer {jwt_token}", "Content-Type": "application/json"}, + stream=True, + timeout=120, + ) as r: + r.raise_for_status() + with open(pending_exe, "wb") as f: + for chunk in r.iter_content(chunk_size=65536): + f.write(chunk) + except Exception as exc: + _log_update_error(f"_self_replace_exe download failed: {exc}") + return + helper = os.path.join(tmp_dir, "uit_update.bat") with open(helper, "w") as f: f.write( @@ -116,7 +100,6 @@ def _self_replace_exe(exe_url: str) -> None: f"del \"%~f0\"\n" ) - # Launch the helper detached, then exit so the running exe is unlocked subprocess.Popen( ["cmd", "/c", helper], creationflags=subprocess.CREATE_NO_WINDOW | subprocess.DETACHED_PROCESS, @@ -127,16 +110,14 @@ def _self_replace_exe(exe_url: str) -> None: def get_release_notes() -> tuple: """ - Returns (version, release_notes) for the current channel from Supabase. + Returns (version, release_notes) for the current channel from the API. Falls back gracefully on any error. """ try: - import license_check - channel = license_check.get_channel() - resp = requests.post( - f"{_SUPABASE_URL}/rest/v1/rpc/get_latest_version", - headers=_HEADERS, - json={"p_channel": channel}, + from license_check import get_api_headers, get_jwt, _API_URL + resp = requests.get( + f"{_API_URL}/update", + headers=get_api_headers(), timeout=8, ) resp.raise_for_status() @@ -148,108 +129,34 @@ def get_release_notes() -> tuple: return APP_VERSION, "Unable to fetch release notes." -def _log_update_error(msg: str) -> None: - """Upload update events/errors to Supabase update_logs table.""" - # Remote upload only — no local file created - try: - hwid = "unknown" - channel = "unknown" - try: - from hwid import get_hwid - hwid = get_hwid() - except Exception: - pass - try: - import license_check - channel = license_check.get_channel() - except Exception: - pass - - requests.post( - f"{_SUPABASE_URL}/rest/v1/update_logs", - headers={ - **_HEADERS, - "Content-Type": "application/json", - "Prefer": "return=minimal", - }, - json={ - "hwid": hwid, - "app_version": APP_VERSION, - "channel": channel, - "message": msg, - }, - timeout=5, - ) - except Exception: - pass # never let logging break the app - - -def _get_github_exe_url(version: str, channel: str) -> str | None: - """ - Query the GitHub Releases API to find the StockScreener.exe asset URL - for the given version and channel. Used as a fallback when Supabase - doesn't return exe_url. - """ - try: - tag = f"v{version}-{channel}" if channel != "stable" else f"v{version}" - resp = requests.get( - f"https://api.github.com/repos/{_GITHUB_REPO}/releases/tags/{tag}", - headers={ - "Authorization": f"Bearer {_GITHUB_TOKEN}", - "Accept": "application/vnd.github+json", - }, - timeout=10, - ) - resp.raise_for_status() - for asset in resp.json().get("assets", []): - if asset["name"] == "StockScreener.exe": - return asset["url"] # API asset URL — requires octet-stream header - _log_update_error(f"GitHub release {tag} found but no StockScreener.exe asset") - return None - except Exception as exc: - _log_update_error(f"_get_github_exe_url({version}, {channel}) failed: {exc}") - return None - - def check_and_apply_update() -> None: """ Silently check for updates and apply them. Errors are swallowed so a failed update check never blocks the app from launching. """ try: - import license_check - channel = license_check.get_channel() + from license_check import get_api_headers, get_jwt, get_channel, _API_URL - # Use the RPC — it runs with elevated DB permissions (SECURITY DEFINER) - # so the anon key can read the table through it even with RLS enabled. - resp = requests.post( - f"{_SUPABASE_URL}/rest/v1/rpc/get_latest_version", - headers=_HEADERS, - json={"p_channel": channel}, + resp = requests.get( + f"{_API_URL}/update", + headers=get_api_headers(), timeout=8, ) resp.raise_for_status() data = resp.json() latest = data.get("version", APP_VERSION) + channel = data.get("channel", "stable") if _parse_version(latest) <= _parse_version(APP_VERSION): return # already up to date - # Always resolve the download URL via the GitHub API — direct - # github.com/releases/download URLs return 404 for private repos - # when accessed with a token outside of a browser session. - exe_url = _get_github_exe_url(latest, channel) - if not exe_url: - _log_update_error( - f"Update available ({APP_VERSION} → {latest}) but could not " - f"locate the asset on GitHub for channel={channel}" - ) + jwt_token = get_jwt() + if not jwt_token: + _log_update_error("No JWT token available for update download") return - # Self-replace exe if a new one is available (exits this process) - _self_replace_exe(exe_url) + _self_replace_exe(jwt_token, channel) except Exception as exc: _log_update_error(f"check_and_apply_update failed: {exc}") - # never crash the app over a failed update check