diff --git a/cache_builder.py b/cache_builder.py index 8cd231a..3f56b2a 100644 --- a/cache_builder.py +++ b/cache_builder.py @@ -43,7 +43,9 @@ load_dotenv("/srv/api/.env") _DB_HOST = "127.0.0.1" _DB_NAME = "verimund" _DB_USER = "verimund_user" -_DB_PASS = os.environ.get("DB_PASS", "Shlevison2k17") +_DB_PASS = os.environ.get("DB_PASS") +if not _DB_PASS: + raise RuntimeError("DB_PASS environment variable is not set") # TABLE_PREFIX is set by the cron job: "" for stable, "beta_" for beta channel. _TABLE_PREFIX = os.environ.get("TABLE_PREFIX", "") @@ -284,6 +286,7 @@ def _fetch_analyst(ticker: str) -> dict | None: return { "ticker": ticker, + "name": info.get("longName") or None, "pe_forward": info.get("forwardPE"), "eps_forward": info.get("forwardEps"), "analyst_norm": analyst_norm, @@ -293,6 +296,8 @@ def _fetch_analyst(ticker: str) -> dict | None: "recommendation": info.get("recommendationKey") or "N/A", "sector": info.get("sector") or None, "industry": info.get("industry") or None, + "short_percent": info.get("shortPercentOfFloat"), + "short_ratio": info.get("shortRatio"), "dividend_yield": info.get("dividendYield"), "updated_at": datetime.now(timezone.utc).isoformat(), } @@ -412,9 +417,10 @@ def _fetch_news_sentiment(ticker: str) -> dict | None: scores.append(_VADER.polarity_scores(title)["compound"]) sentiment = sum(scores) / len(scores) if scores else None return { - "ticker": ticker, - "news_sentiment": sentiment, - "updated_at": datetime.now(timezone.utc).isoformat(), + "ticker": ticker, + "news_sentiment": sentiment, + "news_headline_count": len(scores), + "updated_at": datetime.now(timezone.utc).isoformat(), } except Exception: return None @@ -463,9 +469,9 @@ def run_news_phase(tickers: list[str]) -> None: # Explicit column allow-lists keep extra fields (e.g. dividend_yield from # yfinance) from causing "column does not exist" errors in PostgreSQL. _ANALYST_COLS = ( - "ticker", "pe_forward", "eps_forward", "analyst_norm", "analyst_upside", + "ticker", "name", "pe_forward", "eps_forward", "analyst_norm", "analyst_upside", "analyst_count", "analyst_target", "recommendation", "news_sentiment", - "sector", "industry", "updated_at", + "news_headline_count", "sector", "industry", "short_percent", "short_ratio", "updated_at", ) _FUND_COLS = ( "ticker", "pe_trailing", "pb_ratio", "ev_ebitda", "eps_trailing", @@ -473,6 +479,7 @@ _FUND_COLS = ( "debt_to_equity", "total_debt", "total_cash", "book_value", "current_ratio", "profit_margin", "operating_margin", "fcf_yield", "dividend_yield", "market_cap", "shares_outstanding", "ev_revenue", "short_percent", + "piotroski_score", "accruals_ratio", "filer_type", "updated_at", ) @@ -534,8 +541,11 @@ def _fetch_one_frame(concept: str, unit: str, period: str) -> dict[int, float]: _FRAME_CACHE[key] = result time.sleep(0.15) return result - except Exception: - _FRAME_CACHE[key] = {} + except requests.HTTPError as e: + print(f" [WARN] EDGAR frame {concept}/{unit}/{period}: HTTP {e.response.status_code} — not cached") + return {} + except Exception as e: + print(f" [WARN] EDGAR frame {concept}/{unit}/{period}: {e} — not cached") return {} @@ -547,7 +557,7 @@ def _sum_frames(concepts: list[str], unit: str, periods: list[str]) -> dict[int, ttm.setdefault(cik, {}) if period not in ttm[cik]: ttm[cik][period] = val - return {cik: sum(pv.values()) for cik, pv in ttm.items()} + return {cik: sum(pv.values()) for cik, pv in ttm.items() if len(pv) >= 4} def _best_frame(concepts: list[str], unit: str, periods: list[str]) -> dict[int, float]: @@ -613,6 +623,20 @@ def _compute_fundamentals( if rev is not None and rev_prev and rev_prev != 0 else None) ni_g = ((ni - ni_prev) / abs(ni_prev) if ni is not None and ni_prev and ni_prev != 0 else None) + + # Piotroski partial F-Score (4 signals available at compute time) + p = 0 + if ni is not None and tot_a and tot_a > 0 and ni / tot_a > 0: p += 1 # ROA > 0 + if ocf is not None and ocf > 0: p += 1 # OCF > 0 + if ocf is not None and ni is not None and ocf > ni: p += 1 # OCF > NI (cash-backed) + if rev is not None and rev_prev is not None and rev_prev != 0 and rev > rev_prev: p += 1 # Revenue improving + piotroski = p + + # Sloan accruals ratio: (NI - OCF) / Total Assets — negative = higher earnings quality + accruals = None + if ni is not None and ocf is not None and tot_a and tot_a > 0: + accruals = max(-0.3, min(0.3, (ni - ocf) / tot_a)) + return { "market_cap": mkt_cap, "shares_outstanding": sh, @@ -634,6 +658,8 @@ def _compute_fundamentals( "profit_margin": pm, "operating_margin": om, "fcf_yield": fcf_yield, + "piotroski_score": piotroski, + "accruals_ratio": accruals, } diff --git a/dist/screener_gui.py b/dist/screener_gui.py index 254300c..b160ff8 100644 --- a/dist/screener_gui.py +++ b/dist/screener_gui.py @@ -1365,10 +1365,14 @@ class ScreenerApp(tk.Tk): self._status_var.set(f" Rescoring with strategy: {strategy}…") def _rescore(): - result = score_stocks( - self._df_raw, weights=weights, - sector_stats=self._sector_stats, - ) + try: + result = score_stocks( + self._df_raw, weights=weights, + sector_stats=self._sector_stats, + ) + except Exception: + self.after(0, lambda: setattr(self, "_rescoring", False)) + return def _done(): self._rescoring = False if self._rescore_id != current_id: @@ -1447,9 +1451,9 @@ class ScreenerApp(tk.Tk): price_min = _to_float(self._filter_price_min) price_max = _to_float(self._filter_price_max) if price_min is not None: - df = df[df["price"].fillna(0) >= price_min] + df = df[df["price"].notna() & (df["price"] >= price_min)] if price_max is not None: - df = df[df["price"].fillna(0) <= price_max] + df = df[df["price"].notna() & (df["price"] <= price_max)] # --- Market cap category --- mktcap_cat = self._filter_mktcap.get() diff --git a/dist/stock_screener.py b/dist/stock_screener.py index b616039..94192ba 100644 --- a/dist/stock_screener.py +++ b/dist/stock_screener.py @@ -1298,6 +1298,8 @@ def _fetch_fundamentals( price_now = float(close.iloc[-1]) def pct_return(days_back: int) -> float | None: + if len(close) <= days_back: + return None idx = max(0, len(close) - days_back) past = float(close.iloc[idx]) return (price_now - past) / past if past > 0 else None @@ -1350,7 +1352,7 @@ def _fetch_fundamentals( "ret_12m": pct_return(252), "roe": info.get("returnOnEquity"), "roa": info.get("returnOnAssets"), - "debt_to_equity": info.get("debtToEquity"), + "debt_to_equity": (info["debtToEquity"] / 100.0) if info.get("debtToEquity") is not None else None, "total_debt": info.get("totalDebt"), "total_cash": info.get("totalCash"), "book_value": info.get("bookValue"), @@ -1366,6 +1368,7 @@ def _fetch_fundamentals( "analyst_target": target_price, "recommendation": info.get("recommendationKey", "N/A"), "short_percent": info.get("shortPercentOfFloat"), + "short_ratio": info.get("shortRatio"), "beta": info.get("beta"), "volatility_30d": volatility, "market_cap": info.get("marketCap"), @@ -1500,6 +1503,8 @@ def fetch_all( price_now = float(close.iloc[-1]) def pct_return(days_back: int) -> float | None: + if len(close) <= days_back: + return None idx = max(0, len(close) - days_back) past = float(close.iloc[idx]) return (price_now - past) / past if past > 0 else None @@ -1544,7 +1549,7 @@ def fetch_all( results.append({ "ticker": ticker, - "name": edgar.get("name", ticker), + "name": analyst.get("name") or edgar.get("name") or ticker, "sector": analyst.get("sector") or edgar.get("sector", "N/A"), "industry": analyst.get("industry") or edgar.get("industry", "N/A"), "price": price_now, @@ -1586,8 +1591,12 @@ def fetch_all( "analyst_count": analyst.get("analyst_count"), "analyst_target": analyst_target, "recommendation": analyst.get("recommendation", "N/A"), - # Risk — FINRA cache first, then yfinance shortPercentOfFloat as display fallback - "short_percent": finra_short.get(ticker) or analyst.get("short_percent"), + # yfinance shortPercentOfFloat (% of float) preferred over FINRA short volume ratio + "short_percent": analyst.get("short_percent") or finra_short.get(ticker), + "short_ratio": analyst.get("short_ratio"), + "piotroski_score": edgar.get("piotroski_score"), + "accruals_ratio": edgar.get("accruals_ratio"), + "news_headline_count": analyst.get("news_headline_count"), "beta": beta_map.get(ticker), "volatility_30d": volatility, # Market data @@ -1803,39 +1812,40 @@ _PROFILE_WEIGHTS: dict[str, dict[str, dict[str, float]]] = { "mature": { "value": {"pe": 0.35, "pb": 0.20, "ev_ebitda": 0.45}, "growth": {"revenue_growth": 0.30, "earnings_growth": 0.45, "eps_growth": 0.25}, - "quality": {"roa": 0.40, "roe": 0.20, "debt_to_equity": 0.30, "current_ratio": 0.10}, - "profitability": {"fcf_yield": 0.40, "operating_margin": 0.38, "profit_margin": 0.22}, + "quality": {"roa": 0.40, "roe": 0.20, "debt_to_equity": 0.30, "current_ratio": 0.10, "piotroski": 0.20}, + "profitability": {"fcf_yield": 0.40, "operating_margin": 0.38, "profit_margin": 0.22, "accruals": 0.15}, }, "high_growth": { "value": {"pe": 0.00, "pb": 0.20, "ev_ebitda": 0.35, "ev_revenue": 0.45}, "growth": {"revenue_growth": 0.50, "earnings_growth": 0.30, "eps_growth": 0.20}, - "quality": {"roa": 0.35, "roe": 0.15, "debt_to_equity": 0.15, "current_ratio": 0.35}, - "profitability": {"fcf_yield": 0.15, "operating_margin": 0.50, "profit_margin": 0.35}, + "quality": {"roa": 0.35, "roe": 0.15, "debt_to_equity": 0.15, "current_ratio": 0.35, "piotroski": 0.20}, + "profitability": {"fcf_yield": 0.15, "operating_margin": 0.50, "profit_margin": 0.35, "accruals": 0.15}, }, "financial": { "value": {"pe": 0.35, "pb": 0.50, "ev_ebitda": 0.15}, "growth": {"revenue_growth": 0.30, "earnings_growth": 0.50, "eps_growth": 0.20}, - "quality": {"roa": 0.50, "roe": 0.30, "debt_to_equity": 0.00, "current_ratio": 0.20}, - "profitability": {"fcf_yield": 0.30, "operating_margin": 0.00, "profit_margin": 0.70}, + "quality": {"roa": 0.50, "roe": 0.30, "debt_to_equity": 0.00, "current_ratio": 0.20, "piotroski": 0.15}, + "profitability": {"fcf_yield": 0.30, "operating_margin": 0.00, "profit_margin": 0.70, "accruals": 0.15}, }, "capital_intensive": { "value": {"pe": 0.15, "pb": 0.20, "ev_ebitda": 0.65}, "growth": {"revenue_growth": 0.35, "earnings_growth": 0.40, "eps_growth": 0.25}, - "quality": {"roa": 0.30, "roe": 0.15, "debt_to_equity": 0.40, "current_ratio": 0.15}, - "profitability": {"fcf_yield": 0.35, "operating_margin": 0.45, "profit_margin": 0.20}, + "quality": {"roa": 0.30, "roe": 0.15, "debt_to_equity": 0.40, "current_ratio": 0.15, "piotroski": 0.15}, + "profitability": {"fcf_yield": 0.35, "operating_margin": 0.45, "profit_margin": 0.20, "accruals": 0.15}, }, "early_stage": { "value": {"pe": 0.00, "pb": 0.15, "ev_ebitda": 0.00, "ev_revenue": 0.85}, "growth": {"revenue_growth": 0.70, "earnings_growth": 0.20, "eps_growth": 0.10}, - "quality": {"roa": 0.20, "roe": 0.00, "debt_to_equity": 0.00, "current_ratio": 0.80}, - "profitability": {"fcf_yield": 0.00, "operating_margin": 0.40, "profit_margin": 0.00, - "revenue_growth_proxy": 0.60}, + "quality": {"roa": 0.20, "roe": 0.00, "debt_to_equity": 0.00, "current_ratio": 0.80, "piotroski": 0.10}, + # revenue_growth_proxy removed — revenue growth already carries 70% of the growth score + "profitability": {"fcf_yield": 0.00, "operating_margin": 1.00, "profit_margin": 0.00, + "revenue_growth_proxy": 0.00}, }, "turnaround": { "value": {"pe": 0.00, "pb": 0.30, "ev_ebitda": 0.70}, "growth": {"revenue_growth": 0.15, "earnings_growth": 0.80, "eps_growth": 0.05}, - "quality": {"roa": 0.35, "roe": 0.15, "debt_to_equity": 0.35, "current_ratio": 0.15}, - "profitability": {"fcf_yield": 0.25, "operating_margin": 0.45, "profit_margin": 0.30}, + "quality": {"roa": 0.35, "roe": 0.15, "debt_to_equity": 0.35, "current_ratio": 0.15, "piotroski": 0.20}, + "profitability": {"fcf_yield": 0.25, "operating_margin": 0.45, "profit_margin": 0.30, "accruals": 0.15}, }, } @@ -1871,23 +1881,24 @@ def compute_value_score(df: pd.DataFrame, """ ss = sector_stats or {} sectors = df.get("sector", pd.Series(["N/A"] * len(df), index=df.index)) - pe = df["pe_forward"].combine_first(df["pe_trailing"]).where(lambda x: x > 0) - - # Pre-compute z-scores for each metric - pe_s = _z_series(pe, "pe_trailing", sectors, ss) - pb_s = _z_series(df["pb_ratio"], "pb_ratio", sectors, ss) - ev_s = _z_series(df["ev_ebitda"], "ev_ebitda", sectors, ss) - evr_s = _z_series(df.get("ev_revenue", pd.Series( + # Z-score forward and trailing PE against their own sector distributions, then combine + # Lower valuation ratios are better — invert z-scores so cheap stocks rank high + fwd_s = 100.0 - _z_series(df["pe_forward"].where(df["pe_forward"] > 0), "pe_forward", sectors, ss) + trl_s = 100.0 - _z_series(df["pe_trailing"].where(df["pe_trailing"] > 0), "pe_trailing", sectors, ss) + pe_s = fwd_s.combine_first(trl_s) + pb_s = 100.0 - _z_series(df["pb_ratio"], "pb_ratio", sectors, ss) + ev_s = 100.0 - _z_series(df["ev_ebitda"], "ev_ebitda", sectors, ss) + evr_s = 100.0 - _z_series(df.get("ev_revenue", pd.Series( [None]*len(df), index=df.index)), "ev_revenue", sectors, ss) scores = [] for i, row in df.iterrows(): ms = { - "pe": float(pe_s.iloc[i]), - "pb": float(pb_s.iloc[i]), - "ev_ebitda": float(ev_s.iloc[i]), - "ev_revenue": float(evr_s.iloc[i]), + "pe": float(pe_s.loc[i]), + "pb": float(pb_s.loc[i]), + "ev_ebitda": float(ev_s.loc[i]), + "ev_revenue": float(evr_s.loc[i]), } scores.append(_profile_score_row(row, "value", ms)) return pd.Series(scores, index=df.index) @@ -1914,9 +1925,9 @@ def compute_growth_score(df: pd.DataFrame, scores = [] for i, row in df.iterrows(): ms = { - "revenue_growth": float(rev_s.iloc[i]), - "earnings_growth": float(ear_s.iloc[i]), - "eps_growth": float(eps_s.iloc[i]), + "revenue_growth": float(rev_s.loc[i]), + "earnings_growth": float(ear_s.loc[i]), + "eps_growth": float(eps_s.loc[i]), } scores.append(_profile_score_row(row, "growth", ms)) return pd.Series(scores, index=df.index) @@ -1967,19 +1978,27 @@ def compute_quality_score(df: pd.DataFrame, roa_s = _z_series(df["roa"], "roa", sectors, ss) roe_s = _z_series(df["roe"], "roe", sectors, ss) - de_s = _z_series(df["debt_to_equity"], "debt_to_equity", sectors, ss) cr_s = _z_series(df["current_ratio"], "current_ratio", sectors, ss) + # Negative D/E means negative equity (insolvency risk) — treat as extremely high leverage + de_raw = df["debt_to_equity"].where(df["debt_to_equity"] >= 0, 999.0) # D/E z-score is inverted — lower D/E is better, so flip the z-score - de_s = 100.0 - de_s + de_s = 100.0 - _z_series(de_raw, "debt_to_equity", sectors, ss) + + # Piotroski partial F-Score (0-4) → 0-100 via absolute breakpoints + p_raw = pd.to_numeric(df.get("piotroski_score", + pd.Series([None] * len(df), index=df.index)), errors="coerce") + p_pts = [(0, 5), (1, 30), (2, 50), (3, 72), (4, 95)] + piotroski_s = _score_series(p_raw, p_pts) scores = [] for i, row in df.iterrows(): ms = { - "roa": float(roa_s.iloc[i]), - "roe": float(roe_s.iloc[i]), - "debt_to_equity": float(de_s.iloc[i]), - "current_ratio": float(cr_s.iloc[i]), + "roa": float(roa_s.loc[i]), + "roe": float(roe_s.loc[i]), + "debt_to_equity": float(de_s.loc[i]), + "current_ratio": float(cr_s.loc[i]), + "piotroski": float(piotroski_s.loc[i]), } scores.append(_profile_score_row(row, "quality", ms)) return pd.Series(scores, index=df.index) @@ -1999,13 +2018,21 @@ def compute_profitability_score(df: pd.DataFrame, pm_s = _z_series(df["profit_margin"], "profit_margin", sectors, ss) rev_s = _z_series(df["revenue_growth"], "revenue_growth", sectors, ss) + # Sloan accruals ratio: more negative = earnings are cash-backed = better quality + acc_raw = pd.to_numeric(df.get("accruals_ratio", + pd.Series([None] * len(df), index=df.index)), errors="coerce") + acc_pts = [(-0.25, 95), (-0.10, 78), (-0.03, 62), (0.0, 50), + (0.03, 38), (0.10, 22), (0.25, 5)] + accruals_s = _score_series(acc_raw, acc_pts) + scores = [] for i, row in df.iterrows(): ms = { - "fcf_yield": float(fcf_s.iloc[i]), - "operating_margin": float(om_s.iloc[i]), - "profit_margin": float(pm_s.iloc[i]), - "revenue_growth_proxy": float(rev_s.iloc[i]), # used by early_stage + "fcf_yield": float(fcf_s.loc[i]), + "operating_margin": float(om_s.loc[i]), + "profit_margin": float(pm_s.loc[i]), + "revenue_growth_proxy": float(rev_s.loc[i]), # used by early_stage + "accruals": float(accruals_s.loc[i]), } scores.append(_profile_score_row(row, "profitability", ms)) return pd.Series(scores, index=df.index) @@ -2013,12 +2040,27 @@ def compute_profitability_score(df: pd.DataFrame, def compute_sentiment_score(df: pd.DataFrame) -> pd.Series: """ - News Sentiment Score — VADER compound score mapped to 0–100 (absolute scale). - Sentiment is inherently comparable across all stocks; no z-scoring applied. + News Sentiment Score — VADER compound score mapped to 0–100, confidence-weighted + by headline count. Fewer headlines blend toward neutral (50) to avoid overreacting + to a single article. """ sent_pts = [(-1.0, 0), (-0.40, 20), (-0.15, 36), (0.0, 50), (0.15, 64), (0.40, 80), (1.0, 100)] - return _score_series(df["news_sentiment"], sent_pts) + base = _score_series(df["news_sentiment"], sent_pts) + + # Derive headline count from stored field or from news_headlines list + if "news_headline_count" in df.columns: + count = pd.to_numeric(df["news_headline_count"], errors="coerce").fillna(0) + elif "news_headlines" in df.columns: + count = df["news_headlines"].apply( + lambda h: len(h) if isinstance(h, list) else 0 + ).astype(float) + else: + return base + + # 5+ headlines = full confidence; 0 headlines = 5% confidence (nearly neutral) + confidence = (count.clip(upper=5) / 5.0).clip(lower=0.05) + return base * confidence + 50.0 * (1.0 - confidence) def compute_analyst_score(df: pd.DataFrame, @@ -2092,8 +2134,9 @@ def score_stocks(df: pd.DataFrame, weights: dict | None = None, "ret_1m", "ret_3m", "ret_6m", "ret_12m", "roe", "roa", "debt_to_equity", "current_ratio", "profit_margin", "operating_margin", "fcf_yield", - "news_sentiment", "analyst_norm", "analyst_upside", "analyst_count", - "short_percent", "volatility_30d", "beta", "price", "market_cap", + "news_sentiment", "news_headline_count", "analyst_norm", "analyst_upside", "analyst_count", + "short_percent", "short_ratio", "volatility_30d", "beta", "price", "market_cap", + "piotroski_score", "accruals_ratio", ] for col in _NUMERIC_COLS: if col in df.columns: