
    ]jFO              	         d Z ddlmZ ddlZddlZddlZddlZddlZddl	m
Z
 ddlmZ ddlmZmZ dZddlZddlZ	 ddlZ	 dd	lmZ 	 ddlmZ 	 ddlmZ  ee      j?                         j@                  Z!e!dz  dz  Z"d3dZ#d4d5dZ$	 	 	 	 	 	 	 	 	 	 	 	 d6dZ%d7dZ&d8dZ'ddd	 	 	 	 	 	 	 d9dZ(ddddddd	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d:dZ)dd d!d"	 	 	 	 	 	 	 	 	 d;d#Z*d$d%d<d&Z+d$d'd=d(Z,	 	 	 	 	 	 d>d)Z-d*d!d+d,d-d.d$dd/	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d?d0Z.d@d1Z/e0d2k(  r e/        yy# e$ rZ ed      edZ[ww xY w# e$ rZ ed
      edZ[ww xY w# e$ rZ ed      edZ[ww xY w# e$ rZ ed      edZ[ww xY w)AaR  
Universe acquisition, cached OHLCV downloads, momentum ranking, and cointegration pair discovery.

Requires: pandas, numpy, yfinance, tqdm, statsmodels; optional pyarrow for Parquet cache.

Run from repository root::

    .venv/bin/python universe_scanner.py
    .venv/bin/python universe_scanner.py --universe ndx100 --lookback-years 5
    )annotationsN)StringIO)Path)AnyLiteralzMozilla/5.0 (compatible; universe_scanner/1.0; +https://github.com/) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36z*yfinance is required: pip install yfinance)cointz0statsmodels is required: pip install statsmodels)multipletestszPstatsmodels is required for multiple testing correction: pip install statsmodels)tqdmz"tqdm is required: pip install tqdmdatauniverse_cachec                ^    | j                         j                         j                  dd      S )z-Wikipedia often uses BRK.B; Yahoo uses BRK-B..-)stripupperreplace)symbols    ,/opt/rentech/trading_bot/universe_scanner.py_to_yahoo_symbolr   9   s$    <<>!))#s33    c           	        | dk(  rdnd}t         j                  j                  |dt        i      }t         j                  j	                  |d      5 }|j                         j                  dd	
      }ddd       	 t        j                  t                    }g }|D ]  }d}	|j                  D ]1  }
t        |
      j                         j                         }|dv s/|
}	 n |	H||	   j                         j!                  t              }|D ]a  }|j                         }|r|j                         dv r(|j#                  t%        |j'                  d      d   j                                      c t)        |      dk\  s n t+        t-        |            }t)        |      dk  rt/        dt)        |       d| d      |S # 1 sw Y   RxY w# t        $ r}t        d      |d}~ww xY w)u   
    Scrape current S&P 100 or Nasdaq 100 constituents from Wikipedia.

    Parameters
    ----------
    universe
        ``"sp100"`` — S&P 100; ``"ndx100"`` — Nasdaq 100.
    sp100z'https://en.wikipedia.org/wiki/S%26P_100z(https://en.wikipedia.org/wiki/Nasdaq-100z
User-Agent)headers<   )timeoutzutf-8r   )errorsNz_pandas.read_html needs an HTML parser. Install one of: pip install lxml html5lib beautifulsoup4)r   tickersymbols)r   r   company(r   2      zParsed too few tickers (z) from z$. Wikipedia layout may have changed.)urllibrequestRequest_USER_AGENTurlopenreaddecodepd	read_htmlr   ImportErrorcolumnsstrr   lowerdropnaastypeappendr   splitlensortedsetRuntimeError)universeurlreqresphtmltablesetickerstblsym_colcclrawsouts                  r   get_sp100_tickersrG   >   s    w 	27 
 ..
 
 |[.I
 
JC			R		0 =Dyy{!!')!<=htn-
 G  	AQ%%'B44		
 ?'l!!#**3/ 	FA	A	%DDNN+AGGCLO,A,A,CDE		F
 w<2!$ W
C
3x"}&s3xju<`a
 	
 JA= =  m
	s$   "G=G G	G+G&&G+c                    |j                  dd      j                  dd      }|rdnd}t        |      j                  dd      }| | d| d| d| z  S )	N/r   \parquetcsvr   _zy.)r   r.   )data_dirr   	timeframelookback_yearsuse_parquetsafeextytags           r   _cache_pathrU   p   sc     >>#s#++D#6D")C~&&sC0Da	{!D6C5999r   c                    | j                   j                         dk(  rt        j                  |       S t        j                  | dd      S )N.parquetr   T)	index_colparse_dates)suffixr/   r*   read_parquetread_csv)paths    r   _read_ohlcvr^   }   s9    {{j(t$$;;tqd;;r   c                    |j                   j                  dd       |j                  j                         dk(  r| j	                  |       y | j                  |       y )NTparentsexist_okrW   )parentmkdirrZ   r/   
to_parquetto_csv)dfr]   s     r   _write_ohlcvrh      sE    KKdT2{{j(
d
		$r   g      8@   )max_age_hoursmax_data_stale_calendar_daysc                  | j                         syt        j                         | j                         j                  z
  }||dz  k  ry	 t	        |       }|j                  ryt        j                  |j                  j                               j                         }t        j                  j                         j                         }||z
  j                  }||k  S # t
        $ r Y yw xY w)zu
    Treat cache as fresh if file is recent enough OR last bar is not too old
    (weekends/holidays tolerated).
    Fi  T)existstimestatst_mtimer^   	Exceptionemptyr*   	Timestampindexmax	normalizenowdays)r]   rj   rk   age_secrg   lasttodaygaps           r   _cache_is_freshr}      s     ;;=iikDIIK000G-$&& 
xx<<'113DLL((*E4<

C...  s   C 	C*)C*1d      @T)rO   rP   rN   prefer_parquetrj   auto_adjustc          	        t        |xs t              }|j                  dd       |}|r	 ddl}t        dt        t        j                  |                  }	|	dk\  rdn|	 d	}
i }t        | d
d      D ]  }t        |||||      }t        ||      r,	 t        |      }|j                  sd|j                  v r|||<   K	 t!        j"                  ||
||dd      }||j                  rvt)        |j                  t$        j*                        r |j                  j-                  d      |_        |j/                         }||j0                  j3                  d          }	 t5        ||       |||<    |S # t        $ r d}Y Rw xY w# t        $ r Y w xY w# t        $ r t%        j&                         }Y w xY w# t        $ r" |j7                  d      }t5        ||       |}Y ww xY w)ah  
    Download OHLCV via yfinance with disk cache under ``data_dir``.

    Cache hit: file exists and is considered fresh (recent mtime or recent last bar).

    Parameters
    ----------
    timeframe
        yfinance interval, e.g. ``"1d"``, ``"1wk"``.
    lookback_years
        Passed as ``period=f"{int(ceil(years))}y"`` when using period-based fetch.
    Tr`   r   NF      ru   yzdownload/cacher   descunit)rj   Close)periodintervalr   progressthreadsrz   )keepz.csv)r   DEFAULT_DATA_DIRrd   pyarrowr,   ru   intnpceilr
   rU   r}   r^   rr   r-   rq   yfdownloadr*   	DataFrame
isinstance
MultiIndex	droplevel
sort_indexrt   
duplicatedrh   with_suffix)r?   rO   rP   rN   r   rj   r   rQ   r   	years_intr   rF   tr]   rg   ydfalts                    r   download_and_cache_datar      s   , H0 01HNN4$N/ K	  As277>234I2oUi[?F#%C' 0x@ #8Q	>;O4}= &xxGrzz$9CF
	!++"'C ;#))ckk2==1++//2CKnn399''V'445	d#
 AG#J JY  	 K	     	!,,.C	!  	""6*Cc"D	sG   E4 *F FF84FF	FFF54F58(G#"G#      r   )lookback_daysskip_recent_days	price_colc               f   g }| j                         D ]/  \  }}||j                  s||j                  vr$||   j                         j	                  t
        j                        }t        |      dk  rct        t        |      t        |            dz   }t        |      |k  r|j                  t        |            |j                  t        |            z  dz
  }	t        |	j                  d         }
t        j                  |
      s|j                  ||
t        |      |j                  j                         d       2 t!        j"                  |      }|j                  r|S |j%                  dd      j'                  d	
      }t        j(                  dt        |      dz         |d<   |S )z
    Cross-sectional momentum using AQR "12-minus-1" methodology.

    For each ticker, compute:
        Return = Price_{t-21} / Price_{t-252} - 1

    This explicitly skips the most recent ``skip_recent_days`` to avoid short-term
    mean reversion.
       r         ?)r   
cum_returnn_barsend_dater   F	ascendingTdroprank)itemsrr   r-   r0   r1   r   float64r4   ru   r   shiftfloatilocisfiniter2   rt   r*   r   sort_valuesreset_indexarange)	data_dictr   r   r   rowsr   rg   pxmin_obsaqr_momrrF   s               r   rank_momentumr      sp     "$Doo' 

:Ybjj%@	]!!#**2::6r7Q; c-(#.>*?@1Dr7W ((3/01BHHS=O4PPSVV',,r"#{{1~ b'HHLLN		
)
: ,,t
C
yy

//,%/
8
D
D$
D
OC))As3x!|,CKJr   r"   )r   c               t   | j                         j                  t        j                        }t	        |      |k  rt        d      S |j                  d      j                  dd }|j                  dd }t	        |      |k  rt        d      S |j                         }|j                         }t        j                  t        j                  |      |f      }	 t        j                  j                  ||d      \  }}	}	}	t        |d         }
t        j                  |
      r
|
dk  s|
dk\  rt        d      S t        t        j                  d             t        t        j                  |
            z  }t        j                  |      st        d      S t        |      S # t        $ r t        d      cY S w xY w)u  
    Approximate OU half-life via linear regression:

        s_t = c + phi * s_{t-1} + ε_t

    For mean reversion, ``phi`` should satisfy ``0 < phi < 1`` and:

        hl = -ln(2) / ln(phi)

    Returns half-life in **bars/days** (depending on the input frequency).
    nanr   Nrcondr   infg       @)r0   r1   r   r   r4   r   r   r   to_numpycolumn_stack	ones_likelinalglstsqrq   r   log)spreadr   rE   s_lags_tr   xXcoefrM   phihls               r   calculate_half_lifer   (  sZ    	rzz*A
1vU|GGAJOOABE
&&*C
5zGU|AA
a!,-A		1D9aA Q.C;;ssax3!8U|
s
	uRVVC[1	1B;;r?U|9  U|s   'F   F76F7max_lagsc                  | j                         j                  t        j                        }t	        |      |dz   k  rt        d      S t        t        dt        |      dz               }g }g }|j                         }|D ]  }|t	        |      k\  r n||d |d|  z
  }|j                  dk  r/t        j                  |d      }	|	dk  st        j                  |	      sa|j                  t        t        j                  |	                   |j                  |        t	        |      dk  rt        d      S t        j                  t        j                   |t        j                  	            }
t        j                  t        j                   |t        j                  	            }	 t        j"                  |
|d      \  }}t        |      S # t$        $ r t        d      cY S w xY w)
z
    Estimate Hurst exponent using variance of lagged differences:

        tau(lag) = std(s(t+lag) - s(t))
        log(tau) = H * log(lag) + const

    Returns H in approximately [0, 1], where < 0.5 suggests mean reversion.
       r   r   r   N)ddofr      dtype)r0   r1   r   r   r4   r   listranger   r   sizevarr   r2   sqrtr   asarraypolyfitrq   )r   r   rE   lagstau	used_lagsarrlagdiffvlog_lagslog_tauslope
_intercepts                 r   calculate_hurst_exponentr   P  s    	rzz*A
1v1U|5CMA$567DCI
**,C 
#c(?34y3u:%99q=FF4a 6Q

5$%
 3x!|U|vvbjj"**=>HffRZZ2::67GJJx!<z <  U|s   ,G G('G(c                .   t        j                  | |gdd      j                         }|j                  r$t        j                  t
        j                        S |j                  dddf   j                  t
        j                        }|j                  dddf   j                  t
        j                        }t        j                  t        j                  |      |f      }	 t
        j                  j                  ||d      \  }}}}t        |d         }t        |d         }	|||	|z  z   z
  }
t        j                  |
|j                  t
        j                        S # t        $ r' t        j                  t
        j                        cY S w xY w)	za
    Hedge ratio via OLS: log_y = alpha + beta * log_x + eps
    Spread is the residual eps.
    r   inneraxisjoinr   Nr   r   )rt   r   )r*   concatr0   rr   Seriesr   r   r   r   r   r   r   r   rq   r   rt   )log_ylog_xrg   r   r   r   r   rM   alphabetaresids              r   _ols_hedge_spread_residualr   z  s1    
E5>	8	?	?	AB	xxyyrzz**
1RZZ0A
1RZZ0A
a!,-A+		1D9aA $q'NEa>D!"E99U"(("**==  +yyrzz**+s   &'E$ $-FF皙?r   ?r         .@)p_value_thresholdr   r   	hurst_maxhalf_life_minhalf_life_maxhurst_max_lagstop_kc               t   | j                         D 	
cg c]  \  }	}
|
	||
j                  v s|	 }}	}
t        t        j                  t        |      d            }g }t        |dd      D ]+  \  }}| |   |   j                         }| |   |   j                         }t        j                  ||gdd      j                         }dd	g|_        t        |      |k  rst        j                  |d   j                  t        j                              }t        j                  |d	   j                  t        j                              }	 t        ||d
d      \  }}}|j#                  ||t%        |      t'        t        |            t%        |      d       . |st        j(                  g d      S t        j*                  |D cg c]  }|d   	 c}t        j                        }	 t-        |t%        |      d      \  }}}}t/        ||      D ]  \  }}t%        |      |d<    |D cg c]/  }t        j0                  |d         s|d   t%        |      k  s.|1 }}|st        j(                  g d      S g } t        |dd      D ]  }t3        |d         }t3        |d         }| |   |   j                         }| |   |   j                         }t        j                  ||gdd      j                         }dd	g|_        t        |      |k  rt        j                  |d   j                  t        j                              }!t        j                  |d	   j                  t        j                              }"t5        |!|"      }#|#j6                  s|#j8                  |k  rt;        |#|      }$t=        |#      }%t        j0                  |$      sNt        j0                  |%      se|$t%        |      k  sut%        |      t%        |%      cxk  rt%        |      k  sn | j#                  ||t%        |d         t%        |$      t%        |%      d        | st        j(                  g d      S t        | d       }&g }'t?               }(|&D ]v  }t3        |d         }t3        |d         }||(v s||(v r(|'j#                  |       |(jA                  |       |(jA                  |       |s^t        |'      t'        |      k\  sv n t        j(                  |'      })|)j6                  rt        j(                  g d      S |)jC                  dd      jE                  d      S c c}
}	w # t         $ r Y w xY wc c}w # t         $ r t        j(                  g d      cY S w xY wc c}w )u  
    Discovery of tradable cointegrated pairs with diversification.

    Pipeline:
      1) Raw Engle–Granger p-values for all pairs
      2) Holm-Bonferroni multiple testing correction
      3) Tradability filter on the residual spread:
         - Hurst < ``hurst_max``
         - ``half_life_min`` < half-life < ``half_life_max``
      4) Greedy cluster-risk reduction: select pairs sorted by lowest adjusted
         p-value such that no ticker appears in more than one selected pair.
    r   zcoint pairspairr   r   r   r   abrB   AIC)trendautolag)Ticker_ATicker_Bp_valuen_obscoint_t)r  r  Adjusted_P_ValueHurst	Half_Life)r-   r  r   holm)r   methodr  ztradability filtersr  r  r   c                    | d   S )Nr   )r   s    r   <lambda>z)find_cointegrated_pairs.<locals>.<lambda>  s    Q7I5J r   )keyTr   r   )#r   r-   r   	itertoolscombinationsr5   r
   r0   r*   r   r4   r   r   r1   r   r   rq   r2   r   r   r   r   r	   zipr   r.   r   rr   r   r   r   r6   addr   r   )*r   r  r   r   r  r  r  r  r  r   rg   r?   pairsresultsr
  r  dadbalignedlalbr  pvalue_critr   pvals_reject	pvals_adj_alphacSidak_alphacBonfpadjsurvivedtradablelog_alog_br   hurst	half_lifetradable_sortedfinal_portfolioused_tickersrF   s*                                             r   find_cointegrated_pairsr9    s1   0 (oo/^UQ2>iSUS]S]F]q^G^''w;<E %'GUV< 
1q\)$++-q\)$++-))RH17;BBD*w<'! VVGCL''

34VVGCL''

34	%*2re%L"GVU 	 =S\* >	
#
6 ||V
 	

 JJg6)6bjjIE
8E01&9
5L+ w	* ,4 %d
, #~abkk!4F2G&HQOaMbejk|e}M}~H~||V
 	

 &(H(!6VD "**q\)$++-q\)$++-))RH17;BBD*w<'!ws|**2::67ws|**2::67+E59<<6;;0(.I'/	{{5!{{9%5##m(<uY?O(fRWXeRf(f(fOO ! !(-a0B.C(D"5\!&y!15"H ||V
 	

 X+JKO,.O UL 	**\ 1q!S)SZ7	 ,,
'C
yy||V
 	
 ??-?>JJPTJUUu _&  		& 7
  
||V
 	

 sK   
U3U3U3U9	V	-V /V5V5V59	VV!V21V2c                 p   t        j                  t              } | j                  ddd       | j                  dd       | j                  d	t        d
       | j                  dt
        dd       | j                  dt
        dd       | j                  dt        d       | j                  dt        dd       | j                  dt        dd       | j                  dt        dd       | j                  dt
        dd        | j                  d!t        t        t                     | j                  d"t
        d#d$       | j                         }t        j                  t               t        d%       t        |j                  &      }|j                  r|j                  d#kD  r|d |j                   }t        d'|j                   d(t!        |       d)       t        d*       t#        ||j$                  |j&                  t)        |j*                        +      }t        d,t!        |       d-       t-        ||j.                  |j0                  .      }t        d/       |j2                  rt        d0       n*t        |j5                  d1      j7                  d23             t9        ||j:                  |j<                  |j>                  |j@                  |jB                  4      }t        d5       |j2                  rt        d6       y t        |j5                  d1      j7                  d23             y )7N)descriptionz
--universe)r   ndx100r   )choicesdefaultz--timeframer~   )r>  z--lookback-yearsr   )typer>  z--momentum-daysr   zAQR momentum lookback (t-252).)r?  r>  helpz--momentum-skipr   zAQR momentum skip (t-21).z	--coint-pr   z--hurst-maxr   z7Tradability filter: keep pairs with Hurst < this value.z--half-life-minr   zGTradability filter: keep pairs with half-life > this value (bars/days).z--half-life-maxr  zGTradability filter: keep pairs with half-life < this value (bars/days).z--hurst-max-lagsr"   zFHurst estimation uses variance of lagged differences up to these lags.z
--data-dirz--max-tickersr   zDIf >0, only use the first N tickers (smoke test / faster local runs)u$   Fetching universe from Wikipedia …)r8   z
Universe (z): z tickersu   Downloading / loading cache …)rO   rP   rN   zLoaded OHLCV for z names)r   r   z)
=== Top 10 AQR momentum (12-minus-1) ===z	(no data)
   F)rt   )r  r  r  r  r  z3
=== Top 10 cointegrated pairs (lowest p-value) ===uE   (no pairs under threshold — try looser --coint-p or longer history))"argparseArgumentParser__doc__add_argumentr   r   r.   r   
parse_argsoschdir
_REPO_ROOTprintrG   r8   max_tickersr4   r   rO   rP   r   rN   r   momentum_daysmomentum_skiprr   head	to_stringr9  coint_pr  r  r  r  )pargsr?   r   momr!  s         r   mainrT  )  s   G4ANN<)<gNNNN=$N/NN%E3N? NN$3BbNcNN$3A\N]NN;UDN9NNF	   NNV	   NNV	   NNU	   NN<c37G3HNINNS	   <<>DHHZ	
017GD,,q0,D,,-	Jt}}oSWh
?@	
+,'..**dmm$	I 
c)n-V
45
((++C
 

67
yykchhrl$$5$12#,,..((((**E 

@A{{UVejjn&&U&34r   __main__)r   r.   returnr.   )r   )r8   zLiteral['sp100', 'ndx100']rV  	list[str])rN   r   r   r.   rO   r.   rP   r   rQ   boolrV  r   )r]   r   rV  pd.DataFrame)rg   rY  r]   r   rV  None)r]   r   rj   r   rk   r   rV  rX  )r?   rW  rO   r.   rP   r   rN   zPath | str | Noner   rX  rj   r   r   rX  rV  dict[str, pd.DataFrame])
r   r[  r   r   r   r   r   r.   rV  rY  )r   	pd.Seriesr   r   rV  r   )r   r\  r   r   rV  r   )r   r\  r   r\  rV  r\  )r   r[  r  r   r   r.   r   r   r  r   r  r   r  r   r  r   r  r   rV  rY  )rV  rZ  )1rD  
__future__r   rB  r  rG  rn   urllib.requestr#   ior   pathlibr   typingr   r   r&   numpyr   pandasr*   yfinancer   r,   r>   statsmodels.tsa.stattoolsr   statsmodels.stats.multitestr	   r
   __file__resolverc   rI  r   r   rG   rU   r^   rh   r}   r   r   r   r   r   r9  rT  __name__r  r   r   <module>rj     s2  	 #   	     L 
  KQ/q9C (^##%,,
&)99 4
/d
:
:
: 
: 	
:
 
: 

:<  ()	/
/ / #&	/
 
/< "&II I 	I
  I I I I I^ 3&3 3 	3
 3 3l >@ %P DF 'T>>> >:  $RV&RV RV 	RV
 RV RV RV RV RV RV RVjS5l zF }  K
B
CJK
  Q
H
IqPQ
  q
h
ioppq
  C
:
;BCs_   D% D; E E' %D8*	D33D8;E 	E		EE$	EE$'E:,	E55E: