o
    ð7µhQ  ã                   @  s6  d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dlmZm	Z	 d dl
mZ d dlmZmZmZmZ d dlZd dlmZ d dlmZmZ dZd	Zd
dddœZed dddde› dddœZddddddddœZdZdZdd„ Zd€d d!„Zdddd"œdd)d*„Z dd+œd‚d.d/„Z!dddddd0œdƒd8d9„Z"d„d;d<„Z#d…d?d@„Z$e %dA¡Z&d…dBdC„Z'd†dFdG„Z(d‡dJdK„Z)dˆdMdN„Z*d‰dŠdTdU„Z+d‹dWdX„Z,dŒdYdZ„Z-ddŽd^d_„Z.dd`da„Z/ddgdh„Z0d‘djdk„Z1d’dldm„Z2d“dndo„Z3d”dqdr„Z4ddsedtdufd•dydz„Z5d–d|d}„Z6e7d~kre6ƒ Z8e9e5e8j:e8j;e8j<e8j=e8j>dƒ dS dS )—é    )ÚannotationsN)ÚdatetimeÚtimezone)Úunescape)ÚDictÚOptionalÚTupleÚList)ÚBeautifulSoup)Ú
get_clientÚloggerz8https://industry.na.go.kr/cmmit/mem/cmmitMemList/list.doz=https://industry.na.go.kr/cmmit/mem/cmmitMemList/memList.jsonzCMozilla/5.0 (compatible; govbot/1.0; +https://work.jjickjjicks.com)z?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8zko-KR,ko;q=0.9)ú
User-AgentÚAcceptúAccept-Languager   z.application/json, text/javascript, */*; q=0.01z0application/x-www-form-urlencoded; charset=UTF-8ÚXMLHttpRequestzhttps://industry.na.go.krú?menuNo=2000013)r   r   zContent-TypezX-Requested-WithÚOriginÚRefererr   Ú2000013ÚCMMITÚ )ÚmenuNoÚ	cntsDivCdÚcommitteeCdÚhrCmtIdÚjobResCdÚpolyCdÚhgNmÚnoneÚfullc                   C  s   t ƒ S ©N)r   © r!   r!   ú6/var/www/html/bot/app/crawler/na_industry_committee.pyÚSB8   s   r#   ÚtargetÚstrÚreturnc                 C  s|   t t ¡ ƒ}ztƒ  d¡ || dddd t tj	¡ 
¡ dœ¡ ¡  W |S  ty= } zt d|› ¡ W Y d }~|S d }~ww )NÚcrawler_runÚrunningr   )Úidr$   ÚstatusÚpagesÚrowsÚfail_reasonÚ
started_atz[crawler_run insert] )r%   ÚuuidÚuuid4r#   ÚtableÚinsertr   Únowr   ÚutcÚ	isoformatÚexecuteÚ	Exceptionr   Úwarning)r$   Úrun_idÚer!   r!   r"   Ú
_run_start;   s   ýþ€þr;   )r+   r,   r*   r9   r+   ú
int | Noner,   r*   ú
str | Nonec             
   C  s˜   i }|d ur
||d< |d ur||d< |d ur||d< |rJzt ƒ  d¡ |¡ d| ¡ ¡  W d S  tyI } zt d|› ¡ W Y d }~d S d }~ww d S )Nr+   r,   r*   r'   r)   z[crawler_run update] )r#   r1   ÚupdateÚeqr6   r7   r   r8   )r9   r+   r,   r*   Úpayloadr:   r!   r!   r"   Ú_run_heartbeatG   s   $€ÿýrA   )r-   Úintr-   c             
   C  s†   |t  tj¡ ¡ ||dœ}|d ur||d< ztƒ  d¡ |¡ d| ¡ 	¡  W d S  t
yB } zt d|› ¡ W Y d }~d S d }~ww )N)r*   Úfinished_atr+   r,   r-   r'   r)   z[crawler_run finish] )r   r3   r   r4   r5   r#   r1   r>   r?   r6   r7   r   r8   )r9   r*   r+   r,   r-   r@   r:   r!   r!   r"   Ú_run_finishR   s   ý$€ÿrD   )Úcommittee_cdÚurlÚhtmlr@   Újson_objÚkindrE   rF   rG   r@   údict | NonerH   c          	   
   C  s    | ||dœ}|d ur||d< |d ur||d< |d ur||d< |d ur&||d< zt ƒ  d¡ |¡ ¡  W d S  tyO } zt d|› ¡ W Y d }~d S d }~ww )N)r9   rI   rE   rF   rG   r@   ÚjsonÚcongress_rawz[congress_raw insert] )r#   r1   r2   r6   r7   r   r8   )	r9   rI   rE   rF   rG   r@   rH   Úrecr:   r!   r!   r"   Ú_raw_insert_   s   €ÿrN   Úsc                 C  s"   | sdS d  t| ƒ dd¡ ¡ ¡S )Nr   ú õ   Â )Újoinr   ÚreplaceÚsplit)rO   r!   r!   r"   Ú_cleann   s   rU   ÚvúOptional[str]c                 C  s   t | ƒ}|r|S d S r    )rU   )rV   Útr!   r!   r"   Ú
_norm_textr   s   rY   z(\d{2,4}-\d{3,4}-\d{4})c                 C  s:   | sd S t  ddt| ƒ¡} t | ¡}|r| d¡S |  ¡ S )Nú\s+rP   é   )ÚreÚsubr%   ÚRE_PHONEÚsearchÚgroupÚstrip)rV   Úmr!   r!   r"   Ú_norm_phonew   s   
rc   Ú	committeeÚ	full_namec                 C  s(   t | ƒ› dt |ƒ› }t | ¡ ¡ ¡ S )Nú|)rU   ÚhashlibÚsha256ÚencodeÚ	hexdigest)rd   re   Úbaser!   r!   r"   Ú	_key_hash}   s   rl   rM   Údictc                 C  sâ   |   d¡pd|   d¡pd|   d¡pd|   d¡pd|   d¡pd|   d¡p$d|   d¡p*d|   d	¡p0d|   d
¡p6d|   d¡p<d|   d¡pBd|   d¡pHd|   d¡pNd|   d¡pTdt|   d¡d ura|   d¡ndƒg}t d |¡ ¡ ¡ ¡ S )NÚcommittee_namer   Ú	role_namere   Ú
name_hanjaÚpartyÚdistrictÚphoneÚroom_noÚ	photo_urlÚprofile_urlÚmember_page_urlÚ
aide_staffÚaide_secretaryÚaide_secretary2Úelection_countrf   )Úgetr%   rg   rh   rR   ri   rj   )rM   Úpartsr!   r!   r"   Ú	_row_hash‚   s"   ñr~   úOptional[int]c                 C  sš   t  ddt| dƒjddd¡}dD ]}t  ||¡}|r+z
t| d¡ƒW   S    Y qqt  d|¡}|rKdd	d
d
dddddddddœ}| | d¡¡S d S )NrZ   rP   úhtml.parserT)ra   )u   ë‹¹ì„ íšŸìˆ˜\s*[:ï¼š]?\s*(\d+)u   ì„ ìˆ˜\s*[:ï¼š]?\s*(\d+)r[   uK   (ì´ˆì„ |ìž¬ì„ |ì‚¼ì„ |3ì„ |ì‚¬ì„ |4ì„ |ì˜¤ì„ |5ì„ |ìœ¡ì„ |6ì„ |ì¹ ì„ |7ì„ )é   é   é   é   é   é   )u   ì´ˆì„ u   ìž¬ì„ u   ì‚¼ì„ u   3ì„ u   ì‚¬ì„ u   4ì„ u   ì˜¤ì„ u   5ì„ u   ìœ¡ì„ u   6ì„ u   ì¹ ì„ u   7ì„ )r\   r]   r
   Úget_textr_   rB   r`   r|   )rG   ÚtextÚpatrb   Úm2Úmappingr!   r!   r"   Ú!_extract_election_count_from_htmlš   s   
þrŒ   ç      @Úsessionúrequests.SessionÚtimeoutÚfloatc                 C  sJ   |sd S z| j |dtd i|d}| ¡  t|jƒW S  ty$   Y d S w )Nr   ©Úheadersr   )r|   ÚBASE_HEADERSÚraise_for_statusrŒ   rˆ   r7   )rŽ   rF   r   Úrr!   r!   r"   Ú_fetch_election_count§   s   ÿr—   ú#Tuple[Optional[str], Optional[str]]c              
   C  s¬   z:| j t› dtdd}| ¡  t|jdƒ}| d¡}| d¡}|r)|  d¡ ¡ nd }|r4|  d¡ ¡ nd}||fW S  tyU } zt	 
d	|› ¡ W Y d }~d
S d }~ww )Nr   é   r’   r€   zmeta[name="_csrf"]zmeta[name="_csrf_header"]ÚcontentzX-CSRF-TOKENzCSRF fetch failed: )NN)r|   Ú	LIST_PAGEr”   r•   r
   rˆ   Ú
select_onera   r7   r   r8   )rŽ   r–   ÚsoupÚtoken_elÚ	header_elÚtokenÚheaderr:   r!   r!   r"   Ú_fetch_csrf³   s   


€þr¢   c              	   C  sÈ   t | ƒ\}}ttƒ}|r|||< ||d< tdƒD ]I}z4| jt||dd}|jdkrD|dkrDt | ƒ\}}|r=|||< ||d< t d¡ W q| 	¡  | 
¡ W   S  tya   |dkr`t d¡ Y q‚ w d S )NÚ_csrfr   é   )Údatar“   r   i“  r   gš™™™™™é?)r¢   rm   ÚAJAX_HEADERSÚrangeÚpostÚMEMLIST_APIÚstatus_codeÚtimeÚsleepr•   rK   r7   )rŽ   r@   Ú
csrf_tokenÚcsrf_headerr“   Úattemptr–   r!   r!   r"   Ú_post_memlistÁ   s0   

üôr°   éô  ú
List[dict]Úchunkc                 C  sZ   t ƒ }d}tdt|ƒ|ƒD ]}|||| … }|sq| d¡ |¡ ¡  |t|ƒ7 }q|S )Nr   Úcongress_member_stg)r#   r§   Úlenr1   Úupsertr6   )r9   r,   r³   ÚsbÚtotalÚiÚpartr!   r!   r"   Ú_stg_upsert_rowsÝ   s   r»   c                  C  s<   zt ƒ  d¡jddd ¡ } | jpdW S  ty   Y dS w )NÚcongress_member_curÚkey_hashÚexact)Úcountr   )r#   r1   Úselectr6   r¿   r7   )Úresr!   r!   r"   Ú
_count_curç   s   ÿrÂ   Ú	stg_countÚ	cur_countÚmin_absÚ	min_ratioútuple[bool, str]c                C  s\   | |k rdd| › d|› fS |dkr,| t || ƒk r,dd| › dt || ƒ› d|› dfS dS )	NFztoo_few_rows: z < r   zratio_drop: z (cur=ú))TÚok)rB   )rÃ   rÄ   rÅ   rÆ   r!   r!   r"   Ú	_validateî   s
   $rÊ   údict[str, dict]c                  C  s@   t ƒ  d¡ d¡ dd¡ dd¡ d¡ ¡ jpg } dd	„ | D ƒS )
NÚcongress_member_histÚ*Ú
is_currentTÚvalid_toÚnulléPÃ  c                 S  ó   i | ]}|d  |“qS ©r½   r!   ©Ú.0r–   r!   r!   r"   Ú
<dictcomp>ø   ó    z!_load_cur_map.<locals>.<dictcomp>)r#   r1   rÀ   r?   Úis_Úlimitr6   r¥   )r,   r!   r!   r"   Ú_load_cur_mapõ   s   
"ÿÿrÚ   c                 C  s8   t ƒ  d¡ d¡ d| ¡ d¡ ¡ jpg }dd„ |D ƒS )Nr´   rÍ   r9   rÑ   c                 S  rÒ   rÓ   r!   rÔ   r!   r!   r"   rÖ   ü   r×   z!_load_stg_map.<locals>.<dictcomp>)r#   r1   rÀ   r?   rÙ   r6   r¥   )r9   r,   r!   r!   r"   Ú_load_stg_mapú   s   *rÛ   c                   sÞ  t ƒ }tƒ }t| ƒ‰ t tj¡ ¡ }‡ fdd„| ¡ D ƒ}|r4| 	d¡ 
|ddœ¡ d|¡ dd¡ ¡  g }g }ˆ  ¡ D ]\}}| |¡}	|	s¾| d ¡ |}
| i d|“d	|
 d	¡“d
|
 d
¡“d|
 d¡“d|
 d¡“d|
 d¡“d|
 d¡“d|
 d¡“d|
 d¡“d|
 d¡“d|
 d¡“d|
 d¡“d|
 d¡“d|
 d¡“d|
 d¡“d|
 d¡“d|
 d¡“|d ddœ¥¡ q<|	 d¡| d¡kr>| |¡ |}
| i d|“d	|
 d	¡“d
|
 d
¡“d|
 d¡“d|
 d¡“d|
 d¡“d|
 d¡“d|
 d¡“d|
 d¡“d|
 d¡“d|
 d¡“d|
 d¡“d|
 d¡“d|
 d¡“d|
 d¡“d|
 d¡“d|
 d¡“|d ddœ¥¡ q<dd„ |D ƒ}|r^| 	d¡ 
|ddœ¡ d|¡ dd¡ ¡  |rm| 	d¡ |¡ ¡  d S d S )Nc                   s   g | ]}|ˆ vr|‘qS r!   r!   ©rÕ   Úk©Ústg_mapr!   r"   Ú
<listcomp>  s    z_apply_scd2.<locals>.<listcomp>rÌ   F)rÏ   rÎ   r½   rÎ   Trn   ro   re   rp   rq   rr   rs   rt   ru   rv   rw   rx   ry   rz   r{   Úrow_hash)Ú
valid_fromrÏ   rÎ   c                 S  s   g | ]}|r|‘qS r!   r!   rÜ   r!   r!   r"   rà   H  s    )r#   rÚ   rÛ   r   r3   r   r4   r5   Úkeysr1   r>   Úin_r?   r6   Úitemsr|   Úappendr2   )r9   r·   Úcur_mapÚnow_isoÚto_close_missingÚchanged_keysÚto_addrÝ   rV   Úcurrk   r!   rÞ   r"   Ú_apply_scd2þ   sØ   
ÿ

ÿ
þ
ý
ü
û
ú
ù
ø
	÷

ö
õ
ô
ó
ò
ñ
ð
ïì
ÿ
þ
ý
ü
û
ú
ù
ø
	÷

ö
õ
ô
ó
ò
ñ
ð
ï
ì€
ÿÿrí   Úrowc              
   C  sX   zt ƒ  d¡j| dd ¡  W d S  ty+ } zt d|› ¡ W Y d }~d S d }~ww )NÚcongress_memberr)   )Úon_conflictz[base upsert] )r#   r1   r¶   r6   r7   r   r8   )rî   r:   r!   r!   r"   Ú_upsert_base_memberR  s    €ÿrñ   ç333333Ã?é2   ç333333ã?r¬   ÚelectionúDict[str, int | str]c           '      C  s8  t dƒ}d}d}zEt ¡ }z |jt› dtdd}	|	 ¡  t|d| |	j|	j	d |d7 }W n( t
yV }
 zt|d	| t› dt|
ƒd t d
|
› ¡ W Y d}
~
nd}
~
ww ttƒ}| re| |d< | |d< t||ƒ}t|d| t||d |d7 }|p{i  dg ¡pg }t|ƒ}t d|› ¡ |dkr²t|d|ddd tƒ  d¡ |ddddœ¡ ¡  |dddœW S g }t|dƒD ]þ\}}t| d¡ƒ}t| d¡ƒ}t| d¡ƒ}t| d¡ƒ}t| d¡ƒ}t| d¡ƒ}t| d¡ƒ}t| d¡ƒ}t| d ¡ƒ}t| d!¡ƒ}t| d"¡ƒ}t| d#¡ƒ}t| d$¡ƒ}t| d%¡ƒ}|r%|s;|d& dks1||kr:t||| ¡ d' q¹d} |tkrMt||ƒpLt||ƒ} ||pRd(||||||||||||| d)œ}!t ||ƒ}"t!|!ƒ}#| "d*|i|!¥|"|#d+œ¥¡ t#d,t$ %|› d-|› d-|p‡d(› d-|pd(›  &¡ ¡ '¡ i|!¥d.|i¥ƒ |d& dks©||kr²t||t|ƒd' t( )|¡ q¹|rÁt*||ƒ}nd}tƒ  d¡ ||d/d0|› d1|› dœ¡ ¡  t|d/||d2 t+ƒ }$t,||$||d3\}%}&|%stƒ  d¡ -d|&d4œ¡ .d*|¡ ¡  t|d|||&d t /d5|&› ¡ |d||&d6œW S t0|ƒ tƒ  d¡ -d7d8i¡ .d*|¡ ¡  t|d8||d2 t d9|› d:|› d;|› ¡ |d8||d<œW S  t
y› }
 z@t 1d=|
› ¡ ztƒ  d¡ ||d>t|
ƒdœ¡ ¡  W n
 t
y}   Y nw t|d>||t|
ƒd |d>t|
ƒd?œW  Y d}
~
S d}
~
ww )@ut   
    íŒŒì´í”„ë¼ì¸:
      raw(list/html, memlist/json) â†’ STG(runë³„) â†’ ê²€ì¦ â†’ HIST(SCD2) â†’ snapshot
    rï   r   r   r¤   r’   Úlist)rE   rF   rG   r[   Ú
list_errorz[raw] list fetch failed: Nr   r   Úmemlist)rE   rF   r@   rH   Ú
resultListz[congress] memList size=Úfailedzempty resultList)r*   r+   r,   r-   Úcongress_member_snapshot)r9   r,   r*   Únote)r9   r*   r,   ÚcommitteeNameÚjobResNmr   ÚhjNmÚpolyNmÚorigNmÚassemTelÚroomNoÚ
deptImgUrlÚ	pfLinkUrlÚ
memberPageÚstaffÚ	secretaryÚ
secretary2é
   )r+   r,   r   )rn   ro   re   rp   rq   rr   rs   rt   ru   rv   rw   rx   ry   rz   r{   r9   )r½   rá   r)   rf   Úsource_jsonÚ	collectedzpages=z
, memList=)r*   r+   r,   )rÅ   rÆ   )r*   rý   z[congress] validation failed: )r9   r*   r,   Úreasonr*   Úpassedz[congress] run passed: run_id=z, rows=z, pages=)r9   r*   r,   r+   z[congress] aborted: Úaborted)r9   r*   Úerror)2r;   ÚrequestsÚSessionr|   r›   r”   r•   rN   rF   rˆ   r7   r%   r   r8   rm   ÚDEFAULT_PARAMSr°   r©   rµ   ÚinforD   r#   r1   r2   r6   Ú	enumeraterY   rc   rA   Ú__len__ÚELECTION_FULLr—   rl   r~   ræ   rñ   rg   Úsha1ri   rj   r«   r¬   r»   rÂ   rÊ   r>   r?   r  rí   Ú	exception)'rE   r¬   rõ   rÅ   rÆ   r9   r+   Ústg_rowsrO   Úr_listr:   r@   r¥   Úresult_listr¸   ÚstagingÚidxÚitrd   ÚroleÚnameÚname_hjrq   rr   rs   rt   ÚphotoÚpf_urlÚ	member_pgr  Úsec1Úsec2r{   rk   rÝ   ÚhrÄ   rÉ   r  r!   r!   r"   Úrun_once]  s  €þ
ÿ
ñ
ÿþ
ü4ÿþþþÿÿÿ
ÿ€ör*  úargparse.Namespacec                  C  sh   t jdd} | jddd d | jdtdd | jd	ttgtd
 | jdtdd | jdtdd |  ¡ S )Nz&NA Industry Committee Members pipeline)Údescriptionz--committeeu)   íŠ¹ì • ìœ„ì›íšŒë§Œ ìˆ˜ì§‘ (ì˜ˆ: 9700187))ÚhelpÚdefaultz--sleeprò   )Útyper.  z
--election)Úchoicesr.  z	--min_absró   z--min_ratiorô   )ÚargparseÚArgumentParserÚadd_argumentr‘   ÚELECTION_NONEr  rB   Ú
parse_args)Úpr!   r!   r"   Ú_parse_argsþ  s   r7  Ú__main__)rE   r¬   rõ   rÅ   rÆ   )r$   r%   r&   r%   )r9   r%   r+   r<   r,   r<   r*   r=   )
r9   r%   r*   r%   r+   rB   r,   rB   r-   r=   )r9   r%   rI   r%   rE   r=   rF   r=   rG   r=   r@   rJ   rH   rJ   )rO   r=   r&   r%   )rV   rW   r&   rW   )rd   r%   re   r%   r&   r%   )rM   rm   r&   r%   )rG   r%   r&   r   )r   )rŽ   r   rF   rW   r   r‘   r&   r   )rŽ   r   r&   r˜   )rŽ   r   r@   rm   r&   rm   )r±   )r9   r%   r,   r²   r³   rB   )r&   rB   )
rÃ   rB   rÄ   rB   rÅ   rB   rÆ   r‘   r&   rÇ   )r&   rË   )r9   r%   r&   rË   )r9   r%   )rî   rm   )rE   rW   r¬   r‘   rõ   r%   rÅ   rB   rÆ   r‘   r&   rö   )r&   r+  )?Ú
__future__r   r1  rg   rK   r\   r«   r/   r   r   rG   r   Útypingr   r   r   r	   r  Úbs4r
   Úapp.services.supabase_servicer   r   r›   r©   r”   r¦   r  r4  r  r#   r;   rA   rD   rN   rU   rY   Úcompiler^   rc   rl   r~   rŒ   r—   r¢   r°   r»   rÂ   rÊ   rÚ   rÛ   rí   rñ   r*  r7  Ú__name__ÚargsÚprintrd   r¬   rõ   rÅ   rÆ   r!   r!   r!   r"   Ú<module>   sž   ýù
ù
ÿ















Tû 
"
	ûþ