
    7hQ              	      |   d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dlmZm	Z	 d dl
mZ d dlmZmZmZmZ d dlZd dlmZ d dlmZmZ dZd	Zd
dddZed         dddde dddZddddddddZdZdZd ZdidZdddd djd'Z dd(dkd+Z!dddddd,dld4Z"dmd6Z#dnd9Z$ ej%        d:          Z&dnd;Z'dod>Z(dpdAZ)dqdCZ*drdsdIZ+dtdKZ,dudLZ-dvdwdPZ.dxdQZ/dydWZ0dzdYZ1d{dZZ2d|d[Z3d}d]Z4dd^ed_d`fd~ddZ5ddfZ6e7dgk    r> e6            Z8 e9 e5e8j:        e8j;        e8j<        e8j=        e8j>        h                     dS dS )    )annotationsN)datetimetimezone)unescape)DictOptionalTupleList)BeautifulSoup)
get_clientloggerz8https://industry.na.go.kr/cmmit/mem/cmmitMemList/list.doz=https://industry.na.go.kr/cmmit/mem/cmmitMemList/memList.jsonzCMozilla/5.0 (compatible; govbot/1.0; +https://work.jjickjjicks.com)z?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8zko-KR,ko;q=0.9)
User-AgentAcceptAccept-Languager   z.application/json, text/javascript, */*; q=0.01z0application/x-www-form-urlencoded; charset=UTF-8XMLHttpRequestzhttps://industry.na.go.kr?menuNo=2000013)r   r   zContent-TypezX-Requested-WithOriginRefererr   2000013CMMIT )menuNo	cntsDivCdcommitteeCdhrCmtIdjobResCdpolyCdhgNmnonefullc                     t                      S N)r        6/var/www/html/bot/app/crawler/na_industry_committee.pySBr&   8   s    <<r$   targetstrreturnc                   t          t          j                              }	 t                                          d                              || dddd t          j        t          j	                  
                                d                                           n.# t          $ r!}t          j        d|            Y d }~nd }~ww xY w|S )Ncrawler_runrunningr   )idr'   statuspagesrowsfail_reason
started_atz[crawler_run insert] )r(   uuiduuid4r&   tableinsertr   nowr   utc	isoformatexecute	Exceptionr   warning)r'   run_ides      r%   
_run_startr?   ;   s    F4


=!!((Fi$",x|44>>@@*
 *
 	 	 79999 4 4 42q22333333334Ms   A<B 
C
)CC
)r/   r0   r.   r=   r/   
int | Noner0   r.   
str | Nonec               P   i }|||d<   |||d<   |||d<   |r	 t                                          d                              |                              d|                                            d S # t
          $ r"}t          j        d|            Y d }~d S d }~ww xY wd S )Nr/   r0   r.   r+   r-   z[crawler_run update] )r&   r5   updateeqr:   r;   r   r<   )r=   r/   r0   r.   payloadr>   s         r%   _run_heartbeatrF   G   s    GU''*T'&/v78, 8	8DDJJ}%%,,W5588vFFNNPPPPP 	8 	8 	8N6166777777777	88 8s   AA7 7
B#BB#)r1   intr1   c                  |t          j        t          j                                                  ||d}|||d<   	 t                                          d                              |                              d|           	                                 d S # t          $ r"}t          j        d|            Y d }~d S d }~ww xY w)N)r.   finished_atr/   r0   r1   r+   r-   z[crawler_run finish] )r   r7   r   r8   r9   r&   r5   rC   rD   r:   r;   r   r<   )r=   r.   r/   r0   r1   rE   r>   s          r%   _run_finishrJ   R   s    |HL11;;== G
 !,4


=!!((1144T6BBJJLLLLL 4 4 42q223333333334s   AB 
C$CC)committee_cdurlhtmlrE   json_objkindrK   rL   rM   rE   dict | NonerN   c               6   | ||d}|||d<   |||d<   |||d<   |||d<   	 t                                          d                              |                                           d S # t          $ r"}t          j        d|            Y d }~d S d }~ww xY w)N)r=   rO   rK   rL   rM   rE   jsoncongress_rawz[congress_raw insert] )r&   r5   r6   r:   r;   r   r<   )	r=   rO   rK   rL   rM   rE   rN   recr>   s	            r%   _raw_insertrU   _   s    T<
H
HC
CJ$CKWC	N(CK5


>""))#..6688888 5 5 53334444444445s   AA, ,
B6BBsc                    | sdS d                     t          |                               dd                                                    S )Nr        )joinr   replacesplit)rV   s    r%   _cleanr]   n   s@    RR88HQKK''#66<<>>???r$   vOptional[str]c                ,    t          |           }|r|nd S r"   )r]   )r^   ts     r%   
_norm_textrb   r   s    q		A11tr$   z(\d{2,4}-\d{3,4}-\d{4})c                    | sd S t          j        ddt          |                     } t                              |           }|r|                    d          n|                                 S )N\s+rX      )resubr(   RE_PHONEsearchgroupstrip)r^   ms     r%   _norm_phonerm   w   sW    TT
vsCFF##AA)1771:::		)r$   	committee	full_namec                    t          |            dt          |           }t          j        |                                                                          S )N|)r]   hashlibsha256encode	hexdigest)rn   ro   bases      r%   	_key_hashrw   }   sI    Y55&"3"355D>$++--((22444r$   rT   dictc                t   |                      d          pd|                      d          pd|                      d          pd|                      d          pd|                      d          pd|                      d          pd|                      d          pd|                      d	          pd|                      d
          pd|                      d          pd|                      d          pd|                      d          pd|                      d          pd|                      d          pdt          |                      d          |                      d          nd          g}t          j        d                    |                                                                                    S )Ncommittee_namer   	role_namero   
name_hanjapartydistrictphoneroom_no	photo_urlprofile_urlmember_page_url
aide_staffaide_secretaryaide_secretary2election_countrq   )getr(   rr   rs   rZ   rt   ru   )rT   partss     r%   	_row_hashr      s    	 !!'R""#B
!rB	 b"$"!""(b# !!'R!""(b1A)B)B)NCGG$%%%TVWWE" >#((5//002233==???r$   Optional[int]c                   t          j        ddt          | d                              dd                    }dD ]D}t          j        ||          }|r+t          |                    d                    c S #  Y AxY wEt          j        d|          }|r7dd	d
d
ddddddddd}|                    |                    d                    S d S )Nrd   rX   html.parserT)rk   )u   당선횟수\s*[:：]?\s*(\d+)u   선수\s*[:：]?\s*(\d+)re   uK   (초선|재선|삼선|3선|사선|4선|오선|5선|육선|6선|칠선|7선)                  )u   초선u   재선u   삼선u   3선u   사선u   4선u   오선u   5선u   육선u   6선u   칠선u   7선)rf   rg   r   get_textri   rG   rj   r   )rM   textpatrl   m2mappings         r%   !_extract_election_count_from_htmlr      s    6&#}T=AAJJ3VZJ[[\\DO  Ic4   	AGGAJJ'''DD	 
acg	h	hB	 (q!1aqZ[cdnowx  CD  LM  N  N{{288A;;'''4s   !A99A=      @sessionrequests.Sessiontimeoutfloatc                    |sd S 	 |                      |dt          d         i|          }|                                 t          |j                  S # t
          $ r Y d S w xY w)Nr   headersr   )r   BASE_HEADERSraise_for_statusr   r   r;   )r   rL   r   rs       r%   _fetch_election_countr      sw    ttKKlL4N%OY`Kaa	0888   tts   AA 
A! A!#Tuple[Optional[str], Optional[str]]c                   	 |                      t           dt          d          }|                                 t	          |j        d          }|                    d          }|                    d          }|r'|                     d                                          nd }|r'|                     d                                          nd}||fS # t          $ r"}t          j
        d	|            Y d }~d
S d }~ww xY w)Nr      r   r   zmeta[name="_csrf"]zmeta[name="_csrf_header"]contentzX-CSRF-TOKENzCSRF fetch failed: )NN)r   	LIST_PAGEr   r   r   r   
select_onerk   r;   r   r<   )r   r   souptoken_el	header_eltokenheaderr>   s           r%   _fetch_csrfr      s   KK9555|UWKXX	QV]33??#788OO$?@@	3;EY''--///5>Ry))//111Nf}   0Q00111zzzzzs   CC 
D C;;D c                   t          |           \  }}t          t                    }|r
|||<   ||d<   t          d          D ]}	 |                     t
          ||d          }|j        dk    r9|dk    r3t          |           \  }}|r
|||<   ||d<   t          j        d           e|	                                 |
                                c S # t          $ r |dk    rt          j        d           Y  w xY wd S )N_csrfr      )datar   r   i  r   g?)r   rx   AJAX_HEADERSrangepostMEMLIST_APIstatus_codetimesleepr   rR   r;   )r   rE   
csrf_tokencsrf_headerr   attemptr   s          r%   _post_memlistr      s*   )'22J<  G &)%88  	[wQSTTA}##1*5g*>*>'
K 2+5GK('1GG$
3   6688OOO 	 	 	!||
3		 s   A!C''C$C97C9  
List[dict]chunkc                    t                      }d}t          dt          |          |          D ]^}||||z            }|s|                    d                              |                                           |t          |          z  }_|S )Nr   congress_member_stg)r&   r   lenr5   upsertr:   )r=   r0   r   sbtotaliparts          r%   _stg_upsert_rowsr      s    	BE1c$ii''  AagIX
&''..t44<<>>>TLr$   c                     	 t                                          d                              dd                                          } | j        pdS # t
          $ r Y dS w xY w)Ncongress_member_curkey_hashexact)countr   )r&   r5   selectr:   r   r;   )ress    r%   
_count_curr      sj    ddjj.//66z6QQYY[[y~A   qqs   AA 
A! A!	stg_count	cur_countmin_abs	min_ratiotuple[bool, str]c                   | |k     r
dd|  d| fS |dk    r4| t          ||z            k     rdd|  dt          ||z             d| dfS dS )	NFztoo_few_rows: z < r   zratio_drop: z (cur=))Tok)rG   )r   r   r   r   s       r%   	_validater      s    7>y>>W>>>>1}}SY)>%?%???_Y__3y7J3K3K__S\_____:r$   dict[str, dict]c                 *   t                                          d                              d                              dd                              dd                              d                                          j        pg } d | D             S )	Ncongress_member_hist*
is_currentTvalid_tonullP  c                     i | ]}|d          |S r   r#   .0r   s     r%   
<dictcomp>z!_load_cur_map.<locals>.<dictcomp>       ---!Qz]A---r$   )r&   r5   r   rD   is_limitr:   r   )r0   s    r%   _load_cur_mapr      sx    DDJJ-..VC[[L$//J0G0GeU\U\U^U^_ckhj 	--t----r$   c                   t                                          d                              d                              d|                               d                                          j        pg }d |D             S )Nr   r   r=   r   c                     i | ]}|d          |S r   r#   r   s     r%   r   z!_load_stg_map.<locals>.<dictcomp>   r   r$   )r&   r5   r   rD   r   r:   r   )r=   r0   s     r%   _load_stg_mapr      sn    DDJJ,--44S99<<XvNNTTUZ[[cceejqoqD--t----r$   c                P
   t                      }t                      }t          |           t          j        t
          j                                                  }fd|                                D             }|re|	                    d          
                    |dd                              d|                              dd                                           g }g }                                D ]w\  }}|                    |          }	|	s|                    d            |}
|                    i d|d|
                    d          d	|
                    d	          d
|
                    d
          d|
                    d          d|
                    d          d|
                    d          d|
                    d          d|
                    d          d|
                    d          d|
                    d          d|
                    d          d|
                    d          d|
                    d          d|
                    d          d|
                    d          d|
                    d          |d dd           |	                    d          |                    d          k    r|                    |           |}
|                    i d|d|
                    d          d	|
                    d	          d
|
                    d
          d|
                    d          d|
                    d          d|
                    d          d|
                    d          d|
                    d          d|
                    d          d|
                    d          d|
                    d          d|
                    d          d|
                    d          d|
                    d          d|
                    d          d|
                    d          |d dd           yd |D             }|re|	                    d          
                    |dd                              d|                              dd                                           |r<|	                    d                              |                                           d S d S )Nc                    g | ]}|v|	S r#   r#   )r   kstg_maps     r%   
<listcomp>z_apply_scd2.<locals>.<listcomp>  s#    "Q"Q"Q@P@P1@P@P@Pr$   r   F)r   r   r   r   Trz   r{   ro   r|   r}   r~   r   r   r   r   r   r   r   r   r   row_hash)
valid_fromr   r   c                    g | ]}||S r#   r#   )r   r   s     r%   r   z_apply_scd2.<locals>.<listcomp>H  s    111!q1A111r$   )r&   r   r   r   r7   r   r8   r9   keysr5   rC   in_rD   r:   itemsr   appendr6   )r=   r   cur_mapnow_isoto_close_missingchanged_keysto_addr   r^   currv   r   s              @r%   _apply_scd2r     s   	BooGF##Gl8<((2244G #R"Q"Q"Qgllnn"Q"Q"Q N
'((//u1
 1
 	 	3z+,,RRd-C-CGGIII !LF 5 51kk!nn 3	%%%DMM A $((+;"<"< TXXk22 TXXk22	
 dhh|44 '** DHHZ00 '** 488I.. TXXk22 txx66 "488,=#>#> dhh|44 !$((+;"<"< "488,=#>#>  !$((+;"<"<!" DHHZ00#$ & ")      . wwz""aeeJ&7&777##A&&& $dhh/?&@&@  +!6!6  +!6!6	
 !$((<"8"8 TXXg..  4 4 TXXg.. txx	22  +!6!6 "488M#:#: &txx0A'B'B !$((<"8"8 %dhh/?&@&@ &txx0A'B'B  %dhh/?&@&@!"  4 4#$ #* $"&)     0 21|111L J
'((//u1
 1
 	 	3z<((L$)?)?			 B
'((//77??AAAAAB Br$   rowc                    	 t                                          d                              | d                                           d S # t          $ r"}t          j        d|            Y d }~d S d }~ww xY w)Ncongress_memberr-   )on_conflictz[base upsert] )r&   r5   r   r:   r;   r   r<   )r  r>   s     r%   _upsert_base_memberr	  R  s    -


$%%,,Sd,CCKKMMMMM - - -+++,,,,,,,,,-s   AA 
A8A33A8333333?2   333333?r   electionDict[str, int | str]c                |   t          d          }d}d}	 t          j                    }	 |                    t           dt
          d          }	|	                                 t          |d| |	j        |	j	                   |dz  }nW# t          $ rJ}
t          |d	| t           dt          |
                     t          j        d
|
            Y d}
~
nd}
~
ww xY wt          t                    }| r
| |d<   | |d<   t!          ||          }t          |d| t"          ||           |dz  }|pi                     dg           pg }t%          |          }t          j        d|            |dk    ret)          |d|dd           t+                                          d                              |dddd                                           |dddS g }t3          |d          D ]7\  }}t5          |                    d                    }t5          |                    d                    }t5          |                    d                    }t5          |                    d                    }t5          |                    d                    }t5          |                    d                    }t7          |                    d                    }t5          |                    d                    }t5          |                    d                     }t5          |                    d!                    }t5          |                    d"                    }t5          |                    d#                    }t5          |                    d$                    }t5          |                    d%                    }|r|s5|d&z  dk    s||k    r$t9          |||                                '           d} |t<          k    r t?          ||          pt?          ||          } ||pd(||||||||||||| d)}!tA          ||          }"tC          |!          }#|"                    d*|i|!|"|#d+           tG          d,tI          j%        | d-| d-|pd( d-|pd( &                                          '                                i|!d.|i           |d&z  dk    s||k    rt9          ||t%          |          '           tQ          j)        |           9|rtU          ||          }nd}t+                                          d                              ||d/d0| d1| d                                           t9          |d/||2           tW                      }$tY          ||$||3          \  }%}&|%st+                                          d          -                    d|&d4          .                    d*|                                           t)          |d|||&           t          j/        d5|&            |d||&d6S ta          |           t+                                          d          -                    d7d8i          .                    d*|                                           t)          |d8||2           t          j        d9| d:| d;|            |d8||d<S # t          $ r}
t          j1        d=|
            	 t+                                          d                              ||d>t          |
          d                                           n# t          $ r Y nw xY wt)          |d>||t          |
                     |d>t          |
          d?cY d}
~
S d}
~
ww xY w)@ut   
    파이프라인:
      raw(list/html, memlist/json) → STG(run별) → 검증 → HIST(SCD2) → snapshot
    r  r   r   r   r   list)rK   rL   rM   re   
list_errorz[raw] list fetch failed: Nr   r   memlist)rK   rL   rE   rN   
resultListz[congress] memList size=failedzempty resultList)r.   r/   r0   r1   congress_member_snapshot)r=   r0   r.   note)r=   r.   r0   committeeNamejobResNmr   hjNmpolyNmorigNmassemTelroomNo
deptImgUrl	pfLinkUrl
memberPagestaff	secretary
secretary2
   )r/   r0   r   )rz   r{   ro   r|   r}   r~   r   r   r   r   r   r   r   r   r   r=   )r   r   r-   rq   source_json	collectedzpages=z
, memList=)r.   r/   r0   )r   r   )r.   r  z[congress] validation failed: )r=   r.   r0   reasonr.   passedz[congress] run passed: run_id=z, rows=z, pages=)r=   r.   r0   r/   z[congress] aborted: aborted)r=   r.   error)2r?   requestsSessionr   r   r   r   rU   rL   r   r;   r(   r   r<   rx   DEFAULT_PARAMSr   r   r   inforJ   r&   r5   r6   r:   	enumeraterb   rm   rF   __len__ELECTION_FULLr   rw   r   r   r	  rr   sha1rt   ru   r   r   r   r   r   rC   rD   r*  r  	exception)'rK   r   r  r   r   r=   r/   stg_rowsrV   r_listr>   rE   r   result_listr   stagingidxitrn   rolenamename_hjr}   r~   r   r   photopf_url	member_pgr!  sec1sec2r   rv   r   hr   r   r'  s'                                          r%   run_oncerC  ]  s`    )**FEHMH	<UUi888,XZU[[F##%%%\vzX^XcddddQJEE 	< 	< 	<<PYMjMjMjqtuvqwqwxxxxN:q::;;;;;;;;	<
 ~&& 	.%1GM"!-GIQ((FILk[bmqrrrr
#':2"2"2<"D"D"JK  6u66777A::xu1RdeeeeDDJJ12299 !xI[; ;  wyyy$!DDD ! a00 >	 >	GC"266/#:#:;;I"266*#5#566D"266&>>22D"266&>>22G"266(#3#344E"266(#3#344H#BFF:$6$677E"266(#3#344G"266,#7#788E"266+#6#677F"266,#7#788I"266'??33E"266+#6#677D"266,#7#788D D 8q==C5LL"6W__=N=NOOOO!N=((!6q)!D!D!hH]^_agHhHh #,!ZR!%$""%#,#"&#'"0 D" )T**A$ANN& 	       gli#W#W$#W#W"#W#Wx~SU#W#W#^#^#`#`aakkmm!!%r! !   
 Rx1}}uvUWFFFFJu 	'88HHH 	

-..55h+5U55e557
 7
 	 	 7999vkXNNNN LL	xGyYYY
F 	^DDJJ12299"F; ;  r(F##GGIIIxu8Y_````LB&BBCCC$(V\]]] 	F


-..55h7
 	 	2h			F85xHHHH]V]]H]]V[]]^^^ HhQVWWW 
H 
H 
H333444	DDJJ12299 (iQTUVQWQW; ;  wyyyy 	 	 	D	F9EVYZ[V\V\]]]] IAGGGGGGGG
Hs|   [2 AB [2 
CA C[2 CC<[2 Q8[2 B![2 2
^;<^6A]-,^6-
]:7^69]::6^60^;6^;argparse.Namespacec                    t          j        d          } |                     ddd            |                     dt          d           |                     d	t          t
          gt
          
           |                     dt          d           |                     dt          d           |                                 S )Nz&NA Industry Committee Members pipeline)descriptionz--committeeu)   특정 위원회만 수집 (예: 9700187))helpdefaultz--sleepr
  )typerH  z
--election)choicesrH  z	--min_absr  z--min_ratior  )argparseArgumentParseradd_argumentr   ELECTION_NONEr1  rG   
parse_args)ps    r%   _parse_argsrQ    s    ,TUUUANN='R\`NaaaNN95$N777NN<-)GQ^N___NN;S"N555NN=ucN:::<<>>r$   __main__)rK   r   r  r   r   )r'   r(   r)   r(   )r=   r(   r/   r@   r0   r@   r.   rA   )
r=   r(   r.   r(   r/   rG   r0   rG   r1   rA   )r=   r(   rO   r(   rK   rA   rL   rA   rM   rA   rE   rP   rN   rP   )rV   rA   r)   r(   )r^   r_   r)   r_   )rn   r(   ro   r(   r)   r(   )rT   rx   r)   r(   )rM   r(   r)   r   )r   )r   r   rL   r_   r   r   r)   r   )r   r   r)   r   )r   r   rE   rx   r)   rx   )r   )r=   r(   r0   r   r   rG   )r)   rG   )
r   rG   r   rG   r   rG   r   r   r)   r   )r)   r   )r=   r(   r)   r   )r=   r(   )r  rx   )rK   r_   r   r   r  r(   r   rG   r   r   r)   r  )r)   rD  )?
__future__r   rK  rr   rR   rf   r   r3   r   r   rM   r   typingr   r   r	   r
   r+  bs4r   app.services.supabase_servicer   r   r   r   r   r   r-  rN  r1  r&   r?   rF   rJ   rU   r]   rb   compilerh   rm   rw   r   r   r   r   r   r   r   r   r   r   r  r	  rC  rQ  __name__argsprintrn   r   r  r   r   r#   r$   r%   <module>r[     s   " " " " " "    				   ' ' ' ' ' ' ' '       . . . . . . . . . . . .        < < < < < < < <
 G	M XO'  |,>F(),,,'    
  
 
 
 
 8<PTko 	8 	8 	8 	8 	8 	8 _c 4 4 4 4 4 4 GK"&4X\vz
5 
5 
5 
5 
5 
5@ @ @ @    2:011* * * *5 5 5 5
@ @ @ @0             8          . . . .
. . . .QB QB QB QBh- - - - #'!\H \H \H \H \HB    z;==D	E((^j.        r$   