o
    tJÂh‡/  ã                
   @  sp  U d dl mZ d dlmZmZ d dlmZ d dlmZmZm	Z	m
Z
 d dlZd dlZd dlZd dlmZ d dlmZmZmZmZmZ d dlmZmZmZ d d	lmZmZ eƒ Zg d
¢ZedƒZd^dd„Z d_dd„Z!d`dd„Z"dadd„Z#dbd d!„Z$d"d#„ Z%ej& 'd$¡p…ej(p…d%Z)d&d'd(d)d*d+d,d-d.œZ*d/e+d0< e)d1d2ge)d1d3ge)d1d4ge)d1d5ge)d1d6ge)d1d7ge)d1d8ge)d1d9gd.œZ,d:e+d;< ee-ƒ .¡ j/d< Z0ej& 'd=e1e0ƒ¡Z2ej& 'd>¡Z3ej'd?ed@eddAeddAeddAedBdCdDdEedCdCdFed dAfdcdMdN„ƒZ4ej'dOed@dddQdR„ƒZ5ej6dSed@edTƒeddAed?dAeddAfdedYdZ„ƒZ7e 6d[¡edTƒeddAfdfd\d]„ƒZ8dS )gé    )Úannotations)ÚdatetimeÚtimezone)ÚZoneInfo)ÚAnyÚDictÚListÚOptionalN)ÚPath)Ú	APIRouterÚRequestÚQueryÚFormÚstatus)ÚHTMLResponseÚRedirectResponseÚJSONResponse)Ú	templatesÚsb)ÚrunningÚ	collectedÚpassedÚfailedÚabortedz
Asia/SeoulÚtsúOptional[str]Úreturnc                 C  s`   | sd S z |   dd¡}t |¡}|jd u r|j tjd}| t¡ d¡W S  t	y/   |  Y S w )NÚZú+00:00©Útzinfoz%Y-%m-%d %H:%M:%S)
Úreplacer   Úfromisoformatr    r   ÚutcÚ
astimezoneÚKSTÚstrftimeÚ	Exception)r   ÚsÚdt© r*   ú1/var/www/html/bot/app/web/pages/admin/crawlers.pyÚ_to_kst_str   s   

ÿr,   ÚsecúOptional[float]Ústrc                 C  s`   | d u rdS t | ƒ} t| dƒ\}}t|dƒ\}}|r'|d›d|d›d|d›S |d›d|d›S )Nú-i  é<   Ú02dú:)ÚintÚdivmod)r-   ÚhÚremÚmr(   r*   r*   r+   Ú_sec_to_str"   s   0r9   Ú	start_isoÚend_isoc                 C  sŠ   | r|sd S z4t  |  dd¡¡}t  | dd¡¡}|jd u r%|jtjd}|jd u r1|jtjd}td||  ¡ ƒW S  tyD   Y d S w )Nr   r   r   g        )	r   r"   r!   r    r   r#   ÚmaxÚtotal_secondsr'   )r:   r;   r(   Úer*   r*   r+   Ú_duration_seconds*   s   ÿr?   r   ÚtargetÚqc              
   C  sN   |r|   d|¡} |r|   d|¡} |r%| jj dd|› d|› d|› di¡ | S )Nr   r@   Úorz
id.ilike.*z*,target.ilike.*z*,fail_reason.ilike.*Ú*)ÚeqÚsessionÚparamsÚupdate)Úqbr   r@   rA   r*   r*   r+   Ú_apply_filters6   s   &rI   ú	List[str]c                  C  s”   z>t  d¡ d¡jddd d¡ ¡ } | jpg }tƒ g }}|D ]}| d¡p(d 	¡ }|r;||vr;| 
|¡ | |¡ q |W S  tyI   g  Y S w )NÚcrawler_runr@   Ú
started_atT©Údesciè  Ú )r   ÚtableÚselectÚorderÚlimitÚexecuteÚdataÚsetÚgetÚstripÚaddÚappendr'   )ÚresÚrowsÚseenÚoutÚrÚtr*   r*   r+   Ú_fetch_targets?   s   $
€ÿra   c                 C  s    z| ƒ W S  t y   | Y S w ©N)r'   )ÚfnÚdefaultr*   r*   r+   Ú_safe_qL   s
   ÿre   Ú
PYTHON_BINz/usr/bin/python3u   êµ­íšŒ ì‚°ì—… ê´€ë ¨ ìƒìž„ìœ„u   ì‚°ì—…ë¶€ ì¡°ì§u   í™˜ê²½ë¶€ ì¡°ì§u   ê¸°ìž¬ë¶€ ì¡°ì§u   ì‚°ì—…ë¶€ ì¸ì‚¬ë™ì •u   ê¸°ìž¬ë¶€ ì¸ì‚¬ë™ì •u   í™˜ê²½ë¶€ ì¸ì‚¬ë™ì •u   ì „ë ¥ê·¸ë£¹ì‚¬(ALIO))Úna_industry_committeeÚmotie_org_pipelineÚme_org_pipelineÚmoef_org_pipelineÚmotie_idÚmoef_idÚme_idÚ	group_n8nzDict[str, str]ÚCRAWLER_LABELSz-mz!app.crawler.na_industry_committeezapp.crawler.motie_org_pipelinezapp.crawler.me_org_pipelinezapp.crawler.moef_org_pipelinezapp.crawler.motie_n8nzapp.crawler.moef_n8nzapp.crawler.me_n8nzapp.crawler.group_n8nzDict[str, List[str]]ÚCRAWLER_COMMANDSé   ÚCRAWLER_WORKDIRÚADMIN_LAUNCH_TOKENú/admin/crawlers)Úresponse_class)rd   é2   é   éô  )rd   ÚgeÚle)rd   ry   Úrequestr   rS   r4   ÚpageÚauto_refreshc                 Ã  s^  t ƒ }zt d¡jddd}t||||ƒ}| ¡ jpd}	W n ty)   d}	Y nw td|	| d | ƒ}
t	||
ƒ}|d | }g }z%t d¡ d¡j
dd	d
 ||| d ¡}t||||ƒ}| ¡ jpeg }W n tyr   g }Y nw |D ](}| d¡| d¡}}t||ƒ}t|ƒ|d< t|ƒp’d|d< t|ƒpšd|d< qu|	dddddœ}zcdD ] }t d¡jddd}t||||ƒ d|¡}| ¡ jpÆd||< q©t d¡ d¡j
dd	d
}t||||ƒj dd¡ d¡}| ¡ jpìg }dd„ dd„ |D ƒD ƒ}|rtt|ƒt|ƒ ƒ|d< W n
 ty   Y nw t d| ||t|||p!d|||
|	||tttƒdœ¡S )NrK   ÚidÚexact©Úcountr   rw   rC   rL   TrM   Úfinished_atÚ_duration_strr0   )Útotalr   r   r   Úavg_duration_str)r   r   r   r   zstarted_at,finished_atÚnulléÐ  c                 S  s   g | ]}|d ur|‘qS rb   r*   )Ú.0Údr*   r*   r+   Ú
<listcomp>¦   s    z"admin_crawlers.<locals>.<listcomp>c                 s  s&    | ]}t | d ¡| d¡ƒV  qdS )rL   r‚   N)r?   rW   )rˆ   r_   r*   r*   r+   Ú	<genexpr>¦   s   €$ z!admin_crawlers.<locals>.<genexpr>r…   zadmin/crawlers.htmlrO   )r{   ÚrunsÚtargetsÚSTATUSESr   r@   rA   rS   r|   Úpagesr„   r}   Úkpisro   ÚLAUNCH_TOKEN_ENABLED)ra   r   rP   rQ   rI   rT   r   r'   r<   ÚminrR   ÚrangerU   rW   r?   r9   r,   rD   Únot_Úis_rS   ÚsumÚlenr   ÚTemplateResponserŽ   ro   ÚboolÚLAUNCH_TOKEN)r{   r   r@   rA   rS   r|   r}   r   Úqb_countr„   r   ÚoffsetrŒ   rH   r_   Úst_rawÚfn_rawÚdurr   r(   ÚcqbÚaqbr\   Úsecsr*   r*   r+   Úadmin_crawlerss   sv   €
ÿ
*ÿ
€ÿñr£   z/admin/crawlers/{run_id}Úrun_idc           	      ƒ  s˜  zt  d¡ d¡ dˆ ¡ d¡ ¡ }|jpd gd }W n ty'   d }Y nw |s4t 	d| dd dœ¡S | 
d	¡}| 
d
¡}t||ƒ}t|ƒpHd|d< t|ƒpPd|d< t|ƒ|d< | 
d¡p_d ¡ }g d g g d dœ}|dkr”t‡ fdd„g ƒ|d< t‡ fdd„dƒ|d< t‡ fdd„g ƒ|d< d|d< n.|dkr¾t‡ fdd„g ƒ|d< t‡ fdd„dƒ|d< t‡ fdd„g ƒ|d< d |d< nd!|d< t 	d| ||d"œ¡S )#NrK   rC   r~   rw   r   zadmin/crawler_detail.htmlT)r{   Ú	not_foundÚrunrL   r‚   r0   Ú_started_kstÚ_finished_kstrƒ   r@   rO   )ÚsnapshotÚ	stg_countÚ	raw_pagesÚnotesÚsection_titleÚmoef_orgc                     ó2   t  d¡ d¡ dˆ ¡jddd d¡ ¡ jpg S )NÚmoef_org_snapshotrC   r¤   r~   TrM   r‡   ©r   rP   rQ   rD   rR   rS   rT   rU   r*   ©r¤   r*   r+   Ú<lambda>Þ   ó   þþz'admin_crawlers_detail.<locals>.<lambda>r©   c                     ó&   t  d¡jddd dˆ ¡ ¡ jpdS )NÚmoef_org_stgÚkey_hashr   r€   r¤   r   ©r   rP   rQ   rD   rT   r   r*   r²   r*   r+   r³   â   ó   ÿrª   c                     r¯   )NÚmoef_org_rawzkind,page,url,created_atr¤   r|   T©Úascr‡   r±   r*   r²   r*   r+   r³   å   r´   r«   u   MOEF ìˆ˜ì§‘ ê²°ê³¼r­   rn   c                     r¯   )NÚkepco_org_snapshotzdepartment, run_id, created_atr¤   Ú
departmentTr»   r‡   r±   r*   r²   r*   r+   r³   ì   r´   c                     rµ   )NÚkepco_org_stgr·   r   r€   r¤   r   r¸   r*   r²   r*   r+   r³   ð   r¹   c                     s<   t  d¡ d¡ dˆ ¡jdddjddd d¡ ¡ jpg S )	NÚ	kepco_rawz$apba_id, kind, page, url, created_atr¤   Úapba_idTr»   r|   iˆ  r±   r*   r²   r*   r+   r³   ó   s   þþýu#   ì „ë ¥ê·¸ë£¹ì‚¬(ALIO) ìˆ˜ì§‘ ê²°ê³¼u   ì¶”ê°€ ì •ë³´)r{   r¦   Úextras)r   rP   rQ   rD   rS   rT   rU   r'   r   r˜   rW   r?   r,   r9   rX   re   )	r{   r¤   r[   r¦   Ústarted_rawÚfinished_rawrŸ   r@   rÂ   r*   r²   r+   Úadmin_crawlers_detailÀ   sh   €"ÿÿ


ûýþý
ýþü
ýrÅ   z/admin/crawlers/run.Úlaunch_tokenÚnext_urlÚbackfill_pagesúOptional[int]c           
   
   Ã  sþ   t r| j d¡}|t ks|t kstdddS t |¡}|s'td|› ddS z<tj ¡ }d|d< |d	krA|rA|d
krAt|ƒ|d< t	tj
dƒ}tj|t|||ttdƒrVtjnd d t|p^dtjdW S  ty~ }	 ztd|	› ddW  Y d }	~	S d }	~	ww )Núx-admin-launch-tokenÚUnauthorizedé‘  ©Ústatus_codezUnknown target: é  Ú1ÚCRAWLER_LAUNCHED_FROM_ADMINrm   r   ÚME_ID_BACKFILL_PAGESÚwbÚsetsid©ÚcwdÚenvÚstdoutÚstderrÚ
preexec_fnrt   zFailed to launch: rx   )rš   ÚheadersrW   r   rp   ÚosÚenvironÚcopyr/   ÚopenÚdevnullÚ
subprocessÚPopenÚWORKDIRÚhasattrrÔ   r   r   ÚHTTP_303_SEE_OTHERr'   )
r{   r@   rÆ   rÇ   rÈ   ÚhdrÚcmdr×   rà   r>   r*   r*   r+   Úadmin_crawlers_run  s6   €

ú€ÿrè   z/api/admin/crawlers/runc              
   Ã  sê   t r| j d¡}|t ks|t kstdddœddS t |¡}|s-tdd|› dœddS z*tj ¡ }d	|d
< ttj	dƒ}t
j|t|||ttdƒrLtjnd d td|dœƒW S  tyt } ztdt|ƒdœddW  Y d }~S d }~ww )NrÊ   FÚunauthorized)ÚokÚerrorrÌ   rÍ   zunknown target: rÏ   rÐ   rÑ   rÓ   rÔ   rÕ   T)rê   Úlaunchedrx   )rš   rÛ   rW   r   rp   rÜ   rÝ   rÞ   rß   rà   rá   râ   rã   rä   rÔ   r'   r/   )r{   r@   rÆ   ræ   rç   r×   rà   r>   r*   r*   r+   Úapi_admin_crawlers_run*  s2   €

ú"€ÿrí   )r   r   r   r   )r-   r.   r   r/   )r:   r   r;   r   r   r.   )r   r   r@   r   rA   r   )r   rJ   )r{   r   r   r   r@   r   rA   r   rS   r4   r|   r4   r}   r4   )r{   r   r¤   r/   )
r{   r   r@   r/   rÆ   r   rÇ   r   rÈ   rÉ   )r{   r   r@   r/   rÆ   r   )9Ú
__future__r   r   r   Úzoneinfor   Útypingr   r   r   r	   rÜ   Úsysrá   Úpathlibr
   Úfastapir   r   r   r   r   Úfastapi.responsesr   r   r   Úapp.web.depsr   r   ÚrouterrŽ   r%   r,   r9   r?   rI   ra   re   rÝ   rW   Ú
executableÚPY_BINro   Ú__annotations__rp   Ú__file__ÚresolveÚparentsÚPROJECT_ROOTr/   rã   rš   r£   rÅ   Úpostrè   rí   r*   r*   r*   r+   Ú<module>   s|   




	øø
ùLCû%ý