UHj%%LdZddlmZddlZddlZddlZddlZddlmZddl m Z ddl m Z m Z ddlmZmZmZe ej'j(dZed z Zej.d Zdd Zdd Zdd ZdddZddZddZddZefddZ ddZ!e"dk(re!yy)z@Scan decompressed CNS payloads for CP949 Korean text candidates.) annotationsN)Counter)Path)decompress_cns parse_image) clean_text classify_text is_hanguloutzF^[\u3131-\u318e\uac00-\ud7a3A-Za-z0-9 !?.,:;_+\-/()[\]{}'\"~%&*<>|=]+$cd|dS)N0x08x)values :/home/exedev/hwanse/tools/summarize_cns_text_candidates.pyhex32rs c{ c t|dk\rwtj|ddd}tj|ddd}d||zdzz}d|cxkrdkr/nn,d|cxkrdkr!nnt||k(rd||t|dSt|dk\rP|d}|d}d||zdzz}d|cxkrdkr0nn-d|cxkrdkr"nnt||k(rd||dt|d S t|\}}}}}||zd zd zdz}||z}d|cxkrd krXnnUd|cxkrd krJnnGt||k\r9d t|dzz} d|||t|t t|| |zdSdddS#t $rYwxYw)Nrlittletilemap)kindwidthheight textScanStartr )rrr headerBytesr iimage)rrrbpp paletteColorsrunknown)rr)lenint from_bytesrmin Exception) decodedrr expected_mappalettepixelsr$strideexpected_pixels palette_ends rclassify_decoded_payloadr3s 7|qwq|X6! h756>A-- u  F 1c 1c'll6R%Z]^eZfg g 7|q 56>A-- u  F 1c 1c'll6R!  !$W   .9'.B+vw3;#*a/ 6/ u  V!3t!3F 8Vc'lQ..K !$W!$S\;3P!Q   22    s%BE11 E=<E=c ||}|dvsd|cxkrdkryny|dk\rW|dzt|krF||dzdk7r;|||dz} |jd}t|stj |ryy#t$rYywxYw) N) r!~r rrcp949)r'decodeUnicodeDecodeErrorr PRINTABLE_REmatch)dataindexbytepairchars rvalid_cp949_charrD?s ;D !!TT%9T%9&: t| CI-$uqy/Q2FE%!)$ ;;w'D T?l006 "  sA66 BBc `g}d}|t|kr ||dk(r|dz }|}|t|kr#t||}|sn||z }|t|kr#||z dk\r|||} |jd}t |}t d|D} | dk\redt|cxkrdkrQnnNt j|r9|j||zt||zt||| t|d t|dz|dz}|t|kr |S#t$rd}YwxYw) Nrr rr:c38K|]}t|sdyw)r N)r ).0rCs r z!scan_text_runs..bsETYt_qEsr) decodedOffsetdecodedOffsetHex byteLengthtext hangulCountclassification) r'rDr;r<rsumr=r>appendrr max) r? base_offsetrowsr@startstepblobr,rN hangul_counts rscan_text_runsrZNsW D E #d)  ;!  QJE c$i#D%0D TME c$i 5=A e$D ++g.g&DETEEL!T)c) &&t, %05%8(-kE.A(B"%d) #/&3D&9 EAIuqy)? #d) @ K%&  s#D D-,D-c&|jdddd} t|j}t |}|j t||d|jd|jdd|jd d}||d}||d <t||d <t||}|D]'}|j|d <|d|d <t||d <)t||d<||fS#t$r}t ||d<|gfcYd}~Sd}~wwxYw)Nr&r)name decodedSize payloadKindcandidateCounterrorrrr)r]r^rrr textScanBytesfiler^r]r_) r\r read_bytesr+strr3updater'getrZ) pathrowr,excpayload scan_start scan_data candidates candidates r scan_filerots0    C  !23'w/GJJ7|vW%++h'  _a0J $I%Cy>C :6J0  II &#*6? - #&w< - 0 OC  ?) 3xG BwsC++ D4D D Dcg}g}t|jdD]2}t|\}}|j||j |4|j dt d|D}t d|D}dt|t|td|Dtd|Dt|tt|jtt|jd |Dcgc]}|jd d d kDs|c}|dd d Scc}w)Nz*.cnsc|d|d|dfS)NrPrbrKr)rhs rzbuild_summary..s!  F  O%r)keyc3&K|] }|d yw)r^NrrHrhs rrIz build_summary..sACS/Ac3&K|] }|d yw)rPNrrus rrIz build_summary..sGS3/0Grvz7CP949 Korean string scan over decompressed CNS payloadsc3DK|]}|jddyw)r]Nr rfrus rrIz build_summary..sYccggm6L6XYs  c3LK|]}|jdddkDsdyw)r_rr Nryrus rrIz build_summary..s%!Y@PRS8TWX8X!!Ys$$aThis scan checks text-bearing regions of decompressed CNS payloads for plain CP949 text. Known image pixels and exact tilemap bodies are skipped to avoid pixel-noise false positives. A zero or very small result means story dialogue is probably not stored as simple null-delimited CP949 strings in these payloads; it may be encoded in another script format or generated through executable-side tables.r_ri) scopesourcescannedFileCountdecodedFileCountcandidateFileCountr_payloadKindCountsclassificationCounts conclusionfilesWithCandidatesrm) sortedglobrorRextendsortrrdr'rQdictitemsrf) src_dirfilesrmrgfile_rowfile_candidatespayload_counts class_countsrhs r build_summaryrs- EJw||G,-+$-dO!/ X/*+OOO A5AANGJGGLJg,JYUYY!!Yu!YYj/!&)=)=)?"@A $VL,>,>,@%A B p 05Y@PRS8TWX8XY #&!  Zs D;.D;ctdjd|jdxsgDxsd}djd|jdxsgDxsd}djd d d d d ddddddtj|ddd|dd|dd|ddddd|dd d!d|dd"d#dgS)$NrFc 3K|]b}dtj|ddtj|dd|jdxsdd|jddd dy w) r\ r^r]-r_r N)htmlescaperfrus rrIzhtml_page..s   {{3v;'(){{3}-./0ww}%,-.ww'+,-  sA(A*rzFNo CNS files contained CP949 candidates.c 3K|]k}dtj|ddtj|ddtj|ddtj|dd my w) rrPrrbzrLzrNrN)rrrus rrIzhtml_page..s   {{3/0123{{3v;'()[[%7!89:;{{3v;'()  sA1A3rmz1No text candidates. zzzz zF z$ CNS Text Candidatesz zzz

CNS Text Candidates

z

rz

z

scanned: r}z ; decoded: r~z; candidates: r_z

Files With Candidates

zd z z
filekinddecoded bytescandidates
z

Candidates

zX zz)joinrfrr)summary file_rowscandidate_rowss r html_pagers>;;45;R RWW;;|,2= = 99"P. ~(  GL12348 !345[I[A\@]]klsuEmFlGGK L*n I;./b N##34 + rc|jdd|dz jtj|dddzd |d z jt |d y) NT)parentsexist_okzcns_text_candidates.jsonFr) ensure_asciiindentrzutf-8)encodingcns_text_candidates.html)mkdir write_textjsondumpsr)rout_dirs r write_outputsrsd MM$M. ))55 7q9D@6 ))55i6HSZ5[rc\tj}|jdttdz |jdtt |j }t|j}t||jtd|jdz d|dd y) Nz --src-dir extract_fld)typedefaultz --out-dirzwrote CNS text candidates -> rz (r_z candidates)) argparseArgumentParser add_argumentrROOTOUT parse_argsrrrrprint)parserargsrs rmainrs  $ $ &F  $}8LM  $<    DDLL)G'4<<( ' <<4 4 56 $ % &l 4r__main__)rr(returnrd)r,bytesrr)r?rr@r(rr()r)r?rrTr(rz list[dict])rgrrztuple[dict, list[dict]])rrrr)rrrrd)rrrrrNone)rr)#__doc__ __future__rrrrre collectionsrpathlibr decode_cnsrr summarize_korean_text_candidatesrr r __file__resolverrrcompiler=rr3rDrZrorrrr__name__rrrrsF" 2QQ H~''* Ulrzzcd #3L  #L@D)X25\  zFr
classfileoffsettext