U SW5@sddlZddlZddlZddlZddlZddlZddlZddlZddlZddl Z zddl m Z Wne k rxdZ YnXzddl m Z Wn e k rddlm Z YnXzddlmZWne k rdZYnXddlmZddlmZzddlmZWn6e k r4ddlmZdd lmZd d ZYnXzdd lmZWne k r`dZYnXd ZdZdZdZdZdZ iZ!iZ"iZ#Gddde$Z%e%Z&e%Z'ddZ(ddZ)ddZ*ddZ+ddZ,dd Z-d!d"Z.e*edud#d$Z/e*ed%d&Z0e*edvd(d)Z1e*ed*d+Z2e*ed,d-Z3ej4dd.krbe*edwd0d1Z5e*ed2d3Z6n"e*edxd4d1Z5e*ed5d3Z6e*ed6d7Z7e*edyd8d9Z8e*ed:d;Z9e*edd?Z;e*edzd@dAZe*edFdGZ?e*e dHdIZ@e*e dJdKZAdLdMZBe*e dNdOZCe*e dPdQZDe*e dRdSZEe*e dTdUZFe*e dVdWZGe*e dXdYZHGdZd[d[eIZJGd\d]d]eJZKej4dd.d^krd_d`ZLndadbZLe(eGdcddddeKZMe(eGdedfdfeKZNe(eGdgdhdheIZOe(eGdidjdjeIZPe(eGdkdldleJZQe(eGdmdndneJZRe(eGdodpdpeIZSedk re*e ejTZTe*e ejUZUe*e ejVZVe(eejWZWedk r e)e GdqdrdreZXe)eGdsdtdteZYdS){N)Counter)urlparse) TableFunction)binary_construct) unicode_typeformat_date_time_sqlite)format_date_time)SQLITE_DATETIME_FORMATScCs t|tSN)r r )Z date_valuer /sqlite_udf.pyr!sr) _sqlite_udfZ control_flowdatefileZhelpersmathstringcs@eZdZfddZfddZfddZfddZZS) synchronized_dictcs tt|j||t|_dSr )superr__init__ threadingZLock_lock)selfargskwargs __class__r r r8szsynchronized_dict.__init__c s.|jtt||W5QRSQRXdSr )rrr __getitem__rkeyrr r r<szsynchronized_dict.__getitem__c s0|j tt|||W5QRSQRXdSr )rrr __setitem__)rrvaluerr r r @szsynchronized_dict.__setitem__c s.|jtt||W5QRSQRXdSr )rrr __delitem__rrr r r"Dszsynchronized_dict.__delitem__)__name__ __module__ __qualname__rrr r" __classcell__r r rr r7s   rcsfdd}|S)Ncs(D]}t|gt||q|Sr )AGGREGATE_COLLECTION setdefaultappendklassgroupgroupsr r decoratorNs zaggregate..decoratorr r.r/r r-r aggregateMs r1csfdd}|S)Ncs(D]}t|gt||q|Sr )TABLE_FUNCTION_COLLECTIONr(r)r*r-r r r/Vs z!table_function..decoratorr r0r r-r table_functionUs r3csfdd}|S)Ncs(D]}t|gt||q|Sr )UDF_COLLECTIONr(r))fnr,r-r r r/^s zudf..decoratorr r0r r-r udf]s r6cGsTt}|D]D}t|}|D]2}t|d|j}||kr||||d|qq dS)Nname)setr'getattrr#addZcreate_aggregate)connr.seenr,klassesr+r7r r r register_aggregate_groupsfs r?cGsFt}|D]6}t|}|D]$}|j|kr||j||qq dSr )r9r2r7r;register)r<r.r=r,r>r+r r r register_table_function_groupsps  rAcGsLt}|D]<}t|}|D]*}|j}||kr||||d|qq dS)Nr8)r9r4r#r;Zcreate_function)r<r.r=r,Z functionsZfunctionr7r r r register_udf_groupsys rBcCs.t|ftt|ftt|ftdSr )r?r'rAr2rBr4)r<r r r register_allsrCcCs |r|S|Sr r )ZcondZtruthyZfalseyr r r if_then_elsesrDcCsL|dd}|d}|dkr*|d|S|d}|dkrH|d|S|S)NT +r8- )replacefind)Zdate_strZtz_idx1Ztz_idx2r r r strip_tzs     rL, cCs^d}g}|D]>\}}t||\}}|r |dkr2dp4d}|d|||fq |sTdS||S)N))i3year)i'month)i: Zweek)Qday)ihour)<minute)secondrUsz%s %s%sz 0 seconds)divmodr)join)ZnsecondsZgluepartsZaccumoffsetr7valsuffixr r r human_deltas  r_cCs0ztj|}Wntk r&YdSX|dSNrU)ospathsplitext ValueError)filenameresr r r file_exts rgc Cs<z*t|}|W5QRWSQRXWn YnXdSr )openread)reZfhr r r file_reads   rj cCstt||Sr )rzlibcompressdataZ compressionr r r gzipsrqcCs t|Sr )rm decompressrpr r r gunzipsrtcCstt||Sr )rmrnrror r r rqscCst|dS)Nzutf-8)rmrrdecodersr r r rtscCst|}|r|jSdSr )rZnetloc)ZurlZ parse_resultr r r hostnamesrvcCs2|}|dk r|t|<nt| t|<}|Sr )lowerSTATEget)rZonr r r toggles  rzcGs0|st|St|dkr(|dt|<ndSdS)NrUrF)SETTINGSrylen)rrr r r settings   r}cCs tdSr )r{clearr r r r clear_settingssrcCs tdSr )rxr~r r r r clear_togglessrcCs.|dkrd|}}n |dkr d}t|||SNrrU)randomZ randrange)startendstepr r r randomranges  rcCs*zt||WStk r$YdSXdSr )rZgaussrd)ZmeanZsigmar r r gauss_distributionsrcCs(z t|WStk r"YdSXdSr )rsqrtrd)nr r r rs rc CsDz t|WStk r>zt|WYSYYdSXYnXdSr )intrdfloatrWr r r tonumber s rcCs|r|s dS||SNr)count)haystackZneedler r r substr_countsrcCst||Sr )rstrip)rcharsr r r strip_charssrcGs"|}|D]}||q |Sr )updateZ hexdigest) constructorrZhash_objargr r r _hashs rcGs ttjSr )rhashlibmd5Zvalsr r r r#srcGs ttjSr )rrsha1rr r r r'srcGs ttjSr )rrsha256rr r r r+srcGs ttjSr )rrsha512rr r r r/srcCs t|Sr )rmadler32rr r r r3srcCs t|Sr )rmcrc32rr r r r7src@s$eZdZddZddZddZdS) _heap_aggcCsg|_d|_dSr)heapctrr r r r=sz_heap_agg.__init__cCs|Sr r rr!r r r processAsz_heap_agg.processcCs&|jd7_t|j||dSr`)rheapqheappushrrrr r r rDsz_heap_agg.stepN)r#r$r%rrrr r r r r<src@seZdZddZdS)_datetime_heap_aggcCst|Sr rrr r r rIsz_datetime_heap_agg.processN)r#r$r%rr r r r rHsr)rkcCs|j|jd|jdS)NrPg.A)secondsZdaysZ microsecondstdr r r total_secondsMs rcCs|Sr )rrr r r Rrc@seZdZddZdS)mintdiffcCsnd}}|jrZ|dkr,|dkr,t|j}qt|j}||}|dksP||krT|}|}q|dk rjt|SdSr )rrheappopr)rdtpmin_diffdtdiffr r r finalizeVs  zmintdiff.finalizeNr#r$r%rr r r r rTsrc@seZdZddZdS)avgtdiffcCs|jdkrdS|jdkrdSd}}d}|jrz|dkrL|dkrLt|j}q(t|j}||}|d7}|t|7}|}q(t||SNrUr)rrrrrr)rtotalrrrrr r r rgs"     zavgtdiff.finalizeNrr r r r resrc@s$eZdZddZddZddZdS)durationcCsd|_|_dSr _min_maxrr r r rszduration.__init__cCs@t|}|jdks||jkr"||_|jdks6||jkr<||_dSr )rrr)rr!rr r r rs z duration.stepcCs$|jr |jr |j|j}t|SdSr )rrr)rrr r r rs  zduration.finalizeN)r#r$r%rrrr r r r r}src@sBeZdZer&ddZddZddZnddZddZd dZd S) modecCs t|_dSr )ritemsrr r r rsz mode.__init__cGs|j|dSr )rr)rrr r r rsz mode.stepcCs|jr|jdddSdSr)r most_commonrr r r rsz mode.finalizecCs g|_dSr )rrr r r rscCs|j|dSr )rr))ritemr r r rscCs |jrtt|j|jjdSdS)N)r)rmaxr9rrr r r rsN)r#r$r%rrrrr r r r rs rc@seZdZddZdS)minrangecCsz|jdkrdS|jdkrdSd}}|jrv|dkrH|dkrHt|j}q$t|j}||}|dksl||krp|}|}q$|Sr)rrrr)rprevrcurrrr r r rs     zminrange.finalizeNrr r r r rsrc@seZdZddZdS)avgrangecCs|jdkrdS|jdkrdSd}}d}|jrv|dkrL|dkrLt|j}q(t|j}||}|d7}||7}|}q(t||Sr)rrrrr)rrrrrrr r r rs"    zavgrange.finalizeNrr r r r rsrc@s(eZdZdZddZddZddZdS) _rangerangecCsd|_|_dSr rrr r r rsz_range.__init__cCs8|jdks||jkr||_|jdks.||jkr4||_dSr rrr r r rsz _range.stepcCs$|jdk r |jdk r |j|jSdSr rrr r r rs z_range.finalizeN)r#r$r%r7rrrr r r r rsrc@s0eZdZddgZdgZdZd ddZdd ZdS) RegexSearchregex search_stringmatchZ regex_searchNcCst|||_dSr )refinditer_iter)rrrr r r initializeszRegexSearch.initializecCst|jdfSr)nextrr,)ridxr r r iterateszRegexSearch.iterate)NNr#r$r%Zparamscolumnsr7rrr r r r rs  rc@s2eZdZdddgZdgZdZd ddZd d Zd S) DateSeriesrstop step_secondsrZ date_seriesrPcCst||_t||_t|}tj|d|_|jjdkr^|jjdkr^|jj dkr^|dkr^d|_ n^|jj dkr|jj dkr|jj dkr|jj dkr|jj dkr|jj dkr|dkrd|_ nd|_ dS) N)rrrPz%Y-%m-%dilrUz%H:%M:%Sz%Y-%m-%d %H:%M:%S)rrrrdatetimeZ timedeltarrRrTrVformatrNrOrQ)rrrrr r r rs6           zDateSeries.initializecCs4|j|jkrt|j}|j|j7_||jfSr )rr StopIterationrstrftimer)rrZcurrentr r r rs  zDateSeries.iterateN)rPrr r r r rs   r)N)rM)rl)rl)N)NN)Zrrrrrarrsysrrm collectionsr ImportErrorrZ urllib.parseZvtfuncrZpeeweerrZplayhouse._speedupsrr r Z playhouserZ cython_udfZ CONTROL_FLOWZDATEFILEZHELPERZMATHSTRINGr'r2r4dictrrxr{r1r3r6r?rArBrCrDrLr_rgrj version_inforqrtrvrzr}rrrrrrrrrrrrrrrobjectrrrrrrrrrrZdamerau_levenshtein_distZlevenshtein_distZstr_distZmedianrrr r r r s