
oy,Xc           @` s&  d  d l  m Z m Z m Z d  d l m Z d  d l m Z d  d l m Z d  d l	 Z	 d  d l
 Z
 d  d l Z d  d l Z d  d l Z d  d l Z d  d l Z d  d l Z d  d l Z d  d l Z d  d l Z d  d l Z d  d l Z d  d l Z y" d  d l Z e j e j  Z Wn d Z n Xy d  d l Z e Z Wn e  Z n Xd Z! d Z" e  Z# d Z$ d	 Z% d	 Z& d
 Z' e% e% d e j   7Z% e& e& d e j   7Z& d a( d a) d a* d a+ d a, d a- d a. d a/ d a0 d a1 e2   a3 e j4 d  Z5 e j4 d  Z6 e2   Z7 e j4 d  Z8 e r9e j4 d  Z9 n e j4 d  Z9 d Z: d Z; d Z< d Z= d Z> d Z? d Z@ d ZA i d e: e; f d 6d e: e; f d 6d e: e; f d 6d e: e; f d 6d  e: e; f d! 6d" e> e? f d# 6d$ e@ eA f d% 6d& e: e; f d' 6d( e: e; f d) 6d* e: e; f d+ 6d, e: e; f d- 6d. e: e; f d/ 6d0 e: e; f d1 6d0 e: e; f d2 6d0 e: e; f d3 6d4 e: e; f d5 6d6 e: e; f d7 6d6 e: e; f d8 6d9 e: e; f d: 6d; e< e= f d< 6d= e< e= f d> 6d? e< e= f d@ 6dA e< e= f dB 6dC e: e; f dC 6dC e< e= f dD 6ZB dE   ZC dF   ZD dG   ZE d dH  ZF dI   ZG dJ   ZH d dK  ZI dL   ZJ dM   ZK dN   ZL dO   ZM dP   ZN dQ   ZO dR   ZP dS   ZQ dT   ZR dU   ZS dV   ZT d( d eO f d* d eO f d. d eU f d0 d eJ f d d eN f dA d eP f dA d d[   f d$ d d^   f d\ d eQ f d" d eM f d" d eL f d? d eR f g ZV e reV jW d d di   f  eV jW d d dk   f  n  dl   ZX dm   ZY dn   ZZ do   Z[ dp   Z\ dq   Z] d dr   f d ds   f d e\ f d dt   f d du   f d dv   f d dw   f d dx   f d dy   f d dz   f d d{   f d d|   f d d}   f g Z^ e re^ jW d d~   f  e^ jW d d   f  n  d   Z_ d   Z` d   Za d   Zb d   Zc d   Zd d   Ze d   Zf d   Zg d   Zh d   Zi d   Zj d   Zk d el f d     YZm d d d     YZn d   Zo d   Zp d   Zq d   Zr d   Zs d   Zt d   Zu d   Zv d   Zw d d  Zx d   Zy d   Zz d d  Z{ d   Z| d   Z} d d  e d d  Z~ d   Z d   Z d d  Z d d  Z d   Z d d d d e  d  Z d S(   i    (   t   absolute_importt   divisiont   print_function(   t   copy(   t   md5(   t   callNs   0.7.11s   2016 Nov 16i   i i N  g{Gz?s   ^\d+\.\d+\.\d+\.\d+$s   (?:https?://)?([^/]+)s!   ([a-z]+)(>|<|=|\!=|\!~|\~)([^,]+)sE   (?:(avg|count|dev|iqm|max|min|miqm|sum|var)\(([a-z\*1]+|)\)|([a-z]+))s<   (?:(avg|count|dev|max|min|sum|var)\(([a-z\*1]+|)\)|([a-z]+))s   (\S+)s   \S+s+   ((?:(?<![\\\])(?=[\\\][\\\])*[\\\]"|[^"])*)s)   (?:(?<![\\\])(?=[\\\][\\\])*[\\\]"|[^"])*s   \[?(\S+(?:\s+[\-\+]\d+)?)\]?s   \[?\S+(?:\s+[\-\+]\d+)?\]?s   (\S+ \S+ \S+)s   \S+ \S+ \S+t   ipt   ht   at   lipt   At   autht   lt   usernamet   ut	   timestampt   tt   requestt   rt   methodt   mt   msect   Dt   fbmsect   Ft   queryt   qt   statust   st   bytest   bt   Bt   Ot   bytes_int   It   domaint   vt   Vt   portt   pt   ratios   {ratio}nt   hosts   {host}it   refs
   {referer}it   uas   {user-agent}it   ignoret
   ignorequotc         C` s>   t  d k  r d  St j j |  d  t r: t j j   n  d  S(   Ni   s   
(   t	   LOG_LEVELt   syst   stderrt   writet   LINE_BUFFEREDt   flush(   R   (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt   debugz   s
    c         C` s>   t  d k  r d  St j j |  d  t r: t j j   n  d  S(   Ni   s   
(   R.   R/   R0   R1   R2   R3   (   R   (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt   warn   s
    c          C` s   d   }  t  j j d d  } t t d  rL |  t j |  } | rL | Sn  t  j j d  } | r |  | |  } | r | Sn  |  t j	 |  } | r | S|  t
 j j   |  } | r | S|  d d  } | r | St d   d	 S(
   s   Find cfg_file by searching from most accurate/specific to least.

    1. VirtualEnv + /etc/wtop.cfg
    2. PYTHONUSERBASE + /etc/wtop.cfg
    3. USER_BASE + /etc/wtop.cfg
    4. Python Lib + /etc/wtop.cfg
    5. /etc/wtop.cfg
    c         S` sO   t  j j |  |  } t  j j |  r9 t d |  | St d |  d Sd S(   s7   Concatenate paths and test if resulting path is a file.s   Using cfg_file: %ss(   Cfg_file skipped. Cfg_file not found: %sN(   t   ost   patht   joint   isfileR4   t   None(   t
   cfg_prefixt
   cfg_suffixt   cfg_file(    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt	   test_file   s    t   etcs   wtop.cfgt   real_prefixt   PYTHONUSERBASEt   /s   etc/wtop.cfgs   Cfg_file could not be foundN(   R6   R7   R8   t   hasattrR/   t   prefixt   environt   gett   sitet	   USER_BASEt	   distutilst	   sysconfigt   get_python_libt	   Exception(   R>   t   etc_wtop_pathR=   t	   user_base(    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt   find_cfg_file   s*    		c         C` s  |  d  k r t   }  n  t j   a t j |   t j d d  a t j d d  a t j d d  a t j d d  j	 d  a
 t t j d d   a t t j d d	   a t   } x6 t j d
  D]% } | j | t j d
 |  f  q Wt j t j d d  t j  a t j t j d d   a g  | D]) } | d t j | d t j  f ^ q7a t d k rt j d d  a t t j d d   \ a a n  d  S(   Nt   maint   log_file_typet   log_roott
   log_formatt   default_output_fieldst   ,t   wtopt   max_request_timet   min_rpst   classest   patternst   robotst   generici    i   t   apachet   log_file(   R:   RO   t   ConfigParsert   configt   readRF   t   LOG_FILE_TYPEt   LOG_ROOTt
   LOG_FORMATt   splitt   DEFAULT_OUTPUT_FIELDSt   intt   MAX_REQUEST_TIMEt   floatt   MIN_RPSt   listt   optionst   appendt   ret   compileR"   t	   re_robotst
   re_generict
   re_classest   LOG_FILEt   format2regexpt   LOG_PATTERNt   LOG_COLUMNS(   R=   RY   t   ot   x(    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt	   configure   s*    		#!6c         C` s`   t    } xP |  D]H } t | d  rK t | t  rK | j t |   q | j |  q W| S(   Nt   __iter__(   Rk   RC   t
   isinstancet
   basestringt   extendt   flattenRm   (   Rx   t   resultt   el(    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyR~      s    	c         C` s#   |  j  d  d k r |  j   S|  S(   Nt   {i(   t   findt   lower(   R   (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt   lcase   s    
c         C` sj  t  j d  } t t | j |    } t   } |  } x | D] } | t k rk | j d  d k rk q= n  t | \ } } }	 | }
 | s | | k r | j |  n |	 }
 | j d  d k rt  j d | j	 d d  j	 d d  t  j
  } | j |
 | t  j
  } q= t  j d | |
 |  } q= W| j |  } | rZt d d j |   n  | t |  f S(	   Ns   %[\>\<\!,\d]*([\}\{\w\-]+)R   is   %[\>\<\!,\d]*t   }t   .s$   unrecognized format directives: %%%ss   , %(   Rn   Ro   t   mapR   t   findallRk   t   LOG_DIRECTIVESR   Rm   t   replaceR"   t   subR5   R8   R~   (   t   fmtt   relevant_fieldst   re_dt
   directivest   colnamest   patt   kt   fieldt   patternt   skip_patternt   atomR'   t   leftover(    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyRt      s*    	"c         C` s   t  |  j d d   S(   Nt   -t   0(   Rg   R   (   R   (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt   safeint  s    c         C` sJ   |  d d k r d n d } t  |  d d ! d t  |  d d ! d | S(	   Ni    R   i   ii   i  i   i<   (   Rg   (   R   t	   plusminus(    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt   tz2secs  s    c         C` s2   t  j t j |  d  d d   t |  d d ! S(   Ni   s    GMTs   %d/%b/%Y:%H:%M:%S %Zi   i   (   t   calendart   timegmt   timet   strptimeR   (   R   (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt   apache2unixtime"  s    c      	   C` sm   t  |  d d ! t j |  d d !d  j t  |  d d ! t  |  d d	 ! t  |  d
 d ! t  |  d d ! f S(   Ni   i   i   i   s   %bi    i   i   i   i   i   i   i   (   Rg   R   R   t   tm_mon(   R   (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt   apache2dateparts(  s    9c         C` sI   |  } t  j |   r' t j |   } n  t j | d  d t | <t | S(   Ni    i   (   t   re_ipt   matcht   sockett	   inet_atont   ipcntsRF   (   R   t   ipkey(    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt	   count_ips1  s
    c         C` sR   |  s |  d  k r d S|  j d  d k r@ t t |   d  St |   d Sd  S(   Ni    R   ii  (   R:   R   Rg   Ri   (   R   (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt   fix_usec=  s
    c         C` s6   t  j |   } | r2 d |  | j   | j   !f Sd S(   Ni   i    t    (   i    R   (   Rp   t   searcht   startt   end(   R+   R   (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt
   parse_botsG  s    c         C` sP   x' t  D] \ } } | j |   r | Sq Wt j |   } | rL | j d  Sd S(   Ni   t   UNKNOWN(   Rr   R   Rq   t   group(   t   urlt	   classnameR   R   (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt   classify_urlP  s    c         C` s&   t  j |   } | r" | j d  S|  S(   Ni   (   t	   re_domainR   R   (   R   R   (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyR#   ]  s    c         C` s)   t  j |   r t j |   St j |   S(   N(   R   R   t   geocodert   country_name_by_addrt   country_name_by_name(   R)   (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt   geocode_countrye  s    c         C` s)   t  j |   r t j |   St j |   S(   N(   R   R   R   t   country_code_by_addrt   country_code_by_name(   R)   (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt
   geocode_cck  s    t   ipcntt   bott   botnamet   uasc         C` s   |  d  S(   Ni   (    (   R   (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt   <lambda>{  s    R   t   protoc         C` s   |  j  d  S(   Nt    (   Re   (   R   (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyR   }  s    t   classt   yeart   montht   dayt   hourt   minutet   secondt   tst   refdomt   countryc         C` s   t  t |    S(   N(   t   strR   (   R   (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyR     s    t   ccc         C` s   t  t |    S(   N(   R   R   (   R   (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyR     s    c         C` s   t  |  d  s |  f S|  S(   NRz   (   RC   (   Rx   (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt   listify  s    c         #` s   t    f d   |  } xa |  D]Y } xK | D]C \ } } } | j t t | t | | j | d       q, W| Vq Wd  S(   Nc         ` s     j  |  d  S(   Ni   (   t   intersection(   t   f(   R   (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyR     s    R   (   t   filtert   updatet   dictt   zipR   RF   (   t   logR   t   col_fnst   relevant_col_fnst   recordt
   source_colt   new_colst   fn(    (   R   s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt	   field_map  s    	)c         ` sz   t  |     t  t t d   t   f d   t       t t d   t   f d   t      t    t |    S(   Nc         S` s   |  d d !S(   Ni    i   (    (   R   (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyR     s    c         ` s     j  |  d  S(   Ni   (   R   (   R   (   t   deps(    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyR     s    c         S` s   |  d d !S(   Ni    i   (    (   R   (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyR     s    c         ` s     j  |  d  S(   Ni   (   R   (   R   (   R   (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyR     s    (   t   setR~   R   R   R   Rk   (   t   requested_fields(    (   R   s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt   field_dependencies  s    c         ` sa   t  j |      f d   |  D } d   | D }  f d   | D } t | | t  } | S(   Nc         3` s   |  ] }   j  |  Vq d  S(   N(   R   (   t   .0t   line(   t   logpat(    s0   /usr/local/lib/python2.7/dist-packages/logrep.pys	   <genexpr>  s    c         s` s!   |  ] } | r | j    Vq d  S(   N(   t   groups(   R   t   g(    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pys	   <genexpr>  s    c         3` s$   |  ] } t  t   |   Vq d  S(   N(   R   R   (   R   R   (   Rv   (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pys	   <genexpr>  s    (   Rn   Ro   R   R   (   t   loglinesRu   Rv   R   R   t   tuplesR   (    (   R   Rv   s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt
   apache_log  s    c         C` s/   t  t j t j |  d d |  d d    S(   Nt   dateR   R   s   %Y-%m-%d %H:%M:%S(   Rg   R   t   mktimeR   (   R   (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt   iis2unixtime  s    "c         C` s   |  d d k r d Sd |  S(   Ni    R   R   t   ?(    (   R   (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt	   fix_query  s    c         C` s   t  |  d  S(   NR   (   R   (   Rx   (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyR     s    c         C` s   |  d |  d S(   NR7   R   (    (   Rx   (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyR     s    c         C` s   |  d j  d  S(   NR   R   (   Re   (   Rx   (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyR     s    c         C` s   |  d j  d  S(   NR   t   :(   Re   (   Rx   (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyR     s    c         C` s   t  |  d  S(   NR   (   Rg   (   Rx   (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyR     s    c         C` s   t  |  d  S(   NR   (   Rg   (   Rx   (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyR     s    c         C` s   t  |  d  S(   NR   (   Rg   (   Rx   (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyR     s    c         C` s   t  |  d  S(   NR   (   R   (   Rx   (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyR     s    c         C` s   t  |  d  S(   NR+   (   R   (   Rx   (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyR     s    c         C` s   |  d d  S(   NR+   i   (    (   Rx   (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyR     s    c         C` s   t  |  d  S(   NR   (   R   (   Rx   (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyR     s    c         C` s   t  |  d  S(   NR*   (   R#   (   Rx   (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyR     s    c         C` s   t  t |  d   S(   NR   (   R   R   (   Rx   (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyR     s    c         C` s   t  t |  d   S(   NR   (   R   R   (   Rx   (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyR     s    c      	   #` sq   t    f d   t  } xR |  D]J } x< | D]4 \ } } | j t t | t | |      q, W| Vq Wd  S(   Nc         ` s     j  |  d  S(   Ni    (   R   (   R   (   R   (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyR     s    (   R   t   iis_col_fnsR   R   R   R   (   R   R   R   R   R   R   R   (    (   R   s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt   iis_field_map  s    	,c         ` sB   d   d   |  D }   f d   | D } t  | | t  } | S(   NR   R   t
   s_sitenamet   s_computernamet   s_ipR   R7   R   R&   t   userR   R   R+   t   cookieR*   t   vhostR   t	   substatust   win32_statusR!   R   R   c         s` s   |  ] } | j  d   Vq d S(   R   N(   Re   (   R   R   (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pys	   <genexpr>  s    c         3` s-   |  ]# } t  t   t j |    Vq d  S(   N(   R   R   t   urllibt   unquote_plus(   R   R   (   t   cols(    s0   /usr/local/lib/python2.7/dist-packages/logrep.pys	   <genexpr>  s    (   s   dates   times
   s_sitenames   s_computernames   s_ips   methods   paths   querys   ports   users   ips   protos   uas   cookies   refs   vhosts   statuss	   substatuss   win32_statuss   bytes_ins   bytess   msec(   R   R   (   R   R   R   R   (    (   R   s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt   iis_log  s       c         c` sL   xE |  D]= } | r# | d | k s | r? | d | k r? q n  | Vq Wd  S(   NR   (    (   t   reqst   includet   excludeR   (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt   filter_by_class  s
    c         C` s$   t  t t d |  t  d d   S(   NR   t   keyc         S` s   t  |  j d  d  S(   NR   i(   R   Re   (   Rx   (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyR     s    (   t   sortedt   gen_findRs   Rc   (   t   dt(    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt   logs_for_date  s    c          C` sQ   t  t j   d  d }  d t t |  f } t j j |  rF | g St t g S(   NiQ s   %s%s.%d(   Rg   R   Rc   Rs   R6   R7   R9   (   t   gmt_midnightt   r_file(    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt   todays_logs  s
    c          C` s2   t  t j   d  d d }  d t t |  f g S(   NiQ i   s   %s%s.%d(   Rg   R   Rc   Rs   (   t   gmt_yesterday(    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt   yesterdays_logs  s    c           C` s   t    S(   N(   R  (    (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt
   latest_log  s    c         c` sX   xQ t  j |  D]@ \ } } } x. t j | |   D] } t  j j | |  Vq2 Wq Wd  S(   N(   R6   t   walkt   fnmatchR   R7   R8   (   t   filepatt   topR7   t   dirlistt   filelistt   name(    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyR  %  s    c         c` s0   x) |  D]! } t  |  } | V| j   q Wd  S(   N(   t   opent   close(   t	   filenamesR  t   handle(    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt   gen_open+  s    c         c` s+   x$ |  D] } x | D] } | Vq Wq Wd  S(   N(    (   t   sourcesR   t   item(    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt   gen_cat2  s    c         c` s;   t  j |   } x% | D] } | j |  r | Vq q Wd  S(   N(   Rn   Ro   R   (   R   t   linest   patcR   (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt   gen_grep8  s    c         C` s+   t  |  |  } t |  } t |  } | S(   N(   R  R  R  (   R  t   dirnamet   namest   filesR  (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt   lines_from_dir?  s    c         c` s   t  j    } |  j d d  x t r |  j   } | s t  j    | d k r t   d } t d |  |  j   t |  }  |  j d d  t  j    } n  t  j d  q n  t  j    } | Vq Wd  S(   Ni    i   i   s'   no input for 30 seconds. reopening (%s)g333333?(	   R   t   seekt   Truet   readlineR  R5   R  R  t   sleep(   t   thefilet	   last_seenR   t   logfile(    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt   followF  s     	
t   circular_bufferc           B` s&   e  Z d  d  Z d   Z d   Z RS(   i    c         C` s0   d |  _  |  | g | 7}  | |  _ d |  _ d  S(   Ni    (   t   pointert   lengtht   cnt(   t   selfR/  t   initval(    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt   __init___  s    		c         C` sT   |  j  |  j ;_  |  j |  j  |  |  j  d 7_  t |  j d |  j  |  _ d  S(   Ni   (   R.  R/  t   __setitem__t   minR0  (   R1  R  (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyRm   e  s    c         C` s   |  j  S(   N(   R0  (   R1  (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt   __len__k  s    (   t   __name__t
   __module__R3  Rm   R6  (    (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyR-  ^  s   	t   rrd2c           B` s5   e  Z d    Z d   Z d   Z d   Z d   Z RS(   c         C` s.   | |  _  t |  |  _ t | d  |  _ d  S(   Ni    (   i    i    i    (   R/  Ri   t   windowR-  t   buf(   R1  R/  R:  (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyR3  q  s    	c         C` s#   |  j  j t j   | | f  d  S(   N(   R;  Rm   R   (   R1  R  R   (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyRm   v  s    c         ` sc   t  j     t t    f d     j   } | rR | t  | d d  f St   d f Sd  S(   Nc         ` s    |  d   j  k  S(   Ni    (   R:  (   Rx   (   R1  R   (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyR   {  s    i    g      ?(   R   R  R   R;  Ri   Rk   (   R1  t   items(    (   R1  R   s0   /usr/local/lib/python2.7/dist-packages/logrep.pyRF   y  s
    $c         C` s?   |  j    \ } } d } x | D] } | | d 7} q W| | S(   Ni    i   (   RF   (   R1  R<  t   mwindowR0  R  (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt   avg  s
    c         C` s   |  j    \ } } d } d } d } d } t   } xa | D]Y } | | d 7} | | d 7} t | | d  } t | | d  } | j | d  q: W| s d S| | }	 | t |  }
 t | |
  } t | | |  } |	 |
 | | | | | f S(   Ni    i   i    i   R   I       (   i    i    i    i    R   i    i    (   RF   Rk   R5  t   maxRm   Ri   t   stddevt   hist_sparkline(   R1  R<  R=  R0  t   msec_tott   msec_mnt   msec_mxt   msecsR  t   rpst   msec_avgt   msec_stddevt	   sparkline(    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt   stats  s&    	
(   R7  R8  R3  Rm   RF   R>  RJ  (    (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyR9  p  s
   				c         C` sb   t  |  d } t |   d } t g  |  D] } | | d ^ q*  } t j t | |   | S(   Ni   i   (   Ri   t   lent   sumt   matht   sqrtt   abs(   t   lstR>  t   lst_lenR   t   sumdist(    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyR@    s    'c      	   C` s  d } t  |  } d } t   } t  |   } | | } | | d	 }	 | d
 k r] | d
 | Sg  |  D] }
 t |
 | |	  ^ qd } x( | D]  }
 | j |
 d
  d	 | |
 <q Wd j g  t |  D]: }
 | t t | j |
 d
  t |  | d	   ^ q  S(   NR   R   R   Rw   R    t   @t   #i
   i   i    R   (   R   R   R   Rw   R    RS  RT  (   RK  R   Rg   RF   R8   t   ranget   roundRi   (   RP  t   mnt   mxt   sp_charst   sp_stepst   tilest   cntst   tott   rnt   stepRx   t   steps(    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyRA    s    	
'	c         C` s$   |  d k  r d Sd |  j  d d  S(   Ng{Gz?s   --s   %.2fs   0.R   (   R   (   R   (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt   pretty_float  s    c         c` sA   t  j | t  j  } x% |  D] } | j |  r | Vq q Wd  S(   N(   Rn   Ro   R"   R   (   R  R   R   t   ln(    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt   line_filter  s    c         c` sA   t  j | t  j  } x% |  D] } | j |  s | Vq q Wd  S(   N(   Rn   Ro   R"   R   (   R  R   R   Rb  (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt   line_exclude  s    c         C` s   |  j  d  } t | d  } d } t } t |  d k r\ t t | d j  d   } n  t |  d k r | d d j   d k } n  | | | f S(   NR   i    i   RU   i   t   d(   Re   Rg   R&  RK  R   R   (   t   commandst   ct   limitt   order_byt
   descending(    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt   compile_orderby  s    c         C` s   t  j |   } t   } t   } t   } t } x | D]} } | d d k r{ | j | d  | j | d d ! t } q7 | j | d  | j d  | d f  | j | d  q7 W| | | | f S(   Ni   R   i    i   (   t   re_aggR   Rk   t   FalseRm   R&  R:   (   Rf  t   fieldst   needed_fieldst
   all_fieldst   group_by_fieldst   has_aggR   (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt   compile_aggregates  s    				c         ` s   |  j  d  } g  | D] } t j |  j   ^ q  i d d 6d d 6d d 6d d 6 g   D] } | d ^ q` }  f d	     f d
        f d   } | | f S(   NRU   it   <i   t   >i    t   =s   !=c         ` s   t  g  |  j   D] \ } } | t |  f ^ q  } t   } xi   D]a \ } } } | d d k r | j | | t j |  f  qG | j | | | | |  f  qG W| S(   Nit   ~(   R   t	   iteritemst   typeRk   Rm   Rn   Ro   (   t   exampleR   R$   t   castfnst   retR  t   opt   value(   t
   conditions(    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt   typecast  s    7	"$c         ` s   x | D]y \ } } } | d d k rS | j  t |  |   | d k k r t Sq   | t |  | |  k | d k k r t Sq Wt S(   NiRw  s   !~s   !=(   R   R   Rm  t   cmpR&  (   t   objR  R  R}  R~  (   t   cmp_operators(    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt	   predicate"  s    &)c         3` s[   |  j    }  |  }   | |  r/ | Vn  x% |  D] }   | |  r6 | Vq6 q6 Wd  S(   N(   t   next(   RP  t   firstR  R  (   R  R  (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyR   ,  s    (   Re   t   re_cmpR   R   (   Rf  t   testst   stringRx   Rn  R   (    (   R  R  R  R  s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt   compile_filter  s    ("

c         c` s-   x& t  j d | |  f  D] } | Vq Wd  S(   Ns   tail -%d '%s'(   R6   t   popen(   t   filenamet   numR   (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt   tail_n9  s     i   c      
   c` s  t    } d } | j d t d d  t d d  t d d  t d d  t d d  f  x+|  D]#} | j | d t d d  t d d  t d d  t d d  t d d  f  | d t k  r| | d | d	 d
 d j d | d  | d | d	 d
 d j d | d  n< | | d d j d | d  | d d j d | d  t j   | | k rb t j   } | Vqb qb Wd  S(   Ni    s   (all)i N  i   i  i   R   R   R   id   i   i   i   (   R   t
   setdefaultR9  Rh   Rm   R   (   R   t   everyRJ  t
   last_printR   (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt   gen_top_stats?  s    	-%+,+ c         C` s  xt  |  d d D]} t   } | j d d  | j d  x t | j    D] } | | d j   \ } } } } } }	 }
 | t k  sS | d k  r qS n  t d   | | d	  \ } } } } | j d
 | t |  | |	 | |
 | | | | f
  qS Wt	 d j
 |  d  q Wd  S(   NR  i   sM   % 34s     req/s   avg   min              max     3xx     4xx     5xx     slowR   sk                      ----------------------------------------------------------------------------------------i    i   c         S` s   t  |  j    S(   N(   Ra  R>  (   Rx   (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyR   g  s    i   s1   % 34s % 9s % 5d % 4d  %s % 5d % 7s % 7s % 7s % 7ss   
s   


(   R  Rk   Rm   R  t   keysRJ  Rj   R   Ra  t   printR8   (   R   RJ  R;  Rg  RF  R>  R0  t   stdevRI  RW  RX  t   x3t   x4t   x5t   slow(    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt   apache_top_modeY  s    		)		c         C` s[   xT |  D]L } t  d j g  | D] } t | |  ^ q   t r t j j   q q Wd  S(   Ns   	(   R  R8   R   R2   R/   t   stdoutR3   (   R   Rn  R   R   (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt
   print_modeq  s    0i   c         C` s=   t  d j g  | D] } t |  |  ^ q   j   d | !S(   NRU   i    (   R   R8   R   t   digest(   R   R  t   byte_lenR   (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt   id_from_dict_keys  s    c         ` sa     r9 t     d k r9   f d   }   f d   } n   f d   }   f d   } | | f S(   Ni   c         ` s   g    D] } |  d | ^ q S(   Ni   (    (   R$   t   i(   Ri  (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyR     s    c         ` s   g    D] } |  | ^ q S(   N(    (   R$   R  (   Ri  (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyR     s    c         ` s   |  d   d S(   Ni   i    (    (   R$   (   Ri  (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyR     s    c         ` s   |    d S(   Ni    (    (   R$   (   Ri  (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyR     s    (   RK  (   Ri  t   key_fnt   key_fn2(    (   Ri  s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt   keyfns  s    c         ` s%   t  |   \ }       f d   S(   Nc         ` s#   t  |  j   d  d   d  !S(   NR  t   reversei    (   R  t
   itervalues(   t   table(   Rj  R  Rh  (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyR     s   (   R  (   Ri  Rj  Rh  R  (    (   R  Rj  Rh  s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt   sort_fn  s    c   )   	   ` s  t  r0 t j d   t j d d d t    n  d* } t   } d }	 t }
 | r d d  l } | j | d d	 d
 t } t }
 n  d g t	 |  } d g t	 |  d } t
   } x t |  D] \ } } | \ } } | d k r d | | <q | d+ k r0d, | | d <| j | | d f  d | | <q | d k r`| j | | d f  d | | <q | d k r | | | d <q q Wd j |  } d   }   f d   }  f d   } d   } i d   d  6| d 6d   d 6| d 6d   d 6d   d 6d   d  6| d 6} t  r*| | d <| | d <n  d!   }   f d"   }  f d#   } d$   } i | d 6| d 6} t  r| | d <| | d <n  | rt   } t |  \ } } n  t j   }  xr|  D]j}! |	 d 7}	 |	 t d k rt j   |  }" t j   }  t d% |" |	 f  n  t |! |  }# |# | k rCt |  | |# <n  | |# d c d 7<xL t |  D]> \ }$ \ } } | | |$ d |! | | |#  | |# |$ d <qdW| r| |# | |# <|	 t rt t | j   d& | d' | d | ! } qn  |
 r|	 t d k rt d(  | j   t d)  qqW| r?| }% n | }% | rd }	 x |% j   D] }& xX | D]P \ }' }( | |' |& |( |% |& |( d |% |& |( d |% |& d  |% |& |( <qkW|	 d 7}	 |
 r^|	 t d k r^t d(  | j   q^q^Wn  |% | | | f S(-   Ni  t   round_digitsit   tenth_precisei   i@   i    t   flagt   nt	   writebacks   %sR>  s   %.2ft   devt   miqmt   vart   iqms   %dR5  s   	c         S` sV   | | |  | | d d | | } t  | | d  } | d k rJ d S| | Sd  S(   Ni    i   (   Ri   (   R  R   R   R  R  t	   numeratort   denominator(    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt   agg_avg  s
    &c         ` s%   d | |  f }   | | |  d S(   Ns   %s-%si    (   i    i    (    (   R  R   R   R  R  (   t   diqm(    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt   agg_iqm  s    c         ` s%   d | |  f }   | | |  d S(   Ns   %s-%si    (   i    i    (    (   R  R   R   R  R  (   R  (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt   agg_miqm  s    c         S` sB   | | |  d | | } | | |  d | | d } | | f S(   Ni    i   i   (    (   R  R   R   R  R  t   sumst   sq_sums(    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt   agg_post_prep  s    c         S` s   | | S(   N(    (   R  R   R   R  R  (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyR     s    c         S` s   | | d S(   Ni    (    (   R  R   R   R  R  (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyR     s    t   countc         S` s   t  | | | | |   S(   N(   R?  (   R  R   R   R  R  (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyR     s    R?  c         S` s   t  | | | | |   S(   N(   R5  (   R  R   R   R  R  (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyR     s    c         S` s   | | |  | | S(   N(    (   R  R   R   R  R  (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyR     s    RL  c         S` sQ   t  |  } | | | | } | | d } | d k r< d St j | |  Sd  S(   Ni   i    (   Ri   RM  RN  (   R  t   col_idxR  R  R  R  R  (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt   post_dev  s    c         ` s   d |  | f }    j  |   S(   Ns   %s-%s(   t   report(   R  R  R  R  R  (   R  (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt   post_iqm  s    c         ` s   d |  | f }    j  |   S(   Ns   %s-%s(   R  (   R  R  R  R  R  (   R  (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt	   post_miqm  s    c         S` s    t  |  } | | d | | S(   Ni   (   Ri   (   R  R  R  R  R  (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt   post_var  s    s   %0.2f processed %d records...R  R  s   sync()ing records to disk...s   done.l            (   s   devs   miqms   var(   i    i    (   t   iqm_availableR  t	   MovingIQMt   DictIQMR&  R   Rm  t   shelveR  RK  Rk   t	   enumerateRm   R8   R:   R  R   t   PROGRESS_INTERVALR5   R  R   t   SORT_BUFFER_LENGTHR  Rx  t   DISC_SYNC_CNTt   synct   iterkeys()   R   t
   agg_fieldst   group_byRi  Rh  Rj  t   tmpfilet   MAXINTR  R0  t
   using_diskR  R   t   blankt   needed_post_fnsR  R   R}  R   R  R  R  R  t   agg_fnsR  R  R  R  t   post_fnst   running_listR  R  t   lasttR   R   R  t   idxt   recordsR   R   R  (    (   R  R  s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt   calculate_aggregates  s    					






			

	
0


	

c         C` sC   x< |  D]4 } t  | t | d   t r t j j   q q Wd  S(   Ni   (   R  t   tupleR2   R/   R  R3   (   t   rowsR   t   row(    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt   agg_mode<  s    c         C` s1   g  |  D]& } t  t | t |  |   ^ q S(   N(   Rg   RV  Ri   (   RP  t   totalt   scaleRx   (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt	   normalizeD  s    c         C` s{   t  d |  |  d | } t d d d |  d d | d d	 | d
 d d d d d | d | d | d | d | g  d  S(   Ns   creating rrdiQ t   rrdtoolt   creates   %s.rrds   --steps   %ds   --starts   %ss   DS:rps2xx:GAUGE:5:0:5000s   DS:rps3xx:GAUGE:5:0:5000s   DS:rps4xx:GAUGE:5:0:5000s   DS:rps5xx:GAUGE:5:0:5000s   DS:msec:GAUGE:5:0:10000s   RRA:AVERAGE:0.5:1:%s(   R  R   (   t   klassR   R_  t   rowcnt(    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt
   create_rrdH  s    
t   briefc         C` s]  t  d |  |  d t | |  d t |  d d d d d d d d d d	 d d
 d d d d d d d d d d g } | d k r | d d d d d d g 7} n | d d d d g 7} t d d d |  | f g | d |  d d  |  d! d" |  d# d$ |  d% g  t d d d& |  | f g | d' t |  d( d) d* | | f d+ d, d d- |  d. g
  d  S(/   Ns   creating graphs   -ss   -es   --colors   BACK#ffffff00s   SHADEA#ffffff00s   SHADEB#ffffff00s   CANVAS#eeeeee00s   GRID#eeeeee00s   MGRID#eeeeee00s   AXIS#999999s   ARROW#999999s   -wt   300s   --units-lengtht   4s   --no-legends   --slope-modeR  s   -ht   40s   -at   PNGs   --x-gridt   nonet   750R  t   graphs   %s.%s.rps.pngs   DEF:OK=%s.rrd:rps2xx:AVERAGEs   AREA:OK#44ee00:OKs"   DEF:REDIRECT=%s.rrd:rps3xx:AVERAGEs   AREA:REDIRECT#2a89ed:REDIRECTs"   DEF:NOTFOUND=%s.rrd:rps4xx:AVERAGEs   AREA:NOTFOUND#f59d18:NOTFOUNDs   DEF:ERROR=%s.rrd:rps5xx:AVERAGEs   AREA:ERROR#dd0000:ERRORs   %s.%s.msec.pngs   --upper-limits   --rigids   --y-grids   %s:%ss   --units-exponentR   s   DEF:MSEC=%s.rrd:msec:AVERAGEs   AREA:MSEC#8888cc:MSEC(   R  R   R   (   R  R   R/  t   rpslimt   mseclimRy  t   common(    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt   create_graphU  s.    "

c         C` s   t    } | j d  | j d  d } xI |  d D]= } | j d | d | j d d  | | f  | d 7} q4 Wt d	 d
  j d d j |   d  S(   Ns  
    <table cellspacing="0" width="920">
    <tr class="r1">
        <td align="right" width="120" ><b>All Pages</b>&nbsp;</td>
        <td colspan="2">
        <img src="all.halfday.rps.png"  />
        <img src="all.halfday.msec.png"  />
        </td></tr>
    s   
    <tr>
        <td>&nbsp;</td>
        <td align="center"><b>Traffic Volume (req/sec)</b></td>
        <td align="center"><b>Avg Response Time (msec)</b></td>
    </tr>
    i    i   s   
        <tr class="r%d">
            <td width="120" align="right"><b>%s</b></td>
            <td><img src="%s.brief.rps.png" /></td>
            <td><img src="%s.brief.msec.png" /></td>
        </tr>
        i   t   _R   s   rrd.htmlt   ws   
    <style>
        .r1 {background:#eeeeee;}
        body {font-family: arial;}
        img {padding:2px;}
    </style>
    <meta http-equiv="refresh" content="30" />
    R   (   Rk   Rm   R   R  R1   R8   (   RY   R  R  R;  R  Rg  (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt   create_rrd_pageu  s    	'i  i
   id   c         C` s  d } d } i t  d d d d d g d d d d g |  d 6} t    }	 d }
 d } x||  D]t} | sv | d } n  d |	 k r t |	 d <t d | d d	  n  | d
 j d d  j d d  | d
 <| d
 |	 k rt | d
 | d d	  t |	 | d
 <n  | d | | k r7t |  x |	 D] } | | k r?q'n  | | } t d d d | d | d t | d d t |   t | d d	 t |   t | d d t |   t | d d t |   | d | d d d	 f g  q'W| d } i t  d d d d d g d d d d g |  d 6} n| j | d
 t  d d d d d g d d d d g |   | | d
 d | d d d c d	 7<| d d | d d d c d	 7<| d d k  rS| | d
 d c | d 7<| d d c | d 7<| rSt	 t	 | d |  | | | d	  } | | d
 d | c d	 7<qSn  | | d k r] | } t
 d | d |
 | d  x- |	 j   D] } t
 | | d |
 | d  qWt t |	 j    |
 |  q] q] Wd  S(   Ni    R  t
   total_msect   histt   alli   i  R   i   R   R   R  RB   R   R  R   s   %s.rrds   %d:%s:%s:%s:%s:%si   i   R   id   i+  R   i   i  t   halfdayi  R  (   R   R&  R  R   R  R   Rg   Ri   R  R5  R  R  R  R  (   R   R_  t   msec_maxt
   hist_stepst
   hist_scalet   do_histt
   last_chartt   cur_timeRJ  RY   R  R  R   R   R$   t   min_msec(    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt   rrd_mode  sh    !	
&

"&
!%($"(    (   s   msec(   s   fbmsec(   s   status(   s   bytes(   s   ipcnt(   s   bots   botname(   s   uas(   s   methods   urls   proto(   s   class(   s   years   months   days   hours   minutes   second(   s   ts(   s   refdom(   R   (   s   cc(   s   query(   s   url(   s   ts(   s   years   months   day(   s   hours   minutes   second(   s   msec(   s   status(   s   bytes(   s   ipcnt(   s   bots   botname(   s   uas(   s   class(   s   refdom(   R   (   s   cc(    (   t
   __future__R    R   R   R   t   hashlibR   t
   subprocessR   R_   R   t   distutils.sysconfigRI   R  RM  R6   t   os.patht   randomRn   RG   R   R/   R   R   t   GeoIPt   newt   GEOIP_MEMORY_CACHER   R:   R  R&  R  Rm  t   VERSIONt   VERDATER2   R.   R  R  R  Rc   Rs   Rb   Rd   Rf   Rh   Rj   Rp   Rq   Rr   R   R`   Ro   R   R   R   R  Rl  t   re_strt   re_str_skippedt   re_quott   re_quot_skippedt   re_tst   re_ts_skippedt   re_reqt   re_req_skippedR   R4   R5   RO   Ry   R~   R   Rt   R   R   R   R   R   R   R   R   R#   R   R   Rg   R   Rm   R   R   R   R   R   R   R   R   R   R  R  R  R  R  R  R  R  R   R$  R,  Rk   R-  R9  R@  RA  Ra  Rc  Rd  Rk  Rs  R  R  R  R  R  R  R  R  R  R  R  R  R  R  R  (    (    (    s0   /usr/local/lib/python2.7/dist-packages/logrep.pyt   <module>   sv  


					1$											
						
 				
							
												1								*	+				
			 	*