B
    He9!                 @   s<   d dl mZmZ ddlmZ ddlmZ G dd deZdS )    )ListUnion   )CharSetProber)ProbingStatec                   s  e Zd ZdZdZdZdd fddZdd fdd	Zee	dd
dZ
ee	dddZedddZedddZedddZedddZedddZedddZee ddddZee ddddZeeef ed d!d"Zeedd#d$Zedd%d&Z  ZS )'UTF1632Proberad  
    This class simply looks for occurrences of zero bytes, and infers
    whether the file is UTF16 or UTF32 (low-endian or big-endian)
    For instance, files looking like (       [nonzero] )+
    have a good probability to be UTF32BE.  Files looking like (   [nonzero] )+
    may be guessed to be UTF16BE, and inversely for little-endian varieties.
       gGz?N)returnc                sn   t    d| _dgd | _dgd | _tj| _ddddg| _d| _	d| _
d| _d| _d| _d| _|   d S )Nr      F)super__init__positionzeros_at_modnonzeros_at_modr   	DETECTING_statequadinvalid_utf16beinvalid_utf16leinvalid_utf32beinvalid_utf32le'first_half_surrogate_pair_detected_16be'first_half_surrogate_pair_detected_16lereset)self)	__class__ u/var/www/downstreamdata.science/rtclock/rtclock-venv/lib/python3.7/site-packages/pip/_vendor/chardet/utf1632prober.pyr   )   s    
zUTF1632Prober.__init__c                sf   t    d| _dgd | _dgd | _tj| _d| _d| _	d| _
d| _d| _d| _ddddg| _d S )Nr   r
   F)r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   )r   )r   r   r   r   8   s    
zUTF1632Prober.resetc             C   s4   |   rdS |  rdS |  r$dS |  r0dS dS )Nzutf-32bezutf-32lezutf-16bezutf-16lezutf-16)is_likely_utf32beis_likely_utf32leis_likely_utf16beis_likely_utf16le)r   r   r   r   charset_nameF   s    zUTF1632Prober.charset_namec             C   s   dS )N r   )r   r   r   r   languageS   s    zUTF1632Prober.languagec             C   s   t d| jd S )Ng      ?g      @)maxr   )r   r   r   r   approx_32bit_charsW   s    z UTF1632Prober.approx_32bit_charsc             C   s   t d| jd S )Ng      ?g       @)r%   r   )r   r   r   r   approx_16bit_charsZ   s    z UTF1632Prober.approx_16bit_charsc             C   sj   |   }|| jkoh| jd | | jkoh| jd | | jkoh| jd | | jkoh| jd | | jkoh| j S )Nr   r         )r&   MIN_CHARS_FOR_DETECTIONr   EXPECTED_RATIOr   r   )r   approx_charsr   r   r   r   ]   s    
zUTF1632Prober.is_likely_utf32bec             C   sj   |   }|| jkoh| jd | | jkoh| jd | | jkoh| jd | | jkoh| jd | | jkoh| j S )Nr   r   r(   r)   )r&   r*   r   r+   r   r   )r   r,   r   r   r   r   g   s    
zUTF1632Prober.is_likely_utf32lec             C   sV   |   }|| jkoT| jd | jd  | | jkoT| jd | jd  | | jkoT| j S )Nr   r)   r   r(   )r'   r*   r   r+   r   r   )r   r,   r   r   r   r    q   s    
zUTF1632Prober.is_likely_utf16bec             C   sV   |   }|| jkoT| jd | jd  | | jkoT| jd | jd  | | jkoT| j S )Nr   r(   r   r)   )r'   r*   r   r+   r   r   )r   r,   r   r   r   r!   {   s    
zUTF1632Prober.is_likely_utf16le)r   r	   c             C   s   |d dksL|d dksL|d dkrR|d dkrRd|d   krHdkrRn nd| _ |d dks|d dks|d dkr|d dkrd|d   krdkrn nd| _d	S )
z
        Validate if the quad of bytes is valid UTF-32.

        UTF-32 is valid in the range 0x00000000 - 0x0010FFFF
        excluding 0x0000D800 - 0x0000DFFF

        https://en.wikipedia.org/wiki/UTF-32
        r   r         r(      Tr)   N)r   r   )r   r   r   r   r   validate_utf32_characters   s    
44z'UTF1632Prober.validate_utf32_characters)pairr	   c             C   s   | j sNd|d   krdkr*n nd| _ qxd|d   krBdkrxn qxd| _n*d|d   krfdkrrn nd| _ nd| _| jsd|d   krdkrn nd| _qd|d   krdkrn qd| _n*d|d   krdkrn nd| _nd| _d	S )
a9  
        Validate if the pair of bytes is  valid UTF-16.

        UTF-16 is valid in the range 0x0000 - 0xFFFF excluding 0xD800 - 0xFFFF
        with an exception for surrogate pairs, which must be in the range
        0xD800-0xDBFF followed by 0xDC00-0xDFFF

        https://en.wikipedia.org/wiki/UTF-16
        r.   r      T   r/   Fr   N)r   r   r   r   )r   r1   r   r   r   validate_utf16_characters   s     
z'UTF1632Prober.validate_utf16_characters)byte_strr	   c             C   s   x|D ]}| j d }|| j|< |dkrZ| | j | | jdd  | | jdd  |dkrv| j|  d7  < n| j|  d7  < |  j d7  _ qW | jS )Nr
   r)   r   r(   r   )r   r   r0   r4   r   r   state)r   r5   cZmod4r   r   r   feed   s    


zUTF1632Prober.feedc             C   sF   | j tjtjhkr| j S |  dkr.tj| _ n| jdkr@tj| _ | j S )Ng?i   )r   r   NOT_MEFOUND_ITget_confidencer   )r   r   r   r   r6      s    

zUTF1632Prober.statec             C   s(   |   s |  s |  s |  r$dS dS )Ng333333?g        )r!   r    r   r   )r   r   r   r   r;      s
    zUTF1632Prober.get_confidence) __name__
__module____qualname____doc__r*   r+   r   r   propertystrr"   r$   floatr&   r'   boolr   r   r    r!   r   intr0   r4   r   bytes	bytearrayr   r8   r6   r;   __classcell__r   r   )r   r   r      s*   



 r   N)typingr   r   charsetproberr   enumsr   r   r   r   r   r   <module>   s   