
    zbjd                     x    S SK r S SKrS SKJrJr   " S S\ R                  5      r " S S\R                  5      r	g)    N)common_spec
model_specc                   $    \ rS rSrSrSrSrSrSrg)RotaryScalingType	   zRoPE scaling type.r          N)	__name__
__module____qualname____firstlineno____doc__LinearSuLlama3__static_attributes__r
       ڜ/tmp/claude-0/-home-danvics-docker-quiz/c1e0577a-e42c-4a3d-b1ea-3edd61103a4e/scratchpad/stt/lib/python3.13/site-packages/ctranslate2/specs/attention_spec.pyr   r   	   s    F	
BFr   r   c                   F    \ rS rSr                    SS jrSrg)MultiHeadAttentionSpec   Nc                 h   [         R                  U l        U(       a  [        R                  " US9U l        [        U(       a  SOS5       Vs/ s H  n[        R                  " 5       PM     snU l        U(       a2  [        R                  " US9U l	        [        R                  " US9U l
        U(       a  [        R                  " SS9U l        U(       a  S U l        S U l        U(       a  S U l        S U l        U(       a  S U l        S U l        S U l        US:w  a*  [&        R(                  " S5      R+                  U5      U l        US:w  a*  [&        R(                  " S5      R+                  U5      U l        UGb  [&        R(                  " S5      R+                  U5      U l        Xpl        [&        R(                  " S5      R+                  U
5      U l        Ub*  [&        R(                  " S5      R+                  U5      U l        U[8        R:                  L a+  [&        R(                  " S5      R+                  U	5      U l        OCU[8        R>                  L a  S U l         S U l!        O!U[8        RD                  L a  S U l#        S U l$        Ub*  [&        R(                  " S5      R+                  U5      U l%        Ub*  [&        R(                  " S5      R+                  U5      U l&        Ub*  [&        R(                  " S5      R+                  U5      U l'        U(       a"  SU l(        [        R                  " 5       U l)        g g s  snf )	N)rms_normr	      Tr   int32float32int8)*r   OPTIONALqueries_scaler   LayerNormSpec
layer_normrange
LinearSpeclinearq_normk_normv_normrelative_position_keysrelative_position_valuesrelative_attention_biasrelative_attention_max_distance!relative_asymmetric_position_keysrelative_left_max_positionrelative_right_max_positionnpdtypetype original_max_position_embeddingsmax_position_embeddings
rotary_dimrotary_interleaverotary_baserotary_scaling_typer   r   rotary_scaling_factorr   rotary_scaling_long_factorrotary_scaling_short_factorr   rotary_low_freq_factorrotary_high_freq_factornum_heads_kvhead_dimsliding_windowmerged_encoder_attention	memory_kv)selfself_attentionrelative_positionrelative_asymmetric_positionr+   r   r5   r6   r8   r9   r7   r3   r4   r>   r?   r@   qk_normqk_norm_rmsr(   has_normrA   _s                         r   __init__MultiHeadAttentionSpec.__init__   s   . (00)77JDO.3AQ.O
.OK""$.O
 %33[IDK%33[IDK%33TBDK*.D',0D)"+/D(37D0'59D2.2D+/3D,+q046HHW4E4J4J05D1 #a'+-88G+<+A+A',D( ! hhw/44Z@DO%6"!xx	277DD".+-88F+;+@+@AT+U("&7&>&>>-/XXi-@-E-E).* %(9(<(<<26/370$(9(@(@@.2+/3,# " 1 6 6| DDHHW-228<DM%"$((7"3"8"8"HD#,0D)(335DN $y
s   L/)r?   r'   r"   r%   r4   rB   rA   r>   r3   r&   r    r-   r+   r,   r.   r)   r*   r/   r7   r5   r=   r6   r<   r9   r:   r;   r8   r@   r(   )FFFFFNTNr   i'  r   r   NNNFTFTF)r   r   r   r   rK   r   r
   r   r   r   r      sF     %* % )* !!&+Y6r   r   )
enumnumpyr0   ctranslate2.specsr   r   IntEnumr   	LayerSpecr   r
   r   r   <module>rR      s2      5 Z6Z11 Z6r   