o
    2jd                     @   sH   d dl Z d dlZd dlmZmZ G dd de jZG dd dejZ	dS )    N)common_spec
model_specc                   @   s   e Zd ZdZdZdZdZdS )RotaryScalingTypezRoPE scaling type.r         N)__name__
__module____qualname____doc__LinearSuLlama3 r   r   [/var/www/html/whisper/venv/lib/python3.10/site-packages/ctranslate2/specs/attention_spec.pyr   	   s
    r   c                   @   s>   e Zd Z																				d	ddZdS )
MultiHeadAttentionSpecFNTr   '  r   c                 C   s  t j| _|rtj|d| _dd t|rdndD | _|r+tj|d| _tj|d| _	|r4tjdd| _
|r<d | _d | _|rDd | _d | _|rOd | _d | _d | _|dkr\td|| _|dkritd|| _|d urtd|| _|| _td	|
| _|d urtd
|| _|tju rtd	|	| _n|tju rd | _d | _ n|tj!u rd | _"d | _#|d urtd|| _$|d urtd|| _%|d urtd|| _&|rd| _'t( | _)d S d S )N)rms_normc                 S   s   g | ]}t  qS r   )r   
LinearSpec).0_r   r   r   
<listcomp>-   s    z3MultiHeadAttentionSpec.__init__.<locals>.<listcomp>r      Tr   int32float32int8)*r   OPTIONALqueries_scaler   LayerNormSpec
layer_normrangelinearq_normk_normv_normrelative_position_keysrelative_position_valuesrelative_attention_biasrelative_attention_max_distance!relative_asymmetric_position_keysrelative_left_max_positionrelative_right_max_positionnpdtypetype original_max_position_embeddingsmax_position_embeddings
rotary_dimrotary_interleaverotary_baserotary_scaling_typer   r   rotary_scaling_factorr   rotary_scaling_long_factorrotary_scaling_short_factorr   rotary_low_freq_factorrotary_high_freq_factornum_heads_kvhead_dimsliding_windowmerged_encoder_attentionr   	memory_kv)selfself_attentionrelative_positionrelative_asymmetric_positionr&   r   r0   r1   r3   r4   r2   r.   r/   r9   r:   r;   qk_normqk_norm_rmsr#   has_normr<   r   r   r   __init__   sn   





zMultiHeadAttentionSpec.__init__)FFFFFNTNr   r   r   r   NNNFTFTF)r   r   r	   rE   r   r   r   r   r      s,    r   )
enumnumpyr+   ctranslate2.specsr   r   IntEnumr   	LayerSpecr   r   r   r   r   <module>   s
    