o
    2jn                    @   sJ  d dl Z d dlZd dlZd dlZd dlZd dlmZmZ d dlZ	zd dl
Z
d dlZd dlZW n	 ey7   Y nw d dlmZ d dlmZ d dlmZmZmZmZmZmZmZ ejjejjejjejjejjejjejjejjejjd	Z ej!j"ej!j#ej!j$ej!j#dZ%ej&j'ej&j(dZ)i Z*d	d
 Z+G dd deZ,G dd de j-Z.e+dG dd de.Z/e+dG dd de/Z0e+dG dd de/Z1e+dG dd de/Z2e+dG dd de/Z3e+dG dd  d e/Z4e+d!G d"d# d#e.Z5e+d$G d%d& d&e.Z6e+d'G d(d) d)e.Z7e+d*G d+d, d,e.Z8e+d-G d.d/ d/e.Z9e+d0G d1d2 d2e/Z:e+d3G d4d5 d5e/Z;e+d6G d7d8 d8e/Z<e+d9G d:d; d;e.Z=e+d<G d=d> d>e=Z>e+d?G d@dA dAe.Z?e+dBG dCdD dDe.Z@e+dEG dFdG dGe.ZAe+dHG dIdJ dJe.ZBe+dKG dLdM dMe.ZCe+dNe+dOG dPdQ dQe.ZDe+dRe+dSG dTdU dUe.ZEe+dVG dWdX dXe.ZFe+dYG dZd[ d[e.ZGe+d\G d]d^ d^e.ZHe+d_G d`da dae.ZIe+dbG dcdd dde.ZJe+deG dfdg dge.ZKe+dhG didj dje.ZLe+dkG dldm dmeLZMe+dnG dodp dpe.ZNe+dqG drds dse.ZOe+dtG dudv dve.ZPe+dwG dxdy dye.ZQe+dzG d{d| d|e.ZRd}d~ ZSeTdkrgeS  g dg dg dg dg dg dg dg dg dg dg ddZUe+dG dd de.ZVe+dG dd de.ZWdS )    N)ListOptional)utils)	Converter)attention_speccommon_spec
model_spectransformer_specwav2vec2_specwav2vec2bert_specwhisper_spec)	gelu	gelu_fastgelu_newgelu_pythongelu_pytorch_tanh
quick_gelurelusiluswish)linearsullama3longrope)gemmgemvc                    s    fdd}|S )z5Registers a model loader for this configuration name.c                    s   |  t  < | S N)_MODEL_LOADERS)clsconfig_name ^/var/www/html/whisper/venv/lib/python3.10/site-packages/ctranslate2/converters/transformers.py	decorator<   s   
z"register_loader.<locals>.decoratorr!   )r    r#   r!   r   r"   register_loader9   s   r$   c                   @   st   e Zd ZdZ						ddedee deee  dedee d	ed
efddZdd Z	dd Z
dd Zdd ZdS )TransformersConverterz/Converts models from Hugging Face Transformers.NFmodel_name_or_pathactivation_scales
copy_filesload_as_float16revisionlow_cpu_mem_usagetrust_remote_codec                 C   s.   || _ || _|| _|| _|| _|| _|| _dS )a  Initializes the converter.

        Arguments:
          model_name_or_path: Name of the pretrained model to download, or path to the
            directory containing the pretrained model.
          activation_scales: Path to the pre-computed activation scales. Models may
            use them to rescale some weights to smooth the intermediate activations
            and improve the quantization accuracy. See
            https://github.com/mit-han-lab/smoothquant.
          copy_files: List of filenames to copy from the Hugging Face model to the
            converted model directory.
          load_as_float16: Load the model weights as float16. More precisely, the model
            will be loaded with ``from_pretrained(..., dtype=torch.float16)``.
          revision: Revision of the model to download from the Hugging Face Hub.
          low_cpu_mem_usage: Enable the flag ``low_cpu_mem_usage`` when loading the model
            with ``from_pretrained``.
          trust_remote_code: Allow converting models using custom code.
        N)_model_name_or_path_activation_scales_copy_files_load_as_float16	_revision_low_cpu_mem_usage_trust_remote_code)selfr&   r'   r(   r)   r*   r+   r,   r!   r!   r"   __init__F   s   
zTransformersConverter.__init__c              	   C   s  t   tjj| j| jd}|jj}t	
|}|d u r+td|dtt	 f tt|j}t|dr<|||}tj}d|i}t|drP||| d| jrWt jnt|dd pbt|dd i}| jrl| j|d	< | jrt| j|d
< | jr|| j|d< | j|| jfi ||}i }	| jr| j|	d< | j|| jfi |	}
|||
}| jrt j| jdd}||| | jr| jD ]
}| | !| q|W  d    S 1 sw   Y  d S )N)r,   z]No conversion is registered for the model configuration %s (supported configurations are: %s), get_model_classconfigget_model_kwargsdtypetorch_dtyper*   r+   r,   cpu)map_location)"torchno_gradtransformers
AutoConfigfrom_pretrainedr-   r3   	__class____name__r   get
ValueErrorjoinsortedkeysgetattrarchitecture_namehasattrr7   AutoTokenizerupdater9   r0   float16r1   r2   
load_modelload_tokenizerr.   loadsmooth_activationr/   register_fileget_model_file)r4   r8   r    loadermodel_classtokenizer_classextra_kwargskwargsmodeltokenizer_kwargs	tokenizerspecr'   filenamer!   r!   r"   _loadj   sr   




	





$zTransformersConverter._loadc                 K      |j |fi |S r   rB   )r4   rW   r&   rZ   r!   r!   r"   rP         z TransformersConverter.load_modelc                 K   ra   r   rb   )r4   rX   r&   rZ   r!   r!   r"   rQ      rc   z$TransformersConverter.load_tokenizerc                 C   sz   t j| jrt j| j|}nz
tj| j|d}W n tjjy'   d }Y nw |d u s2t j	|s;t
d|| jf |S )N)repo_idr_   z"File %s does not exist in model %s)ospathisdirr-   rG   huggingface_hubhf_hub_downloadr   EntryNotFoundErrorisfilerF   )r4   r_   rf   r!   r!   r"   rU      s    
z$TransformersConverter.get_model_file)NNFNFF)rD   
__module____qualname____doc__strr   r   boolr5   r`   rP   rQ   rU   r!   r!   r!   r"   r%   C   s6    

$Cr%   c                   @   s   e Zd ZdZedd Zejdd Zdd Z	dd	 Z
d
d Zdd Zdd ZejjfddZdd Zdd Zdd Zdd ZdS )ModelLoaderzRBase class for loading Transformers models into a CTranslate2 model specification.c                 C      d S r   r!   r4   r!   r!   r"   rK         zModelLoader.architecture_namec                 C   s   t  r   NotImplementedErrorr4   r[   r!   r!   r"   get_model_spec   s   zModelLoader.get_model_specc                 C   s6   |  |}| |j|| | ||}| || |S r   )rx   
set_configr8   get_vocabularyset_vocabulary)r4   r[   r]   r^   tokensr!   r!   r"   __call__   s
   
zModelLoader.__call__c                 C   s"   dd t |  dd dD S )Nc                 S   s   g | ]\}}|qS r!   r!   ).0token_r!   r!   r"   
<listcomp>   s    z.ModelLoader.get_vocabulary.<locals>.<listcomp>c                 S   s   | d S N   r!   )itemr!   r!   r"   <lambda>   s    z,ModelLoader.get_vocabulary.<locals>.<lambda>)key)rH   	get_vocabitemsr4   r[   r]   r!   r!   r"   rz      s
   zModelLoader.get_vocabularyc                 C   rr   r   r!   r4   r^   r|   r!   r!   r"   r{         zModelLoader.set_vocabularyc                 C   rr   r   r!   r4   r8   r[   r]   r!   r!   r"   ry      r   zModelLoader.set_configc                 C   s   |j |_|j|_d S r   weightgammabiasbetar4   r^   moduler!   r!   r"   set_layer_norm      zModelLoader.set_layer_normc                 C   sr   |t jjkr|j|_n|j|_|j|_|j|_t	|t
jr%|jdd|_t|dr5|jd ur7|j|_d S d S d S )Nr   r   r   )r   QuantizationCT2r   qweightscalesweight_scaleqzerosweight_zero
isinstancer@   Conv1D	transposerL   r   )r4   r^   r   
quant_typer!   r!   r"   
set_linear   s   
zModelLoader.set_linearc                 C   s   |j |_ d S r   )r   r   r!   r!   r"   set_embeddings      zModelLoader.set_embeddingsc                 C   s4   |j |_t|dd}|dkr|j|d  |_d S d S )Noffsetr   r   	encodingsrJ   r4   r^   r   r   r!   r!   r"   set_position_encodings   s
   z"ModelLoader.set_position_encodingsc                 C   s   t d)Nz7No activation smoothing logic is defined for this modelru   )r4   r^   r'   r!   r!   r"   rS     s   zModelLoader.smooth_activationc                 C   s   t |dd }|r<|dp|d}|dkrd }nt|}|d u r/td|dt f |dd}|d	|}n
d }d}t |d	|}|||fS )
Nrope_scalingtype	rope_typedefaultkRoPE scaling type '%s' is not yet implemented. The following RoPE scaling types are currently supported: %sr6   factorr   
rope_theta)rJ   rE   _SUPPORTED_ROPE_SCALINGrv   rG   rI   )r4   r8   default_rope_thetar   r   rotary_scaling_typerotary_scaling_factorr   r!   r!   r"   get_rotary_params  s$   

zModelLoader.get_rotary_paramsN)rD   rl   rm   rn   propertyrK   abcabstractmethodrx   r}   rz   r{   ry   r   r   r   r   r   r   r   rS   r   r!   r!   r!   r"   rq      s     

	rq   
BartConfigc                       sb   e Zd Zedd Zdd Z fddZdd Zd	d
 Zdd Z	dd Z
dddZdd Z  ZS )
BartLoaderc                 C      dS )NBartForConditionalGenerationr!   rs   r!   r!   r"   rK   #  rt   zBartLoader.architecture_namec              	   C   s   t jj|jj|jjf|jj|jjt|jj	 t
|jddd}| |j|jj | |j|jj | |jj|j t
|dd }|d urR|  dkrR| |jj_|S )Nnormalize_embeddingTpre_norm
activationlayernorm_embeddingfinal_logits_biasr   )r	   TransformerSpecfrom_configr8   encoder_layersdecoder_layersencoder_attention_headsnormalize_before_SUPPORTED_ACTIVATIONSactivation_functionrJ   set_encoderencoderr[   set_decoderdecoderr   
projectionlm_headnonzeronumelsqueezer   )r4   r[   r^   r   r!   r!   r"   rx   '  s   
zBartLoader.get_model_specc                    s2   t  ||}|jjt|k r|d |jj }|S r   )superrz   r8   
vocab_sizelenr4   r[   r]   r|   rC   r!   r"   rz   :  s   zBartLoader.get_vocabularyc                 C      | | || d S r   register_source_vocabularyregister_target_vocabularyr   r!   r!   r"   r{   @     
zBartLoader.set_vocabularyc                 C   s,   |j |_ |j|_|j|_||jj|_d S r   )	bos_token	eos_token	unk_tokenconvert_ids_to_tokensr8   decoder_start_token_iddecoder_start_tokenr   r!   r!   r"   ry   D  s   
zBartLoader.set_configc                 C   s   |  || t|j|jD ]2\}}| j|j|jdd | |jj|j	 | 
|jj|j | 
|jj|j | |jj|j qd S NTself_attention)set_common_layersziplayerlayersset_attentionr   	self_attnr   
layer_normself_attn_layer_normr   ffnlinear_0fc1linear_1fc2final_layer_norm)r4   r^   r   
layer_specr   r!   r!   r"   r   L  s   zBartLoader.set_encoderc                 C   s   |  || t|j|jD ]J\}}| j|j|jdd | |jj|j	 t
|dr<| j|j|jdd | |jj|j | |jj|j | |jj|j | |jj|j qd S )NTr   encoder_attnF)r   r   r   r   r   r   r   r   r   r   rL   	attentionr   encoder_attn_layer_normr   r   r   r   r   r   r   )r4   r^   r   r   r   r!   r!   r"   r   ^  s2   
zBartLoader.set_decoderFc                 C   s   dd t dD }| |d |j | |d |j | |d |j |r0t|jd | nt|jd |d d  t|jd |dd   | |jd |j d S )Nc                 S      g | ]}t  qS r!   r   
LinearSpecr~   r   r!   r!   r"   r   |      z,BartLoader.set_attention.<locals>.<listcomp>   r   r      )	ranger   q_projk_projv_projr   fuse_linearr   out_projr4   r^   r   r   split_layersr!   r!   r"   r   {  s   zBartLoader.set_attentionc                 C   s   dd l }t|ds|jjr||jjnd}n|j}||_| |j	|j
 | t|jtr2|jd n|j|j t|drE| |j|j t|drT| |j|j d S d S )Nr   embed_scale      ?r   r   )mathrL   r8   scale_embeddingsqrtd_modelr  scale_embeddingsr   position_encodingsembed_positionsr   r   
embeddingslistembed_tokensr   r   r   )r4   r^   r   r
  r  r!   r!   r"   r     s(   


	
zBartLoader.set_common_layersF)rD   rl   rm   r   rK   rx   rz   r{   ry   r   r   r   r   __classcell__r!   r!   r   r"   r   !  s    

r   MarianConfigc                       sP   e Zd Zedd Z fddZdd Z fddZ fd	d
Zdd Z	  Z
S )MarianMTLoaderc                 C   r   )NMarianMTModelr!   rs   r!   r!   r"   rK     rt   z MarianMTLoader.architecture_namec                    s*   d|j _d|j _t |}| | |S NF)r8   r   r   r   rx   _remove_pad_weightsr4   r[   r^   r   r!   r"   rx     s
   
zMarianMTLoader.get_model_specc                 C   s   |j |_ |j|_|j |_d S r   )r   r   r   r   r!   r!   r"   ry     s   zMarianMTLoader.set_configc                    s   d|_ t || d S NT)start_from_zero_embeddingr   r   r4   r^   r   r   r!   r"   r     s   zMarianMTLoader.set_decoderc                    s&   t  ||}|d dkr|  |S )Nr   z<pad>)r   rz   popr   r   r!   r"   rz     s   zMarianMTLoader.get_vocabularyc                 C   s   |j jd |jj|jjg}|d jjd d }|D ]0}|jjd |d kr-|jd d |_t|tjrI|	 rI|j
jd |d krI|j
d d |_
qd S )Nr   r   r   )r   r  r   r   r   shaper   r   r   has_biasr   )r4   r^   vocab_specsnew_vocab_size
vocab_specr!   r!   r"   r    s    

z"MarianMTLoader._remove_pad_weights)rD   rl   rm   r   rK   rx   ry   r   rz   r  r  r!   r!   r   r"   r    s    
	r  M2M100Configc                       s<   e Zd Zedd Z fddZdd Z fddZ  ZS )	M2M100Loaderc                 C   r   )NM2M100ForConditionalGenerationr!   rs   r!   r!   r"   rK     rt   zM2M100Loader.architecture_namec                    s   d|j _d|j _t |S )NTF)r8   r   r   r   rx   rw   r   r!   r"   rx        zM2M100Loader.get_model_specc                 C   s   |j |jd  |_d S r   )weightsr   r   r   r!   r!   r"   r        z#M2M100Loader.set_position_encodingsc                    s   t  ||}|d |jkr||j|  |jdg D ]}||vr)|| qt	|d|j
jt| }|dkrE|dd t|D 7 }|S )Nr   additional_special_tokensnum_madeup_wordsr   c                 S   s   g | ]}d | qS )zmadeupword%dr!   r~   ir!   r!   r"   r     r   z/M2M100Loader.get_vocabulary.<locals>.<listcomp>)r   rz   r   insertunk_token_idr  special_tokens_maprE   appendrJ   r8   r   r   r   )r4   r[   r]   r|   r   r,  r   r!   r"   rz     s   
zM2M100Loader.get_vocabulary)	rD   rl   rm   r   rK   rx   r   rz   r  r!   r!   r   r"   r&    s    
r&  MBartConfigc                   @       e Zd Zedd Zdd ZdS )MBartLoaderc                 C   r   )NMBartForConditionalGenerationr!   rs   r!   r!   r"   rK     rt   zMBartLoader.architecture_namec                 C   s@   |j |_ |j|_|j|_t|jdd dv rd |_d S |j|_d S )NrX   )MBartTokenizerN)r   r   r   rJ   r8   r   r   r!   r!   r"   ry     s   
zMBartLoader.set_configNrD   rl   rm   r   rK   ry   r!   r!   r!   r"   r5        
r5  PegasusConfigc                   @   r4  )PegasusLoaderc                 C   r   )NPegasusForConditionalGenerationr!   rs   r!   r!   r"   rK     rt   zPegasusLoader.architecture_namec                 C   s$   |j |_|j|_|j|_|j |_d S r   )	pad_tokenr   r   r   r   r   r!   r!   r"   ry     s   zPegasusLoader.set_configNr8  r!   r!   r!   r"   r;    r9  r;  	OPTConfigc                       s\   e Zd Zedd Zdd Zdd Zdd Zd	d
 Z fddZ	dd Z
 fddZ  ZS )	OPTLoaderc                 C   r   )NOPTForCausalLMr!   rs   r!   r!   r"   rK      rt   zOPTLoader.architecture_namec                 C   s^   t jj|jj|jj|jjt|jj |jj	|jj
kd}| |j|jj | |jj|j |S )N)r   r   project_in_out)r	   TransformerDecoderModelSpecr   r8   num_hidden_layersnum_attention_headsdo_layer_norm_beforer   r   word_embed_proj_dimhidden_sizer   r   r[   r   r   r   r  r!   r!   r"   rx   $  s   
zOPTLoader.get_model_specc                 C   sb   t |jjD ](\}}d| }t|jj|jjd |d|   t|jj|jj	|d|   qd S )Nzmodel.decoder.layers.%dr   z%s.self_attn.q_projz%s.fc1)
	enumerater   r   r   rS   r   r   r   r   r   )r4   r^   r'   r.  r   layer_scoper!   r!   r"   rS   1  s   


zOPTLoader.smooth_activationc                 C      | | d S r   register_vocabularyr   r!   r!   r"   r{   A     zOPTLoader.set_vocabularyc                 C      |j |_ |j|_|j|_d S r   r   r   r   r   r!   r!   r"   ry   D  r(  zOPTLoader.set_configc                    sd   t  || |jd ur| |j|j |jd ur!| |j|j |jd ur0| |j|j d S d S r   )r   r   
project_inr   project_outr   r   r   r  r   r!   r"   r   I  s   


zOPTLoader.set_decoderc                 C   s*   d|_ | |j|j | |j|j d S r  )r  r   r  r  r   r  r  r   r!   r!   r"   r   S  s   zOPTLoader.set_common_layersc                    sZ   t  ||}d}t|d dkr+d|}||vr|| |d7 }t|d dks|S )Nr      zmadeupword{:04d}r   )r   rz   r   formatr2  )r4   r[   r]   r|   r.  symbolr   r!   r"   rz   X  s   

zOPTLoader.get_vocabulary)rD   rl   rm   r   rK   rx   rS   r{   ry   r   r   rz   r  r!   r!   r   r"   r?    s    

r?  GPTBigCodeConfigc                       sH   e Zd Zedd Zdd Zdd Z fddZd	d
 Zdd Z	  Z
S )GPTBigCodeMHALoaderc                 C   r   )NGPTBigCodeForCausalLMr!   rs   r!   r!   r"   rK   g  rt   z%GPTBigCodeMHALoader.architecture_namec                 C   sL   t jj|jj|jjdt|jj dd}| |j	|j
 | |j	j|j |S )NT)r   r   multi_query_attentionr	   rB  r   r8   n_layern_headr   r   r   r   transformerr   r   r   r  r!   r!   r"   rx   k  s   
z"GPTBigCodeMHALoader.get_model_specc                 C   rJ  r   rK  r   r!   r!   r"   r{   x  rM  z"GPTBigCodeMHALoader.set_vocabularyc                    >   t  ||}|jjt| }t|D ]	}|d|  q|S N<extra_id_%d>r   rz   r8   r   r   r   r2  r4   r[   r]   r|   	extra_idsr.  r   r!   r"   rz   {  
   z"GPTBigCodeMHALoader.get_vocabularyc                 C   rN  r   rO  r   r!   r!   r"   ry     r(  zGPTBigCodeMHALoader.set_configc                 C      d|_ | |j|j | |j|j | |j|j	 t
|j|jD ]B\}}| |jj|j | |jjd |jj | |jjd |jj | |jj|j | |jj|jj | |jj|jj q"d S NFr   r   r  r   r  wter   r  wper   r   ln_fr   r   hr   ln_1r   r   attnc_attnc_projr   ln_2r   mlpc_fcr   r4   r^   r   r   r   r!   r!   r"   r        zGPTBigCodeMHALoader.set_decoder)rD   rl   rm   r   rK   rx   r{   rz   ry   r   r  r!   r!   r   r"   rV  e  s    
	rV  
GPT2Configc                   @   8   e Zd Zedd Zdd Zdd Zdd Zd	d
 ZdS )
GPT2Loaderc                 C   r   )NGPT2LMHeadModelr!   rs   r!   r!   r"   rK     rt   zGPT2Loader.architecture_namec                 C   sJ   t jj|jj|jjdt|jj d}| |j	|j
 | |j	j|j |S )NT)r   r   rY  r  r!   r!   r"   rx     s   
zGPT2Loader.get_model_specc                 C   rJ  r   rK  r   r!   r!   r"   r{     rM  zGPT2Loader.set_vocabularyc                 C   rN  r   rO  r   r!   r!   r"   ry     r(  zGPT2Loader.set_configc                 C   rd  re  rf  rr  r!   r!   r"   r     rs  zGPT2Loader.set_decoderN	rD   rl   rm   r   rK   rx   r{   ry   r   r!   r!   r!   r"   rv    s    
rv  
GPTJConfigc                   @   ru  )
GPTJLoaderc                 C   r   )NGPTJForCausalLMr!   rs   r!   r!   r"   rK     rt   zGPTJLoader.architecture_namec              
   C   sb   t jj|jj|jjdt|jj |jjdddd}| 	|j
|j|jj|jj | |j
j|j |S NTFr   r   
rotary_dimrotary_interleaveparallel_residualshared_layer_norm)r	   rB  r   r8   rZ  r[  r   r   r~  r   r   r\  r   r   r   r  r!   r!   r"   rx     s$   
zGPTJLoader.get_model_specc                 C   rJ  r   rK  r   r!   r!   r"   r{     rM  zGPTJLoader.set_vocabularyc                 C   rN  r   rO  r   r!   r!   r"   ry     r(  zGPTJLoader.set_configc           
      C   s   d|_ | |j|j | |j|j t|j|j	D ]V\}}| |j
|j |jjj}|jjj}|jjj}	t|||}t|||}t|||	f|jjd _| |jjd |jj | |jj|jj | |jj|jj qd S re  )r  r   r  rg  r   r   ri  r   r   rj  r  rk  rl  r  r   r  r  r   permute_for_sliced_rotaryr>   catr   r   r   r  r   r   rp  fc_inr   fc_out)
r4   r^   r   r~  	num_headsr   r   qwkwvwr!   r!   r"   r     s   


zGPTJLoader.set_decoderNrx  r!   r!   r!   r"   rz    s    
rz  CodeGenConfigc                       H   e Zd Zedd Zdd Z fddZdd Zd	d
 Zdd Z	  Z
S )CodeGenLoaderc                 C   r   )NCodeGenForCausalLMr!   rs   r!   r!   r"   rK     rt   zCodeGenLoader.architecture_namec              
   C   s   t jj|jj|jjdt|jj |jjdddd}d}t	|jdr(|jj
dv r(d}| j|j|j|jj|jj|jj|d | |jj|j |S )	NTFr}     head_dim)      rR  )mp_num)r	   rB  r   r8   rZ  r[  r   r   r~  rL   r  r   r   r\  n_embdr   r   r   )r4   r[   r^   r  r!   r!   r"   rx      s.   
zCodeGenLoader.get_model_specc                    r]  r^  r`  ra  r   r!   r"   rz     s
   zCodeGenLoader.get_vocabularyc                 C   rJ  r   rK  r   r!   r!   r"   r{   (  rM  zCodeGenLoader.set_vocabularyc                 C   rN  r   rO  r   r!   r!   r"   ry   +  r(  zCodeGenLoader.set_configc                    s8  d|_ | |j|j | |j|j td|d 	ddj
  }||  t fdd|D }t|j|jD ]^\}	}
| |	j|
j |
jjj}||d d f }|jddd\}}}t|||}t|||}t|||f|	jjd _| |	jjd |
jj | |	jj|
j j! | |	jj"|
j j# q;d S )	NFr   r   r   c                    s$   g | ]}t |  |d    qS )r   )r>   aranger-  	local_dimr!   r"   r   8  s   $ z-CodeGenLoader.set_decoder.<locals>.<listcomp>dimr   )$r  r   r  rg  r   r   ri  npr  reshapeTflattentolistr>   r  r   r   rj  r  rk  rl  qkv_projr   chunkr   r  r   r   r   r  r   r   rp  r  r   r  )r4   r^   r   r~  r  	embed_dimr  base_permutationpermutationr   r   r  new_qkv_projr  r  r  r!   r  r"   r   0  s(   "
zCodeGenLoader.set_decoderrD   rl   rm   r   rK   rx   rz   r{   ry   r   r  r!   r!   r   r"   r    s    

r  GPTNeoXConfigc                       r  )GPTNeoXLoaderc                 C   r   )NGPTNeoXForCausalLMr!   rs   r!   r!   r"   rK   V  rt   zGPTNeoXLoader.architecture_namec              
   C   st   t jj|jj|jjdt|jj t|jj	|jj
|jj  d|jjdd}| |j|j|jj | |jj|j |S r|  )r	   rB  r   r8   rC  rD  r   
hidden_actint
rotary_pctrG  use_parallel_residualr   r   gpt_neoxr   r   	embed_outr  r!   r!   r"   rx   Z  s"   
zGPTNeoXLoader.get_model_specc                    r]  r^  r`  ra  r   r!   r"   rz   m  rc  zGPTNeoXLoader.get_vocabularyc                 C   rJ  r   rK  r   r!   r!   r"   r{   v  rM  zGPTNeoXLoader.set_vocabularyc                 C   rN  r   rO  r   r!   r!   r"   ry   y  r(  zGPTNeoXLoader.set_configc                 C   sJ  d|_ | |j|j | |j|j t|j|j	D ]\}}t
|dr4| |j|j | |j|j n| |jj|j | |jj|j |jjj}|jjj}||dd|jd ddd|jd }||ddddd}||jjd _||jjd _| |jjd |jj | |jj|jj | |jj|jj qd S )NFinput_layer_normr   r   r   r   ) r  r   r  embed_inr   r   r   r   r   r   rL   r  input_layernormpost_attention_layer_normpost_attention_layernormr   r   r   query_key_valuer   r   r  r   swapaxesr   r   denser   rp  dense_h_to_4hr   dense_4h_to_h)r4   r^   r   r  r   r   qkv_wqkv_br!   r!   r"   r   ~  s8   




zGPTNeoXLoader.set_decoderr  r!   r!   r   r"   r  T  s    
	r  WhisperConfigc                       sp   e Zd Zedd Zdd Zdd Zdd Z fd	d
Zdd Z	 fddZ
 fddZdd Zdd Z  ZS )WhisperLoaderc                 C   r   )NWhisperForConditionalGenerationr!   rs   r!   r!   r"   rK     rt   zWhisperLoader.architecture_namec                 C   sZ   t |jj|jj|jj|jj}| |j|j	j | 
|j|j	j | |jj|j |S r   )r   WhisperSpecr8   r   r   r   decoder_attention_headsr   r   r[   r   r   r   r   proj_outr  r!   r!   r"   rx     s   zWhisperLoader.get_model_specc                    s0   g d t dg }|sg S  fdd|D S )N)z<|endoftext|>z<|startoftranscript|>z<|translate|>z<|transcribe|>z<|startoflm|>z<|startofprev|>z<|nocaptions|>z<|notimestamps|>r+  c                    s   g | ]}| vr |qS r!   )convert_tokens_to_ids)r~   r   non_lang_special_tokensr]   r!   r"   r     s
    z>WhisperLoader._get_lang_ids_from_tokenizer.<locals>.<listcomp>)rJ   )r4   r]   additional_tokensr!   r  r"   _get_lang_ids_from_tokenizer  s   z*WhisperLoader._get_lang_ids_from_tokenizerc                 C   s   t |dd }|d ur)|j|_|j|_t|dr|j|_t|dr(t|j	 |_
n|jj|_|jj|_t|j|_t |dd d u rH| ||_
|jd u rg|jj}|jj}ttt|d |t||_d S d S )Ngeneration_configalignment_heads
lang_to_idlang_idsr   )rJ   suppress_tokenssuppress_idsbegin_suppress_tokenssuppress_ids_beginrL   r  rH   r  valuesr  r8   _WHISPER_ALIGNMENT_HEADSrE   name_or_pathr  r   r  r  	itertoolsproductr   )r4   r8   r[   r]   
gen_config
num_layersr  r!   r!   r"   ry     s0   





zWhisperLoader.set_configc                    s6   t  ||}|dd t|jjt| D  |S )Nc                 s   s    | ]	}d |d  V  qdS )z<|%.2f|>g{Gz?Nr!   r-  r!   r!   r"   	<genexpr>  s
    

z/WhisperLoader.get_vocabulary.<locals>.<genexpr>)r   rz   extendr   r8   r   r   r   r   r!   r"   rz     s
   
zWhisperLoader.get_vocabularyc                 C   rJ  r   rK  r   r!   r!   r"   r{     rM  zWhisperLoader.set_vocabularyc                    s2   |  |j|j |  |j|j t || d S r   )
set_conv1dconv1conv2r   r   )r4   r^   r   r   r!   r"   r     s   zWhisperLoader.set_encoderc                    s"   |  |j|j t || d S r   )r   r  r  r   r   r  r   r!   r"   r     s   zWhisperLoader.set_decoderc                 C   s$   |  |j|j | |j|j d S r   )r   r  r  r   r   r   r!   r!   r"   r        zWhisperLoader.set_common_layersc                 C   s   |j |_ |j|_d S r   r   r   r   r!   r!   r"   r    r   zWhisperLoader.set_conv1d)rD   rl   rm   r   rK   rx   r  ry   rz   r{   r   r   r   r  r  r!   r!   r   r"   r    s    
r  Wav2Vec2Configc                       sh   e Zd Zedd Zdd Zdd Zdd Zd	d
 Zdd Z	dd Z
dd Z fddZdd Z  ZS )Wav2Vec2Loaderc                 C   r   )NWav2Vec2ForCTCr!   rs   r!   r!   r"   rK     rt   z Wav2Vec2Loader.architecture_namec                 C   s   t |jjdd}t|jjj|jjjj|jjjj|j	j
jd |}|jjjD ]}|j|_|j|_|jj|_|jj|_|jj|_q&| |j||jj |S Nreturn_hiddenFr   )rJ   wav2vec2r8   r
   Wav2Vec2Specnum_feat_extract_layersr   rC  rD  r   r   r   r   r   r   r   r   feed_forwardintermediate_act_fnactivation_fnintermediate_denser   output_denser   r   )r4   r[   r  r^   r   r!   r!   r"   rx     s    

	

zWav2Vec2Loader.get_model_specc                 C   rr   r   r!   r   r!   r!   r"   ry   &  r   zWav2Vec2Loader.set_configc                 C      |  S r   r   r   r!   r!   r"   rz   )     zWav2Vec2Loader.get_vocabularyc                 C   rJ  r   rK  r   r!   r!   r"   r{   ,  rM  zWav2Vec2Loader.set_vocabularyc                 C   s   |j d jj|jj_|j d jj|jj_| |jj|j d j t|j|j dd  D ]\}}|jj|j_|jj|j_| |j|j q+d S )Nr   r   )	conv_layersconvr   feat_layer0r   r   r   r   
feat_layer)r4   r^   feature_extractor
spec_layermodule_layerr!   r!   r"   set_feature_extractor/  s   z$Wav2Vec2Loader.set_feature_extractorc                 C   $   |  |j|j | |j|j d S r   r   fp_layer_normr   r   fp_projectionr   r4   r^   feature_projectionr!   r!   r"   set_feature_projection<  r  z%Wav2Vec2Loader.set_feature_projectionc                 C   s   |j jjj |j jj_|j jj |j jj_|j  D ]}|j |_q| tdd|j	f |j jj|j j_|j jj|j j_d S r   )
pos_conv_embedr  r   datafloatr   
parametersr>   randnrG  )r4   r^   r   r8   paramr!   r!   r"   set_pos_conv_embed@  s   
z!Wav2Vec2Loader.set_pos_conv_embedc                    sp   |  ||jj | ||jj | ||jj| t ||jj t	|jj
dd}|s6| |j|j d S d S Nr  F)r  r  r  r  r  r   r   r   r   rJ   r8   r   r   )r4   r^   r[   r8   r  r   r!   r"   r   M  s   zWav2Vec2Loader.set_encoderc                 C   s   |  |j|j d S r   )r   r   r   r!   r!   r"   r   V  s   z Wav2Vec2Loader.set_common_layers)rD   rl   rm   r   rK   rx   ry   rz   r{   r  r  r   r   r   r  r!   r!   r   r"   r    s    
	r  Wav2Vec2BertConfigc                   @   st   e Zd Zedd Zdd Zdd Zdd Zd	d
 Zdd Z		dddZ
dd Zdd Zdd Zdd Zdd ZdS )Wav2Vec2BertLoaderc                 C   r   )NWav2Vec2BertForCTCr!   rs   r!   r!   r"   rK   \  rt   z$Wav2Vec2BertLoader.architecture_namec                 C   sH   t |jjdd}t|jjj|jjj|jjj	d |}| 
|j| |S r  )rJ   wav2vec2_bertr8   r   Wav2Vec2BertSpecnum_adapter_layersrC  r   r   r   r   r   )r4   r[   r  r^   r!   r!   r"   rx   `  s   z!Wav2Vec2BertLoader.get_model_specc                 C   rr   r   r!   r   r!   r!   r"   ry   k  r   zWav2Vec2BertLoader.set_configc                 C   r  r   r  r   r!   r!   r"   rz   n  r  z!Wav2Vec2BertLoader.get_vocabularyc                 C   rJ  r   rK  r   r!   r!   r"   r{   q  rM  z!Wav2Vec2BertLoader.set_vocabularyc                 C   r  r   r  r  r!   r!   r"   r  t  r  z)Wav2Vec2BertLoader.set_feature_projectionNc                 C   s   dd t dD }| |d |j | |d |j | |d |j t|jd | | |jd |j |s;|rT|j	j
|_td||_td||_d S d S )	Nc                 S   r   r!   r   r   r!   r!   r"   r   {  r   z4Wav2Vec2BertLoader.set_attention.<locals>.<listcomp>r   r   r   r   r   int32)r   r   linear_qlinear_klinear_vr   r  r   
linear_outdistance_embeddingr   !relative_asymmetric_position_keysr  r:   r   relative_left_max_positionrelative_right_max_position)r4   r^   r   left_max_positionright_max_positionr  r!   r!   r"   r   x  s   


z Wav2Vec2BertLoader.set_attentionc                 C   s  t ||D ]\}}| |j|j | |jj|jj | |jj	|jj
 | |j|j|| | |j|j | |j|jj | |j|jj | |j|jj | |j|jj | |j|jj | |j|j | |jj|jj | |jj	|jj
 | |j |j! qd S r   )"r   r   enc_ffn1_layer_normffn1_layer_normr   enc_ffn1r   ffn1r  r   r  r   enc_attnr   enc_attn_layer_normr   enc_conv_layer_normconv_moduler   r  enc_conv_pointwise_conv1pointwise_conv1enc_conv_depthwise_convdepthwise_convenc_conv_depthwise_layer_normdepthwise_layer_normenc_conv_pointwise_conv2pointwise_conv2enc_ffn2_layer_normffn2_layer_normenc_ffn2ffn2enc_final_layer_normr   )r4   spec_layersr   r  r  slayerr   r!   r!   r"   set_wav2vec2bert_encoder  s:   



z+Wav2Vec2BertLoader.set_wav2vec2bert_encoderc                 C   s   t ||D ]H\}}| |j|j | |j|j | |j|j | |j	|j
 | |j|j | |j|j | |jj|jj | |jj|jj qd S r   )r   r   adpt_residual_layer_normresidual_layer_normr  adpt_residual_convresidual_convadpt_attn_layer_normr   adpt_attn_convself_attn_convr   adpt_attn_layerr   adpt_ffn_layer_normffn_layer_normr   adpt_ffnr   r   r  r   r  )r4   r(  r   r)  r   r!   r!   r"   set_wav2vec2bert_adapter  s   z+Wav2Vec2BertLoader.set_wav2vec2bert_adapterc                 C   st   |  ||jj | |j|jjj|jjj|jjj	 | 
|j|jjj t|jjdd}|s8| |j|j d S d S r  )r  r  r  r*  r   r   r   r8   left_max_position_embeddingsright_max_position_embeddingsr6  adapter_layersadapterrJ   r   r   )r4   r^   r[   r  r!   r!   r"   r     s   zWav2Vec2BertLoader.set_encoderc                 C   s"   |j |_ |jd ur|j|_d S d S r   r  r   r!   r!   r"   r       
zWav2Vec2BertLoader.set_conv1dc                 C   s"   |j |_|jd ur|j|_d S d S r   r   r   r!   r!   r"   r     r;  z!Wav2Vec2BertLoader.set_layer_norm)NN)rD   rl   rm   r   rK   rx   ry   rz   r{   r  r   r*  r6  r   r  r   r!   r!   r!   r"   r  Z  s    

 r  T5Configc                       st   e Zd Zedd Zdd Z fddZdd Zd	d
 ZdddZ	dd Z
dd Zdd ZdddZdd Z  ZS )T5Loaderc                 C   r   )NT5ForConditionalGenerationr!   rs   r!   r!   r"   rK     rt   zT5Loader.architecture_namec              	   C   s   t jj|jj|jjf|jjdt|jj |jj	ddd}| 
|j|j | j
|j|jdd | |jj|j |jjrB|jjd |j_|S )NT)r   r   ffn_glurelative_attention_biasrms_norm)
is_decoderg      )r	   r   r   r8   r  num_decoder_layersr  r   dense_act_fnis_gated_act	set_stackr   r   r   r   r   tie_word_embeddingsr  scale_outputsr  r!   r!   r"   rx     s   

zT5Loader.get_model_specc                    r]  r^  r`  ra  r   r!   r"   rz     rc  zT5Loader.get_vocabularyc                 C   r   r   r   r   r!   r!   r"   r{     r   zT5Loader.set_vocabularyc                 C   sD   |j |_|j|_|j|_t|jdr||jj|_d S |j |_d S )Nr   )	r=  r   r   r   rL   r8   r   r   r   r   r!   r!   r"   ry     s   
zT5Loader.set_configFc                 C   s   |  |j|j | t|jtr|jd n|j|j d|_t	t
|j|jD ]:\}\}}| |j|jd  |dkrK|jd j}|j|j_|j|j_|rW| |j|jd  | |j|jd  q'd S )Nr   Fr   r   )r   r   r   r   r   r  r  r  r  rH  r   r   blockset_self_attentionr   r@  relative_attention_max_distanceset_cross_attentionr   set_ffnr   )r4   r^   r   rB  r.  r   rI  first_self_attentionr!   r!   r"   rF    s(   
	zT5Loader.set_stackc                 C   sh   t |dr| |j|jj | |j|jj n	| |j|jj | |j|jj	 | 
|j|j d S )Nlinear_0_noact)rL   r   r   DenseReluDensewi_0rO  wi_1wir   wor   r   r   r!   r!   r"   rM     s   
zT5Loader.set_ffnc                 C   s&   | j ||jdd | |j|j d S r   )r   SelfAttentionr   r   r   r!   r!   r"   rJ  *  s   zT5Loader.set_self_attentionc                 C   s"   |  ||j | |j|j d S r   )r   EncDecAttentionr   r   r   r!   r!   r"   rL  .  s   zT5Loader.set_cross_attentionc                 C   s   d|_ dd tdD }| |d |j | |d |j | |d |j |r3t|jd | nt|jd |d d  t|jd |dd   | |jd |j	 |j
rk|jj|_td	|j|_d S d S )
Nr	  c                 S   r   r!   r   r   r!   r!   r"   r   5  r   z*T5Loader.set_attention.<locals>.<listcomp>r   r   r   r   r   r  )queries_scaler   r   qkvr   r  r   ohas_relative_attention_biasr@  r   r  r:   r   rK  r  r!   r!   r"   r   2  s    


zT5Loader.set_attentionc                 C      |j |_d S r   r   r   r4   r^   r   r!   r!   r"   r   H  r   zT5Loader.set_layer_normr  )rD   rl   rm   r   rK   rx   rz   r{   ry   rF  rM  rJ  rL  r   r   r  r!   r!   r   r"   r=    s    
	


r=  	MT5Configc                   @   s   e Zd Zedd ZdS )	MT5Loaderc                 C   r   )NMT5ForConditionalGenerationr!   rs   r!   r!   r"   rK   N  rt   zMT5Loader.architecture_nameN)rD   rl   rm   r   rK   r!   r!   r!   r"   ra  L  s    ra  BloomConfigc                       P   e Zd Zedd Zdd Z fddZdd Zd	d
 Zdd Z	dd Z
  ZS )BloomLoaderc                 C   r   )NBloomForCausalLMr!   rs   r!   r!   r"   rK   U  rt   zBloomLoader.architecture_namec              	   C   sL   t jj|jj|jjdtjjdddd}| 	|j
|j | |j
j|j |S )NT)r   r   r   alibialibi_use_positive_positions)r	   rB  r   r8   rZ  r[  r   
ActivationGELUTanhr   r   r\  r   r   r   r  r!   r!   r"   rx   Y  s   
zBloomLoader.get_model_specc                    r]  r^  r`  ra  r   r!   r"   rz   h  rc  zBloomLoader.get_vocabularyc                 C   rJ  r   rK  r   r!   r!   r"   r{   q  rM  zBloomLoader.set_vocabularyc                 C   rN  r   rO  r   r!   r!   r"   ry   t  r(  zBloomLoader.set_configc                 C   s   d|_ | |j|j | |j|j | |j|j t	|j
|jD ]E\}}| |jj|j | |jjd |jj|jj | |jjd |jj | |jj|j | |jj|jj | |jj|jj q"d S re  )r  r   r  word_embeddingsr   r   word_embeddings_layernormr   ri  r   r   rj  r   r  set_qkv_linearr   r  r  r   r  r   r  r   rp  r  r   r  rr  r!   r!   r"   r   y  s,   


zBloomLoader.set_decoderc                 C   st   |j }||dd|jd }|dd}|d|jd }|j}||dd}|dd}|d}||_ ||_d S )Nr   r   r   r   )r   r  r   r   r   )r4   r^   r   r  r   r   r!   r!   r"   rm    s   

zBloomLoader.set_qkv_linear)rD   rl   rm   r   rK   rx   rz   r{   ry   r   rm  r  r!   r!   r   r"   re  S  s    
	re  	MPTConfigc                       rd  )	MPTLoaderc                 C   r   NAutoModelForCausalLMr!   rs   r!   r!   r"   rK     rt   zMPTLoader.architecture_namec                 C   s6   t jj|jj|jjdtjjdd}| 	|j
|j |S )NT)r   r   rg  )r	   rB  r   r8   n_layersn_headsr   ri  GELUr   r   r\  r  r!   r!   r"   rx     s   zMPTLoader.get_model_specc                    r]  r^  r`  ra  r   r!   r"   rz     rc  zMPTLoader.get_vocabularyc                 C   rJ  r   rK  r   r!   r!   r"   r{     rM  zMPTLoader.set_vocabularyc                 C   rN  r   rO  r   r!   r!   r"   ry     r(  zMPTLoader.set_configc                 C   s   |  |j|j | |j|j d|_|jj|j_t	|j
|jD ]B\}}| |jj|j | |jjd |jj | |jjd |jj | |jj|j | |jj|jj | |jj|jj q d S re  )r   r  rg  r   r   norm_fr  r   r   r   r   blocksr   norm_1r   r   rl  Wqkvr  r   norm_2r   up_projr   	down_projrr  r!   r!   r"   r     s   zMPTLoader.set_decoderc                 C   s   |j |_t|j|_d S r   )r   r   r>   
zeros_liker   r   r!   r!   r"   r     s   zMPTLoader.set_layer_norm)rD   rl   rm   r   rK   rx   rz   r{   ry   r   r   r  r!   r!   r   r"   ro    s    
	ro  GemmaConfigc                       rd  )GemmaLoaderc                 C   r   )NGemmaForCausalLMr!   rs   r!   r!   r"   rK     rt   zGemmaLoader.architecture_namec                 C   s   |j j}|j j}t|j d|}||krd }t|j dd}tjj|||dkr)tjj	ntjj
dddddt|j dd	||j jd
}| |j|j | |jj|j |j jd |jj_|S )Nnum_key_value_headshidden_activationr   r   Tr   Fr   '  )	r   r   r?  rA  r~  r  rotary_basenum_heads_kvr        ?r8   rC  rD  rJ   r	   rB  r   r   ri  rt  rj  r  r   r   r[   r   r   r   rG  r  multiply_by_sqrt_depthr4   r[   r  r  r  activation_configr^   r!   r!   r"   rx     s6   zGemmaLoader.get_model_specc                    ^   t  ||}|jjt| }t|D ]	}|d|  q|jjt|k r-|d |jj }|S r^  r`  ra  r   r!   r"   rz        zGemmaLoader.get_vocabularyc                 C   rJ  r   rK  r   r!   r!   r"   r{     rM  zGemmaLoader.set_vocabularyc                 C   &   |j |_ |j|_|j|_|jj|_d S r   r   r   r   r8   rms_norm_epslayer_norm_epsilonr   r!   r!   r"   ry        zGemmaLoader.set_configc                 C      |j |_d|_d S r  r   r   layer_norm_use_residualr_  r!   r!   r"   r        
zGemmaLoader.set_layer_normc           	      C   s  d|_ d|_| |j|j | |j|j t|j	|j
D ]j\}}| |jj|j | |jj|j |jjj}|jjj}|jjj}|jjj}t|||g|jjd _||jjd _| |jj|jj | |jj|jj | |jj|jj t |d t |d t!"  qd S NTFr   r   r   rp  )#r  r  r   r  r  r   r   normr   r   r   r   r  r   r  r   r  r   r  r  o_projr>   r  r   r   r   rp  	gate_projrO  rz  r   r{  delattrgccollect	r4   r^   r   r   r   wqwkwvrT  r!   r!   r"   r     s0   








zGemmaLoader.set_decoderrD   rl   rm   r   rK   rx   rz   r{   ry   r   r   r  r!   r!   r   r"   r~    s    
#r~  Gemma2Configc                       rd  )Gemma2Loaderc                 C   r   )NGemma2ForCausalLMr!   rs   r!   r!   r"   rK   ;  rt   zGemma2Loader.architecture_namec                 C   s   |j j}|j j}t|j d|}||krd }t|j dd}tjj|||dkr)tjj	ntjj
dddddt|j dd	||j jdd
}| |j|j | |jj|j |j jd |jj_|S )Nr  r  r   r   Tr   Fr   r  )
r   r   r?  rA  r~  r  r  r  r  pre_post_layer_normr  r  r  r!   r!   r"   rx   ?  s8   zGemma2Loader.get_model_specc                    r  r^  r`  ra  r   r!   r"   rz   c  r  zGemma2Loader.get_vocabularyc                 C   rJ  r   rK  r   r!   r!   r"   r{   n  rM  zGemma2Loader.set_vocabularyc                 C   r  r   r  r   r!   r!   r"   ry   q  r  zGemma2Loader.set_configc                 C   r  r  r  r_  r!   r!   r"   r   w  r  zGemma2Loader.set_layer_normc           	      C   s0  d|_ d|_| |j|j | |j|j t|j	|j
D ]x\}}| |j|j | |j|j | |j|j | |j|j |jjj}|jjj}|jjj}|jjj}t|||g|jjd _||jjd _| |jj|j j! | |jj"|j j# | |jj$|j j% t&|d t&|d t'(  qd S r  ))r  r  r   r  r  r   r   r  r   r   r   r  r  r  r  pre_feedforward_layer_normpre_feedforward_layernormpost_feedforward_layer_normpost_feedforward_layernormr   r  r   r  r  r  r>   r  r   r   r   r   r   rp  r  rO  rz  r   r{  r  r  r  r  r!   r!   r"   r   {  s8   






zGemma2Loader.set_decoderr  r!   r!   r   r"   r  9  s    
$r  LlamaConfigc                       X   e Zd Zedd Zdd Z fddZdd Zd	d
 Zdd Z	e
jjfddZ  ZS )LlamaLoaderc                 C   r   )NLlamaForCausalLMr!   rs   r!   r!   r"   rK     rt   zLlamaLoader.architecture_namec                 C   sB  |j j}|j j}t|j d|}||krd }| |j d\}}}t|j dd }|rNd }	|jdkr5t|j}	|	d u rGt	d|jd
t f |j}
|j}ntjj}	d }
d }tjj||tjjddddd	|||||	|
|d
}| |j|j|	 | |jj|j t|j dd }|tjjkr|jjD ]}|d |j _!|d |j _"q|S )Nr  r  quantization_configawqkQuantization type '%s' is not yet implemented. The following Quantization types are currently supported: %sr6   Tr   Fr   r   r?  rA  r~  r  r   r   r  r  r   quant_group_size
quant_bitsr   low_freq_factorhigh_freq_factor)#r8   rC  rD  rJ   r   quant_method_SUPPORTED_QUANTIZATIONrE   versionrv   rG   rI   
group_sizebitsr   r   r   r	   rB  r   ri  SWISHr   r   r[   r   r   r   r   RotaryScalingTypeLlama3r   r   rotary_low_freq_factorrotary_high_freq_factor)r4   r[   r  r  r  r   r   r   r  r   r  r  r^   r   r   r!   r!   r"   rx     sn   


zLlamaLoader.get_model_specc                    r  r^  r`  ra  r   r!   r"   rz     r  zLlamaLoader.get_vocabularyc                 C   rJ  r   rK  r   r!   r!   r"   r{     rM  zLlamaLoader.set_vocabularyc                 C   s4   |j |_ |j|_|jd ur|jnd|_|jj|_d S N r  r   r!   r!   r"   ry     s
   zLlamaLoader.set_configc                 C   r]  r   r^  r_  r!   r!   r"   r     r   zLlamaLoader.set_layer_normc                 C     d|_ | |j|j | |j|j t|j|j	D ]\}}| |j
j|j | |jj|j dd tdD }| j|d |jj|d | j|d |jj|d | j|d |jj|d |tjjkrnt|j
jd | n|tjjkrvdnd}t|j
jd || | j|j
jd |jj|d | j|jj|jj|d | j|jj |jj!|d | j|jj"|jj#|d t$|d	 t$|d
 t%&  qd S )NFc                 S   r   r!   r   r   r!   r!   r"   r     r   z+LlamaLoader.set_decoder.<locals>.<listcomp>r   r   r   r   r   r   rp  'r  r   r  r  r   r   r  r   r   r   r   r  r   r  r   r   r   r  r  r  r   r   r   r   r  r   AWQ_GEMMfuse_linear_prequantr  r   rp  r  rO  rz  r   r{  r  r  r  r4   r^   r   r   r   r   r  cc_dimr!   r!   r"   r     sX   





zLlamaLoader.set_decoderrD   rl   rm   r   rK   rx   rz   r{   ry   r   r   r   r   r   r  r!   r!   r   r"   r    s    
Br  Gemma3TextConfigGemma3Configc                       `   e Zd Zedd Zdd Zdd Z fddZd	d
 Zdd Z	dd Z
ejjfddZ  ZS )Gemma3Loaderc                 C   r   )NGemma3ForCausalLMr!   rs   r!   r!   r"   rK   9  rt   zGemma3Loader.architecture_namec                 C      |j jdkr	tjS |S )Nr  )rC   rD   r@   Gemma3ForConditionalGenerationr4   r8   default_classr!   r!   r"   r7   =  s   zGemma3Loader.get_model_classc                    s  t |jd|j}|j}|j}t |d|}||krd }|j}t |dd}t |dd}t |dd}	t |d	d
}
t |dd }|d u rTt |dd   d urT fddt|D }t |dd }|ry|jdkrgt|j	}|d u rrt
d|j |j}|j}ntjj}d }d }tjj||f|dkrtjjntjjddd|d|	|||
d|||dd}|| _t|D ];\}}|jj| }|dkrtd||j_tdd|j_q|dkrtd|	|j_td|
|j_qt |j d|j }| !|j|| | "|jj#|j$ |S )Ntext_configr  r  r   r   @B rope_local_base_freqr  sliding_windowi   layer_types_sliding_window_patternc                    s$   g | ]}|d    dkrdndqS )r   r   full_attentionsliding_attentionr!   r-  sliding_window_patternr!   r"   r   `  s    z/Gemma3Loader.get_model_spec.<locals>.<listcomp>r  r  .Quantization type '%s' is not yet implemented.r   TF)r   r   r?  rA  r~  r  r  r  r  r  r  r   r  r  qk_normr  float32r  r   r  language_model)%rJ   r8   rC  rD  r  r   r  r  rE   r  rv   r  r  r   r   r   r	   rB  r   ri  rt  rj  _layer_typesrH  r   r   r  r:   r   r   r  r  r[   r   r   r   r   )r4   r[   r  r  r  r  r  r  r   r  r  r  r  r   r  r  r^   r.  
layer_typer   
text_modelr!   r  r"   rx   D  s   



zGemma3Loader.get_model_specc                    h   t  ||}t|jd|j}|jt| }t|D ]	}|d|  q|jt|k r2|d |j }|S Nr  r_  r   rz   rJ   r8   r   r   r   r2  r4   r[   r]   r|   r  rb  r.  r   r!   r"   rz     s   zGemma3Loader.get_vocabularyc                 C   rJ  r   rK  r   r!   r!   r"   r{     rM  zGemma3Loader.set_vocabularyc                 C   F   |j |_ |j|_t|drt|jtr|j rd|_d S |j|_d S Nchat_templatez<end_of_turn>r   r   rL   r   r  ro   stripr   r   r!   r!   r"   ry     s   

zGemma3Loader.set_configc                 C   r  r  r  r_  r!   r!   r"   r     r  zGemma3Loader.set_layer_normc                 C   s  d|_ d|_| |j|j | |j|j t|j	|j
D ]\}}| |j|j | |j|j | |j|j | |j|j | |jj|jj | |jj|jj dd tdD }| j|d |jj|d | j|d |jj|d | j|d	 |jj|d |tjjkrt |jj!d | n|tjj"krdnd}t#|jj!d || | j|jj!d |jj$|d | j|j%j&|j'j(|d | j|j%j)|j'j*|d | j|j%j+|j'j,|d t-|d
 t-|d t./  qd S )NTFc                 S   r   r!   r   r   r!   r!   r"   r     r   z,Gemma3Loader.set_decoder.<locals>.<listcomp>r   r   r  r   r   r   rp  )0r  r  r   r  r  r   r   r  r   r   r   r  r  r  r  r  r  r  r  r   q_normr   k_normr   r   r  r  r  r   r   r   r   r  r   r  r  r  r   r   rp  r  rO  rz  r   r{  r  r  r  r  r!   r!   r"   r     sn   



zGemma3Loader.set_decoderrD   rl   rm   r   rK   r7   rx   rz   r{   ry   r   r   r   r   r   r  r!   r!   r   r"   r  6  s    
br  Gemma4TextConfigGemma4Configc                       r  )Gemma4Loaderc                 C   r   )NGemma4ForCausalLMr!   rs   r!   r!   r"   rK     rt   zGemma4Loader.architecture_namec                 C   r  )Nr  )rC   rD   r@   Gemma4ForConditionalGenerationr  r!   r!   r"   r7     s   zGemma4Loader.get_model_classc                    sb  t |jd|j}|j}|j}t |d|}||krd }t |dd}|dkr*td| |j}t |d|}t |dd }	t |dd	}
t |d
d}t |dd pLi }|di }|di }t|dd}t|dd}t|dd}t|| }t |dd}t |dd }|d u rd  fddt	|D }t |dd }|r|j
dkrt|j}|d u rtd|j
 |j}|j}ntjj}d }d }tjj||fi d|dkrtjjntjjdd d!d d"d d#|d$d	d%|d&|d'|d|d(d d)|d*|d+|d,d d-d }|| _|
| _t|D ]l\}}|jj| }td.d|j _!|dkr`td/||j _"td.||j _#td/d|j _$td/||j _|	d ur^td/|	|j _%q|dkrytd.||j _#td/||j _$qt |jd|j}t |d0d }|rtd.||j_&t |j'd1|j'}| (|j|| | )|jj*|j+ |S )2Nr  r  num_kv_shared_layersr   ztGemma 4 KV-shared layers (num_kv_shared_layers=%d) are not yet supported. Use the 31B model which has no KV sharing.global_head_dimnum_global_key_value_headsattention_k_eq_vFr  r   rope_parametersr  r  r   r  r  partial_rotary_factorr	  r  i   r     c                    s$   g | ]}t |d    rdndqS )r   r  r  )rp   r-  r  r!   r"   r   H  s    z/Gemma4Loader.get_model_spec.<locals>.<listcomp>r  r  r  r   r   r   Tr?  rA  r~  r  r  r  r  r  r   r  r  r  v_normr  r  final_logit_softcappingr  ),rJ   r8   rC  rD  rv   r  rE   r  r  r   r  r  r  r  r  r   r   r   r	   rB  r   ri  rt  rj  r  _attention_k_eq_vrH  r   r   r  r:   r   r   rW  r~  r  r  r  r  r[   r   r   r   r   )r4   r[   r  r  r  r  r  r  r  num_global_kv_headsr  r  rope_paramssliding_ropeglobal_roper  r   global_partial_factorglobal_rotary_dimr  r  r  r   r  r  r^   r.  r  r   final_softcapr  r!   r  r"   rx     s   

	








zGemma4Loader.get_model_specc                    r  r  r  r  r   r!   r"   rz     s   zGemma4Loader.get_vocabularyc                 C   rJ  r   rK  r   r!   r!   r"   r{     rM  zGemma4Loader.set_vocabularyc                 C   r  r  r  r   r!   r!   r"   ry     s   

zGemma4Loader.set_configc                 C   r]  r   r^  r_  r!   r!   r"   r     r   zGemma4Loader.set_layer_normc                 C   sd  d|_ d|_| |j|j | |j|j t| dd}t	|j
|jD ]\}}| |j|j | |j|j | |j|j | |j|j | |jj|jj | |jj|jj t|jjj  |jj_|jjdk}|ot|}dd t dD }	| j!|	d |jj"|d	 | j!|	d
 |jj#|d	 |r| j!|	d |jj#|d	 n| j!|	d |jj$|d	 |t%j&j'krt()|jj*d |	 n|t%j&j+krd
nd}
t(,|jj*d |	|
 | j!|jj*d
 |jj-|d	 | j!|j.j/|j0j1|d	 | j!|j.j2|j0j3|d	 | j!|j.j4|j0j5|d	 t|dd }|d ur!t67d8| 9 |_:t;|d t;|d t<=  q#d S )NTFr  r  c                 S   r   r!   r   r   r!   r!   r"   r     r   z,Gemma4Loader.set_decoder.<locals>.<listcomp>r   r   r  r   r   layer_scalarr  r   rp  )>r  r  r   r  r  r   r   r  rJ   r   r   r   r  r  r  r  r  r  r  r  r   r  r   r  r>   	ones_liker   r  numpyr  r   r  r   r   r  r  r  r   r   r   r   r  r   r  r  r  r   r   rp  r  rO  rz  r   r{  r  r:   r   r   r  r  r  r  )r4   r^   r   r   r  r   r   is_full_attn
use_k_as_vr  r  lsr!   r!   r"   r     s   




zGemma4Loader.set_decoderr  r!   r!   r   r"   r    s    
 
r  MistralConfigc                       r  )MistralLoaderc                 C   r   )NMistralForCausalLMr!   rs   r!   r!   r"   rK   	  rt   zMistralLoader.architecture_namec                 C   s  |j j}|j j}t|j d|}||krd }t|j dd}| |j d\}}}t|j dd }	|	rS|	jdkr:t|	j}
|
d u rLt	d|	jd
t f |	j}|	j}ntjj}
d }d }tjj||ftjjd	d	d	dd
||||||
|||j jd}| j|j|j|
d | |jj|j |S )Nr  r  r   r  r  r  r  r6   TF)r   r   r?  rA  r~  r  r   r   r  r  r  r   r  r  r  r  )r8   rC  rD  rJ   r   r  r  rE   r  rv   rG   rI   r  r  r   r   r   r	   rB  r   ri  r  r  r   r   r[   r   r   r   )r4   r[   r  r  r  r  r   r   r   r  r   r  r  r^   r!   r!   r"   rx   	  sb   

zMistralLoader.get_model_specc                    r]  r^  r`  ra  r   r!   r"   rz   R	  rc  zMistralLoader.get_vocabularyc                 C   rJ  r   rK  r   r!   r!   r"   r{   [	  rM  zMistralLoader.set_vocabularyc                 C   r  r   r  r   r!   r!   r"   ry   ^	  r  zMistralLoader.set_configc                 C   r]  r   r^  r_  r!   r!   r"   r   d	  r   zMistralLoader.set_layer_normc                 C   r  )NFc                 S   r   r!   r   r   r!   r!   r"   r   s	  r   z-MistralLoader.set_decoder.<locals>.<listcomp>r   r   r  r   r   r   rp  r  r  r!   r!   r"   r   g	  sX   





zMistralLoader.set_decoderr  r!   r!   r   r"   r  	  s    
:	r  Qwen2Configc                       r  )Qwen2Loaderc                 C   r   )NQwen2ForCausalLMr!   rs   r!   r!   r"   rK   	  rt   zQwen2Loader.architecture_namec                 C   s  |j j}|j j}t|j d|}||krd }| |j d\}}}t|j dd }|rNd }	|jdkr5t|j}	|	d u rGt	d|jd
t f |j}
|j}ntjj}	d }
d }tjj||tjjddddd	|||||	|
|d
}| |j|j|	 | |jj|j |S )Nr  r  r  r  r  r6   Tr   Fr  )r8   rC  rD  rJ   r   r  r  rE   r  rv   rG   rI   r  r  r   r   r   r	   rB  r   ri  r  r   r   r[   r   r   r   )r4   r[   r  r  r  r   r   r   r  r   r  r  r^   r!   r!   r"   rx   	  s\   

zQwen2Loader.get_model_specc                    r]  r^  r`  ra  r   r!   r"   rz   	  s
   zQwen2Loader.get_vocabularyc                 C   rJ  r   rK  r   r!   r!   r"   r{   	  rM  zQwen2Loader.set_vocabularyc                 C   D   |j d ur|j n|j|_ |j|_|jd ur|jnd|_|jj|_d S r  r   r=  r   r   r8   r  r  r   r!   r!   r"   ry   	     
zQwen2Loader.set_configc                 C   r]  r   r^  r_  r!   r!   r"   r   	  r   zQwen2Loader.set_layer_normc                 C   r  )NFc                 S   r   r!   r   r   r!   r!   r"   r   	  r   z+Qwen2Loader.set_decoder.<locals>.<listcomp>r   r   r  r   r   r   rp  r  r  r!   r!   r"   r   	  sX   





zQwen2Loader.set_decoderr  r!   r!   r   r"   r  	  s    
8r  Qwen3Configc                       r  )Qwen3Loaderc                 C   r   )NQwen3ForCausalLMr!   rs   r!   r!   r"   rK   )
  rt   zQwen3Loader.architecture_namec                 C   s$  |j j}|j j}t|j d|}t|j d|j j| }||kr d }| |j d\}}}t|j dd }	|	rYd }
|	jdkr@t|	j	}
|
d u rRt
d|	jdt f |	j}|	j}ntjj}
d }d }tjj||ftjjddd|j jd	|||||d|
||d
}| |j|j|
 | |jj|j |S )Nr  r  r  r  r  r  r6   TF)r   r   r?  rA  r~  r  r   r   r  r  r  r  r   r  r  )r8   rC  rD  rJ   rG  r   r  r  rE   r  rv   rG   rI   r  r  r   r   r   r	   rB  r   ri  r  r  r   r   r[   r   r   r   )r4   r[   r  r  r  r  r   r   r   r  r   r  r  r^   r!   r!   r"   rx   -
  sh   

zQwen3Loader.get_model_specc                    r]  r^  r`  ra  r   r!   r"   rz   i
  
   zQwen3Loader.get_vocabularyc                 C   rJ  r   rK  r   r!   r!   r"   r{   p
  rM  zQwen3Loader.set_vocabularyc                 C   r  r  r  r   r!   r!   r"   ry   s
  r  zQwen3Loader.set_configc                 C   r]  r   r^  r_  r!   r!   r"   r   
  r   zQwen3Loader.set_layer_normc           	      C   s  d|_ | |j|j | |j|j tt|j	|j
D ]\}\}}| |jj|j | |jj|j | |jj|jj | |jj|jj dd tdD }| j|d |jj|d | j|d |jj|d | j|d |jj|d |tjjkrt|jjd | n|tjjkrdnd}t|jjd || | j|jjd |jj|d | j|jj |j!j"|d | j|jj#|j!j$|d | j|jj%|j!j&|d t'|d	 t'|d
 t()  qd S )NFc                 S   r   r!   r   r   r!   r!   r"   r   
  r   z+Qwen3Loader.set_decoder.<locals>.<listcomp>r   r   r  r   r   r   rp  )*r  r   r  r  r   r   r  rH  r   r   r   r   r  r   r  r  r   r  r   r   r  r  r  r   r   r   r   r  r   r  r  r  r   rp  r  rO  rz  r   r{  r  r  r  )	r4   r^   r   r   	layer_idxr   r   r  r  r!   r!   r"   r   
  sd   





zQwen3Loader.set_decoderr  r!   r!   r   r"   r  '
  s    
<r  MixFormerSequentialConfigc                       r  )MixFormerSequentialLoaderc                 C   r   rp  r!   rs   r!   r!   r"   rK   
  rt   z+MixFormerSequentialLoader.architecture_namec              
   C   s\   t jj|jj|jjdt|jj |jjdddd}| 	|j
|j | |j
j|jd j |S )NTFr  r  r   r   r~  r  r  r  r   )r	   rB  r   r8   rZ  r[  r   r   r~  r   r   r   r   r   r   r  r!   r!   r"   rx   
  s   
z(MixFormerSequentialLoader.get_model_specc                    r]  r^  r`  ra  r   r!   r"   rz   
  rc  z(MixFormerSequentialLoader.get_vocabularyc                 C   rJ  r   rK  r   r!   r!   r"   r{   
  rM  z(MixFormerSequentialLoader.set_vocabularyc                 C   rN  r   rO  r   r!   r!   r"   ry   
  r(  z$MixFormerSequentialLoader.set_configc                 C   s   d|_ | |j|d j | |j|d j t|j|dd D ]8\}}| |j	|j | 
|jjd |jj | 
|jjd |jj | 
|jj|jj | 
|jj|jj q!d S )NFr   r   r   )r  r   r  rg  r   r   lnr   r   r  r   r   r   mixerrx  r  r   r   rp  r   r   r   rr  r!   r!   r"   r   
  s   z%MixFormerSequentialLoader.set_decoderr  r!   r!   r   r"   r  
  s    
	r  	PhiConfigc                       r  )	PhiLoaderc                 C   r   rp  r!   rs   r!   r!   r"   rK   
  rt   zPhiLoader.architecture_namec              
   C   sl   t jj|jj|jjdt|jj |jjdddd}| 	|j
|j | |j
j|jj | |j
j|jj |S )NTFr  )r	   rB  r   r8   rZ  r[  r   r   r~  r   r   r\  r   r   r   r   r   r   r  r  r!   r!   r"   rx   
  s   
zPhiLoader.get_model_specc                    r]  r^  r`  ra  r   r!   r"   rz   	  rc  zPhiLoader.get_vocabularyc                 C   rJ  r   rK  r   r!   r!   r"   r{     rM  zPhiLoader.set_vocabularyc                 C   rN  r   rO  r   r!   r!   r"   ry     r(  zPhiLoader.set_configc                 C   s   d|_ | |j|jj t|j|jD ]8\}}| |j	|j
 | |jjd |jj | |jjd |jj | |jj|jj | |jj|jj qd S re  )r  r   r  embdrg  r   r   rj  r   r  r  r   r   r   r  rx  r  r   r   rp  r   r   r   rr  r!   r!   r"   r     s   zPhiLoader.set_decoderr  r!   r!   r   r"   r  
  s    
	r  
Phi3Configc                       s`   e Zd Zedd Zdd Z fddZdd Zd	d
 Zdd Z	dd Z
ejjfddZ  ZS )
Phi3Loaderc                 C   r   rp  r!   rs   r!   r!   r"   rK   (  rt   zPhi3Loader.architecture_namec                 C   sx  |j j}|j j}t|j d|}||krd }t|j dd}t|j dd}t|j dd }|rMt|d }|dd}	|d u rLtd	|d d
t f nd }d}	t|j dd }
|
rd }|
j	dkrgt
|
j}|d u rytd|
j	d
t
 f |
j}|
j}ntjj}d }d }tjj||ftjjddddd||	t|j dd||||||d}| |j|j| | |jj|j |S )Nr   original_max_position_embeddingsr   max_position_embeddingsr   r   r   r   r   r6   r  r  r  TFr   r  )r   r   r?  rA  r~  r  r   r   r  r#  r$  r  r   r  r  )r8   rC  rD  rJ   r   rE   rv   rG   rI   r  r  r  r  r  r   r   r   r	   rB  r   ri  r  r   r   r[   r   r   r   )r4   r[   r  r  r  r#  r$  r   r   r   r  r   r  r  r^   r!   r!   r"   rx   ,  s~   
zPhi3Loader.get_model_specc                    r]  r^  r`  ra  r   r!   r"   rz   u  rc  zPhi3Loader.get_vocabularyc                 C   rJ  r   rK  r   r!   r!   r"   r{   ~  rM  zPhi3Loader.set_vocabularyc                 C   rN  r   rO  r   r!   r!   r"   ry     r(  zPhi3Loader.set_configc                 C   r]  r   r^  r_  r!   r!   r"   r     r   zPhi3Loader.set_layer_normc                 C   s(   t j|t jd|_t j|t jd|_d S )N)r:   )r>   tensorr  rotary_scaling_long_factorrotary_scaling_short_factor)r4   r^   r&  r'  r!   r!   r"   set_rotary_embeddings  s   z Phi3Loader.set_rotary_embeddingsc                 C   s  d|_ | |j|j | |j|j t|j|j	D ]\}}| |j
j|j | |jj|j | j|j
jd |jj|d | j|j
jd |jj|d |jjjd urh|jjjd urh| |j
|jjj|jjj |tjjkr|jjjjddd\}}||jj_||jj_nB|jjj jddd\}}	|jjj!jddd\}
}|jjj"jddd\}}||jj_|
|jj_#||jj_$|	|jj_||jj_#||jj_$| j|jj%|jj&|d t'|d t'|d t()  qd S )	NFr   r  r   r   r  r   rp  )*r  r   r  r  r   r   r  r   r   r   r   r  r   r  r   r   r   r  r  
rotary_emblong_factorshort_factorr(  r   r   r   rp  gate_up_projr   r  r   rO  r   r   r   r   r   r   r{  r  r  r  )r4   r^   r   r   r   r   r  rz  gate_qweight
up_qweightgate_scales	up_scalesgate_qzeros	up_qzerosr!   r!   r"   r     sb   















zPhi3Loader.set_decoder)rD   rl   rm   r   rK   rx   rz   r{   ry   r   r(  r   r   r   r   r  r!   r!   r   r"   r"  &  s    
I	
r"  RWConfigc                       sZ   e Zd Zedd Zdd Zdd Z fddZd	d
 Zdd Z	dd Z
dddZ  ZS )RWLoaderc                 C   r   rp  r!   rs   r!   r!   r"   rK     rt   zRWLoader.architecture_namec                 C   s.   |j j| _|j j| _t|j dd | _d| _d S )N	n_head_kvnum_kv)r8   rZ  _num_layersr[  
_num_headsrJ   _num_heads_kv_num_kv_attrrw   r!   r!   r"   get_falcon_spec     


zRWLoader.get_falcon_specc                 C   s   |  | t|jddrd}n| j}tjj| j| jdt	j
j|jjdd|jjr(dnd d|jj|dk|d}| |j|j | |jj|j |S )Nmulti_queryFr   Tr   )
r   r   rg  rh  scale_alibir~  r  r  r  r  )r;  rJ   r8   r9  r	   rB  r   r7  r8  r   ri  rt  rg  rotaryparallel_attnr   r   r\  r   r   r   )r4   r[   r  r^   r!   r!   r"   rx     s*   
zRWLoader.get_model_specc                    r]  r^  r`  ra  r   r!   r"   rz     rc  zRWLoader.get_vocabularyc                 C   rJ  r   rK  r   r!   r!   r"   r{     rM  zRWLoader.set_vocabularyc                 C   s   |j |_|j |_ |j |_d S r   )r   r   r   r   r!   r!   r"   ry     r(  zRWLoader.set_configc                 C   sN  d|_ | |j|j | |j|j t|j|j	D ]\}}t
|dr4| |j|j | |j|j n t
|drB| |j|j n| |jj|j | |jj|j t|j| j}|dkrl| |jjd |jj n| |jjd |jj|jj||jjk r|nd  | |jjd |jj | |jj|jj | |jj|jj  qd S )NFln_attnr  r   r   )!r  r   r  rk  r   r   ri  r   r   rj  rL   r  rA  r  ln_mlpr  r  r   r   r  rJ   r:  r   r   r  rm  r  r  r   rp  r  r   r  )r4   r^   r   r   r   r6  r!   r!   r"   r   
  sB   





zRWLoader.set_decoderNc                 C   s~  |j }|d u r"||dd|jd }|dd}|d|jd }nC|jd ||d   }|d|| d ||jd }|j|| ddgdd\}}}	t||| d||| d|	|| dg}||_ |jd ur|j}
|d u r|
|dd}
|
dd}
|
d}
n1|
d|| d |}
|
j|| ddgdd\}}}	t||| ||| |	|| g}
|
|_d S d S )Nr   r   r   r   r   r  )r   r  r   r   splitr>   r  r   )r4   r^   r   r  r6  r   r  rX  rY  rZ  r   r!   r!   r"   rm  2  sB   

zRWLoader.set_qkv_linearr   )rD   rl   rm   r   rK   r;  rx   rz   r{   ry   r   rm  r  r!   r!   r   r"   r4    s    
	(r4  FalconConfigc                   @   s   e Zd Zdd ZdS )FalconLoaderc                 C   s.   |j j| _|j j| _t|j dd | _d| _d S )Nnum_kv_heads)r8   rC  r7  rD  r8  rJ   r9  r:  rw   r!   r!   r"   r;  `  r<  zFalconLoader.get_falcon_specN)rD   rl   rm   r;  r!   r!   r!   r"   rE  ^  s    rE  DistilBertConfigc                   @   s0   e Zd Zedd Zdd Zdd Zdd Zd	S )
DistilBertLoaderc                 C   r   )NDistilBertModelr!   rs   r!   r!   r"   rK   i  rt   z"DistilBertLoader.architecture_namec                 C   s\  t j|jj|jjdt|jj dd}t |}d|j_	| 
|jjd |jj | |jj|jj | |jj|jj t|jj|jjD ]g\}}dd tdD }| |d |jj | |d |jj | |d	 |jj t|jjd | | |jjd |jj  | |jj!|j" | |j#j$|j#j% | |j#j&|j#j' | |j#j!|j( qD|S )
NFTr   r   c                 S   r   r!   r   r   r!   r!   r"   r     r   z3DistilBertLoader.get_model_spec.<locals>.<listcomp>r   r   r   ))r	   TransformerEncoderSpecr8   rr  rs  r   r   TransformerEncoderModelSpecr   r  r   r  rk  r   r  position_embeddingsr   r   	LayerNormr   r   r\  r   r   r   q_link_linv_linr   r  r   r   out_linr   sa_layer_normr   r   lin1r   lin2output_layer_normr4   r[   encoder_specr^   r   r   r  r!   r!   r"   rx   m  sH   

zDistilBertLoader.get_model_specc                 C   rJ  r   rK  r   r!   r!   r"   r{     rM  zDistilBertLoader.set_vocabularyc                 C   s   |j |_ d|_d S )Ng-q=)r   r  r   r!   r!   r"   ry     r  zDistilBertLoader.set_configN)rD   rl   rm   r   rK   rx   r{   ry   r!   r!   r!   r"   rH  g  s    
,rH  
BertConfigc                       s@   e Zd Zedd Zdd Z fddZdd Zd	d
 Z  Z	S )
BertLoaderc                 C   r   )N	BertModelr!   rs   r!   r!   r"   rK     rt   zBertLoader.architecture_namec              	   C   s  |j jdksJ tj|j j|j jdt|j j ddtj	j
d}tj|dtjjd}d|j_| |jjd |jj | |jjd |jj | |jj|jj | |jj|jj | |j|jj t|jj|jjD ]n\}}d	d
 t dD }| |d |j!j"j# | |d |j!j"j$ | |d |j!j"j% t&'|j(j)d | | |j(j)d |j!j*j | |j(j+|j!j*j | |j,j-|j.j | |j,j/|j*j | |j,j+|j*j qj|S )NabsoluteFTr   r   r   r   num_source_embeddingsembeddings_mergepooling_layerpooling_activationr   r   c                 S   r   r!   r   r   r!   r!   r"   r     r   z-BertLoader.get_model_spec.<locals>.<listcomp>r   )0r8   position_embedding_typer	   rJ  rC  rD  r   r  r   EmbeddingsMergeADDrK  ri  Tanhr   r  r   r  rk  token_type_embeddingsr   r  rL  r   r   rM  r   pooler_densepoolerr  r   r   r   r   r4   queryr   valuer   r  r   r   outputr   r   r   intermediater   rV  r!   r!   r"   rx     sZ   

zBertLoader.get_model_specc                    r]  r^  r`  ra  r   r!   r"   rz     rc  zBertLoader.get_vocabularyc                 C   rJ  r   rK  r   r!   r!   r"   r{     rM  zBertLoader.set_vocabularyc                 C      |j |_ |jj|_d S r   r   r8   layer_norm_epsr  r   r!   r!   r"   ry        zBertLoader.set_config)
rD   rl   rm   r   rK   rx   rz   r{   ry   r  r!   r!   r   r"   rY    s    
8	rY  XLMRobertaConfigc                   @   ru  )XLMRobertaLoaderc                 C   r   )N#XLMRobertaForSequenceClassificationr!   rs   r!   r!   r"   rK     rt   z"XLMRobertaLoader.architecture_namec              	   C   s  |j jdksJ tj|j j|j jdt|j j ddtj	j
d}|jjd u r'd}nd}tj||tjjd}d|j_| |jjd |jjj | |jjd |jjj | |jj|jjj | |jj|jjj |rs| |j|jjj t|jj |jjj D ]n\}}d	d
 t!dD }| |d |j"j#j$ | |d |j"j#j% | |d |j"j#j& t'(|j)j*d | | |j)j*d |j"j+j | |j)j,|j"j+j | |j-j.|j/j | |j-j0|j+j | |j-j,|j+j q}|S )Nr[  FTr   r\  r_  r   r   c                 S   r   r!   r   r   r!   r!   r"   r   !  r   z3XLMRobertaLoader.get_model_spec.<locals>.<listcomp>r   )1r8   rb  r	   rJ  rC  rD  r   r  r   rc  rd  robertarh  rK  ri  re  r   r  r   r  rk  rf  r   r  rL  r   r   rM  r   rg  r  r   r   r   r   r4   ri  r   rj  r   r  r   r   rk  r   r   r   rl  r   r4   r[   rW  r`  r^   r   r   r  r!   r!   r"   rx     sd   

zXLMRobertaLoader.get_model_specc                 C   rJ  r   rK  r   r!   r!   r"   r{   4  rM  zXLMRobertaLoader.set_vocabularyc                 C   rm  r   rn  r   r!   r!   r"   ry   7  rp  zXLMRobertaLoader.set_configc                 C   8   |j |_t|dd}|dkr|j|d d  |_d S d S Npadding_idxr   r   r   r   r!   r!   r"   r   ;  
   z'XLMRobertaLoader.set_position_encodingsN	rD   rl   rm   r   rK   rx   r{   ry   r   r!   r!   r!   r"   rr        
>rr  RobertaConfigc                   @   ru  )RobertaLoaderc                 C   r   )NRobertaModelr!   rs   r!   r!   r"   rK   D  rt   zRobertaLoader.architecture_namec              	   C     |j jdksJ tj|j j|j jdt|j j ddtj	j
d}|jd u r&d}nd}tj||tjjd}d|j_| |jjd |jj | |jjd |jj | |jj|jj | |jj|jj |rm| |j|jj t|jj|jjD ]n\}}d	d
 t dD }| |d |j!j"j# | |d |j!j"j$ | |d |j!j"j% t&'|j(j)d | | |j(j)d |j!j*j | |j(j+|j!j*j | |j,j-|j.j | |j,j/|j*j | |j,j+|j*j qv|S )Nr[  FTr   r\  r_  r   r   c                 S   r   r!   r   r   r!   r!   r"   r   s  r   z0RobertaLoader.get_model_spec.<locals>.<listcomp>r   0r8   rb  r	   rJ  rC  rD  r   r  r   rc  rd  rh  rK  ri  re  r   r  r   r  rk  rf  r   r  rL  r   r   rM  r   rg  r  r   r   r   r   r4   ri  r   rj  r   r  r   r   rk  r   r   r   rl  r   ru  r!   r!   r"   rx   H  d   


zRobertaLoader.get_model_specc                 C   rJ  r   rK  r   r!   r!   r"   r{     rM  zRobertaLoader.set_vocabularyc                 C   rm  r   rn  r   r!   r!   r"   ry     rp  zRobertaLoader.set_configc                 C   rv  rw  r   r   r!   r!   r"   r     ry  z$RobertaLoader.set_position_encodingsNrz  r!   r!   r!   r"   r}  B  r{  r}  CamembertConfigc                   @   ru  )CamembertLoaderc                 C   r   )NCamembertModelr!   rs   r!   r!   r"   rK     rt   z!CamembertLoader.architecture_namec              	   C   r  )Nr[  FTr   r\  r_  r   r   c                 S   r   r!   r   r   r!   r!   r"   r     r   z2CamembertLoader.get_model_spec.<locals>.<listcomp>r   r  ru  r!   r!   r"   rx     r  zCamembertLoader.get_model_specc                 C   rJ  r   rK  r   r!   r!   r"   r{     rM  zCamembertLoader.set_vocabularyc                 C   rm  r   rn  r   r!   r!   r"   ry     rp  zCamembertLoader.set_configc                 C   rv  rw  r   r   r!   r!   r"   r     ry  z&CamembertLoader.set_position_encodingsNrz  r!   r!   r!   r"   r    r{  r  c               	   C   s   t jt jd} | jdddd | jddd | jd	d
dd | jddd | jdddd | jdddd t|  |  }t|j|j	|j
|jdv |j|j|jd}|| d S )N)formatter_classz--modelTzaName of the pretrained model to download, or path to a directory containing the pretrained model.)requiredhelpz--activation_scaleszPath to the pre-computed activation scales. Models may use them to rescale some weights to smooth the intermediate activations and improve the quantization accuracy. See https://github.com/mit-han-lab/smoothquant.)r  z--copy_files+zWList of filenames to copy from the Hugging Face model to the converted model directory.)nargsr  z
--revisionz<Revision of the model to download from the Hugging Face Hub.z--low_cpu_mem_usage
store_truezNEnable the flag low_cpu_mem_usage when loading the model with from_pretrained.)actionr  z--trust_remote_codez*Allow converting models using custom code.)rO   int8_float16)r'   r(   r)   r*   r+   r,   )argparseArgumentParserArgumentDefaultsHelpFormatteradd_argumentr   declare_arguments
parse_argsr%   r[   r'   r(   quantizationr*   r+   r,   convert_from_args)parserargs	converterr!   r!   r"   main  sV   	
	r  __main__))r   r   )r   r   )r      r   r   r   r   r   r   r   r   r   r  ))r   r   r  r  r  r  )r   r  )r  r     r  r   )r  r  )r  r  )r  )r  r   )r  r   r  r  )r  r   )r  r  )r  r  ))r  r  r  r   )r  r   )r  rR  )rR  r   )rR  r  rR  r  	   r   )r  r  )r  rR  )r  
   )r  r   )r  r   )r  r   )r  r   )r  r  )r     r  r   r  r  )
)r  r   )r  r  )rR  r   )rR  r  r  )rR  rR  r  )r  r  )r  r  )r  r  )r  )   r   )r     )r  r     r  )   r   )r  r  )r  r  )   r  r  r  )   r  )r  r  )r  r     r   )r  r   )r  r  )r  r  )   r  ))   r  r  )r  r  r  r   r  )   r  )	)r     r  r  )r  r  )   r  )r  r  )r  r  )r  r   )r  r  ))r  r  )r  r  )r  r  )r  r  )r  r  )r  r  )r  r  )r  r  )r  r  )r  r  r  r  )r  r   )r  r   )r  r   )r  r  )r  r  )r  r  )r  r  )r  r  )   r  )   r   )r  r  )   r  )
r  )r  r  )r  r  )r  r  r  r  r  )r  r  )   r   )r  r  )zopenai/whisper-tiny.enzopenai/whisper-tinyzopenai/whisper-base.enzopenai/whisper-basezopenai/whisper-small.enzopenai/whisper-smallzopenai/whisper-medium.enzopenai/whisper-mediumzopenai/whisper-largezopenai/whisper-large-v2zopenai/whisper-large-v3T5GemmaConfigc                   @   sX   e Zd Zedd Zdd Zdd Zdd Zd	d
 Ze	j
jfddZe	j
jfddZdS )T5GemmaLoaderc                 C   r   )NT5GemmaForConditionalGenerationr!   rs   r!   r!   r"   rK     rt   zT5GemmaLoader.architecture_namec                 C      |j jd  |_d S Nr	  r   r  r  r   r_  r!   r!   r"   r     r*  zT5GemmaLoader.set_layer_normc                 C   s&  |j j}|j j}t|j dd}|j}t|d|}||krd }tj|j|jdt|j	 dd|j
dt|dd|d||j
d}|j}t|d|}	|	|krJd }	tj|j|jdt|j	 ddd|j
dt|dd|dd|	|j
d	}
t||
}| |j|jj| | |j|jj|tjj | |jj|jjj |S )
Nr  i   r  TFr   r  )r   r   r?  rA  r~  r  r  r  r  r  r  )r   r   r?  rA  with_encoder_attentionr~  r  r  r  r   external_pre_post_encoder_layersr  r  )r8   r   r   rJ   rD  r	   rJ  rC  r   r  r  TransformerDecoderSpecr   r   r[   r   r   r   r   r   r   r  )r4   r[   encoder_configdecoder_configr  encoder_num_headsencoder_num_heads_kvr   decoder_num_headsdecoder_num_heads_kvr   r^   r!   r!   r"   rx     sr   

zT5GemmaLoader.get_model_specc                 C   r   r   r   r   r!   r!   r"   r{     r   zT5GemmaLoader.set_vocabularyc                 C   s\   |j |_ |j|_|j|_t|jdr|jjj|_nt|jdr%|jj|_nd|_|j |_d S )Nr   r  gư>)	r   r   r   rL   r8   r   r  r  r   r   r!   r!   r"   ry     s   zT5GemmaLoader.set_configc                 C   s  d|_ t|jtr|jd n|j}| ||j |jd |_| |j	|j
 |}tt|j|jD ]\}\}}	| |j|	j | |j|	j dd tdD }
| j|
d |	jj|d | j|
d |	jj|d | j|
d	 |	jj|d t|jjd |
 | j|jjd |	jj|d | |j|	j | |j |	j! | j|j"j#|	j$j%|d | j|j"j&|	j$j'|d | j|j"j(|	j$j)|d t*|	d
 t*|	d t+,  q1d S )NTr   r  c                 S   r   r!   r   r   r!   r!   r"   r     r   z-T5GemmaLoader.set_encoder.<locals>.<listcomp>r   r  r   r   r   rp  )-r  r   r  r  r   r  rG  r  r   r   r  rH  r   r   r   r  pre_self_attn_layernormr  post_self_attn_layernormr   r   r   r  r  r  r   r  r   r   r  r  r  r  r  r   r   rp  r  rO  rz  r   r{  r  r  r  )r4   r^   r   r  r   encoder_emb_specr   r.  r   r   qkv_split_layersr!   r!   r"   r     sb   



zT5GemmaLoader.set_encoderc           
      C   sD  d|_ d|_| |j|j |jd |j_| |j|j	 t
t|j|jD ]\}\}}| |j|j | |j|j dd tdD }| j|d |jj|d | j|d	 |jj|d | j|d
 |jj|d t|jjd | | j|jjd	 |jj|d | |j|j | |j|j  | j|j!jd |j"j|d dd td
D }	| j|	d |j"j|d | j|	d	 |j"j|d t|j!jd	 |	 | j|j!jd
 |j"j|d | |j#|j$ | |j%|j& | j|j'j(|j)j*|d | j|j'j+|j)j,|d | j|j'j-|j)j.|d t/|d t/|d t/|d t01  q&d S )NTFr  c                 S   r   r!   r   r   r!   r!   r"   r   \  r   z-T5GemmaLoader.set_decoder.<locals>.<listcomp>r   r   r  r   r   c                 S   r   r!   r   r   r!   r!   r"   r     r   r   
cross_attnrp  )2r  r  r   r  r  rG  r  r   r   r  rH  r   r   r   r  r  r  r  r   r   r   r  r  r  r   r  r   r   r  )external_pre_encoder_attention_layer_normpre_cross_attn_layernorm*external_post_encoder_attention_layer_normpost_cross_attn_layernormr   r  r  r  r  r  r   r   rp  r  rO  rz  r   r{  r  r  r  )
r4   r^   r   r  r   r.  r   r   r  kv_split_layersr!   r!   r"   r   H  s   






zT5GemmaLoader.set_decoderN)rD   rl   rm   r   rK   r   rx   r{   ry   r   r   r   r   r   r!   r!   r!   r"   r    s    
D
Br  T5Gemma2Configc                       s   e Zd Zedd Zdd Zdd Zdd Zd	d
 Z fddZ	dd Z
dd ZejjfddZejjfddZdd Z  ZS )T5Gemma2Loaderc                 C   r   )N T5Gemma2ForConditionalGenerationr!   rs   r!   r!   r"   rK     rt   z T5Gemma2Loader.architecture_namec                 C   s   |j }t|d|}||krd }|j}t|di pi }|di dt|dd}tdi d|jd|dd	d
t|j dd	dd	d|dddtj	j
ddd|dt|ddd|d|dd	dd	S )Nr  r  r  r   r  r  r  r   Tr   r?  rA  r~  r  Fr   r   r   r  r  r   r  r  r  r  r!   )rD  rJ   r  rE   dictrC  r   r  r   r  Linear)r4   side_configr  r  r  r  global_thetar!   r!   r"   _side_kwargs  sT   
	
zT5Gemma2Loader._side_kwargsc                 C   s  t |dd }|s
d S t |di pi }|di dd}|di dd}t |dd	}|di }|d
d}	|dd}
t||D ]=\}}|j}|dkrntd||_tdd	|_|	dkrmtd|
|_	qCtd||_td||_qCd S )Nr  r  r  r   r  r  r  r  r   r   r   r   r	  r  r  r   )
rJ   rE   r   r   r  r:   r   r  r  r   )r4   r  r(  r  r  r  local_thetar  full_attn_paramsfull_rope_typefull_rope_factorr  r   rl  r!   r!   r"   _apply_layer_types  s2   
z!T5Gemma2Loader._apply_layer_typesc                 C   s   |j jj}|j j}tjdi | |}tjdi | |ddd}t||}| 	|j|j
jj | ||jj | |j|j
j | ||jj t|jdr^| |jj|j |S | |jj|j
jj |S )NT)r  merged_encoder_attentionr   r!   )r8   r   r  r   r	   rJ  r  r  r   r   r[   r  r  r   r   rL   r   r   r   r  )r4   r[   r  r  r   r   r^   r!   r!   r"   rx     s*   

zT5Gemma2Loader.get_model_specc                 C   r   r   r   r   r!   r!   r"   r{     r   zT5Gemma2Loader.set_vocabularyc                    r]  r^  r`  ra  r   r!   r"   rz   	  r  zT5Gemma2Loader.get_vocabularyc                 C   sJ   t |dd |_t |dd |_t |dd |_t |dd |_|jjjj|_	d S )Nr   r   r   )
rJ   r   r   r   r   r8   r   r  r  r  r   r!   r!   r"   ry     s
   zT5Gemma2Loader.set_configc                 C   s   t |jtr|jd n|j}| ||j | |j|j t|jdd }|d ur,t	|nd|_
t|j|jD ]W\}}| |j|j | |j|j | |j| | |j|j | |j|j | |jj|jj | |jj|jj | |jj|jj  t!|d t!|d t"#  q6d S )Nr   r  Fr   rp  )$r   r  r  r   r  r   r   r  rJ   r  r  r   r   r   r  r  r  r  _set_self_attentionr   r  r  r  r  r   r   r   rp  r  rO  rz  r   r{  r  r  r  )r4   r^   r   r  r  r   r   r!   r!   r"   r     s6   


zT5Gemma2Loader.set_encoderc           	      C   sp  t |jdd }|d urt|nd|_d|_| |j|j | |j|j	 t
|j|jD ]\}}|j}| |j|j | |j|j dd tdD }| j|d |jj|d | j|d |jj|d t|j| | j|||d | |j|j | |j|j | j|jj |j!j"|d | j|jj#|j!j$|d | j|jj%|j!j&|d t'|d	 t'|d
 t()  q,d S )Nr  Fc                 S   r   r!   r   r   r!   r!   r"   r   G  r   z.T5Gemma2Loader.set_decoder.<locals>.<listcomp>r   r   r  r   r   rp  )*rJ   r  r  r  r  r   r  r   r   r  r   r   r   r   r  r  r  r  r   r   r   r  r  r   r  	memory_kvr  r  r  r  r  r   r   rp  r  rO  rz  r   r{  r  r  r  )	r4   r^   r   r   r  r   r   	attn_speckv_splitr!   r!   r"   r   5  sJ   


zT5Gemma2Loader.set_decoderc                 C   s   dd t dD }| j|d |jj|d | j|d |jj|d | j|d |jj|d t|jd | | j|jd |jj	|d | 
|j|jj | 
|j|jj d S )Nc                 S   r   r!   r   r   r!   r!   r"   r   f  r   z6T5Gemma2Loader._set_self_attention.<locals>.<listcomp>r   r   r  r   r   )r   r   r   r  r  r  r   r  r   r  r   r  r  )r4   r  r   r   	qkv_splitr!   r!   r"   r  a  s   z"T5Gemma2Loader._set_self_attentionc                 C   r  r  r  r_  r!   r!   r"   r   q  r*  zT5Gemma2Loader.set_layer_norm)rD   rl   rm   r   rK   r  r  rx   r{   rz   ry   r   r   r   r   r   r  r   r  r!   r!   r   r"   r    s    
-
r  )Xr   r  r  r  re   typingr   r   r  r  rh   r>   r@   ImportErrorctranslate2.convertersr    ctranslate2.converters.converterr   ctranslate2.specsr   r   r   r	   r
   r   r   ri  rt  rj  GELUSigmoidRELUr  r   r  r  Sur  r   r   r  AWQ_GEMVr  r   r$   r%   ABCrq   r   r  r&  r5  r;  r?  rV  rv  rz  r  r  r  r  r  r=  ra  re  ro  r~  r  r  r  r  r  r  r  r  r  r"  r4  rE  rH  rY  rr  r}  r  r  rD   r  r  r  r!   r!   r!   r"   <module>   s$   $
 Z 7!F2(8YOfNt|M7_g  V     33 , 9NQQQ
:

   