
    si*                     p    d dl mZmZmZ d dlZd dlmZmZ d dlmZ d dl	m
Z
  G d dej                  ZdS )    )AnyDictIterableN)Tensornn)Transformer)SentenceTransformerc            	            e Zd Z	 ddedededdf fdZded	edefd
Zdee	e
ef                  dedefdZde	e
ef         fdZede
fd            Z xZS )GISTEmbedLoss{Gz?modelguidetemperaturereturnNc                    t          t          |                                            || _        || _        || _        t          j        d          | _        t          |d         t                    rt          |d         t                    st          d          |j        j        |j        j        k    p|j        |j        k     | _        dS )a	  
        This loss is used to train a SentenceTransformer model using the GISTEmbed algorithm.
        It takes a model and a guide model as input, and uses the guide model to guide the
        in-batch negative sample selection. The cosine similarity is used to compute the loss
        and the temperature parameter is used to scale the cosine similarities.

        Args:
            model: SentenceTransformer model based on a `transformers`
                model.
            guide: SentenceTransformer model to guide the in-batch
                negative sample selection.
            temperature: Temperature parameter to scale the cosine
                similarities.

        References:
            - For further details, see: https://arxiv.org/abs/2402.16829

        Requirements:
            1. (anchor, positive, negative) triplets
            2. (anchor, positive) pairs

        Relations:
            - :class:`MultipleNegativesRankingLoss` is similar to this loss, but it does not use
              a guide model to guide the in-batch negative sample selection. `GISTEmbedLoss` yields
              a stronger training signal at the cost of some training overhead.

        Inputs:
            +---------------------------------------+--------+
            | Texts                                 | Labels |
            +=======================================+========+
            | (anchor, positive, negative) triplets | none   |
            +---------------------------------------+--------+
            | (anchor, positive) pairs              | none   |
            +---------------------------------------+--------+

        Example:
            ::

                from sentence_transformers import SentenceTransformer, SentenceTransformerTrainer, losses
                from datasets import Dataset

                model = SentenceTransformer("microsoft/mpnet-base")
                guide = SentenceTransformer("all-MiniLM-L6-v2")
                train_dataset = Dataset.from_dict({
                    "anchor": ["It's nice weather outside today.", "He drove to work."],
                    "positive": ["It's so sunny.", "He took the car to the office."],
                })
                loss = losses.GISTEmbedLoss(model, guide)

                trainer = SentenceTransformerTrainer(
                    model=model,
                    train_dataset=train_dataset,
                    loss=loss,
                )
                trainer.train()
        dimr   z_Both the training model and the guiding model must be based on the `transformers` architecture.N)superr   __init__r   r   r   r   CosineSimilaritysimilarity_fct
isinstancer   
ValueError	tokenizervocabmax_seq_lengthmust_retokenize)selfr   r   r   	__class__s       ]/var/www/icac/venv/lib/python3.11/site-packages/sentence_transformers/losses/GISTEmbedLoss.pyr   zGISTEmbedLoss.__init__   s    | 	mT""++---

& 1b999%(K00 	
58[8Y8Y 	q   O!U_%::ie>RUZUi>i 	    embed1embed2c                 z    |                      |                    d          |                    d                    S )N   r   )r   	unsqueeze)r   r#   r$   s      r!   
sim_matrixzGISTEmbedLoss.sim_matrixV   s4    ""6#3#3A#6#68H8H8K8KLLLr"   sentence_featureslabelsc                 v     fd|D             }t          j                    5   j        r* fd|D             } fd|D             } fd|D             } fd|D             }d d d            n# 1 swxY w Y   d }d }t          |          dk    r|\  }}	|\  }
}nOt          |          dk    r|\  }}	}|\  }
}}n/t	          d                    t          |                                                   ||	          }                     ||          }                     |	|	          }                     |
|          }                     |
|
          }                     ||          }|                                                    d	d
          }t           j	         |||k    <   t           j	         |||k    <   t           j	         |||k    <   |||g}|U                     ||          }                     |
|          }t           j	         |||k    <   |
                    |           t          j        |d
           j        z  }t          j        |                    d                                                                        |j                  } t%          j                    ||          S )Nc                 F    g | ]}                     |          d          S sentence_embedding)r   .0sentence_featurer   s     r!   
<listcomp>z)GISTEmbedLoss.forward.<locals>.<listcomp>Z   s-    sssM]djj!1223GHsssr"   c                 ^    g | ])}j         j                            |d          d          *S )	input_idsT)skip_special_tokens)r   r   batch_decoder/   s     r!   r2   z)GISTEmbedLoss.forward.<locals>.<listcomp>]   sH       ( J(556F{6Sim5nn  r"   c                 D    g | ]}j                             |          S  )r   tokenize)r0   	sentencesr   s     r!   r2   z)GISTEmbedLoss.forward.<locals>.<listcomp>a   s)    $]$]$]	TZ%8%8%C%C$]$]$]r"   c                 P    g | ]"}fd |                                 D             #S )c                 V    i | ]%\  }}||                     j        j                  &S r8   )tor   device)r0   keyvaluer   s      r!   
<dictcomp>z4GISTEmbedLoss.forward.<locals>.<listcomp>.<dictcomp>c   s0    aaa*#uS%((4:#455aaar"   )itemsr/   s     r!   r2   z)GISTEmbedLoss.forward.<locals>.<listcomp>b   sK     % % %( baaaHXH^H^H`H`aaa% % %r"   c                 F    g | ]}                     |          d          S r-   )r   r/   s     r!   r2   z)GISTEmbedLoss.forward.<locals>.<listcomp>g   s9          GW

+,,-AB     r"         z"Expected 2 or 3 embeddings, got {}r   r&   r   r   )torchno_gradr   lenr   formatr(   diagonalviewinfappendcatr   arangesizelongr=   r>   r   CrossEntropyLoss)r   r)   r*   
embeddingsdecodedguide_embeddingsnegativenegative_guideanchorpositiveanchor_guidepositive_guideap_simaa_simpp_simguided_ap_simguided_aa_simguided_pp_sim
guided_simscoresan_simguided_an_sims   `                     r!   forwardzGISTEmbedLoss.forwardY   sC   ssssarsss
]__ 	 	# 	   ,=   %^$]$]$]U\$]$]$]!% % % %,=% % %!
       [l     	 	 	 	 	 	 	 	 	 	 	 	 	 	 	  z??a)FH+;(L..__!!)3&FHh;K8L...AHHZYYZZZ 2200844 nEElCCGG #++--222q99
 /4iZ}z)*.3iZ}z)*.3iZ}z)*&&) __VX66F OOL.IIM27)F=:-.MM&!!!6q)))D,<< fkk!nn--224477FF$r"$$VV444s   A A//A36A3c                      | j         | j        dS )Nr   r   rh   r   s    r!   get_config_dictzGISTEmbedLoss.get_config_dict   s    Z+
 
 	
r"   c                     dS )Na	  
@misc{solatorio2024gistembed,
    title={GISTEmbed: Guided In-sample Selection of Training Negatives for Text Embedding Fine-tuning}, 
    author={Aivin V. Solatorio},
    year={2024},
    eprint={2402.16829},
    archivePrefix={arXiv},
    primaryClass={cs.LG}
}
r8   ri   s    r!   citationzGISTEmbedLoss.citation   s    	 	r"   )r   )__name__
__module____qualname__r	   floatr   r   r(   r   r   strrf   r   rj   propertyrl   __classcell__)r    s   @r!   r   r   
   s3       
 "	I
 I
"I
 #I
 	I

 
I
 I
 I
 I
 I
 I
VM M MF M M M MC5$sF{2C)D C5f C5Y_ C5 C5 C5 C5J
c3h 
 
 
 
 
# 
 
 
 X
 
 
 
 
r"   r   )typingr   r   r   rF   r   r   sentence_transformers.modelsr   )sentence_transformers.SentenceTransformerr	   Moduler   r8   r"   r!   <module>rx      s    & & & & & & & & & &          4 4 4 4 4 4 I I I I I Ie e e e eBI e e e e er"   