
    si                     0    d dl Z d dlZ G d d          ZdS )    Nc                        e Zd Zd Zd Zd ZdS )NoDuplicatesDataLoaderc                 p    || _         d| _        d| _        || _        t	          j        | j                   dS )z
        A special data loader to be used with MultipleNegativesRankingLoss.
        The data loader ensures that there are no duplicate sentences within the same batch
        r   N)
batch_sizedata_pointer
collate_fntrain_examplesrandomshuffle)selfr	   r   s      h/var/www/icac/venv/lib/python3.11/site-packages/sentence_transformers/datasets/NoDuplicatesDataLoader.py__init__zNoDuplicatesDataLoader.__init__   s;    
 %,t*+++++    c              #     K   t          |                                           D ]V}g }t                      }t          |          | j        k     r
| j        | j                 }d}|j        D ].}|                                	                                |v rd} n/|rX|
                    |           |j        D ];}|                    |                                	                                           <| xj        dz  c_        | j        t          | j                  k    r d| _        t          j        | j                   t          |          | j        k     
| j        |                     |          n|V  Xd S )NTF   r   )range__len__setlenr   r	   r   textsstriplowerappendaddr
   r   r   )r   _batchtexts_in_batchexamplevalid_exampletexts          r   __iter__zNoDuplicatesDataLoader.__iter__   s     t||~~&& 	S 	SAE UUNe**t..-d.?@ $#M  Dzz||))++~==(- > ! ALL))) ' A A&**4::<<+=+=+?+?@@@@!!Q&!!$D,?(@(@@@()D%N4#6777# e**t..& -1O,G$//%(((URRRR/	S 	Sr   c                 ^    t          j        t          | j                  | j        z            S )N)mathfloorr   r	   r   )r   s    r   r   zNoDuplicatesDataLoader.__len__+   s$    z#d122T_DEEEr   N)__name__
__module____qualname__r   r!   r    r   r   r   r      sI        	, 	, 	,S S S4F F F F Fr   r   )r#   r
   r   r(   r   r   <module>r)      sS     'F 'F 'F 'F 'F 'F 'F 'F 'F 'Fr   