
    ui:                        d dl Z d dlmZmZmZ d dlmZmZmZm	Z	m
Z
 d dlmZmZmZmZmZ d dlmZ d dlmZmZmZ d dlmZ d dlmZ d d	lmZmZmZmZm Z m!Z! d d
l"m#Z# d dl$m%Z%m&Z& d dl"Z"d dl'Z'd dl(m)Z)  e'j*        e+          Z,ej-        d ej.        dej/        dej0        diZ1d e12                                D             Z3dZ4 G d deee          Z5dS )    N)SqlDBParameterValueget_sql)ProducerConsumerConsumerCallbackFndecode_vectorencode_vector)OperationRecord	LogRecordScalarEncodingSeqId	Operation)System)OpenTelemetryClientOpenTelemetryGranularitytrace_method)override)defaultdict)SequenceOptionalDictSetTuplecast)UUID)Table	functions)create_topic_name         c                     i | ]\  }}||	S  r$   ).0kvs      V/var/www/icac/venv/lib/python3.11/site-packages/chromadb/db/mixins/embeddings_queue.py
<dictcomp>r)   )   s    BBBA1BBB    Fc                   H    e Zd ZU dZ G d d          Zeeee         f         ed<   e	e
         ed<   eed<   eed<   dZd	ef fd
Z edej                  ed4 fd                        Z edej                  ededdfd                        Z edej                  edededefd                        Z edej                  ededee         dee         fd                        Z edej                  e	 	 	 d5dedede	e         de	e         de	e         defd                        Z edej                  ededdfd                         Zedefd!            Zedefd"            Ze ed#ej                  ede
fd$                                    Z  ed%ej                  dede!e	e"         e	e         e	e         f         fd&            Z# ed'ej                  d(eddfd)            Z$ ed*ej                  de	e         de	e         de!e
e
f         fd+            Z% ed,ej                  de
fd-            Z& ed.ej                  d/edee'         ddfd0            Z( ed1ej                  d2edee'         ddfd3            Z) xZ*S )6SqlEmbeddingsQueuea  A SQL database that stores embeddings, allowing a traditional RDBMS to be used as
    the primary ingest queue and satisfying the top level Producer/Consumer interfaces.

    Note that this class is only suitable for use cases where the producer and consumer
    are in the same process.

    This is because notifiaction of new embeddings happens solely in-process: this
    implementation does not actively listen to the the database for new records added by
    other processes.
    c            
       ^    e Zd ZU eed<   eed<   eed<   eed<   eed<   dededededef
dZdS )SqlEmbeddingsQueue.Subscriptionid
topic_namestartendcallbackc                 L    || _         || _        || _        || _        || _        d S N)r/   r0   r1   r2   r3   )selfr/   r0   r1   r2   r3   s         r(   __init__z(SqlEmbeddingsQueue.Subscription.__init__C   s+     DG(DODJDH$DMMMr*   N)	__name__
__module____qualname__r   __annotations__strintr   r7   r$   r*   r(   Subscriptionr.   <   s         


$$$$	%	% 	% 		%
 	% )	% 	% 	% 	% 	% 	%r*   r>   _subscriptions_max_batch_size_tenant_topic_namespace   systemc                 D   t          t                    | _        d | _        |                    t
                    | _        |j                            d          | _        |j                            d          | _	        t                                          |           d S )N	tenant_idtopic_namespace)r   setr?   r@   requirer   _opentelemetry_clientsettingsrA   rB   superr7   )r6   rD   	__class__s     r(   r7   zSqlEmbeddingsQueue.__init__X   s~    )#..#%+^^4G%H%H"..{;; & 7 78I J J     r*   zSqlEmbeddingsQueue.reset_statereturnNc                 z    t                                                       t          t                    | _        d S r5   )rL   reset_stater   rH   r?   )r6   rM   s    r(   rP   zSqlEmbeddingsQueue.reset_state`   s1     	)#..r*   zSqlEmbeddingsQueue.delete_topiccollection_idc                    t          | j        | j        |          }t          d          }|                                                     |                              |j        t          |          k              	                                }| 
                                5 }t          ||                                           \  }}|                    ||           d d d            d S # 1 swxY w Y   d S )Nembeddings_queue)r   rA   rB   r   querybuilderfrom_wheretopicr   deletetxr   parameter_formatexecute)r6   rQ   r0   tqcursqlparamss           r(   
delete_logzSqlEmbeddingsQueue.delete_logf   s    'L$/
 

 $%%U1XXU17nZ88899VXX	 	
 WWYY 	%#!!T%:%:%<%<==KCKKV$$$	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	%s   !<C**C.1C.z#SqlEmbeddingsQueue.submit_embedding	embeddingc                 h    | j         st          d          |                     ||g          d         S )NComponent not runningr   )_runningRuntimeErrorsubmit_embeddings)r6   rQ   rb   s      r(   submit_embeddingz#SqlEmbeddingsQueue.submit_embeddingw   s:    
 } 	86777%%mi[AA!DDr*   z$SqlEmbeddingsQueue.submit_embeddings
embeddingsc                    | j         st          d          t          |          dk    rg S t          |          | j        k    r"t	          d| j        dd| j        dd          t          | j        | j        |          }t          d          }| 	                                
                    |                              |j        |j        |j        |j        |j        |j                  }i }|D ]}|                     |          \  }}	}
|                    t)          t*          |d                            t)          |          t)          |d	                   t)          |          t)          |	          t)          |
                    }t          |          ||d	         <   |                                 5 }t/          ||                                           \  }}| d
}|                    ||                                          }t7          t8          d           gt          |          z  }g }|D ]n\  }}||||         <   |||                  }t;          |t=          ||d         |d         |d         |d                             }|                    |           o|                      ||           |cd d d            S # 1 swxY w Y   d S )Nrd   r   z-
                    Cannot submit more than ,zn embeddings at once.
                    Please submit your embeddings in batches of size
                    z or less.
                    rS   	operationr/   z RETURNING seq_id, idrb   encodingmetadata)r/   rb   rm   rn   rl   
log_offsetoperation_record)!re   rf   lenmax_batch_size
ValueErrorr   rA   rB   r   rT   intocolumnsrl   rW   r/   vectorrm   rn   !_prepare_vector_encoding_metadatainsertr   _operation_codesrY   r   rZ   r[   fetchallr   r   r   r   append_notify_all)r6   rQ   ri   r0   r\   ry   	id_to_idxrb   embedding_bytesrm   rn   r^   r_   r`   resultsseq_idsembedding_recordsseq_idr/   submit_embedding_recordembedding_records                        r(   rg   z$SqlEmbeddingsQueue.submit_embeddings   sP   
 } 	86777z??aIz??T000-1-@   (      'L$/
 

 $%%T!WWWQ[!'141:qzRR 	
 %'	# 	8 	8I
 66yAA	]]/	+0FGHHz**y////x((x(( F *-YIio&&WWYY 	#!&$*?*?*A*ABBKC ///Ckk#v..7799GE4(()C- - G !#% ; ;
)/	"&*4Yr]*C' $-%%4"9+"F!8!D!8!D"9+"F& & &	$ 	$ 	$  "(()9::::Z):;;;9	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	s   8DKKKzSqlEmbeddingsQueue.subscribe
consume_fnr1   r2   r/   c                 `   | j         st          d          t          | j        | j        |          }|pt          j                    }|                     ||          \  }}|                     |||||          }| 	                    |           | j
        |                             |           |S )Nrd   )re   rf   r   rA   rB   uuiduuid4_validate_ranger>   	_backfillr?   add)	r6   rQ   r   r1   r2   r/   r0   subscription_idsubscriptions	            r(   	subscribezSqlEmbeddingsQueue.subscribe   s     } 	86777&L$/
 

 ,
))%55
s((ZZ
 

 	|$$$J'++L999r*   zSqlEmbeddingsQueue.unsubscriber   c                     | j                                         D ]I\  }}|D ]A}|j        |k    r4|                    |           t	          |          dk    r| j         |=   d S BJd S )Nr   )r?   itemsr/   removerr   )r6   r   r0   subscriptionsr   s        r(   unsubscribezSqlEmbeddingsQueue.unsubscribe   s     *.)<)B)B)D)D 	 	%J -  ?o55!((666=))Q.. /
;FFF	 6	 	r*   c                     dS )Nr$   r6   s    r(   	min_seqidzSqlEmbeddingsQueue.min_seqid   s    rr*   c                     dS )Nl    r$   r   s    r(   	max_seqidzSqlEmbeddingsQueue.max_seqid   s    yr*   z!SqlEmbeddingsQueue.max_batch_sizec                    | j         |                                 5 }|                    d           |                                }|D ]G}d|d         v r;t	          |d                             d          d                   | j        z  | _         H| j         d| j        z  | _         d d d            n# 1 swxY w Y   | j         S )NzPRAGMA compile_options;MAX_VARIABLE_NUMBERr   =r    i  )r@   rY   r[   r{   r=   splitVARIABLES_PER_RECORD)r6   r^   compile_optionsoptions       r(   rs   z!SqlEmbeddingsQueue.max_batch_size   s    ' Lc5666"%,,..-  F,q	99/26!9??33G3G3J/K/K 50, '/ ,/$2K+KD(L L L L L L L L L L L L L L L  ##s   B
B22B69B6z4SqlEmbeddingsQueue._prepare_vector_encoding_metadatac                     |d         r9t          t          |d                   }|j        }t          |d         |          }nd }d }|d         rt	          j        |d                   nd }|||fS )Nrb   rm   rn   )r   r   valuer
   jsondumps)r6   rb   encoding_typerm   r   rn   s         r(   rx   z4SqlEmbeddingsQueue._prepare_vector_encoding_metadata  s~     [! 	 :1FGGM$*H+Ik,BMRROO"OH8A*8MW4:i
3444SW(22r*   zSqlEmbeddingsQueue._backfillr   c                    t          d          }|                                                     |                              |j        t          |j                  k                                  |j        t          |j                  k                                  |j        t          |j	                  k              
                    |j        |j        |j        |j        |j        |j                                      |j                  }|                                 5 }t%          ||                                           \  }}|                    ||           |                                }|D ]}|d         r,t-          |d                   }	t/          |d         |	          }
nd}	d}
|                     |t3          |d         t5          t6          |d                  |d         |
|	|d         rt9          j        |d                   nd	          
          g           	 ddd           dS # 1 swxY w Y   dS )zUBackfill the given subscription with any currently matching records in the
        DBrS   r"      Nr   r    r!      )rl   r/   rb   rm   rn   ro   )r   rT   rU   rV   rW   r   r0   r   r1   r2   selectrl   r/   rw   rm   rn   orderbyrY   r   rZ   r[   r{   r   r	   _notify_oner   r   _operation_codes_invr   loads)r6   r   r\   r]   r^   r_   r`   rowsrowrm   rw   s              r(   r   zSqlEmbeddingsQueue._backfill&  sI    $%%U1XXU17n\-DEEEFFU18n\-?@@@AAU18~l.>???@@VAHak141:qzRRWQX 	
 WWYY 	#!!T%:%:%<%<==KCKKV$$$<<>>D  q6 "-c!f55H*3q68<<FF#H!F   !'*1v-<*>s1v*F#&q6*0)1?B1v)OCF);););4. . .	 	 	   		 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	s   #DH55H9<H9z"SqlEmbeddingsQueue._validate_rangec                 
   |p|                                  }|p|                                 }t          |t                    rt          |t                    st	          d          ||k    rt          d| d|           ||fS )z[Validate and normalize the start and end SeqIDs for a subscription using this
        impl.z2SeqIDs must be integers for sql-based EmbeddingsDBzInvalid SeqID range: z to )_next_seq_idr   
isinstancer=   	TypeErrorrt   )r6   r1   r2   s      r(   r   z"SqlEmbeddingsQueue._validate_rangeO  s     ,**,,%T^^%%%%% 	RZS-A-A 	RPQQQC<<EUEEEEFFFczr*   zSqlEmbeddingsQueue._next_seq_idc                    t          d          }|                                                     |                              t	          j        |j                            }|                                 5 }|                    |	                                           t          |                                d                   dz   cddd           S # 1 swxY w Y   dS )z%Get the next SeqID for this database.rS   r   r    N)r   rT   rU   r   r   Maxr   rY   r[   r   r=   fetchone)r6   r\   r]   r^   s       r(   r   zSqlEmbeddingsQueue._next_seq_id]  s     $%%%%a((//	ah0G0GHHWWYY 	.#KK		$$$s||~~a())A-	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	.s   5ACCCzSqlEmbeddingsQueue._notify_allrW   c                 d    | j         r&| j        |         D ]}|                     ||           dS dS )z:Send a notification to each subscriber of the given topic.N)re   r?   r   )r6   rW   ri   subs       r(   r}   zSqlEmbeddingsQueue._notify_allf  sR     = 	2*51 2 2  j1111	2 	22 2r*   zSqlEmbeddingsQueue._notify_oner   c                    d}g }|D ]>}|d         |j         k    r|d         |j        k    rd} n|                    |           ?	 t          |          dk    r|                    |           |r|                     |j                   dS dS # t          $ rU}t          	                    d|j        j
         d|j         dz   t          |                     t          r|Y d}~dS d}~ww xY w)	z+Send a notification to a single subscriber.Frp   Tr   z6Exception occurred invoking consumer for subscription z	to topic z %sN)r1   r2   r|   rr   r3   r   r/   BaseExceptionloggererrorhexr0   r<   _called_from_test)r6   r   ri   should_unsubscribefiltered_embeddingsrb   es          r(   r   zSqlEmbeddingsQueue._notify_onem  sQ    # # 	2 	2I&#)33&00%)"&&y1111	&''!++0111! )  ((((() ) 	 	 	LLUUU1cn1112A  
 !      	s   AB 
C.A
C))C.)rN   N)NNN)+r8   r9   r:   __doc__r>   r   r<   r   r;   r   r=   r   r   r7   r   r   ALLr   rP   r   ra   r   r   rh   r   rg   r   r   r   r   r   propertyrs   r   bytesrx   r   r   r   r   r}   r   __classcell__)rM   s   @r(   r,   r,   0   s`        	 	% % % % % % % %* c,//0000c]"""LLL!v ! ! ! ! ! ! \24L4PQQ/ / / / / X RQ/ \35M5QRR% % % % % X SR% \79Q9UVVE!E.=E	E E E X WVE \8:R:VWWH!H/7/HH	%H H H X XWHT \02J2NOO
 "&#!  ' 	
 e_ TN 
   X PO8 \24L4PQQ4 D    X RQ 5    X 5    X \57O7STT$ $ $ $ X UT X$( \> $ 3(3	xx}<	=3 3 3	 3 \02J2NOO&l &t & & & PO&P \68P8TUUe_+3E?	sCx   VU \35M5QRR.c . . . SR. \24L4PQQ2 2(92E 2$ 2 2 2 RQ2 \24L4PQQ| )9L QU    RQ    r*   r,   )6r   chromadb.db.baser   r   r   chromadb.ingestr   r   r   r	   r
   chromadb.typesr   r   r   r   r   chromadb.configr    chromadb.telemetry.opentelemetryr   r   r   	overridesr   collectionsr   typingr   r   r   r   r   r   r   r   pypikar   r   loggingchromadb.ingest.impl.utilsr   	getLoggerr8   r   ADDUPDATEUPSERTDELETErz   r   r   r   r,   r$   r*   r(   <module>r      sM    ; ; ; ; ; ; ; ; ; ;                           # " " " " "         
       # # # # # # = = = = = = = = = = = = = = = =       # # # # # # # #   8 8 8 8 8 8 
	8	$	$ M1aaa	  CB)9)?)?)A)ABBB   Y Y Y Y Y( Y Y Y Y Yr*   