
    uiv(                        d dl mZ d dlmZmZmZmZmZmZ d dl	Z	d dl
mZmZmZ d dlZd dlmZmZ d dlmZ d dlmZ d dlmZ d d	lmZmZmZ d d
lmZmZmZm Z m!Z! d dl"m#Z#m$Z$m%Z%m&Z&m'Z' d dl(m)Z)m*Z* d dl+Z+d dl,m-Z- d dl.m/Z/  e+j0                    dk    rd dl1Z1n e+j0                    dk    rd dl2Z2ej3        dej4        dej5        diZ6 G d de          Z7dede dedefdZ8dS )    )Lock)SegmentImplementationSegmentManagerMetadataReaderSegmentTypeVectorReaderSN)SegmentLRUCache
BasicCacheSegmentCache)System	get_class)SysDB)override)PersistentLocalHnswSegment)OpenTelemetryClientOpenTelemetryGranularitytrace_method)
Collection	OperationSegmentSegmentScopeMetadata)DictTypeSequenceOptionalcast)UUIDuuid4)LRUCache)get_directory_sizeWindowsz;chromadb.segment.impl.metadata.sqlite.SqliteMetadataSegmentz8chromadb.segment.impl.vector.local_hnsw.LocalHnswSegmentzMchromadb.segment.impl.vector.local_persistent_hnsw.PersistentLocalHnswSegmentc                       e Zd ZU eed<   eed<   eed<   eee	f         ed<   e
eef         ed<   ej        Zeed<   eed<   eed<   d	ef fd
Z edej                  defd            Zed% fd            Zed% fd            Zed% fd            Z edej                  ededee         fd                        Z edej                  ededee         fd                        ZdedefdZ edej                  dede fd            Z! edej                  edede"e#         de#fd                        Z$ ed ej                  eded!e%ddfd"                        Z&dede"e	         fd#Z'dede	fd$Z( xZ)S )&LocalSegmentManager_sysdb_system_opentelemetry_client
_instances#_vector_instances_file_handle_cache_vector_segment_type_lock_max_file_handlessystemc                     t                                          |                                t                     _        | _        |                    t                     _        t          j	        t                     _        i  _        t          j        t                      i _        |j        j        dk    rE|j        j        dk    r5t)          |j        j         fd fd           j        t          j        <   n t                       j        t          j        <   t-                       _         j        j                            d          rt0          j         _        t7          j                    dk    r*t;          j        t:          j                  d          _         n(tB          j"        j#        $                                 _          j         tK          j&                    z  }tO          |d 	           _(        d S d S )
NLRUr   c                 .                         |          S N)callback_cache_evict)kvselfs     V/var/www/icac/venv/lib/python3.11/site-packages/chromadb/segment/impl/manager/local.py<lambda>z.LocalSegmentManager.__init__.<locals>.<lambda>N   s    d&?&?&B&B     c                 .                         |           S r2   )_get_segment_disk_size)r4   r6   s    r7   r8   z.LocalSegmentManager.__init__.<locals>.<lambda>O   s    D$?$?$B$B r9   )capacitycallback	size_funcis_persistentr#   c                 *    |                                 S r2   )close_persistent_index)_r5   s     r7   r8   z.LocalSegmentManager.__init__.<locals>.<lambda>c   s    Q5M5M5O5O r9   )r=   ))super__init__requirer   r&   r'   r   r(   logging	getLogger__name__loggerr)   r   METADATAr   segment_cachesettingschroma_segment_cache_policychroma_memory_limit_bytesr
   VECTORr   r,   r   HNSW_LOCAL_PERSISTEDr+   platformr.   resource	getrlimitRLIMIT_NOFILEr-   ctypeswindllmsvcrt_getmaxstdior   get_file_handle_countr!   r*   )r6   r.   segment_limit	__class__s   `  r7   rD   zLocalSegmentManager.__init__>   s      ll5))%+^^4G%H%H"'11!:<<@
 O75@@9A==6EBBBBBBBBB7 7 7D|233 7AllD|23VV
 < ((99 	(3(HD%  I--)1);H<R)S)STU)V&&)/)=)J)J)L)L&&-CEEF  8@(O(O8 8 8D444	 	r9   z(LocalSegmentManager.callback_cache_evictsegmentc                     |d         }| j                             d|            |                     |          }|                                 | j        |d         = d S )N
collectionzLRU cache evict collection id)rI   info	_instancestopr)   )r6   r\   collection_idinstances       r7   r3   z(LocalSegmentManager.callback_cache_evictf   s_    
  -F}FFGGG>>'**OGDM***r9   returnNc                     | j                                         D ]}|                                 t                                                       d S r2   )r)   valuesstartrC   r6   rd   r[   s     r7   rh   zLocalSegmentManager.startq   sH    ..00 	 	HNNr9   c                     | j                                         D ]}|                                 t                                                       d S r2   )r)   rg   rb   rC   ri   s     r7   rb   zLocalSegmentManager.stopw   sD    ..00 	 	HMMOOOOr9   c                 0   | j                                         D ]*}|                                 |                                 +i | _         | j        t
          j                                                  t                                                       d S r2   )	r)   rg   rb   reset_staterK   r   rO   resetrC   ri   s     r7   rl   zLocalSegmentManager.reset_state}   s    ..00 	# 	#HMMOOO  """"<./55777r9   z#LocalSegmentManager.create_segmentsr^   c                     t          | j        t          j        |          }t          t          j        t          j        |          }||gS r2   )_segmentr+   r   rO   r   SQLITErJ   )r6   r^   vector_segmentmetadata_segments       r7   create_segmentsz#LocalSegmentManager.create_segments   sM     "%|':J
 
 $ 5z
 
  011r9   z#LocalSegmentManager.delete_segmentsrc   c                    | j                             |          }|D ]1}|d         | j        v r|d         t          j        j        k    r0|                     |t                    }|                                 nJ|d         t          j	        j        k    r/|                     |t                    }|                                 | j        |d         = |d         t          j        u r*| j        t          j                                     |           |d         t          j        u r*| j        t          j                                     |           3d |D             S )N)r^   r_   typescopec                     g | ]
}|d          S )r_    ).0ss     r7   
<listcomp>z7LocalSegmentManager.delete_segments.<locals>.<listcomp>   s    ***A$***r9   )r&   get_segmentsr)   r   rP   valueget_segmentr   deleterp   r   r   rO   rK   poprJ   )r6   rc   segmentsr\   rd   s        r7   delete_segmentsz#LocalSegmentManager.delete_segments   s<    ;++}+EE 	M 	MGt}//6?k&F&LLL#//|LLHOO%%%%V_(:(@@@#//~NNHOO%%%OGDM2w<#666"<#67;;MJJJw<#888"<#89==mLLL******r9   c           	      B   | j                             |t          j                  }t	          |          dk    rdS t          t          j                            | j	        j
                            d          t          |d         d                                       }|S )Nr^   rv   r   persist_directoryr_   )r&   r|   r   rO   lenr"   ospathjoinr'   rL   rE   str)r6   rc   r   sizes       r7   r;   z*LocalSegmentManager._get_segment_disk_size   s    ;++$L,? , 
 
 x==A1!GLL%--.ABBHQK%&& 
 
 r9   z&LocalSegmentManager._get_segment_sysdbrv   c                     | j                             ||          }t          d t                                          D                       t          t          fd|                    }|S )Nr   c                     g | ]	}|j         
S rx   )r}   )ry   r4   s     r7   r{   z:LocalSegmentManager._get_segment_sysdb.<locals>.<listcomp>   s    FFFq17FFFr9   c                     | d         v S Nru   rx   )rz   known_typess    r7   r8   z8LocalSegmentManager._get_segment_sysdb.<locals>.<lambda>   s    &	[(@ r9   )r&   r|   setSEGMENT_TYPE_IMPLSkeysnextfilter)r6   rc   rv   r   r\   r   s        @r7   _get_segment_sysdbz&LocalSegmentManager._get_segment_sysdb   sn    
 ;++}E+RRFF,>,C,C,E,EFFFGGv@@@@(KKLLr9   zLocalSegmentManager.get_segmentru   c                    |t           k    rt          j        }n*|t          k    rt          j        }nt          d|           | j        |                             |          }|7|                     ||          }| j        |         	                    ||           | j
        5  |                     |          }d d d            n# 1 swxY w Y   t          t          |          S )NzInvalid segment type: )r   r   rJ   r   rO   
ValueErrorrK   getr   r   r,   ra   r   r	   )r6   rc   ru   rv   r\   rd   s         r7   r~   zLocalSegmentManager.get_segment   s    >!! )EE\!! 'EE<d<<===$U+//>>?--mUCCGu%))-AAA Z 	/ 	/~~g..H	/ 	/ 	/ 	/ 	/ 	/ 	/ 	/ 	/ 	/ 	/ 	/ 	/ 	/ 	/Ax   s   #CC	C	z'LocalSegmentManager.hint_use_collection	hint_typec                 0   t           t          fD ]}|                     ||          }|t          k    rc| j        j                            d          rDt          t          |          }|                                 | j	        
                    ||           d S )Nr?   )r   r   r~   r'   rL   rE   r   r   open_persistent_indexr*   r   )r6   rc   r   ru   rd   s        r7   hint_use_collectionz'LocalSegmentManager.hint_use_collection   s     $\2 	V 	VD''t<<H |##(=(E(Eo(V(V# :HEE..0008<<]HUUU	V 	Vr9   c                 p    t           t          |d                            }t          |t                    }|S r   )r   r   r   r   )r6   r\   	classnameclss       r7   _clszLocalSegmentManager._cls   s-    &{76?'C'CD		#899
r9   c                     |d         | j         vrJ|                     |          } || j        |          }|                                 || j         |d         <   | j         |d                  S )Nr_   )r)   r   r'   rh   )r6   r\   r   rd   s       r7   ra   zLocalSegmentManager._instance   sh    4=//))G$$Cs4<11HNN-5DOGDM*wt}--r9   )re   N)*rH   
__module____qualname__r   __annotations__r   r   r   r   r   r!   r   r   HNSW_LOCAL_MEMORYr+   r   intrD   r   r   OPERATION_AND_SEGMENTr   r3   r   rh   rb   rl   r   r   rs   r   r;   r   r   r   r	   r~   r   r   r   ra   __classcell__)r[   s   @r7   r%   r%   2   s        MMMOOO....T001111)1((*    )4(E+EEEKKK&v & & & & & &P \2 6 +G + + +	 +      X
      X
      X \- 6  2* 2'9J 2 2 2 X	 
2 \- 6  +T +htn + + + X	 
+"D S     \0 6  \   	  \) 6  ! !T!W ! ! ! ! X	 
!& \1 6  V V) VPT V V V X	 
VG -B(C    
. .-B . . . . . . . .r9   r%   ru   rv   r^   re   c                     t          t          |          t                    }|                    dd          }d}|r|                    |          }t          t                      | j        ||d         |          S )zRCreate a metadata dict, propagating metadata correctly for the given segment type.metadataNr_   )r_   ru   rv   r^   r   )r   r   r   r   propagate_collection_metadatar   r    r}   )ru   rv   r^   r   collection_metadatar   s         r7   ro   ro      s    
&t,.C
D
DC$..T::#'H J445HII77Zd#   r9   )9	threadingr   chromadb.segmentr   r   r   r   r   r	   rF   )chromadb.segment.impl.manager.cache.cacher
   r   r   r   chromadb.configr   r   chromadb.db.systemr   	overridesr   2chromadb.segment.impl.vector.local_persistent_hnswr    chromadb.telemetry.opentelemetryr   r   r   chromadb.typesr   r   r   r   r   typingr   r   r   r   r   uuidr   r    rQ   chromadb.utils.lru_cacher!   chromadb.utils.directoryr"   r.   rR   rU   rp   r   rP   r   r%   ro   rx   r9   r7   <module>r      s                                 
 
			 - - - - - - - - $ $ $ $ $ $                    
 R Q Q Q Q Q Q Q Q Q Q Q Q Q 7 7 7 7 7 7 7 7 7 7 7 7 7 7          - - - - - - 7 7 7 7 7 7 8?	!!OOOOX_)##MMM U!#]$&u G. G. G. G. G.. G. G. G.T; |  PW      r9   