
    hL                        S SK Jr  S SKrS SKrS SKrS SKrS SKJr  S SKJ	r	  S SK
Jr  S SKJr  S SKrS SKrS SKJr  S SKJrJrJrJr  S S	KJrJrJrJrJr  S S
KJr   " S S\5      r g)    )annotationsN)deepcopy)
ThreadPool)Path)Any)Dataset)FORMATS_HELP_MSGHELP_URLIMG_FORMATScheck_file_speeds)DEFAULT_CFG
LOCAL_RANKLOGGERNUM_THREADSTQDM)imreadc                     ^  \ rS rSrSrSSS\SSSSS	SS
SS4                           SU 4S jjjrSS jrSS jrS S!S jjr	S"S jr
S#S jrS$S%S jjrS$S%S jjrS"S jrS&S jrS&S jrS'S jrS(S jrS)S*S jjrS+S jrSrU =r$ ),BaseDataset   aj	  
Base dataset class for loading and processing image data.

This class provides core functionality for loading images, caching, and preparing data for training and inference
in object detection tasks.

Attributes:
    img_path (str): Path to the folder containing images.
    imgsz (int): Target image size for resizing.
    augment (bool): Whether to apply data augmentation.
    single_cls (bool): Whether to treat all objects as a single class.
    prefix (str): Prefix to print in log messages.
    fraction (float): Fraction of dataset to utilize.
    channels (int): Number of channels in the images (1 for grayscale, 3 for RGB).
    cv2_flag (int): OpenCV flag for reading images.
    im_files (list[str]): List of image file paths.
    labels (list[dict]): List of label data dictionaries.
    ni (int): Number of images in the dataset.
    rect (bool): Whether to use rectangular training.
    batch_size (int): Size of batches.
    stride (int): Stride used in the model.
    pad (float): Padding value.
    buffer (list): Buffer for mosaic images.
    max_buffer_length (int): Maximum buffer size.
    ims (list): List of loaded images.
    im_hw0 (list): List of original image dimensions (h, w).
    im_hw (list): List of resized image dimensions (h, w).
    npy_files (list[Path]): List of numpy file paths.
    cache (str): Cache images to RAM or disk during training.
    transforms (callable): Image transformation function.
    batch_shapes (np.ndarray): Batch shapes for rectangular training.
    batch (np.ndarray): Batch index of each image.

Methods:
    get_img_files: Read image files from the specified path.
    update_labels: Update labels to include only specified classes.
    load_image: Load an image from the dataset.
    cache_images: Cache images to memory or disk.
    cache_images_to_disk: Save an image as an *.npy file for faster loading.
    check_cache_disk: Check image caching requirements vs available disk space.
    check_cache_ram: Check image caching requirements vs available memory.
    set_rectangle: Set the shape of bounding boxes as rectangles.
    get_image_and_label: Get and return label information from the dataset.
    update_labels_info: Custom label format method to be implemented by subclasses.
    build_transforms: Build transformation pipeline to be implemented by subclasses.
    get_labels: Get labels method to be implemented by subclasses.
i  FT              ?Ng      ?   c                P  > [         TU ]  5         Xl        X l        X@l        Xl        X`l        Xl        Xl        US:X  a  [        R                  O[        R                  U l        U R                  U R                  5      U l        U R                  5       U l        U R#                  US9  [%        U R                   5      U l        Xpl        Xl        Xl        Xl        U R(                  (       a  U R*                  c   eU R1                  5         / U l        U R                  (       a%  [5        U R&                  U R*                  S-  S45      OSU l        S/U R&                  -  S/U R&                  -  S/U R&                  -  sU l        U l        U l        U R                   Vs/ s H  n[?        U5      RA                  S5      PM     snU l!        [E        U[F        5      (       a  URI                  5       OUSL a  S	OSU l%        U RJ                  S	:X  aM  U RM                  5       (       a8  URN                  (       a  [P        RR                  " S
5        U RU                  5         O5U RJ                  S:X  a%  U RW                  5       (       a  U RU                  5         U RY                  US9U l-        gs  snf )au  
Initialize BaseDataset with given configuration and options.

Args:
    img_path (str | list[str]): Path to the folder containing images or list of image paths.
    imgsz (int): Image size for resizing.
    cache (bool | str): Cache images to RAM or disk during training.
    augment (bool): If True, data augmentation is applied.
    hyp (dict[str, Any]): Hyperparameters to apply data augmentation.
    prefix (str): Prefix to print in log messages.
    rect (bool): If True, rectangular training is used.
    batch_size (int): Size of batches.
    stride (int): Stride used in the model.
    pad (float): Padding value.
    single_cls (bool): If True, single class training is used.
    classes (list[int], optional): List of included classes.
    fraction (float): Fraction of dataset to utilize.
    channels (int): Number of channels in the images (1 for grayscale, 3 for RGB).
   )include_classN   i  r   z.npyTramzcache='ram' may produce non-deterministic training results. Consider cache='disk' as a deterministic alternative if your disk space allows.disk)hyp).super__init__img_pathimgszaugment
single_clsprefixfractionchannelscv2IMREAD_GRAYSCALEIMREAD_COLORcv2_flagget_img_filesim_files
get_labelslabelsupdate_labelslennirect
batch_sizestridepadset_rectanglebufferminmax_buffer_lengthimsim_hw0im_hwr   with_suffix	npy_files
isinstancestrlowercachecheck_cache_ramdeterministicr   warningcache_imagescheck_cache_diskbuild_transforms
transforms)selfr$   r%   rF   r&   r!   r(   r6   r7   r8   r9   r'   classesr)   r*   f	__class__s                   O/home/james-whalen/.local/lib/python3.13/site-packages/ultralytics/data/base.pyr#   BaseDataset.__init__H   s   H 	 
$  08A,,3CSCS**4==9oo'1dkk"	$99??...  NRlldggt/BD%I!J`a .2FTWW,<tftww>NQUPVY]Y`Y`P`)$+tz?C}}M}!$q'--f5}M&0&<&<U[[]5TX=%^b
::4#7#7#9#9  f ZZ6!d&;&;&=&= //C/8 Ns   $J#c           
         / n[        U[        5      (       a  UOU/ GH  n[        U5      nUR                  5       (       a)  U[        R                  " [        US-  S-  5      SS9-  nMM  UR                  5       (       a  [        USS9 nUR                  5       R                  5       R                  5       n[        UR                  5      [        R                  -   nX$ Vs/ s H,  ofR                  S5      (       a  UR                  SU5      OUPM.     sn-  nSSS5        GM  [!        U R"                   U S	35      e   [%        S
 U 5       5      nU(       d   U R"                   SU S[&         35       e U R,                  S:  a$  US[/        [1        U5      U R,                  -  5       n[3        XpR"                  S9  U$ s  snf ! , (       d  f       GM  = f! [(         a'  n[!        U R"                   SU S[*         35      UeSnAff = f)a  
Read image files from the specified path.

Args:
    img_path (str | list[str]): Path or list of paths to image directories or files.

Returns:
    (list[str]): List of image file paths.

Raises:
    FileNotFoundError: If no images are found or the path doesn't exist.
z**z*.*T)	recursivezutf-8)encodingz./Nz does not existc              3     #    U  HP  oR                  S 5      S   R                  5       [        ;   d  M.  UR                  S[        R
                  5      v   MR     g7f)./N)
rpartitionrE   r   replaceossep).0xs     rR   	<genexpr>,BaseDataset.get_img_files.<locals>.<genexpr>   sD     pa<<PSCTUWCXC^C^C`doCo4aiiRVV44as
   -A'AzNo images found in z. zError loading data from 
r   )r(   )rC   listr   is_dirglobrD   is_fileopenreadstrip
splitlinesparentr]   r^   
startswithr\   FileNotFoundErrorr(   sortedr	   	Exceptionr
   r)   roundr4   r   )	rN   r$   rP   ptrl   r`   r0   es	            rR   r/   BaseDataset.get_img_files   s   	kA!+Hd!;!;X(KG88::3q4x%'7#8DIIAYY[[a'2aFFHNN,779!$QXX!7\]^\]WXd9K9Kaiif5QRR\]^^ 32 ,t{{mA3o,NOO L pappH^},?zL\K]^^8 ==1 F%H(E"FGH(;;7 _ 32  	k#t{{m3KH:UWX`Wa$bcijj	ksD   BG 
AG!3G 
GAG  G
G		G 
H	""HH	c                   [         R                  " U5      R                  SS5      n[        [	        U R
                  5      5       GH  nUb  U R
                  U   S   nU R
                  U   S   nU R
                  U   S   nU R
                  U   S   nXB:H  R                  S5      nXH   U R
                  U   S'   XX   U R
                  U   S'   U(       a;  [        U5       V	V
s/ s H  u  pU
(       d  M  Xi   PM     sn
n	U R
                  U   S'   Ub  Xx   U R
                  U   S'   U R                  (       d  M  SU R
                  U   S   SS2S4'   GM     gs  sn
n	f )	z
Update labels to include only specified classes.

Args:
    include_class (list[int], optional): List of classes to include. If None, all classes are included.
r   rY   Nclsbboxessegments	keypointsr   )	nparrayreshaperanger4   r2   any	enumerater'   )rN   r   include_class_arrayirw   rx   ry   rz   jsiidxs              rR   r3   BaseDataset.update_labels   s7    !hh}5==aDs4;;'(A(kk!nU+Q1;;q>*5 KKN;7	/44Q7(+Au%+19Ax(NWXYl1bl72^a,(,l1bDKKN:.(2;,DKKN;/./Au%ad+ ) 2cs   $E5Ec                d   U R                   U   U R                  U   U R                  U   pTnUGcj  UR                  5       (       a   [        R
                  " U5      nO[        X@R                  S9nUc  [        SU 35      eUR                  SS u  pxU(       a  U R                   [#        Xx5      -  n	U	S	:w  a|  [%        [&        R(                  " X-  5      U R                   5      [%        [&        R(                  " Xy-  5      U R                   5      p[*        R,                  " X:U4[*        R.                  S
9nOQXxs=:X  a  U R                   :X  d;  O  [*        R,                  " X0R                   U R                   4[*        R.                  S
9nUR0                  S:X  a  US   nU R2                  (       a  X7U4UR                  SS sU R                   U'   U R4                  U'   U R6                  U'   U R8                  R;                  U5        S	[=        U R8                  5      s=:  a  U R>                  :  a[  O  OXU R8                  RA                  S5      nU RB                  S:w  a-  Su  U R                   U'   U R4                  U'   U R6                  U'   X7U4UR                  SS 4$ U R                   U   U R4                  U   U R6                  U   4$ ! [         a^  n[        R                  " U R                   SU SU 35        [        U5      R                  SS9  [        X@R                  S9n SnAGNSnAff = f)a  
Load an image from dataset index 'i'.

Args:
    i (int): Index of the image to load.
    rect_mode (bool): Whether to use rectangular resizing.

Returns:
    im (np.ndarray): Loaded image as a NumPy array.
    hw_original (tuple[int, int]): Original image dimensions in (height, width) format.
    hw_resized (tuple[int, int]): Resized image dimensions in (height, width) format.

Raises:
    FileNotFoundError: If the image file is not found.
Nz"Removing corrupt *.npy image file z	 due to: T)
missing_ok)flagszImage Not Found    r   )interpolation).Nr   r   )NNN)"r>   r0   rB   existsr{   loadrp   r   rI   r(   r   unlinkr   r.   rn   shaper%   maxr<   mathceilr+   resizeINTER_LINEARndimr&   r?   r@   r;   appendr4   r=   poprF   )rN   r   	rect_modeimrP   fnrt   h0w0rwhr   s                rR   
load_imageBaseDataset.load_image   s}     HHQKq!14>>!3Dr:yy{{8B A]]3z'*:1#(>??XXbq\FBJJR,6		"& 14::>DIIbfDUW[WaWa@bqBAc>N>NOB,$**,ZZZZ$<CL\L\]ww!|	] ||=?b288TVUV<:T[[^TZZ]""1%s4;;'A4+A+AA*AzzU*EUBT[[^TZZ]Bx"1--xx{DKKNDJJqM99? ! 8NNdkk]2TUWTXXabcad#efHOOtO47B8s   K 
L/AL**L/c                   Su  pU R                   S:X  a  U R                  S4OU R                  S4u  p4[        [        5       nUR                  U[        U R                  5      5      n[        [        U5      U R                  [        S:  S9nU H  u  pU R                   S:X  a*  XR                  U   R                  5       R                  -  nOHU	u  U R                  U'   U R                  U'   U R                   U'   XR                  U   R"                  -  nU R$                   SX-  S S	U S
3Ul        M     UR)                  5         SSS5        g! , (       d  f       g= f)z3Cache images to memory or disk for faster training.r   i   @r    DiskRAMr   )totaldisablezCaching images (.1fzGB )N)rF   cache_images_to_diskr   r   r   imapr~   r5   r   r   r   rB   statst_sizer>   r?   r@   nbytesr(   descclose)
rN   bgbfcnstoragepoolresultspbarr   r`   s
             rR   rJ   BaseDataset.cache_images
  s    >BjjF>R116:Y]YhYhjoXp$iiU477^4G	'*$'':PQ>RD::'*//1999AAB>DHHQKQA!+++A#{{m+;AF3<s7)STU	  JJL %$$s   DE
Ec                    U R                   U   nUR                  5       (       d;  [        R                  " UR	                  5       [        U R                  U   5      SS9  gg)z2Save an image as an *.npy file for faster loading.F)allow_pickleN)rB   r   r{   saveas_posixr   r0   )rN   r   rP   s      rR   r    BaseDataset.cache_images_to_disk  sD    NN1xxzzGGAJJL&q)9":O     c                   SSK nSu  p4[        U R                  S5      n[        U5       H  n[        R
                  " U R                  5      n[        U5      nUc  M3  X8R                  -  n[        R                  " [        U5      R                  [        R                  5      (       a  M  SU l        [        R                   " U R"                   S35          g   X0R                  -  U-  SU-   -  n	UR%                  [        U R                  S   5      R                  5      u  pnX:  aK  SU l        [        R                   " U R"                   X-  S S	['        US
-  5       SX-  S SX-  S S3	5        gg)z
Check if there's enough disk space for caching images.

Args:
    safety_margin (float): Safety margin factor for disk space calculation.

Returns:
    (bool): True if there's enough disk space, False otherwise.
r   Nr      z8Skipping caching images to disk, directory not writeableFr   r   zGB disk space required, with d   % safety margin but only rZ   z#GB free, not caching images to diskT)shutilr<   r5   r~   randomchoicer0   r   r   r]   accessr   rl   W_OKrF   r   rI   r(   
disk_usageint)rN   safety_marginr   r   r   n_im_filer   disk_requiredr   usedfrees                rR   rK   BaseDataset.check_cache_disk   sH    	qAmmDMM2GBzNA99T']11277;;!
$++.fgh  GGa1}+<="--d4==3C.D.K.KLTDJNN;;- 237 8MC/011J9S/5:c"22UW
 r   c                   Su  p#[        U R                  S5      n[        U5       Hx  n[        [        R
                  " U R                  5      5      nUc  M1  U R                  [        UR                  S   UR                  S   5      -  nX&R                  US-  -  -  nMz     X R                  -  U-  SU-   -  n[        S5      R                  5       n	XR                  :  aa  SU l        [        R                   " U R"                   X-  S S	[%        US
-  5       SU	R                  U-  S SU	R&                  U-  S S3	5        gg)z
Check if there's enough RAM for caching images.

Args:
    safety_margin (float): Safety margin factor for RAM calculation.

Returns:
    (bool): True if there's enough RAM, False otherwise.
r   r   Nr   r   r   psutilr   z%GB RAM required to cache images with r   r   rZ   z GB available, not caching imagesFT)r<   r5   r~   r   r   r   r0   r%   r   r   r   
__import__virtual_memory	availablerF   r   rI   r(   r   r   )
rN   r   r   r   r   r   r   ratiomem_requiredmems
             rR   rG   BaseDataset.check_cache_ramD  s+    qAdmm45BzJJRXXa["((1+!>>EUAX%%A  77{Q!m*;<"113--'DJNN;;- 1#6 7MC/011J==2%c*!CIIN3+??_a
 r   c                   [         R                  " [         R                  " U R                  5      U R                  -  5      R                  [        5      nUS   S-   n[         R                  " U R                   Vs/ s H  o3R                  S5      PM     sn5      nUSS2S4   USS2S4   -  nUR                  5       nU Vs/ s H  opR                  U   PM     snU l        U Vs/ s H  opR                  U   PM     snU l        XV   nSS//U-  n[        U5       HH  nXQU:H     n	U	R                  5       U	R                  5       pUS:  a  US/X'   M7  U
S:  d  M?  SSU
-  /X'   MJ     [         R                  " [         R                  " U5      U R                   -  U R"                  -  U R$                  -   5      R                  [        5      U R"                  -  U l        Xl        gs  snf s  snf s  snf )zBSet the shape of bounding boxes for YOLO detections as rectangles.rY   r   r   Nr   )r{   flooraranger5   r7   astyper   r|   r2   r   argsortr0   r~   r<   r   r   r%   r8   r9   batch_shapesbatch)rN   binbr`   sarirectr   shapesariminimaxis               rR   r:   BaseDataset.set_rectangleb  s   XXbii(4??:;BB3GVaZHHdkk:keeGnk:;q!tWqAw

3895aq)59/45u!{{1~u5Y a&BrA1W+CCGGI$ax!1I	DM	  GGBHHV$4tzz$ADKK$ORVRZRZ$Z[bbcfgjnjujuu
% ; :5s   ;G'G,'G1c                B    U R                  U R                  U5      5      $ )z5Return transformed label information for given index.)rM   get_image_and_label)rN   indexs     rR   __getitem__BaseDataset.__getitem__{  s    t77>??r   c                ^   [        U R                  U   5      nUR                  SS5        U R                  U5      u  US'   US'   US'   US   S   US   S   -  US   S   US   S   -  4US'   U R                  (       a  U R
                  U R                  U      US	'   U R                  U5      $ )
z
Get and return label information from the dataset.

Args:
    index (int): Index of the image to retrieve.

Returns:
    (dict[str, Any]): Label dictionary with image and metadata.
r   Nimg	ori_shaperesized_shaper   r   	ratio_pad
rect_shape)r   r2   r   r   r6   r   r   update_labels_info)rN   r   labels      rR   r   BaseDataset.get_image_and_label  s     U+,		'4 CG??SXCY@eeK(%*@/"1%k(:1(==/"1%k(:1(==
k 99"&"3"3DJJu4E"FE,&&u--r   c                ,    [        U R                  5      $ )z5Return the length of the labels list for the dataset.)r4   r2   rN   s    rR   __len__BaseDataset.__len__  s    4;;r   c                    U$ )zCustom your label format here. )rN   r   s     rR   r   BaseDataset.update_labels_info  s    r   c                    [         e)z
Users can customize augmentations here.

Examples:
    >>> if self.augment:
    ...     # Training transforms
    ...     return Compose([])
    >>> else:
    ...    # Val transforms
    ...    return Compose([])
NotImplementedError)rN   r!   s     rR   rL   BaseDataset.build_transforms  s
     "!r   c                    [         e)a  
Users can customize their own format here.

Examples:
    Ensure output is a dictionary with the following keys:
    >>> dict(
    ...     im_file=im_file,
    ...     shape=shape,  # format: (height, width)
    ...     cls=cls,
    ...     bboxes=bboxes,  # xywh
    ...     segments=segments,  # xy
    ...     keypoints=keypoints,  # xy
    ...     normalized=True,  # or False
    ...     bbox_format="xyxy",  # or xywh, ltwh
    ... )
r  r   s    rR   r1   BaseDataset.get_labels  s
    " "!r   )r&   r   r   r7   r;   rF   r*   r.   r)   r0   r@   r?   r$   r%   r>   r2   r=   r5   rB   r9   r(   r6   r'   r8   rM   )r$   str | list[str]r%   r   rF   z
bool | strr&   boolr!   dict[str, Any]r(   rD   r6   r  r7   r   r8   r   r9   floatr'   r  rO   list[int] | Noner)   r
  r*   r   )r$   r  returnz	list[str])r   r  r  None)T)r   r   r   r  r  z3tuple[np.ndarray, tuple[int, int], tuple[int, int]])r  r  )r   r   r  r  )r   )r   r
  r  r  )r   r   r  r	  )r  r   )r   r	  r  r	  )N)r!   zdict[str, Any] | None)r  zlist[dict[str, Any]])__name__
__module____qualname____firstlineno____doc__r   r#   r/   r3   r   rJ   r   rK   rG   r:   r   r   r   r   rL   r1   __static_attributes____classcell__)rQ   s   @rR   r   r      s'   .f !) $(L9!L9 L9 	L9
 L9 L9 L9 L9 L9 L9 L9 L9 "L9 L9 L9 L9\$L004:l P"H<2@.* "" "r   r   )!
__future__r   rf   r   r]   r   copyr   multiprocessing.poolr   pathlibr   typingr   r+   numpyr{   torch.utils.datar   ultralytics.data.utilsr	   r
   r   r   ultralytics.utilsr   r   r   r   r   ultralytics.utils.patchesr   r   r   r   rR   <module>r     sH    #   	   +   
  $ ] ] P P ,d"' d"r   