
    cCio                        S r SSKJr  SSKJrJrJr  SSKrSSK	r	SSK
Jr  SSKJs  Jr  SSKJr  SSKJrJr  SSKJrJr  SS	KJrJrJrJrJrJrJr  SS
KJ r   SSK!J"r"J#r#  SSK$J%r%  SSK&J'r'J(r(J)r)J*r*  SSK+J,r,J-r-  SSK.J/r/  SSK0J1r1  SSK2J3r3  SSK4J5r5J6r6J7r7J8r8J9r9J:r:J;r;J<r<  SSK=J>r>J?r?  SSK@JArAJBrBJCrCJDrDJErE  \*R                  " \G5      rH " S S\5      rI\) " S S\35      5       rJ\\)" SS9 " S S\'5      5       5       rK\\)" SS9 " S  S!\'5      5       5       rL " S" S#\R                  5      rN " S$ S%\15      rO " S& S'\R                  5      rPSPS(\	R                  S)\\R   S*\	R                  4S+ jjrS " S, S-\R                  5      rT " S. S/\R                  5      rU " S0 S1\ 5      rV\\)" S2S9 " S3 S4\'5      5       5       rW\) " S5 S6\#5      5       rX " S7 S8\X5      rY\)" S9S9 " S: S;\X5      5       rZ " S< S=\R                  5      r[ " S> S?\75      r\ " S@ SA\95      r] " SB SC\R                  5      r^ " SD SE\:\ 5      r_ " SF SG\;5      r` " SH SI\55      ra " SJ SK\65      rb\)" SLS9 " SM SN\85      5       rc/ SOQrdg)QzPyTorch SAM 2 model.    )	dataclass)CallableOptionalUnionN   )ACT2FN)BatchFeatureget_size_dict)BaseImageProcessorFastDefaultFastImageProcessorKwargs)IMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STDChannelDimension
ImageInputPILImageResamplingSizeDictpil_torch_interpolation_mapping)GradientCheckpointingLayer)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)ModelOutput
TensorTypeauto_docstringlogging)TransformersKwargscheck_model_inputs   )	AutoModel)MaskFormerSinePositionEmbedding)SamImageProcessorFast)SamLayerNormSamMaskDecoderSamMaskEmbeddingSamModelSamPromptEncoderSamTwoWayAttentionBlockSamTwoWayTransformereager_attention_forward)window_partitionwindow_unpartition   )
Sam2ConfigSam2HieraDetConfigSam2MaskDecoderConfigSam2PromptEncoderConfigSam2VisionConfigc                   4    \ rS rSr% Sr\\\\4      \	S'   Sr
g)Sam2FastImageProcessorKwargsI   z|
mask_size (`dict[str, int]`, *optional*):
    The size `{"height": int, "width": int}` to resize the segmentation maps to.
	mask_size N)__name__
__module____qualname____firstlineno____doc__r   dictstrint__annotations____static_attributes__r6       _/home/james-whalen/.local/lib/python3.13/site-packages/transformers/models/sam2/modular_sam2.pyr3   r3   I   s    
 S#X''rA   r3   c                      \ rS rSr\R
                  r\r\	r
SSS.rSSS.rSrSrSrSr\rSrSrSrS\\   4S jrS	 rS
 rS rS\S   S\\\\4      SS4S jr  S"S\!S\\!   S\"S\#S\\\S4      S\\   S\$4S jjr%      S#S\\&   S\\&   S\\"   S\\\'\\'   4      S\\\'\\'   4      S\\#   S\(4S jjr)S\*RV                  S\*RV                  4S jr,     S$S  jr-S!r.g)%Sam2ImageProcessorFastR   i   )heightwidth   TNkwargsc                 2    [         R                  " U 40 UD6  g N)r   __init__)selfrI   s     rB   rL   Sam2ImageProcessorFast.__init__e   s    ''77rA   c                     [        S5      e)NzNo pad_image for SAM 2.NotImplementedErrorrM   s    rB   	pad_image Sam2ImageProcessorFast.pad_imageh   s    !";<<rA   c                     [        S5      e)Nz#No _get_preprocess_shape for SAM 2.rP   rR   s    rB   _get_preprocess_shape,Sam2ImageProcessorFast._get_preprocess_shapek   s    !"GHHrA   c                     [        S5      e)Nz%No need to override resize for SAM 2.rP   rR   s    rB   resizeSam2ImageProcessorFast.resizen   s    !"IJJrA   imagesztorch.Tensorreturn_tensorsreturnc                 H    [         R                  " X4SU0UD6R                  $ )Nr\   )r   _preprocesspixel_values)rM   r[   r\   rI   s       rB   r_   "Sam2ImageProcessorFast._preprocessq   s&     &11$h~haghuuurA   segmentation_mapsdo_convert_rgbinput_data_formatdeviceztorch.devicec                 |   U R                  XXES9nU Vs/ s H  owR                  SS PM     nnUR                  5       n	U R                  " U40 U	D6n
U Vs/ s H  owR                  SS PM     nnU
UUS.nUb  U R                  USS[        R
                  S9nUR                  5       nUR                  SS[        [        R                     UR                  S5      S	.5        U R                  " SS
U0UD6nUR                  S5      R                  [        R                  5      US'   [        XS   S9$ s  snf s  snf )z
Preprocess image-like inputs.
)r[   rc   rd   re   N)r`   original_sizesreshaped_input_sizesr   F)r[   expected_ndimsrc   rd   r5   )do_normalize
do_rescaleinterpolationsizer[   r,   labelsr\   )datatensor_typer6   )_prepare_image_like_inputsshapecopyr_   r   FIRSTupdater   r   NEARESTpopsqueezetotorchint64r	   )rM   r[   rb   rc   rd   re   rI   imagerh   images_kwargsr`   ri   rp   processed_segmentation_mapssegmentation_maps_kwargss                  rB   _preprocess_image_like_inputs4Sam2ImageProcessorFast._preprocess_image_like_inputsy   sg    00L] 1 
 9??u++bc*?''@-@>DEfUBC 0fE(,$8
 (*.*I*I( $"2"8"8	 +J +' (.{{}$$++$)"'%DEWE_E_%`488E	 +/*:*: +2+6N+' 9@@CFFu{{SDN:J3KLLA @  Fs   D4D9rn   r5   default_to_square
image_mean	image_stddata_formatc           	         Uc  0 nUb  [        S
0 [        XS9D6nUb  [        S
0 [        USS9D6n[        U[        5      (       a  [	        U5      n[        U[        5      (       a  [	        U5      nUc  [
        R                  nXS'   X'S'   XGS'   XWS'   XgS'   UR                  S5      n[        U[        [        45      (       a	  [        U   OUUS	'   U$ )z
Update kwargs that need further processing before being validated
Can be overridden by subclasses to customize the processing of kwargs.
)rn   r   r5   )
param_namern   r   r   r   resamplerm   r6   )r   r
   
isinstancelisttupler   ru   rx   r   r>   r   )	rM   rn   r5   r   r   r   r   rI   r   s	            rB   _further_process_kwargs.Sam2ImageProcessorFast._further_process_kwargs   s     >F\m[\D  T={#STIj$''z*Ji&&i(I*00Kv'{)|'{ +} ::j)9CHOacfNg9h9h+H5nv 	 rA   
pred_masksc           	         UR                  S5      nUS:X  a  U$ UR                  n[        R                  " USSS9n[        R                  " X#S9SS2SSS4   nXE:H  n[        R
                  " Xa[        R                  " USS95      nU$ )	z
Apply non-overlapping constraints to the object scores in pred_masks. Here we
keep only the highest scoring object at each spatial location in pred_masks.
r   r,   T)dimkeepdim)re   Ng      $)max)rn   re   r{   argmaxarangewhereclamp)rM   r   
batch_sizere   max_obj_indsbatch_obj_indskeeps          rB   "_apply_non_overlapping_constraints9Sam2ImageProcessorFast._apply_non_overlapping_constraints   s    
  __Q'
?""||JAtDj@D$PTATU- [[5;;zu3UV
rA   c                    [        U[        R                  [        R                  45      (       a  UR                  5       n/ n	[        U5       H  u  p[        X   [        R                  5      (       a  [        R                  " X   5      X'   O,[        X   [        R                  5      (       d  [        S5      e[        R                  " X   USSS9nU(       a  U R                  U5      nU(       a  X:  nU	R                  U5        M     U	$ )a  
Remove padding and upscale masks to the original image size.

Args:
    masks (`Union[torch.Tensor, List[torch.Tensor], np.ndarray, List[np.ndarray]]`):
        Batched masks from the mask_decoder in (batch_size, num_channels, height, width) format.
    original_sizes (`Union[torch.Tensor, List[Tuple[int,int]]]`):
        The original sizes of each image before it was resized to the model's expected input shape, in (height,
        width) format.
    mask_threshold (`float`, *optional*, defaults to 0.0):
        Threshold for binarization and post-processing operations.
    binarize (`bool`, *optional*, defaults to `True`):
        Whether to binarize the masks.
    max_hole_area (`float`, *optional*, defaults to 0.0):
        The maximum area of a hole to fill.
    max_sprinkle_area (`float`, *optional*, defaults to 0.0):
        The maximum area of a sprinkle to fill.
    apply_non_overlapping_constraints (`bool`, *optional*, defaults to `False`):
        Whether to apply non-overlapping constraints to the masks.

Returns:
    (`torch.Tensor`): Batched masks in batch_size, num_channels, height, width) format, where (height, width)
    is given by original_size.
zIInput masks should be a list of `torch.tensors` or a list of `np.ndarray`bilinearF)modealign_corners)r   r{   Tensornpndarraytolist	enumerate
from_numpy
ValueErrorFinterpolater   append)rM   masksrh   mask_thresholdbinarizemax_hole_areamax_sprinkle_area!apply_non_overlapping_constraintsrI   output_masksioriginal_sizeinterpolated_masks                rB   post_process_masks)Sam2ImageProcessorFast.post_process_masks   s    F nu||RZZ&@AA+224N ). 9A%(BJJ// ++EH5%,,77 !lmm !ehJfk l0$($K$KL]$^!$5$F! 12 !: rA   r6   rK   )NNNNNN)        Tr   r   F)/r7   r8   r9   r:   r   BILINEARr   r   r   r   r   rn   r5   	do_resizerl   rk   rc   r3   valid_kwargsdo_padpad_sizemask_pad_sizer   rL   rS   rV   rY   r   r   r   r=   r   r_   r   boolr   r	   r   r   floatr<   r   r{   r   r   r   r@   r6   rA   rB   rD   rD   R   s   !**H&J$IT*D-IIJLN/L FHM8(D!E 8=IKv^$v !sJ!78v
 
v 8</M/M $J//M 	/M
 ,/M sN234/M 56/M 
/Mf $((,,0:>9=26*x * H%* $D>	*
 U5$u+#567* E%e"456* ./* 
*XU\\ ell 0 */3rA   rD   z,Base class for the vision encoder's outputs.)custom_introc                       \ rS rSr% SrSr\\R                     \	S'   Sr
\\R                     \	S'   Sr\\R                     \	S'   Sr\\\R                  S4      \	S'   Sr\\\R                  S4      \	S	'   S
rg)Sam2VisionEncoderOutputi   a  
last_hidden_state (`torch.FloatTensor` of shape `(batch_size, height, width, hidden_size)`):
    Sequence of hidden-states at the output of the last layer of the model.
fpn_hidden_states (`tuple(torch.FloatTensor)`):
    Tuple of `torch.FloatTensor` (one for each feature level, from high to low resolution) of shape
    `(batch_size, hidden_size, height, width)`. Feature maps from the Feature Pyramid Network neck.
fpn_position_encoding (`tuple(torch.FloatTensor)`):
    Tuple of `torch.FloatTensor` (one for each feature level, from high to low resolution) of shape
    `(batch_size, hidden_size, height, width)`. Positional encodings corresponding to the `fpn_hidden_states`.
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
    Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +
    one for the output of each stage) of shape `(batch_size, height, width, hidden_size)`. Hidden-states of the
    model at the output of each stage.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
    Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
    sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in
    the self-attention heads.
Nlast_hidden_statefpn_hidden_statesfpn_position_encoding.hidden_states
attentionsr6   )r7   r8   r9   r:   r;   r   r   r{   FloatTensorr?   r   r   r   r   r   r@   r6   rA   rB   r   r      s    & 6:x 1 12959x 1 1299=8E$5$56==AM8E%"3"3S"89:A:>Ju00#567>rA   r   z'Base class for the Sam2 model's output.c                   T   \ rS rSr% SrSr\\R                     \	S'   Sr
\\R                     \	S'   Sr\\R                     \	S'   Sr\\R                  S4   \	S'   Sr\\\R                  S4      \	S	'   Sr\\\R                  S4      \	S
'   Sr\\\R                  S4      \	S'   Srg)Sam2ImageSegmentationOutputi=  ay  
iou_scores (`torch.FloatTensor` of shape `(batch_size, point_batch_size, num_masks)`):
    The Intersection over Union (IoU) scores of the predicted masks.
pred_masks (`torch.FloatTensor` of shape `(batch_size, point_batch_size, num_masks, height, width)`):
    The predicted low-resolution masks. This is an alias for `low_res_masks`. These masks need to be post-processed
    by the processor to be brought to the original image size.
object_score_logits (`torch.FloatTensor` of shape `(batch_size, point_batch_size, 1)`):
    Logits for the object score, indicating if an object is present.
image_embeddings (`tuple(torch.FloatTensor)`):
    The features from the FPN, which are used by the mask decoder. This is a tuple of `torch.FloatTensor` where each
    tensor has shape `(batch_size, channels, height, width)`.
vision_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True`):
    Tuple of `torch.FloatTensor` (one for the output of each stage) of shape `(batch_size, height, width, hidden_size)`.
    Hidden-states of the vision model at the output of each stage.
vision_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True`):
    Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
    Attentions weights of the vision model.
mask_decoder_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True`):
    Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
    Attentions weights of the mask decoder.
N
iou_scoresr   object_score_logits.image_embeddingsvision_hidden_statesvision_attentionsmask_decoder_attentionsr6   )r7   r8   r9   r:   r;   r   r   r{   r   r?   r   r   r   r   r   r   r   r@   r6   rA   rB   r   r   =  s    , /3J**+2.2J**+27;%"3"34;6:eE--s23:DH(5):):C)?#@AHAExe&7&7&< =>EGKXeE,=,=s,B&CDKrA   r   c                   :   ^  \ rS rSrSrS\4U 4S jjrS rSrU =r	$ )Sam2PatchEmbeddingsi_  a  
Turns pixel values into patch embeddings for transformer consumption.

Args:
    pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
        Pixel values. Pixel values can be obtained using
        [`AutoImageProcessor`]. See [`Sam2ImageProcessorFast.__call__`] for details.

Returns:
    embeddings (`torch.FloatTensor`):
        Patch embeddings depend on image_size, patch_kernel_size, patch_stride and patch_padding
configc                    > [         TU ]  5         UR                  nUR                  n[        R
                  " UUUR                  UR                  UR                  S9U l	        g )N)kernel_sizestridepadding)
superrL   num_channelshidden_sizennConv2dpatch_kernel_sizepatch_stridepatch_padding
projection)rM   r   r   r   	__class__s       rB   rL   Sam2PatchEmbeddings.__init__m  sU    **(())00&&((
rA   c                 j    UR                   u  p#pEU R                  U5      R                  SSSS5      nU$ )Nr   r   r   r,   )rs   r   permute)rM   r`   _r   rF   rG   
embeddingss          rB   forwardSam2PatchEmbeddings.forwardz  s7    )5););&__\2::1aAF
rA   )r   )
r7   r8   r9   r:   r;   r.   rL   r   r@   __classcell__r   s   @rB   r   r   _  s    
1 
 rA   r   c                       \ rS rSrSrg)Sam2SinePositionEmbeddingi  r6   Nr7   r8   r9   r:   r@   r6   rA   rB   r   r         rA   r   c                      ^  \ rS rSrS\4U 4S jjrS\R                  S\\\R                  S4   \\R                  S4   4   4S jr	Sr
U =r$ )	Sam2VisionNecki  r   c                   > [         TU ]  5         Xl        [        UR                  S-  SS9U l        [        R                  " 5       U l        UR                   H\  nU R                  R                  [        R                  " UUR                  UR                  UR                  UR                  S95        M^     UR                  U l        g )Nr   T)num_pos_feats	normalize)in_channelsout_channelsr   r   r   )r   rL   r   r   fpn_hidden_sizeposition_encodingr   
ModuleListconvsbackbone_channel_listr   r   fpn_kernel_size
fpn_stridefpn_paddingfpn_top_down_levels)rM   r   r   r   s      rB   rL   Sam2VisionNeck.__init__  s    !:I_I_cdIdpt!u]]_
!77KJJ		 +!'!7!7 & 6 6!,,".. 8 $*#=#= rA   r   r]   .c           	      8   SnSn[        U R                  5      S-
  n[        USS5       H  nX   R                  SSSS5      nU R                  XE-
     " U5      nXPR                  ;  d  XT:X  a  UnOP[
        R                  " WR                  [        R                  S9SS	S S
S9R                  UR                  5      nXh-   nU R                  UR                  UR                  UR                  5      R                  UR                  5      n	X'4-  nX94-  nM     X#4$ )Nr6   r,   r   r   r   )dtypeg       @nearestF)scale_factorr   r   	antialias)lenr   ranger   r   r   r   rz   r{   float32r  r   rs   re   )
rM   r   r   r   nr   lateral_featuresprev_featurestop_down_featuresprev_position_encodings
             rB   r   Sam2VisionNeck.forward  s+    " 

Oaq"b!A,/771aC#zz!%01AB000AF 0$%MM!$$5==$9!$""&#% "%++, " !1 D%)%;%;##]%9%9=;N;N&b$$% # !11!%>>!) ", !77rA   )r   r   r   r   )r7   r8   r9   r:   r1   rL   r{   r   r   r   r@   r   r   s   @rB   r   r     sX    >/ >$8U\\ 8eE%,,PSBS<TV[\a\h\hjm\mVn<n6o 8 8rA   r   xquery_strider]   c                     Uc  U $ U R                  SSSS5      n [        R                  R                  XUSS9n U R                  SSSS5      n U $ )Nr   r   r,   r   F)r   r   	ceil_mode)r   r   
functional
max_pool2d)r  r  s     rB   do_poolr    sV    			!Q1A
  \]b cA			!Q1AHrA   c                      ^  \ rS rSr SS\S\S\S\S\\\\4      4
U 4S jjjrS\	R                  S	\	R                  4S
 jrSrU =r$ )Sam2MultiScaleAttentioni  r   r   dim_outnum_attention_headsr  c                    > [         TU ]  5         Xl        X l        X0l        XPl        X@l        X4-  nUS-  U l        [        R                  " X#S-  5      U l
        [        R                  " X35      U l        SU l        g )N      r   F)r   rL   r   r   r  r  r  scaler   Linearqkvproj	is_causal)rM   r   r   r  r  r  head_dimr   s          rB   rL    Sam2MultiScaleAttention.__init__  si     	(#6 1t^
99SA+.IIg/	rA   r   r]   c                    UR                   u  p4pVU R                  U5      R                  X4U-  SU R                  S5      n[        R
                  " US5      u  pn
XR                  -  U	R                  SS5      -  n[        R                  R                  R                  U[        R                  SS9R                  UR                  5      nU R                  (       aX  [        UR                  X4US5      U R                  5      nUR                   SS u  pEUR                  X4U-  U R                  S5      nUR                  SS5      nU	R                  SS5      n	U
R                  SS5      n
[         nU R"                  R$                  S:w  a  [&        U R"                  R$                     nU" U UU	U
4S U R(                  U R                  S.UD6u  pUR                  X4US5      nU R+                  U5      nU$ )	Nr   r  r   rg   )r  r   r,   eager)attention_maskr!  scaling)rs   r  reshaper  r{   unbindr  	transposer   r  softmaxr	  rz   r  r  r  r)   r   _attn_implementationr   r!  r   )rM   r   rI   r   rF   rG   r   r  querykeyvalueattn_weightsattention_interfaceattn_outputs                 rB   r   Sam2MultiScaleAttention.forward  s   '4':':$
Ehh}%--j5.!TMeMegij!LLa0E

*cmmB.CCxx**22<u}}Z\2]``afalalm EMM*eRH$J[J[\E!KK!,MFMM*und>V>VXZ[E 1%mmAq!1%(?;;++w6"9$++:Z:Z"[,		

  nnJJ	
 	
 "))*eRHii,rA   )	r   r   r  r!  r  r   r  r  r  rK   )r7   r8   r9   r:   r.   r>   r   r   rL   r{   r   r   r@   r   r   s   @rB   r  r    so     37"  	
 ! uS#X/ 0&U\\ & & &rA   r  c                   R   ^  \ rS rSr  SS\S\S\S\S\S\4U 4S jjjrS	 rS
r	U =r
$ )Sam2FeedForwardi  	input_dim
hidden_dim
output_dim
num_layers
activationsigmoid_outputc           	      d  > [         TU ]  5         X@l        [        U   U l        [
        R                  " X5      U l        [
        R                  " X#5      U l        [
        R                  " [        US-
  5       Vs/ s H  n[
        R                  " X"5      PM     sn5      U l        X`l        g s  snf )Nr   )r   rL   r9  r   r:  r   r  proj_inproj_outr   r  layersr;  )	rM   r6  r7  r8  r9  r:  r;  r   r   s	           rB   rL   Sam2FeedForward.__init__  s     	$ ,yy7		*9mmPUV`cdVdPe$fPe1RYYz%FPe$fg, %gs   : B-c                    U R                  U5      nU R                  U5      nU R                   H  nU R                  U" U5      5      nM     U R                  U5      nU R                  (       a  [
        R                  " U5      nU$ rK   )r=  r:  r?  r>  r;  r   sigmoid)rM   r   layers      rB   r   Sam2FeedForward.forward  sl    ]36[[E OOE-,@AM ! m4IIm4MrA   )r:  r?  r9  r=  r>  r;  )reluF)r7   r8   r9   r:   r>   r=   r   rL   r   r@   r   r   s   @rB   r5  r5    sY     !$-- - 	-
 - - - -"	 	rA   r5  c                      ^  \ rS rSrS\S\S\S\4U 4S jjrS\R                  S\	\
   S	\R                  4S
 jrSrU =r$ )Sam2MultiScaleBlocki!  r   	stage_idx	block_idxtotal_block_idxc                 $  > [         TU ]  5         US:  a  US:X  a  UR                  US-
     OUR                  U   U l        UR                  U   U l        [
        R                  " U R                  UR                  S9U l        US:  a  US:X  a  UR                  US-
     OUR                  U   U l
        XAR                  ;   a  SOU R                  U l
        SUs=:  a  UR                  ::  a  O  OUS:X  a  UR                  OS U l        [        UU R                  U R                  UR                  U   U R                  S9U l        [
        R                  " U R                  UR                  S9U l        [%        U R                  ['        U R                  UR(                  -  5      U R                  SUR*                  S9U l        U R                  U R                  :w  a1  [
        R.                  " U R                  U R                  5      U l        g g )Nr   r,   )eps)r  r  r   )r9  r:  )r   rL   embed_dim_per_stager   r  r   	LayerNormlayer_norm_epslayer_norm1window_size_per_stagewindow_sizeglobal_attention_blocksnum_query_pool_stagesr  r  num_attention_heads_per_stageattnlayer_norm2r5  r>   	mlp_ratio
hidden_actmlpr  r   )rM   r   rH  rI  rJ  r   s        rB   rL   Sam2MultiScaleBlock.__init__"  s    	
 1}a &&y1}5++I6 	
 11)<<<f6K6KL 1}a ((Q7--i8 	
 !03Q3Q Q1W[WgWg $%y#PF4P4P#PU^bcUcFim 	 ,HHLL & D DY O**
	 <<&:O:OP"LLv///0LL((
 88t||#		$((DLL9DI $rA   r   rI   r]   c                    UnU R                  U5      nU R                  U R                  :w  a%  [        U R	                  U5      U R
                  5      nU R                  nU R                  S:  a*  UR                  S   UR                  S   pe[        X5      u  pU R                  " SSU0UD6nUnU R
                  (       aI  U R                  U R
                  S   -  nUR                  SS u  pVXEU-  -
  U-  n	XFU-  -
  U-  n
XY-   Xj-   4nU R                  S:  a  [        XWWW45      nX1-   nU R                  U5      nXR                  U5      -   nU$ )Nr   r,   r   r   r   r6   )rP  r   r  r  r   r  rR  rs   r*   rV  r+   rW  rZ  )rM   r   rI   residualrR  HWpad_hwr2  pad_hpad_wlayernorm_outputs               rB   r   Sam2MultiScaleBlock.forwardQ  sn   
 !((7 88t||#tyy79J9JKH &&a &&q)=+>+>q+Aq$4]$P!M ii 
'

 $**d.?.?.BBK>>!A&DA {?2kAE {?2kAEi+F a.}6TUWXSYZM 0++M:%1A(BBrA   )	rV  r   r  rP  rW  rZ  r   r  rR  )r7   r8   r9   r:   r.   r>   rL   r{   r   r   r   r   r   r@   r   r   s   @rB   rG  rG  !  se    -:"-: -: 	-:
 -:^*||* +,* 
			* *rA   rG  zW
    Hiera model's outputs that also contains a pooling of the last hidden states.
    c                   t    \ rS rSr% SrSr\\R                     \	S'   Sr
\\\R                  S4      \	S'   Srg)Sam2HieraDetModelOutputi~  a`  
last_hidden_state (`torch.FloatTensor` of shape `(batch_size, height, width, hidden_size)`):
    hidden-states at the output of the last layer of the model.
intermediate_hidden_states (`tuple[torch.FloatTensor]` of shape `(batch_size, height, width, hidden_size)`):
    Sequence of hidden-states at the output of the intermediate layers of the model.
Nr   .intermediate_hidden_statesr6   )r7   r8   r9   r:   r;   r   r   r{   r   r?   rg  r   r@   r6   rA   rB   rf  rf  ~  s?     6:x 1 129JNu/@/@#/E)F GNrA   rf  c                   2    \ rS rSr\rSrSrSrSr	Sr
S rSrg)Sam2PreTrainedModeli  sam2r`   Tc                    U R                   R                  n[        U[        R                  [        R
                  [        R                  45      (       aV  UR                  R                  R                  SUS9  UR                  b$  UR                  R                  R                  5         O[        U[        R                  5      (       ac  UR                  R                  R                  SUS9  UR                  b1  UR                  R                  UR                     R                  5         On[        U[        R                  [        45      (       aI  UR                  R                  R!                  S5        UR                  R                  R                  5         [        U["        5      (       ab  UR$                  b$  UR$                  R                  R                  5         UR&                  b$  UR&                  R                  R                  5         [        U[(        5      (       a3  UR*                  b%  UR*                  R                  R                  5         g g g )Nr   )meanstd      ?)r   initializer_ranger   r   r  r   ConvTranspose2dweightrp   normal_biaszero_	Embeddingpadding_idxrN  Sam2LayerNormfill_Sam2HieraDetModel	pos_embedpos_embed_window	Sam2Modelno_memory_embedding)rM   modulerm  s      rB   _init_weights!Sam2PreTrainedModel._init_weights  s   kk++fryy"))R5G5GHIIMM&&CS&9{{&  &&(--MM&&CS&9!!-""6#5#56<<>} =>>MM$$S)KK""$f/00+  %%++-&&2'',,224fi(())5**//557 6 )rA   r6   N)r7   r8   r9   r:   r-   config_classbase_model_prefixmain_input_name_supports_sdpa_supports_flash_attn_2_supports_attention_backendr  r@   r6   rA   rB   ri  ri    s(    L$ON!"&8rA   ri  c            
          ^  \ rS rSr\rSr\\S.r	S\4U 4S jjr
S rS\\\4   S\R                  4S	 jr\" 5        SS\\R&                     S
\\   S\\\4   4S jj5       rSrU =r$ )ry  i  r`   r   r   r   c           	        > [         TU ]  U5        [        U5      U l        [        R
                  " [        R                  " SUR                  /UR                  Q76 5      U l
        [        R
                  " [        R                  " SUR                  UR                  S   UR                  S   5      5      U l        [        R                  " UR                  5      S-
  R!                  5       U l        [        R$                  " 5       U l        Sn[)        UR                  5       HA  u  p4[+        U5       H-  n[-        XXRS9nU R&                  R/                  U5        US-  nM/     MC     g )Nr,   r   )r   rH  rI  rJ  )r   rL   r   patch_embedr   	Parameterr{   zerosr   +window_positional_embedding_background_sizerz  rQ  r{  r   cumsumblocks_per_stager   
stage_endsr   blocksr   r  rG  r   )rM   r   rJ  rH  r  rI  blockr   s          rB   rL   Sam2HieraDetModel.__init__  s    .v6KK6--c0b0bc
 !#KK6--v/K/KA/NPVPlPlmnPop!
 99V%<%<=AIIKmmo+4V5L5L+M'I"#34	+!) ""5)1$ 5 ,NrA   c                     U R                   $ rK   )r  rR   s    rB   get_input_embeddings&Sam2HieraDetModel.get_input_embeddings  s    rA   hwr]   c           	      ,   Uu  p#U R                   n[        R                  " U R                  X#4SS9nXTR	                  [        UR                  UR                  5       VVs/ s H	  u  pgXg-  PM     snn5      -   nUR                  SSSS5      nU$ s  snnf )Nbicubic)rn   r   r   r   r   r,   )r{  r   r   rz  tileziprs   r   )rM   r  hwwindow_embedrz  r  ys           rB   _get_pos_embed Sam2HieraDetModel._get_pos_embed  s    ,,MM$..vIN	 1 1c)//[g[m[mFn2oFnda16Fn2o pp	%%aAq1	 3ps    BrI   c                    Uc  [        S5      eU R                  U5      nX0R                  UR                  SS 5      -   nSn[	        U R
                  5       H$  u  pVU" U40 UD6nXPR                  ;   d  M  XC4-   nM&     [        UUS9$ )N You have to specify pixel_valuesr,   r   r6   )r   rg  )r   r  r  rs   r   r  r  rf  )rM   r`   rI   r   rg  r   block_modules          rB   r   Sam2HieraDetModel.forward  s     ?@@((6%(;(;M<O<OPQRS<T(UU%'"(5OA(A&AMOO#-GJZ-Z*	  6 '+'A
 	
rA   )r  r  rz  r{  r  rK   )r7   r8   r9   r:   r.   r  r  rG  r  _can_record_outputsrL   r  r   r>   r{   r   r  r   r   r   r   r   r   rf  r   r@   r   r   s   @rB   ry  ry    s    %L$O,-
%1 %, sCx U\\   59
u001
 +,
 
u--	.	
 
rA   ry  zJ
    The vision model from Sam without any head or projection on top.
    c            
          ^  \ rS rSr\rSr\\S.r	S\4U 4S jjr
S r\" 5        SS\\R                     S\\   S\\\4   4S	 jj5       rS
rU =r$ )Sam2VisionModeli  r`   r  r   c                    > [         TU ]  U5        Xl        [        R                  " UR
                  5      U l        [        U5      U l        UR                  U l	        U R                  5         g rK   )r   rL   r   r   from_configbackbone_configbackboner   necknum_feature_levels	post_initrM   r   r   s     rB   rL   Sam2VisionModel.__init__  sQ     !--f.D.DE"6*	"(";";rA   c                 6    U R                   R                  5       $ rK   )r  r  rR   s    rB   r  $Sam2VisionModel.get_input_embeddings
  s    }}1133rA   rI   r]   c                    Uc  [        S5      eU R                  " U40 UD6nUR                  nUR                  nU R	                  U5      u  pgX`R
                  * S  S S S2   nXpR
                  * S  S S S2   n[        UUUS9$ )Nr  r  )r   r   r   )r   r  r   rg  r  r  r   )rM   r`   rI   backbone_outputr   rg  r   r   s           rB   r   Sam2VisionModel.forward  s     ?@@ --??'99%4%O%O"3799=W3X0-/F/F.F.HI$B$O 57N7N6N6P QRVTVRV W&+/"7
 	
rA   )r  r   r  r  rK   )r7   r8   r9   r:   r1   r  r  rG  r  r  rL   r  r   r   r{   r   r   r   r   r   r   r   r@   r   r   s   @rB   r  r    s     $L$O,-
	/ 	4  59
u001
 +,
 
u--	.	
 
rA   r  c                   :   ^  \ rS rSrS\4U 4S jjrSS jrSrU =r$ )Sam2PositionalEmbeddingi'  r   c                    > [         TU ]  5         UR                  U l        U R                  [        R                  " SUR
                  S-  45      -  nU R                  SU5        g )Nr   positional_embedding)r   rL   r  r{   randnr   register_buffer)rM   r   r  r   s      rB   rL    Sam2PositionalEmbedding.__init__(  sT    \\
#zzEKKF<N<NRS<S8T,UU35IJrA   c                    UR                  5       nUbD  USS2SS2SS2S4   US   -  USS2SS2SS2S4'   USS2SS2SS2S4   US   -  USS2SS2SS2S4'   UR                  [        R                  5        SU-  S-
  nUR                  U R                  R
                  5      nX0R                  -  nS[        R                  -  U-  n[        R                  " [        R                  " U5      [        R                  " U5      /SS9$ )z8Positionally encode points that are normalized to [0,1].Nr   r,   r   r  r   )clonerz   r{   r	  r  r  r   picatsincos)rM   input_coordsinput_shapecoordinatess       rB   r   Sam2PositionalEmbedding.forward.  s    "((*"&1!Q1*&=A&NK1a
#&1!Q1*&=A&NK1a
#u}}% +o)!nnT%>%>%D%DE!$=$=="%%i+-yy%))K0%))K2HIrRRrA   )r  rK   )	r7   r8   r9   r:   r0   rL   r   r@   r   r   s   @rB   r  r  '  s    K6 KS SrA   r  c                       \ rS rSrSrg)Sam2MaskEmbeddingi@  r6   Nr   r6   rA   rB   r  r  @  r   rA   r  c                       \ rS rSrS\4S jrS\R                  S\R                  S\S\R                  4S jr	S	\R                  S\R                  4S
 jr
Srg)Sam2PromptEncoderiD  r   c                    [         R                  R                  U 5        [        U5      U l        [        U5      U l        [         R                  " SUR                  5      U l	        UR                  UR                  -  UR                  UR                  -  4U l        SUR                  -  UR                  -  SUR                  -  UR                  -  4U l        UR                  U l        [         R                  " UR                  UR                  5      U l        UR                  U l        [         R                  " SUR                  5      U l        g )Nr,      )r   ModulerL   r  shared_embeddingr  
mask_embedru  r   no_mask_embed
image_size
patch_sizeimage_embedding_sizemask_input_sizeinput_image_sizenum_point_embeddingspoint_embednot_a_point_embedrM   r   s     rB   rL   Sam2PromptEncoder.__init__E  s   
		4  7 ?+F3\\!V-?-?@%+%6%6&:K:K%KVM^M^bhbsbsMs$t! !F$5$5 59J9J JAPVPaPaLaekevevLvw & 1 1<<(C(CVEWEWX!--!#a1C1C!DrA   pointsro   padr]   c                 J   US-   nU(       aT  [         R                  R                  R                  USSSS9n[         R                  R                  R                  USSSS9nU R                  U R                  4nU R                  X5      n[         R                  " US   S:H  U R                  R                  U5      n[         R                  " US   S	:g  U[         R                  " U5      5      nXPR                  UR                  SS
95      US:  R                  S5      -  -   nU$ )zEmbeds point prompts.      ?r   r   r   r,   constantr   r   r/  )r   r,   r  ).Ni)min)r{   r   r  r  r  r  r   r  rq  
zeros_liker  r   	unsqueeze)rM   r  ro   r  r  point_embeddings         rB   _embed_pointsSam2PromptEncoder._embed_pointsS  s   #XX((,,V\
Z[,\FXX((,,VV*TV,WF,,d.C.CD//D  ++fY&72&=t?U?U?\?\^mn  ++9$_-
 *,<,<V\\a\=P,QU[_`U`TkTklnTo,oorA   boxesc                 .   US-  nUR                   " / UR                  SS QSPSP76 n[        R                  R                  R                  USSSS9nU R                  X R                  U R                  45      nUSS2SS2SSS24==   U R                  R                  S   -  ss'   USS2SS2SSS24==   U R                  R                  S	   -  ss'   U R                  R                  R                  USS2SS2SSS24   5      USS2SS2SSS24'   U$ )
zEmbeds box prompts.r  Nr   r  r  r   r  r,   r   )viewrs   r{   r   r  r  r  r  r  rq  r  	expand_as)rM   r  coordscorner_embeddings       rB   _embed_boxesSam2PromptEncoder._embed_boxesl  s   3U[[!_3a33$$((JVW(X00:O:OQUQfQf9ghAq!$(8(8(?(?(BB$Aq!$(8(8(?(?(BB$'+'='='D'D'N'NO_`acdfgij`jOk'lAq!$rA   )	r   r  r  r  r  r  r  r  r  N)r7   r8   r9   r:   r0   rL   r{   r   r   r  r  r@   r6   rA   rB   r  r  D  s\    E6 EELL %,, T V[VbVb 2
 %,, 
 5<< 
 rA   r  c                      ^  \ rS rSrSrSU 4S jjr SS\R                  S\R                  S\R                  S\\R                     S\	\
   S	\\R                  \R                  4   4S
 jjrSrU =r$ )Sam2Attentioniy  z}
SAM2's attention layer that allows for downscaling the size of the embedding after projection to queries, keys, and
values.
c                   > [         TU ]  5         Uc  UR                  OUnXl        UR                  U l        UR                  U-  U l        UR                  U l        U R
                  UR                  -  U l        U R                  S-  U l        SU l	        [        R                  " U R                  U R
                  5      U l        [        R                  " U R                  U R
                  5      U l        [        R                  " U R                  U R
                  5      U l        [        R                  " U R
                  U R                  5      U l        g )Nr  F)r   rL   attention_downsample_rater   r   internal_dimr  r"  r'  r!  r   r  q_projk_projv_projo_proj)rM   r   downsample_rater   s      rB   rL   Sam2Attention.__init__  s    >M>U&::[j!--"../A#)#=#= ))V-G-GG}}d*ii 0 0$2C2CDii 0 0$2C2CDii 0 0$2C2CDii 1 143C3CDrA   r-  r.  r/  attention_similarityrI   r]   c                    UR                   S S u  pgXg-  SU R                  U R                  4nU R                  U5      R                  " U6 R                  SS5      nU R                  U5      R                  " U6 R                  SS5      nU R                  U5      R                  " U6 R                  SS5      n[        n	U R                  R                  S:w  a  [        U R                  R                     n	U	" U UUU4USU R                  U R                  S.UD6u  pU
R                  XgSU R                  U R                  -  5      R                  5       n
U R!                  U
5      n
X4$ )Nr   r  r,   r%  r   )r&  dropoutr'  r!  )rs   r  r"  r  r  r*  r  r  r)   r   r,  r   r'  r!  r(  
contiguousr  )rM   r-  r.  r/  r  rI   r   point_batch_size	new_shaper1  r2  r0  s               rB   r   Sam2Attention.forward  s\    (-{{2A$
2B8P8PRVR_R_`	E"''3==aCkk###Y/99!Q?E"''3==aC(?;;++w6"9$++:Z:Z"[$7	
%

 0LLnn
%
 
%
! "))"d.F.F.V

*, 	 kk+.((rA   )r   r"  r   r  r!  r  r  r  r  r'  r  rK   )r7   r8   r9   r:   r;   rL   r{   r   r   r   r   r   r   r@   r   r   s   @rB   r  r  y  s    
E* 8<%)||%) \\%) ||	%)
 'u||4%) +,%) 
u||U\\)	*%) %)rA   r  c                   *    \ rS rSrSS\S\4S jjrSrg)Sam2TwoWayAttentionBlocki  r   skip_first_layer_pec                 F   [         R                  R                  U 5        [        USS9U l        [         R
                  " UR                  5      U l        [        U5      U l        [         R
                  " UR                  5      U l	        [        UR                  UR                  UR                  UR                  S9U l        [         R
                  " UR                  5      U l        [         R
                  " UR                  5      U l        [        U5      U l        X l        g )Nr,   )r  )r9  )r   r  rL   r  	self_attnrN  r   rP  cross_attn_token_to_imagerW  r5  mlp_dimnum_hidden_layersrZ  layer_norm3layer_norm4cross_attn_image_to_tokenr  )rM   r   r  s      rB   rL   !Sam2TwoWayAttentionBlock.__init__  s    
		4 &vqA<<(:(:;)6v)>&<<(:(:;"0B0BvOgOg
 <<(:(:;<<(:(:;)6v)>&#6 rA   )	r  r	  rP  rW  r  r  rZ  r  r  N)F)r7   r8   r9   r:   r/   r   rL   r@   r6   rA   rB   r  r    s    74 74 7 7rA   r  c                       \ rS rSrSrg)Sam2TwoWayTransformeri  r6   Nr   r6   rA   rB   r  r    r   rA   r  c                       \ rS rSrSrg)rw  i  r6   Nr   r6   rA   rB   rw  rw    r   rA   rw  c                   x  ^  \ rS rSrS\4U 4S jjrS rS r  SS\R                  S\R                  S\R                  S	\R                  S
\
S\\R                     S\\R                     S\\R                     S\\   S\\R                  \R                  \R                  \R                  4   4S jjrSrU =r$ )Sam2MaskDecoderi  r   c                 h  > [         TU ]  U5        U ?[        U R                  UR
                  U R                  UR                  SS9U l        [        R                  " UR                  UR                  S-  SSS9U l
        [        R                  " UR                  UR                  S-  SSS9U l        [        R                  " SU R                  5      U l        [        U R                  U R                  SS5      U l        UR                  U l        UR                   U l        UR"                  U l        g )NT)r;     r,   )r   r   r  r   )r   rL   iou_prediction_headr5  r   iou_head_hidden_dimnum_mask_tokensiou_head_depthr   r   conv_s0conv_s1ru  obj_score_tokenpred_obj_score_headdynamic_multimask_via_stability!dynamic_multimask_stability_delta"dynamic_multimask_stability_threshr  s     rB   rL   Sam2MaskDecoder.__init__  s    $#2&&  !!$
  yy!3!3V5G5G15LZ[defyy!3!3V5G5G15LZ[def!||At/?/?@#243C3CTEUEUWXZ[#\ /5/U/U,171Y1Y.282[2[/rA   c                    UR                  S5      nU R                  n[        R                  " X:  SS9R	                  5       n[        R                  " X* :  SS9R	                  5       n[        R
                  " US:  X4-  S5      nU$ )zb
Compute stability scores of the mask logits based on the IoU between upper and
lower thresholds.
rg   r  r  r   rn  )flattenr   r{   sumr   r   )rM   mask_logitsstability_deltaarea_iarea_ustability_scoress         rB   _get_stability_scores%Sam2MaskDecoder._get_stability_scores  sy    
 "))"-@@;8bAGGI;)99rBHHJ ;;vz6?CHrA   c           	         USS2SS2SS2SS2SS24   nUSS2SS2SS24   n[         R                  " USS9nUR                  S5      R                  S5      R                  S5      nUR                  SSSUR	                  S5      UR	                  S5      5      n[         R
                  " USU5      n[         R
                  " USUR                  S5      5      nUSS2SS2SS2SS2SS24   n	USS2SS2SS24   n
U R                  U	5      nXR                  :  n[         R                  " US   R                  U	5      U	U5      n[         R                  " UR                  U
5      U
U5      nX4$ )	aK  
When outputting a single mask, if the stability score from the current single-mask
output (based on output token 0) falls below a threshold, we instead select from
multi-mask outputs (based on output token 1~3) the mask with the highest predicted
IoU score. This is intended to ensure a valid mask for both clicking and tracking.
Nr,   r  r  rg   r   r   ).NN)
r{   r   r  expandrn   gatherr+  r!  r   r  )rM   all_mask_logitsall_iou_scoresmultimask_logitsmultimask_iou_scoresbest_scores_indsbest_scores_inds_expandedbest_multimask_logitsbest_multimask_iou_scoressinglemask_logitssinglemask_iou_scoresr*  	is_stablemask_logits_outiou_scores_outs                  rB    _dynamic_multimask_via_stability0Sam2MaskDecoder._dynamic_multimask_via_stability  s~    +1aQ>:-aABh7 <<(<"E$4$>$>r$B$L$LR$P$Z$Z[]$^!$=$D$DA',,R02B2G2G2K%
! !&-=qB[ \$)LL1EqJZJdJdegJh$i! ,Aq!A#q!O< .q!QqSy 9556GH$(O(OO	  ++o&001BC!

  56!%

 ..rA   r   image_positional_embeddingssparse_prompt_embeddingsdense_prompt_embeddingsmultimask_outputhigh_resolution_featuresr  target_embeddingrI   r]   c	           
         UR                   u  ppUR                   S   n[        R                  " U R                  R                  U R
                  R                  U R                  R                  /SS9nUR                  XSS5      nUR                   S   S:w  a  [        R                  " X4SS9nOUnUR                  U R
                  R                  R                  5      nX-   nUR                  USS9nUR                  US5      nU R                  " S	UUUUUS.U	D6u  nnUSS2SS2SSS24   nUSS2SS2SSU R                  -   2SS24   nUR                  SS5      R                  X-  XU5      nUu  nnUR                  USS9nUR                  USS9nU R                  U5      U-   nU R!                  U R#                  U5      5      nU R!                  U R%                  U5      U-   5      n/ n['        U R                  5       H+  nU R(                  U   nUU" USS2SS2USS24   5      /-  nM-     [        R*                  " USS9nUR                   u  npnUR                  XXU-  5      nUU-  R                  XSX5      nU R-                  U5      nU R/                  USS2SS2SSS24   5      nU(       a+  [1        SS5      nUSS2SS2USS2SS24   nUSS2SS2U4   nOnU R2                  (       a3  U R4                  (       d"  [1        SS5      nU R7                  UU5      u  nnO*[1        SS5      nUSS2SS2USS2SS24   nUSS2SS2U4   nUSS2SS2U4   n UUU U4$ )
aC  
Predict masks given image and prompt embeddings.

Args:
    image_embeddings (`torch.Tensor`):
        The embeddings from the image encoder.
    image_positional_embeddings (`torch.Tensor`):
        Positional encoding with the shape of image_embeddings.
    sparse_prompt_embeddings (`torch.Tensor`):
        The embeddings of the points and boxes.
    dense_prompt_embeddings (`torch.Tensor`):
        The embeddings of the mask inputs.
    multimask_output (`bool`):
        Whether to return multiple masks or a single mask.
    high_resolution_features (`list[torch.Tensor]`, *optional*):
        The high-resolution features from the vision encoder.
    attention_similarity (`torch.Tensor`, *optional*):
        The attention similarity tensor.
    target_embedding (`torch.Tensor`, *optional*):
        The target embedding.
r,   r   r  r   )point_embeddingsr   r?  r  rD  Nr   r  r6   )rs   r{   r  r  rq  	iou_tokenmask_tokensrepeatrz   r  repeat_interleavetransformerr  r*  r  upscale_conv1r:  upscale_layer_normupscale_conv2r  output_hypernetworks_mlpsstackr  r  slicer  trainingr=  )!rM   r   r?  r@  rA  rB  rC  r  rD  rI   r   r   rF   rG   r  output_tokenstokensrF  iou_token_outmask_tokens_outfeat_s0feat_s1upscaled_embeddinghyper_in_listr   current_mlphyper_inr   r   iou_predr   
mask_slicesam_tokens_outs!                                    rB   r   Sam2MaskDecoder.forward  s   B 3C2H2H/
&399!<		$$++%%  ''
 
 &,,Z1aP#))!,1YYHaPF"F!99T^^%:%:%@%@A ,E+==>NTU=V&A&S&STdfg&h#-1-=-= .
--(C!5-.
 .
** )Aq!4*1aa$:N:N6N1OQR+RS ,55a;@@)<
 4++,<!+D++,<!+D!//0@AGK!__T-D-DEW-XY!__T-?-?@R-SV]-]^,.t++,A88;Kk/!Q1**EFGGM - ;;}!4);)A)A&</44ZS_joaop..44ZSUW]e ++M:"667G1aQR
7ST q$J!Q
Aq01E1j 01H11$--q!J"CCE8TOE8q!J!Q
Aq01E1j 01H(Az)9:h0CCCrA   )r  r  r   r!  r  r  r  r  )NN)r7   r8   r9   r:   r/   rL   r+  r=  r{   r   r   r   r   r   r   r   r   r@   r   r   s   @rB   r  r    s    \4 \*
 #/Z 8<37mD,,mD &+\\mD #(,,	mD
 "'mD mD #'u||"4mD 'u||4mD #5<<0mD +,mD 
u||U\\5<<E	FmD mDrA   r  z
    Segment Anything Model 2 (SAM 2) for generating segmentation masks, given an input image and
    input points and labels, boxes, or masks.
    c                      \ rS rSr/ SQrS\4S jrS\R                  4S jr	\R                  " 5       S\R                  S\\   S\\R                     4S	 j5       rS\R                  S\\   S\\\R                     \\R                     \\\R                  S
4      \\\R                  S
4      4   4S jr\" 5       \         SS\\R                     S\\R                     S\\R*                     S\\R                     S\\R*                     S\\R                     S\S\\R                     S\\R                     S\\   S\4S jj5       5       rSrg)r|  i  )z
^memory_.*z^mask_downsample.*z^object_pointer_proj.*z0^temporal_positional_encoding_projection_layer.*no_memory_positional_encodingno_object_pointer%occlusion_spatial_embedding_parameterr   c                    [         R                  " X5        [        UR                  5      U l        [
        R                  " UR                  5      U l        [        UR                  5      U l
        UR                  UR                  l        [        UR                  5      U l        UR                  R                  U l        UR                  R                   U l        UR                  R"                  U l        [&        R(                  R+                  [&        R,                  " SSU R$                  5      5      U l        U R1                  5         g )Nr,   )r   rL   r  prompt_encoder_configshared_image_embeddingr   r  vision_configvision_encoderr  prompt_encoderr,  mask_decoder_configr  mask_decoderr  backbone_feature_sizesr   r7  r{   r   r  r  r}  r  r  s     rB   rL   Sam2Model.__init__  s      .&=f>Z>Z&[#'33F4H4HI/0L0LM:@:U:U""7+F,F,FG"("6"6"I"I&,&:&:&Q&Q# ..>>#(88#5#5ekk!Q6X#Y rA   r]   c                    U R                   R                  nU R                  R                  R                  nU R                  R                  R
                  n[        R                  " XUS9nUR                  SS9S-
  nUR                  SS9S-
  nXQS   -  nXaS   -  nU R                  [        R                  " Xe/SS95      nUR                  SSS5      R                  S5      $ )N)re   r  r   r  r  r,   r  r   )rj  r  rg  r  re   r  r{   onesr  rP  r   r  )rM   rn   target_devicetarget_dtypegridy_embedx_embedr  s           rB   $get_image_wide_positional_embeddings.Sam2Model.get_image_wide_positional_embeddings  s    ""7733HHOO22GGMMzz$LI++!+$s*++!+$s*G#G##::5;;GY_a;bc#++Aq!4>>qAArA   r`   rI   c           	      "   UR                   S   nU R                  " U40 UD6u  n    nUS   U R                  -   US'   [        X@R                  5       VVs/ s H*  u  pgUR                  SSS5      R                  " US/UQ76 PM,     nnnU$ s  snnf )z
Returns the image embeddings by passing the pixel values through the vision encoder.

Args:
    pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
        Input pixel values
r   r  r,   r   )rs   get_image_featuresr}  r  rm  r   r  )	rM   r`   rI   r   feature_mapsr   feat	feat_sizer   s	            rB   get_image_embeddingsSam2Model.get_image_embeddings  s     "''*
 $ 7 7 O OaA (+d.F.FFR
 $'|5P5P#Q
#Q LLAq!&&z2B	B#Q 	 

  
s   1B.c                    U R                   " U40 UD6nUR                  nUR                  n[        U5      nU R                  R                  US   5      US'   U R                  R                  US   5      US'   U Vs/ s H$  ofR                  S5      R                  SSS5      PM&     nnU Vs/ s H%  nUR                  S5      R                  SSS5      PM'     nnXEUR                  UR                  4$ s  snf s  snf )a  
Extract and preprocess image features using the vision encoder.

Args:
    pixel_values (`torch.FloatTensor`):
        Input pixel values of shape `(batch_size, num_channels, height, width)`.

Returns:
    `tuple`: A tuple containing:
        - feature_maps (`list[torch.Tensor]`): List of feature maps from different levels.
        - feature_maps_position_embeddings (`list[torch.Tensor]`): List of positional embeddings for each feature level.
        - vision_hidden_states (`tuple[torch.FloatTensor]`, *optional*): Hidden states from the vision encoder.
        - vision_attentions (`tuple[torch.FloatTensor]`, *optional*): Attention weights from the vision encoder.
r   r,   r   )ri  r   r   r   rl  r  r  r$  r   r   r   )rM   r`   rI   vision_outputsrz   feature_maps_position_embeddingsfeature_mapfeature_map_position_embeddings           rB   ry  Sam2Model.get_image_features  s   0 372E2E3
3

 &77+9+O+O( L)++33LODQ++33LODQ T``S_K++A.66q!Q?S_` 3S,
2R. +2215==aAF2R 	) ,

 ~?[?[]k]v]vvv a,
s   =+C5.,C:Ninput_pointsinput_labelsinput_boxesinput_masksr   rB  r  rD  c
                 \   USL USL -  (       d  [        S5      eUbO  UbL  UR                  S   UR                  S   :w  a,  [        SUR                  S    SUR                  S    S35      eU R                  5       nUb  UR                  S   OUS   R                  S   nUR                  USSS5      nSnSnUby  U R                  " U40 U
D6u  nnpUS   U R
                  -   US'   [        XR                  5       VVs/ s H+  u  nnUR                  SS	S5      R                  " US/UQ76 PM-     nnnUb>  Uc;  [        R                  " USS2SS2SS2S4   [        R                  UR                  S
9nUcj  Ucg  [        R                  " USSS	US   R                  US   R                  S
9n[        R                   " USS[        R"                  US   R                  S
9* nUbz  UR                  SS U R$                  R&                  :w  aS  [(        R*                  " UR-                  5       U R$                  R&                  SSSS9R/                  UR                  5      nU R%                  UUUUS9u  nnU R0                  " SUS   UUUUUSS UU	S.U
D6u  nnnn[3        UUUUUUS9$ s  snnf )a  
input_points (`torch.FloatTensor` of shape `(batch_size, num_points, 2)`):
    Input 2D spatial points, this is used by the prompt encoder to encode the prompt. Generally yields to much
    better results. The points can be obtained by passing a list of list of list to the processor that will
    create corresponding `torch` tensors of dimension 4. The first dimension is the image batch size, the
    second dimension is the point batch size (i.e. how many segmentation masks do we want the model to predict
    per input point), the third dimension is the number of points per segmentation mask (it is possible to pass
    multiple points for a single mask), and the last dimension is the x (vertical) and y (horizontal)
    coordinates of the point. If a different number of points is passed either for each image, or for each
    mask, the processor will create "PAD" points that will correspond to the (0, 0) coordinate, and the
    computation of the embedding will be skipped for these points using the labels.
input_labels (`torch.LongTensor` of shape `(batch_size, point_batch_size, num_points)`):
    Input labels for the points, this is used by the prompt encoder to encode the prompt. According to the
    official implementation, there are 3 types of labels

    - `1`: the point is a point that contains the object of interest
    - `0`: the point is a point that does not contain the object of interest
    - `-1`: the point corresponds to the background

    We added the label:

    - `-10`: the point is a padding point, thus should be ignored by the prompt encoder

    The padding labels should be automatically done by the processor.
input_boxes (`torch.FloatTensor` of shape `(batch_size, num_boxes, 4)`):
    Input boxes for the points, this is used by the prompt encoder to encode the prompt. Generally yields to
    much better generated masks. The boxes can be obtained by passing a list of list of list to the processor,
    that will generate a `torch` tensor, with each dimension corresponding respectively to the image batch
    size, the number of boxes per image and the coordinates of the top left and bottom right point of the box.
    In the order (`x1`, `y1`, `x2`, `y2`):

    - `x1`: the x coordinate of the top left point of the input box
    - `y1`: the y coordinate of the top left point of the input box
    - `x2`: the x coordinate of the bottom right point of the input box
    - `y2`: the y coordinate of the bottom right point of the input box
input_masks (`torch.FloatTensor` of shape `(batch_size, image_size, image_size)`):
    SAM model also accepts segmentation masks as input. The mask will be embedded by the prompt encoder to
    generate a corresponding embedding, that will be fed later on to the mask decoder. These masks needs to be
    manually fed by the user, and they need to be of shape (`batch_size`, `image_size`, `image_size`).
image_embeddings (`torch.FloatTensor` of shape `(batch_size, output_channels, window_size, window_size)`):
    Image embeddings, this is used by the mask decoder to generate masks and iou scores. For more memory
    efficient computation, users can first retrieve the image embeddings using the `get_image_embeddings`
    method, and then feed them to the `forward` method instead of feeding the `pixel_values`.
multimask_output (`bool`, *optional*):
    In the original implementation and paper, the model always outputs 3 masks per image (or per point / per
    bounding box if relevant). However, it is possible to just output a single mask, that corresponds to the
    "best" mask, by specifying `multimask_output=False`.
attention_similarity (`torch.FloatTensor`, *optional*):
    Attention similarity tensor, to be provided to the mask decoder for target-guided attention in case the
    model is used for personalization as introduced in [PerSAM](https://huggingface.co/papers/2305.03048).
target_embedding (`torch.FloatTensor`, *optional*):
    Embedding of the target concept, to be provided to the mask decoder for target-semantic prompting in case
    the model is used for personalization as introduced in [PerSAM](https://huggingface.co/papers/2305.03048).

Example:

```python
>>> from PIL import Image
>>> import requests
>>> from transformers import AutoModel, AutoProcessor

>>> model = AutoModel.from_pretrained("danelcsb/sam2.1_hiera_tiny")
>>> processor = AutoProcessor.from_pretrained("danelcsb/sam2.1_hiera_tiny")

>>> img_url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/model_doc/sam-car.png"
>>> raw_image = Image.open(requests.get(img_url, stream=True).raw).convert("RGB")
>>> input_points = [[[400, 650]]]  # 2D location of a window on the car
>>> inputs = processor(images=raw_image, input_points=input_points, return_tensors="pt")

>>> # Get segmentation mask
>>> outputs = model(**inputs)

>>> # Postprocess masks
>>> masks = processor.post_process_masks(
...     outputs.pred_masks, inputs["original_sizes"], inputs["reshaped_input_sizes"]
... )
```
NzAExactly one of pixel_values or image_embeddings must be provided.r,   zGYou should provide as many bounding boxes as input points per box. Got z and .r   r  r   )r  re   rg   Fr   T)rn   r   r   r  )r  r  r  r  )r   r?  r@  rA  rB  rC  r  rD  )r   r   r   r   r   r   r6   )r   rs   rv  rI  ry  r}  r  rm  r   r  r{   	ones_liker>   re   r  r  rp  int32rj  r  r   r   r   rz   rl  r   )rM   r`   r  r  r  r  r   rB  r  rD  rI   r?  r   r   r   rz  r   r{  r|  sparse_embeddingsdense_embeddingslow_res_multimasksr   r   s                           rB   r   Sam2Model.forward  sZ   z %*:d*BC`aa#(?!!!$(9(9!(<< ]^j^p^pqr^s]tty  {F  {L  {L  MN  {O  zP  PQ  R  '+&O&O&Q#.:.F\''*L\]_L`LfLfghLi
&A&H&HUVXY[\&]# ##GKG^G^HHDL!1  ,B/$2J2JJL
 (+<9T9T'U 'UOD) Q1%**:rFIF'U   
 #(< ??<1a
+C599]i]p]pqLK$7 ;;Aq!+;B+?+E+EN^_aNbNiNiL "JJz1au{{ScdfSgSnSnooL"   %)<)<)L)LLmm%%',,<<"'#" "[&&'  /3.A.A%%##	 /B /
++ BFARAR 
B
-b1(C%6$4-%5cr%:!5-
B
 
B
>J+> +!) 3-!5/
 	
[ s   ?2J()rm  r7  rl  r}  r  rj  rg  ri  )	NNNNNNTNN)r7   r8   r9   r:   "_keys_to_ignore_on_load_unexpectedr-   rL   r{   r   rv  no_gradr   r   r   r   r}  r   r   ry  r   r   
LongTensorr   r   r   r@   r6   rA   rB   r|  r|    s
   *&z "Bell B ]]_ ''  +,  
ell		   4-w''-w +,-w 
U\\U\\u((#-./u((#-./	1
	-w^  59483737268<!%<@8<i
u001i
 u001i
 u//0	i

 e//0i
 e../i
 #5#4#45i
 i
 'u'8'89i
 #5#4#45i
 +,i
 
%i
  i
rA   r|  )r|  r  ri  rD   ry  rK   )er;   dataclassesr   typingr   r   r   numpyr   r{   torch.nnr   torch.nn.functionalr  r   activationsr   image_processing_utilsr	   r
   image_processing_utils_fastr   r   image_utilsr   r   r   r   r   r   r   modeling_layersr   modeling_utilsr   r   processing_utilsr   utilsr   r   r   r   utils.genericr   r   autor   maskformer.modeling_maskformerr    sam.image_processing_sam_fastr!   sam.modeling_samr"   r#   r$   r%   r&   r'   r(   r)   vitdet.modeling_vitdetr*   r+   configuration_sam2r-   r.   r/   r0   r1   
get_loggerr7   loggerr3   rD   r   r   r  r   r   r   r   r>   r  r  r5  rG  rf  ri  ry  r  r  r  r  r  r  r  rw  r  r|  __all__r6   rA   rB   <module>r     s    ! , ,      ! A b   : F &  D  L A	 	 	 J  
		H	%(#B ( J2 J JZ KL?k ? M ?6 FGL+ L H L@")) B	 ? 	/8RYY /8du|| 8C= ELL ?bii ?Dbii <Z4 Zz 
	Ok 	O 	O 8/ 8 8>?
+ ?
D 
-
) -

-
`Sbii S2	( 	2 ( 2 j;)BII ;)|768R 7(	0 		L 	tDn tDn ^
 ^
^
B	rA   