
    cCiF                     8   S SK Jr  S SKJrJrJr  S SKrS SKrS SKJ	r	  S SK
Jr  SSKJr  SSKJr  SS	KJrJr  SS
KJr  SSKJrJrJr  SSKJr  SSKJr  SSKJr  SSKJ r   \\" SS9 " S S\5      5       5       r! " S S\	RD                  5      r#S r$SAS jr%S\RL                  S\'S\RL                  4S jr( SBS\	RD                  S\RL                  S \RL                  S!\RL                  S"\\RL                     S#\)S$\)S%\\   4S& jjr* " S' S(\	RD                  5      r+ " S) S*\	RD                  5      r, " S+ S,\	RD                  5      r-S-\RL                  S.\RL                  S/\RL                  S\RL                  4S0 jr. " S1 S2\	RD                  5      r/ " S3 S4\	RD                  5      r0\ " S5 S6\5      5       r1S7\RL                  S8\)S\2\RL                  \RL                  4   4S9 jr3S:\RL                  S;\'S<\'S\RL                  4S= jr4\" S>S9 " S? S@\15      5       r5S6S@/r6g)C    )	dataclass)CallableOptionalUnionN)nnpad_sequence   )ACT2FN)FlashAttentionKwargs)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)ModelOutputTransformersKwargsauto_docstring)deprecate_kwarg)can_return_tuple   )AutoModelForKeypointDetection   )LightGlueConfiga  
    Base class for outputs of LightGlue keypoint matching models. Due to the nature of keypoint detection and matching,
    the number of keypoints is not fixed and can vary from image to image, which makes batching non-trivial. In the
    batch of images, the maximum number of matches is set as the dimension of the matches and matching scores. The mask
    tensor is used to indicate which values in the keypoints, matches, matching_scores and prune tensors are keypoint
    matching information.
    )custom_introc                   f   \ rS rSr% SrSr\\R                     \	S'   Sr
\\R                     \	S'   Sr\\R                     \	S'   Sr\\R                     \	S'   Sr\\R                     \	S'   Sr\\R                     \	S	'   Sr\\\R                        \	S
'   Sr\\\R                        \	S'   Srg)LightGlueKeypointMatchingOutput'   aW  
loss (`torch.FloatTensor` of shape `(1,)`, *optional*):
    Loss computed during training.
matches (`torch.FloatTensor` of shape `(batch_size, 2, num_matches)`):
    Index of keypoint matched in the other image.
matching_scores (`torch.FloatTensor` of shape `(batch_size, 2, num_matches)`):
    Scores of predicted matches.
keypoints (`torch.FloatTensor` of shape `(batch_size, num_keypoints, 2)`):
    Absolute (x, y) coordinates of predicted keypoints in a given image.
prune (`torch.IntTensor` of shape `(batch_size, num_keypoints)`):
    Pruning mask indicating which keypoints are removed and at which layer.
mask (`torch.BoolTensor` of shape `(batch_size, num_keypoints)`):
    Mask indicating which values in matches, matching_scores, keypoints and prune are keypoint matching
    information.
hidden_states (`Tuple[torch.FloatTensor, ...]`, *optional*):
    Tuple of `torch.FloatTensor` (one for the output of each stage) of shape `(batch_size, 2, num_channels,
    num_keypoints)` returned when `output_hidden_states=True` is passed or when
    `config.output_hidden_states=True`
attentions (`Tuple[torch.FloatTensor, ...]`, *optional*):
    Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, 2, num_heads, num_keypoints,
    num_keypoints)` returned when `output_attentions=True` is passed or when
    `config.output_attentions=True`
Nlossmatchesmatching_scores	keypointsprunemaskhidden_states
attentions )__name__
__module____qualname____firstlineno____doc__r   r   torchFloatTensor__annotations__r   r   r    r!   	IntTensorr"   r#   tupler$   __static_attributes__r%       j/home/james-whalen/.local/lib/python3.13/site-packages/transformers/models/lightglue/modeling_lightglue.pyr   r   '   s    0 )-D(5$$
%,+/GXe''(/37OXe//07-1Ix))*1'+E8EOO$+(,D(5$$
%,8<M8E%"3"345<59Ju00129r1   r   c                      ^  \ rS rSrS\4U 4S jjr S	S\R                  S\\	   S\
\\R                     \\R                  \R                  4   4   4S jjrSrU =r$ )
LightGluePositionalEncoderT   configc                    > [         TU ]  5         [        R                  " SUR                  UR
                  -  S-  SS9U l        g )Nr   Fbias)super__init__r   Lineardescriptor_dimnum_attention_heads	projectorselfr6   	__class__s     r2   r;   #LightGluePositionalEncoder.__init__U   s:    1f&;&;v?Y?Y&Y]^&^ejkr1   r    output_hidden_statesreturnc                     U R                  U5      nUR                  SSS9n[        R                  " U5      n[        R                  " U5      nXV4nU(       a  XC4nU$ U4nU$ )Nr   dim)r?   repeat_interleaver+   cossin)rA   r    rD   projected_keypoints
embeddingscosinessinesoutputs           r2   forward"LightGluePositionalEncoder.forwardY   sl     #nnY7(::1":E
))J'		*%%
6J*2 R\P]r1   )r?   F)r&   r'   r(   r)   r   r;   r+   Tensorr   boolr   r/   rR   r0   __classcell__rB   s   @r2   r4   r4   T   sg    l l
 OT		=Ed^		uU\\"E%,,*D$EE	F	 	r1   r4   c                 |    U SS S S24   nU SSS S24   n[         R                  " U* U/SS9R                  S5      nU$ )N.r   r   rG   rH   )r+   stackflatten)xx1x2rot_xs       r2   rotate_halfra   e   sL    	
3!8B	
319BKK"b	r*2226ELr1   c                 &   U R                   nU R                  5       n UR                  5       nUR                  U5      nUR                  U5      nX-  [        U 5      U-  -   nX-  [        U5      U-  -   nUR	                  US9UR	                  US94$ )a  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    position_ids (`torch.Tensor`, *optional*):
        Deprecated and unused.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
dtype)rd   float	unsqueezera   to)	qkrK   rL   position_idsunsqueeze_dimrd   q_embedk_embeds	            r2   apply_rotary_pos_embrn   m   s    ( GGE		A		A
--
&C
--
&Cw;q>C/0Gw;q>C/0G::E:"GJJUJ$;;;r1   r#   n_reprE   c                     U R                   u  p#pEUS:X  a  U $ U SS2SS2SSS2SS24   R                  X#XU5      n U R                  X#U-  XE5      $ )z
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
r   N)shapeexpandreshape)r#   ro   batchnum_key_value_headsslenhead_dims         r2   	repeat_kvrx      s_    
 2?1D1D.Ez!!Qa"23::5W\dlmM  e(CTTTr1   modulequerykeyvalueattention_maskscalingdropoutkwargsc                 @   [        X R                  5      n[        X0R                  5      n	[        R                  " XR	                  SS5      5      U-  n
Ub"  US S 2S S 2S S 2S UR
                  S   24   nX-   n
[        R                  R                  U
S[        R                  S9R                  UR                  5      n
[        R                  R                  XU R                  S9n
[        R                  " X5      nUR	                  SS5      R                  5       nX4$ )Nr   r
   rZ   rG   )rI   rd   )ptrainingr   )rx   num_key_value_groupsr+   matmul	transposerq   r   
functionalsoftmaxfloat32rg   rd   r   r   
contiguous)ry   rz   r{   r|   r}   r~   r   r   
key_statesvalue_statesattn_weightscausal_maskattn_outputs                r2   eager_attention_forwardr      s     3 ; ;<JU$?$?@L<<';';Aq'ABWLL!$Q1.D
0@0@0D.D%DE#1==((2U]](SVVW\WbWbcL==((6??([L,,|:K''1-88:K$$r1   c                   X  ^  \ rS rSrSrS\S\4U 4S jjr\" SSSS	9    SS
\	R                  S\\\	R                  \	R                  4      S\\	R                     S\\	R                     S\\	R                     S\\   S\\	R                  \\	R                     4   4S jj5       rSrU =r$ )LightGlueAttention   z=Multi-headed attention from 'Attention Is All You Need' paperr6   	layer_idxc                 P  > [         TU ]  5         Xl        X l        [	        USUR
                  UR                  -  5      U l        UR                  UR                  -  U l	        U R                  S-  U l
        UR                  U l        SU l        [        R                  " UR
                  UR                  U R                  -  UR                  S9U l        [        R                  " UR
                  UR                  U R                  -  UR                  S9U l        [        R                  " UR
                  UR                  U R                  -  UR                  S9U l        [        R                  " UR                  U R                  -  UR
                  UR                  S9U l        g )Nrw   g      Tr8   )r:   r;   r6   r   getattrhidden_sizer>   rw   ru   r   r~   attention_dropout	is_causalr   r<   attention_biasq_projk_projv_projo_projrA   r6   r   rB   s      r2   r;   LightGlueAttention.__init__   sI   "
F4F4F&JdJd4de$*$>$>&B\B\$\!}}d*!'!9!9ii : :T]] JQWQfQf
 ii : :T]] JQWQfQf
 ii : :T]] JQWQfQf
 ii&&68J8JQWQfQf
r1   past_key_valuepast_key_valuesz4.58)new_nameversionr#   position_embeddingsr}   encoder_hidden_statesencoder_attention_maskr   rE   c                 $   UR                   S S n/ UQSPU R                  P7nU R                  U5      R                  U5      R	                  SS5      n	US Ln
U
(       a  UOUnU
(       a  UOUnU R                  U5      R                  U5      R	                  SS5      nU R                  U5      R                  U5      R	                  SS5      nUb  Uu  nn[        XUU5      u  p[        nU R                  R                  S:w  a  [        U R                  R                     nU" U U	UUU4U R                  (       d  SOU R                  U R                  S.UD6u  nnUR                  " / UQSP76 R!                  5       nU R#                  U5      nUU4$ )NrG   r   r   eager        )r   r~   )rq   rw   r   viewr   r   r   rn   r   r6   _attn_implementationr   r   r   r~   rs   r   r   )rA   r#   r   r}   r   r   r   input_shapehidden_shapequery_statesis_cross_attentioncurrent_statescurrent_attention_maskr   r   rK   rL   attention_interfacer   r   s                       r2   rR   LightGlueAttention.forward   s    $))#2.88b8$--8{{=166|DNNqRST2$>2D.-;M!7Sa[[055lCMMaQRS
{{>277EOOPQSTU**HC';LVY[^'_$L(?;;++w6"9$++:Z:Z"[$7"	%
  $}}C$2H2HLL	%
 	%
!\ "));;;;FFHkk+.L((r1   )r   r6   rw   r   r   r   r   r   r   r~   r   )NNNN)r&   r'   r(   r)   r*   r   intr;   r   r+   rU   r   r/   r   r   rR   r0   rW   rX   s   @r2   r   r      s    G
 
3 
. %0A6R LP158<9=*)||*) &eELL%,,,F&GH*) !.	*)
  (5*) !) 6*) -.*) 
u||Xell33	4*) S*)r1   r   c                   j   ^  \ rS rSrS\4U 4S jjrS\R                  S\R                  4S jrSr	U =r
$ )LightGlueMLP   r6   c                 f  > [         TU ]  5         Xl        [        UR                     U l        [        R                  " UR                  UR                  5      U l	        [        R                  " UR                  UR                  5      U l        [        R                  " UR                  SS9U l        g )NT)elementwise_affine)r:   r;   r6   r   
hidden_actactivation_fnr   r<   intermediate_sizefc1r   fc2	LayerNorm
layer_normr@   s     r2   r;   LightGlueMLP.__init__   s{    #F$5$5699V55v7O7OP99V55v7I7IJ,,v'?'?TXYr1   r#   rE   c                     U R                  U5      nU R                  U5      nU R                  U5      nU R                  U5      nU$ N)r   r   r   r   )rA   r#   s     r2   rR   LightGlueMLP.forward  sB    /6**=9/r1   )r   r6   r   r   r   r&   r'   r(   r)   r   r;   r+   rU   rR   r0   rW   rX   s   @r2   r   r      s1    Z ZU\\ ell  r1   r   c                     ^  \ rS rSrS\S\4U 4S jjr  SS\R                  S\R                  S\R                  S\	\
   S	\	\
   S
\\R                  \	\\R                        \	\\R                        4   4S jjrSrU =r$ )LightGlueTransformerLayeri
  r6   r   c                    > [         TU ]  5         [        X5      U l        [	        U5      U l        [        X5      U l        [	        U5      U l        g r   )r:   r;   r   self_attentionr   self_mlpcross_attention	cross_mlpr   s      r2   r;   "LightGlueTransformerLayer.__init__  s@    0C$V,1&D%f-r1   descriptorsr    r}   rD   output_attentionsrE   c                    U(       a  SOS nU(       a  SOS nU(       a  Xa4-   nUR                   u  pn
U R                  UUUUS9u  p[        R                  " X/SS9nU R	                  U5      nX-   nU(       a  X4nUR                  SSX5      R                  S5      R                  XU
5      nUb6  UR                  SSSSU	5      R                  S5      R                  USSU	5      OS nU R                  UUUUS9u  nn[        R                  " UU/SS9nU R                  U5      nUU-   nU(       a4  UU4nUUR                  XU
5      4-   W-   UR                  XU
5      4-   U-   nU(       a	  X|4-   U4-   nXU4$ )Nr%   )r   r}   r   rG   rH   r   r   )r   r   r   )	rq   r   r+   catr   rs   flipr   r   )rA   r   r    r}   rD   r   all_hidden_statesall_attentions
batch_sizenum_keypointsr=   attention_outputself_attentionsintermediate_statesoutput_statesself_attention_descriptorsself_attention_hidden_statesr   r   cross_attention_outputcross_attentionscross_intermediate_statescross_output_statescross_attention_hidden_statess                           r2   rR   !LightGlueTransformerLayer.forward  s    #7BD0d 1N B4?4E4E1
> -1,?,? ))/	 -@ -
) $ii(GRP&9:%0%@",?+O( '..r1mTT!WWZ? 	 ) ""2q!Q>CCAFNNz[\^_ano 	 483G3G&"7#9/	 4H 4
0 0 %*II/IKa.bhj$k!"nn-FG03FF-FH[,\)!-55jQ_`bc./ &&z.QST 0	0  +.@@DTCVVN~==r1   )r   r   r   r   )FF)r&   r'   r(   r)   r   r   r;   r+   rU   r   rV   r/   rR   r0   rW   rX   s   @r2   r   r   
  s    . .3 . 05,1H>\\H> <<H> 	H>
 'tnH> $D>H> 
u||XeELL&9:HU5<<EX<YY	ZH> H>r1   r   
similaritymatchability0matchability1c                    U R                   u  p4n[        R                  R                  U5      [        R                  R                  U5      R	                  SS5      -   n[        R                  R                  U S5      n[        R                  R                  U R	                  SS5      R                  5       S5      R	                  SS5      nU R                  X4S-   US-   4S5      n	Xx-   U-   U	SS2SU2SU24'   [        R                  R                  UR                  S5      * 5      U	SS2SS2S4'   [        R                  R                  UR                  S5      * 5      U	SS2SSS24'   U	$ )z;create the log assignment matrix from logits and similarityr   r   rG   rZ   r   N)	rq   r   r   
logsigmoidr   log_softmaxr   new_fullsqueeze)
r   r   r   r   num_keypoints_0num_keypoints_1certaintiesscores0scores1scoress
             r2   sigmoid_log_double_softmaxr   ]  sM    4>3C3C0J--**=9BMM<T<TUb<c<m<mnoqr<ssKmm''
A6Gmm''
(<(<R(D(O(O(QSTU__`bdfgG  *.A?UVCV!WYZ[F4;4E4SF1 0 00111=3H3H3L2LMF1crc2:11=3H3H3L2LMF1b#2#:Mr1   c                      ^  \ rS rSrS\4U 4S jjrS\R                  S\R                  S\R                  4S jrS\R                  S\R                  4S jr	S	r
U =r$ )
LightGlueMatchAssignmentLayeril  r6   c                    > [         TU ]  5         UR                  U l        [        R                  " U R                  U R                  SS9U l        [        R                  " U R                  SSS9U l        g )NTr8   r   )r:   r;   r=   r   r<   final_projectionmatchabilityr@   s     r2   r;   &LightGlueMatchAssignmentLayer.__init__m  sY    $33 "		$*=*=t?R?RY] ^IId&9&914Hr1   r   r"   rE   c                    UR                   u  p4nU R                  U5      nU[        R                  " U R                  UR
                  S9S-  -  nUR                  US-  SXE5      nUS S 2S4   nUS S 2S4   nXxR                  SS5      -  n	Ub  UR                  US-  SU5      nUS S 2S4   R                  S5      n
US S 2S4   R                  S5      R                  SS5      nX-  nU	R                  US:H  [        R                  " U	R                  5      R                  5      n	U R                  U5      nUR                  US-  SUS5      nUS S 2S4   nUS S 2S4   n[        XU5      nU$ )Ndeviceg      ?r   r   r   rG   rZ   )rq   r   r+   tensorr=   r   rs   r   rf   masked_fillfinford   minr   r   )rA   r   r"   r   r   r=   m_descriptorsm_descriptors0m_descriptors1r   mask0mask1r   matchability_0matchability_1r   s                   r2   rR   %LightGlueMatchAssignmentLayer.forwardt  sy   4?4E4E1
>--k:%T5H5HQ^QeQe(fjn(nn%--jAoq-`&q!t,&q!t,#&>&>r2&FF
<<
aMBDAJ((,EAJ((,66r2>E=D#//	5;;zGWGW;X;\;\]J ((5#++J!OQqQ%ad+%ad+ ,JWr1   c                     U R                  U5      n[        R                  R                  U5      R	                  S5      nU$ )z0Get matchability of descriptors as a probabilityrG   )r   r   r   sigmoidr   )rA   r   r   s      r2   get_matchability.LightGlueMatchAssignmentLayer.get_matchability  s7    ((5}},,\:BB2Fr1   )r=   r   r   )r&   r'   r(   r)   r   r;   r+   rU   rR   r
  r0   rW   rX   s   @r2   r   r   l  sW    I I5<< u||  4ELL U\\  r1   r   c                   j   ^  \ rS rSrS\4U 4S jjrS\R                  S\R                  4S jrSr	U =r
$ )LightGlueTokenConfidenceLayeri  r6   c                 n   > [         TU ]  5         [        R                  " UR                  S5      U l        g Nr   )r:   r;   r   r<   r=   tokenr@   s     r2   r;   &LightGlueTokenConfidenceLayer.__init__  s&    YYv44a8
r1   r   rE   c                     U R                  UR                  5       5      n[        R                  R	                  U5      R                  S5      nU$ )NrG   )r  detachr   r   r	  r   )rA   r   r  s      r2   rR   %LightGlueTokenConfidenceLayer.forward  s=    

;--/0%%e,44R8r1   )r  r   rX   s   @r2   r  r    s/    9 9
5<< ELL  r1   r  c                   8    \ rS rSr% Sr\\S'   SrSrSr	Sr
SrSrg	)
LightGluePreTrainedModeli  zz
An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
models.
r6   	lightgluepixel_valuesFTr%   N)r&   r'   r(   r)   r*   r   r-   base_model_prefixmain_input_namesupports_gradient_checkpointing_supports_flash_attn_supports_sdpar0   r%   r1   r2   r  r    s+    
 #$O&+#Nr1   r  r   	thresholdc                 (   U R                   u  n  nU SS2SS2SS24   R                  S5      nU SS2SS2SS24   R                  S5      nUR                  nUR                  n[        R                  " UR                   S   UR
                  S9S   n[        R                  " UR                   S   UR
                  S9S   n	XR                  SU5      :H  n
XR                  SU5      :H  nUR                  R                  5       nUR                  S5      n[        R                  " XU5      n[        R                  " XR                  SU5      U5      nXU:  -  nXR                  SU5      -  n[        R                  " XS5      n[        R                  " UUS5      n[        R                  " Xg/5      R                  SS5      R                  US-  S5      n[        R                  " X/5      R                  SS5      R                  US-  S5      nUU4$ )z1obtain matches from a score matrix [Bx M+1 x N+1]NrG   r   r   r   r   )rq   maxindicesr+   aranger   gathervaluesexp
new_tensorwherer[   r   rs   )r   r  r   _max0max1matches0matches1indices0indices1mutual0mutual1zeromatching_scores0matching_scores1valid0valid1r   r   s                      r2   get_matches_from_scoresr6    s   ||J1!SbS#2#+""1%D!SbS#2#+""1%D||H||H ||HNN1-hooFtLH||HNN1-hooFtLH//!X66G//!X66G ;;??D??1D{{7$7{{7,C,CAx,PRVW945F}}Q11F {{6R0H{{68R0Hkk8./99!Q?GG
UVXZ[Gkk#3"FGQQRSUVW__`jmn`nprsOO##r1   r    heightwidthc                     [         R                  " X!/U R                  U R                  S9S   nUS-  nUR	                  S5      R
                  S-  nXSSSS24   -
  US   -  n U $ )a}  
Normalize keypoints locations based on image image_shape

Args:
    keypoints (`torch.Tensor` of shape `(batch_size, num_keypoints, 2)`):
        Keypoints locations in (x, y) format.
    height (`int`):
        Image height.
    width (`int`):
        Image width.

Returns:
    Normalized keypoints locations of shape (`torch.Tensor` of shape `(batch_size, num_keypoints, 2)`).
r   rd   Nr   rG   .).NN)r+   r   r   rd   r   r$  )r    r7  r8  sizeshiftscales         r2   normalize_keypointsr>    sl     <<	0@0@	XY]^D1HEHHRL!#E3a<00E/4JJIr1   zV
    LightGlue model taking images as inputs and outputting the matching of them.
    c                   D  ^  \ rS rSrSrS\4U 4S jjrS\S\4S jr	 S$S\
R                  S	\
R                  S
\\   S\\
R                  \\
R                  \
R                  4   4   4S jjrS\
R                  S\S\
R                  S\
R                  S\
R                  4
S jrS%S jrS\
R                  S\
R                  S\S\
R                  4S jrS\
R                  S	\
R                  S\
R                  S\
R                  S\
R                  S\
R                  S\4S jrS rS\
R                  S\
R                  S\
R                  S\
R                  S\\
R                  \
R                  4   4
S jr   S&S	\
R                  S\
R                  S\S\S\\
R                     S\\   S
\\   S\\
R                  \
R                  \
R                  \\4   4S jjr\\   S&S \
R2                  S!\\
R4                     S\\   S
\\   S\\\4   4
S" jj5       5       rS#rU =r$ )'LightGlueForKeypointMatchingi  aJ  
LightGlue is a model matching keypoints in images by leveraging detections from a keypoint detector such as
SuperPoint. It is based on the SuperGlue architecture and is designed to be lightweight and efficient.
It consists of :
    1. Keypoint Encoder
    2. A Graph Neural Network with self and cross attention layers
    3. Matching Assignment layers

The correspondence ids use -1 to indicate non-matching points.

Philipp Lindenberger, Paul-Edouard Sarlin and Marc Pollefeys. LightGlue: Local Feature Matching at Light Speed.
In ICCV 2023. https://huggingface.co/papers/2306.13643
r6   c           
      8  > [         TU ]  U5        [        R                  " UR                  UR
                  S9U l        UR                  R                  U l        UR                  U l	        UR                  U l        UR                  U l        UR                  U l        UR                  U l        U R                  U R                  :w  a0  [        R                   " U R                  U R                  SS9U l        O[        R$                  " 5       U l        ['        U5      U l        [        R*                  " [-        UR                  5       Vs/ s H  n[/        XS9PM     sn5      U l        [        R*                  " [-        UR                  5       Vs/ s H  n[3        U5      PM     sn5      U l        [        R*                  " [-        UR                  S-
  5       Vs/ s H  n[7        U5      PM     sn5      U l        U R;                  5         g s  snf s  snf s  snf )N)trust_remote_codeTr8   )r   r   )r:   r;   r   from_configkeypoint_detector_configrB  keypoint_detectordescriptor_decoder_dim keypoint_detector_descriptor_dimr=   num_hidden_layers
num_layersfilter_thresholddepth_confidencewidth_confidencer   r<   input_projectionIdentityr4   positional_encoder
ModuleListranger   transformer_layersr   match_assignment_layersr  token_confidence	post_init)rA   r6   ir(  rB   s       r2   r;   %LightGlueForKeypointMatching.__init__  s    !>!J!J++v?W?W"
 170O0O0f0f-$33 22 & 7 7 & 7 7 & 7 7$"G"GG$&IId.S.SUYUhUhos$tD!$&KKMD!"<V"D"$--EJ6KcKcEdeEd&v;Ede#
 (*}}<A&BZBZ<[\<[q*62<[\(
$ !#<A&BZBZ]^B^<_`<_q*62<_`!
 	 f ] as   
HHHlayer_indexrE   c                     SS[         R                  " SU-  U R                  -  5      -  -   n[         R                  " USS5      $ )z-scaled confidence threshold for a given layerg?g?g      r   r   )npr%  rI  clip)rA   rX  r  s      r2   _get_confidence_threshold6LightGlueForKeypointMatching._get_confidence_threshold  s;    #tk'9DOO'K LLL	wwy!Q''r1   r   r    rD   c                     UR                  5       R                  5       nU R                  U5      nU R                  X#S9nXE4$ )NrD   )r  r   rM  rO  )rA   r   r    rD   projected_descriptorskeypoint_encoding_outputs         r2   _keypoint_processing1LightGlueForKeypointMatching._keypoint_processing  sI     "((*557 $ 5 5k B#'#:#:9#:#p $>>r1   keypoint_confidencesr"   
num_pointsc                 l   UR                   u  pVX R                  S-
  :  ap  UR                  US:H  S5      nUR                  US-  S5      nU R	                  U5      nSX:  R                  5       R                  SS9U-  -
  nXR                  :  n	U	$ [        R                  " U[        R                  S9n	U	$ )zRevaluate whether we should stop inference based on the confidence of the keypointsr   r   r   rG   g      ?rH   rc   )rq   rI  r   rs   r\  re   sumrK  r+   onesrV   )
rA   rd  rX  r"   re  r   r(  r  ratio_confidentearly_stopped_pairss
             r2   _get_early_stopped_image_pairs;LightGlueForKeypointMatching._get_early_stopped_image_pairs'  s     


1,, $8#C#CDAIq#Q #7#?#?
aQS#T 66{CI!%9%E$L$L$N$R$RWX$R$Y\f$ffO"14I4I"I
 #" #(**Zuzz"J""r1   c                 v    Ub  X   nX$   nU R                   U   " X5      n[        XPR                  5      u  pgXg4$ r   )rS  r6  rJ  )rA   r   r"   rX  early_stopsr   r   r   s           r2   _get_keypoint_matching3LightGlueForKeypointMatching._get_keypoint_matching:  sI    "%2K$D--k:;M#:6CXCX#Y ''r1   confidencesr   c                 \    USU R                   -
  :  nUb  XAU R                  U5      :*  -  nU$ )z#mask points which should be removedr   )rL  r\  )rA   rq  r   rX  keeps        r2   _get_pruning_mask.LightGlueForKeypointMatching._get_pruning_maskB  s:    T2223"4#A#A+#NNNDr1   r!  prune_outputc                   ^ UR                   u  n  n	U R                  U   R                  U5      n
U R                  XjU5      mTR	                  US:H  [
        R                  " S5      5      mU4S jXS   US   TU4 5       u  ppn[        U5       H  nUUUU   4==   S-  ss'   M     S XX4 5       u  ppX4n[        USSS9nUUXU4$ )	zv
For a given layer, prune keypoints based on the confidence of the keypoints and the matchability of the
descriptors.
r   Fc              3   v   >#    U  H(  n[        UT5       VVs/ s H	  u  p#X#   PM     snnv   M*     g s  snnf 7fr   )zip).0r   tr"   pruned_keypoints_masks       r2   	<genexpr>JLightGlueForKeypointMatching._do_layer_keypoint_pruning.<locals>.<genexpr>]  s<      c
c %(0E$FG$FQW$FGc Hs   939r   c              3   6   #    U  H  n[        US S9v   M     g7f)T)batch_firstNr   )rz  pruned_tensors     r2   r}  r~  e  s      S
!j D9!js   TrG   r  padding_value)	rq   rS  r
  rt  r   r+   r   rQ  r	   )rA   r   r    r"   r!  rv  rd  rX  r   r(  descriptors_matchabilitypruned_descriptorspruned_keypoints_0pruned_keypoints_1pruned_maskpruned_indicesrV  pruned_keypointsr|  s                     @r2   _do_layer_keypoint_pruning7LightGlueForKeypointMatching._do_layer_keypoint_pruningI  s    ',,
Aq#'#?#?#L#]#]^i#j  $ 6 67Kgr s 5 A A$!)U\\Z_M` ac
&!ilDY[bcc
_0BQ_ z"AN1--.!3. #S
"4J\!jS
O0B /C%n$VXY!#3^R^^^r1   c                    ^ [         R                  " T5      m[         R                  " TR                  S   5      nTU   nTU   mS XB4 5       u  pBS XS4 5       u  pSU4S jUUUU4 5       u  pEp#X#XE4$ )Nr   c              3   8   #    U  H  n[        US SS9v   M     g7f)TrG   r  Nr   rz  r   s     r2   r}  MLightGlueForKeypointMatching._concat_early_stopped_outputs.<locals>.<genexpr>{  s       3
C TDC   c              3   8   #    U  H  n[        US SS9v   M     g7f)Tr   r  Nr   r  s     r2   r}  r    s       >
N TCNr  c              3   .   >#    U  H
  nUT   v   M     g 7fr   r%   )rz  r   early_stops_indicess     r2   r}  r    s#      g
 &'s   )r+   r[   r"  rq   )rA   r  final_pruned_keypoints_indices!final_pruned_keypoints_iterationsr   r   idsorder_indicess    `      r2   _concat_early_stopped_outputs:LightGlueForKeypointMatching._concat_early_stopped_outputsn  s     $kk*=>ll.44Q78+C01-@3
"C3
/>
*N>
:g
 .1	g
c"@ .RYjjr1   r   r   r   c                   ^ UR                   u  mnU4S jXU4 5       u  pnUS S 2S4   nUS S 2S4   nUS S 2S4   nUS S 2S4   n	US S 2S4   n
US S 2S4   n[        R                  " TS-  SU4SUR                  UR                  S9n[        R
                  " TS-  SU4UR                  UR                  S9n[        TS-  5       H  n[        R                  " X   S:H  SX~   R                  SX   R                  SS95      5      XSXn   4'   [        R                  " X   S:H  SXn   R                  SX   R                  SS95      5      XSX~   4'   X   XSXn   4'   X   XSX~   4'   M     X4$ )Nc              3   N   >#    U  H  oR                  TS -  S S5      v   M     g7f)r   rG   N)rs   )rz  r   r   s     r2   r}  JLightGlueForKeypointMatching._do_final_keypoint_pruning.<locals>.<genexpr>  s'      -
AdvNN:?Ar22Ads   "%r   r   r   rG   r:  )r   )
rq   r+   fullr   rd   zerosrQ  r'  r#  clamp)rA   r!  r   r   r   r(  r-  r.  r+  r,  r2  r3  _matches_matching_scoresrV  r   s                  @r2   _do_final_keypoint_pruning7LightGlueForKeypointMatching._do_final_keypoint_pruning  s     
A-
BITcAd-
)/ 1a4=1a4=1a4=1a4=*1a40*1a40 ::zQ=A2gnndkdqdqr ;;1_a/oNcNc
 zQ'A*/++r!2x{'9'9!X[=N=NST=N=U'V+H8;&' +0++r!2x{'9'9!X[=N=NST=N=U'V+H8;&' 3C2E8;./2B2E8;./ ( ))r1   r7  r8  r   c           
      p	  ^( U(       a  SOS nU(       a  SOS n	UR                   S   S:X  aQ  UR                   S S n
UR                  U
S[        R                  S9UR	                  U
5      UR	                  U
5      UU	4$ UR
                  nUR                   u  pp[        R                  " UR                  US5      SS9nUR                  US-  US5      nUb  UR                  US-  U5      OS nUR                  US-  XR                  5      n[        R                  " US-  US9n[        XU5      nU R                  X!US	9u  nnUS   nU R                  S:  nU R                  S:  n/ n/ n/ n/ n/ n[        R                  " SXS9R                  US-  S5      n[        R                  " U5      n[!        U R"                  5       GHR  nUR%                  5       nUb  U R'                  UU5      nO$[        R(                  " UUS
   4UR
                  S9nU R*                  U   " UUUUUS9nUu  nnn U(       a  UU-   nU(       a  U	U -   n	U(       Ga  UU R"                  S-
  :  a'  U R,                  U   " U5      n!U R/                  U!UX_S9n"O#[        R(                  " U[        R0                  S9n"[        R2                  " U"5      (       Ga  U"R5                  S5      m(UT(   n#U R7                  X%UT(S9u  n$n%UR9                  [;        U#5      5        UR9                  [;        U$5      5        UR9                  [;        U%5      5        U(       a:  UR9                  [;        UT(   5      5        UR9                  [;        UT(   5      5        UU")    n[=        U(4S jX!S   US   UU4 5       5      u  nn&n'nnU&U'4nU(       a  [=        U(4S jUUW!4 5       5      u  nnn![        R>                  " U"5      (       a    O+U(       d  GM6  U RA                  UUUUUW!U5      u  p!nnnGMU     U(       a9  U(       a2  U RC                  UUUUU5      u  nnnnU RE                  UUUU5      u  nnOEU R7                  X%U R"                  S-
  5      u  nn[        R                  " U5      U R"                  -  nUR                  USU5      nUUUUU	4$ )Nr%   r   r   rG   rc   r   rH   r   r_  rZ   )r}   rD   r   )re  )rn  c              3   0   >#    U  H  nUT)    v   M     g 7fr   r%   rz  r   rn  s     r2   r}  ALightGlueForKeypointMatching._match_image_pair.<locals>.<genexpr>  s       V&dF |,&d   c              3   0   >#    U  H  nUT)    v   M     g 7fr   r%   r  s     r2   r}  r  #  s$      l+ #K<0+r  )#rq   r   r+   r   	new_zerosr   rg  rs   rG  r"  r>  rb  rK  rL  rr   	ones_likerQ  rI  r;  get_extended_attention_maskrh  rR  rT  rk  rV   anyrJ   ro  extendlistr/   allr  r  r  ))rA   r    r   r7  r8  r"   r   rD   r   r   rq   r   r   r(  initial_num_keypointsnum_points_per_pairimage_indicesra  do_early_stopdo_keypoint_pruningr  r   r   r  r  pruned_keypoints_indicespruned_keypoints_iterationsrX  r   extended_attention_masklayer_outputr#   	attentionrd  rj  early_stopped_image_indicesearly_stopped_matchesearly_stopped_matching_scoreskeypoints_0
keypoint_1rn  s)                                           @r2   _match_image_pair.LightGlueForKeypointMatching._match_image_pair  sK    #7BD0d??1"OOCR(E""5"EII">##E*##E*!  !!2;///
,#iiZ(D!L%%j1n6KQO	FJFVt||JN,AB\`!))*q.:OQvQvwZ!^FC'	5A	040I0I9M 1J 1
-- -Q/	 --1 #33a7 )+&,.)#(<<3H#X#_#_`jmn`npr#s &+oo6N&O# 1K%**,K*.*J*J4Q\*]'*/**j+b/5R[d[k[k*l'22;?6%9"3L 5A1K	#$5$E! !/)!;1!44+/+@+@+Mk+Z( +/*M*M,k4 +N +'
 +0**Zuzz*R'99011 #6"G"G"JK2?2L/KOKfKf#;K Lg LH)+H (..t4O/PQNN4(=#>?#**40M+NO*6==dC[\gCh>ij9@@FabmFnAop +>?R>R*S'PU V'2aL)A,PTVc&dV QMKj$ "-j 9I*fk l !9 ; 4+l gc02MOc 99011"" 33#!03,# d(@$HcQ 2h 0 22'25# h*,MwXg (,'F'F.%	($G_ (,'B'B;VZVeVehiVi'j$G_050PSWSbSb0b-,M,U,U0-
)
 -
 	
r1   r  labelsc                 $   S nUb  [        S5      eUb  UOU R                  R                  nUb  UOU R                  R                  nUR                  S:w  d  UR                  S5      S:w  a  [        S5      eUR                  u  pgpn
UR                  US-  XU
5      nU R                  U5      nUS S u  ppUR                  USSS5      R                  U5      nUR                  USSU R                  5      R                  U5      nUR                  USS5      nUR                  5       nUS S 2S S 2S S 2S4   U
-  US S 2S S 2S S 2S4'   US S 2S S 2S S 2S4   U	-  US S 2S S 2S S 2S4'   U R                  UUU	U
UUUS	9u  nnnnn[        UUUUUUUUS
9$ )Nz9LightGlue is not trainable, no labels should be provided.   r   r   zOInput must be a 5D tensor of shape (batch_size, 2, num_channels, height, width)   rG   r   )r"   r   rD   )r   r   r   r    r!   r"   r#   r$   )
ValueErrorr6   r   rD   ndimr;  rq   rs   rE  rg   rG  cloner  r   )rA   r  r  r   rD   r   r   r(  channelsr7  r8  keypoint_detectionsr    r   r"   absolute_keypointsr   r   r!   r#   r$   s                        r2   rR   $LightGlueForKeypointMatching.forward`  s    XYY1B1N-TXT_T_TqTq$8$D $++JjJj 	 !\%6%6q%9Q%>noo1=1C1C.
x#++JNHeT"44\B*=bq*A'	k%%j!R;>>|L	!))*aT=b=bcffgst||J2.&__.);Aq!QJ)G%)O1aA:&);Aq!QJ)G&)P1aA:&EIE[E[/!5 F\ F
B%
 /+'!	
 		
r1   )rK  r=   rJ  rM  rE  rG  rS  rI  rO  rT  rR  rL  rT   r   )NNN) r&   r'   r(   r)   r*   r   r;   r   re   r\  r+   rU   r   rV   r/   rb  rk  ro  rt  r  r  r  r  r   r   r,   
LongTensorr   r   rR   r0   rW   rX   s   @r2   r@  r@    s    @(S (U ( jo? <<?49LL?X`aeXf?	u||U5<<#=>>	??#$)LL#?B#JO,,#didpdp#	#&(U\\ 5<< ^a fkfrfr #_\\#_ <<#_ ll	#_
 #_ ll#_ $ll#_ #_Jk@#*#* #* 	#*
 ||#* 
u||U\\)	*#*V (,,0/3k
<<k
 \\k
 	k

 k
 u||$k
 $D>k
 'tnk
 
u||U\\5<<E	Fk
Z  .2,0/33
''3
 ))*3
 $D>	3

 'tn3
 
u55	63
  3
r1   r@  r  )r   )7dataclassesr   typingr   r   r   numpyrZ  r+   r   torch.nn.utils.rnnr	   activationsr   modeling_flash_attention_utilsr   modeling_utilsr   r   processing_utilsr   utilsr   r   r   utils.deprecationr   utils.genericr   auto.modeling_autor   configuration_lightgluer   r   Moduler4   ra   rn   rU   r   rx   re   r   r   r   r   r   r   r  r  r/   r6  r>  r@  __all__r%   r1   r2   <module>r     sX  ( " , ,    + ! B F & D D 0 - > 4  :k  :  :F "<<	UU\\ 	U# 	U%,, 	U& %II%<<% 
% <<	%
 U\\*% % % '(%4E) E)P299 "P>		 P>f-2\\JO,,
\\&BII &R	BII 	   $ELL $U $uU\\[`[g[gMgGh $@5<<  S U\\ , 
j
#; j

j
Z &'E
Fr1   