
    h                       S r SSKJr  SSKrSSKrSSKrSSKrSSKrSSKrSSK	J
r
JrJr   SSKrSSKrSSKrSSKrSSKrSSKrSSKrSSKJr  \(       a  SSKJr  SSKJr  \R<                  " \5      r S	r! S
r" Sr# Sr$Sr%Sr&Sr'Sr(Sr) " S S5      r*S\+" 5       ;  a  \*" 5       r, " S S5      r-S r.S r/S r0S\'\"SSSSS4S jr1S r2S r3 " S S\45      r5S  r6 " S! S"\Rn                  5      r8 " S# S$\Rn                  5      r9 " S% S&\Rn                  5      r:S' r;     S.S) jr<S(S* 4S+ jr=S/S, jr>S- r?g! \ a    Sr GNf = f)0zDImplements file-like objects for reading and writing from/to AWS S3.    )annotationsN)CallableListTYPE_CHECKINGT)	constants)S3Client)Bufferi  P i   l        )s3s3ns3us3ai  zs3.amazonaws.comi   )zs3://my_bucket/my_keyz&s3://my_key:my_secret@my_bucket/my_keyz8s3://my_key:my_secret@my_server:my_port@my_bucket/my_keyInvalidRangec                  $    \ rS rSrS rSS jrSrg)RetryR   c                p    SU l         SU l        [        R                  R                  /U l        S/U l        g )N   
   NoSuchUpload)attemptssleep_secondsbotocore
exceptionsEndpointConnectionErrorclient_error_codesselfs    G/home/james-whalen/.local/lib/python3.13/site-packages/smart_open/s3.py__init__Retry.__init__S   s2    "$,4,?,?,W,W+X.<-=    c                    [        U R                  5       H  n U" 5       s  $    [        R                  S5        [        SUR                   U R                  5      e! [        U R                  5       ag  n[        R                  SUU R                  U-
  S-
  5        [        R                  U5        [        R                  " U R                  5         S nAM  S nAf[        R                  R                   a  nUR                  S   R                  S5      nX@R                  ;  a  e [        R                  SUU R                  U-
  S-
  5        [        R                  U5        [        R                  " U R                  5         S nAGMr  S nAff = f)Nz+Caught non-fatal %s, retrying %d more times   ErrorCodez9Caught non-fatal ClientError (%s), retrying %d more timesz0encountered too many non-fatal errors, giving upz%s failed after %d attempts)ranger   tupler   loggercritical	exceptiontimesleepr   r   ClientErrorresponsegetr   IOErrorfunc)r   fnattempterr
error_codes        r   _do	Retry._doY   s0   T]]+G/t ,. OONO7$--PP+ ) /AMMG+a/
   %

4--..&&22 
/ \\'266v>
%<%<<OMMG+a/
   %

4--..
/s%   AF4AC!F7B
FF)r   r   r   r   N)r2   r   )__name__
__module____qualname____firstlineno__r   r6   __static_attributes__ r!   r   r   r   R   s    >Qr!   r   MISSING_DEPSc                  $    \ rS rSrSrS rS rSrg)_ClientWrapper}   ad  Wraps a client to inject the appropriate keyword args into each method call.

The keyword args are a dictionary keyed by the fully qualified method name.
For example, S3.Client.create_multipart_upload.

See https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#client

This wrapper behaves identically to the client otherwise.
c                    Xl         X l        g Nclientkwargs)r   rE   rF   s      r   r   _ClientWrapper.__init__   s    r!   c                    [        U R                  U5      nU R                  R                  SU-  0 5      n[        R
                  " U40 UD6$ )NzS3.Client.%s)getattrrE   rF   r/   	functoolspartial)r   method_namemethodrF   s       r   __getattr___ClientWrapper.__getattr__   sA    k2+!=rB  2622r!   rD   N)r8   r9   r:   r;   __doc__r   rN   r<   r=   r!   r   r@   r@   }   s    3r!   r@   c                X   [         R                  R                  U 5      nUR                  [        ;   d   e[
        n[        nSnSu  pVUR                  UR                  -   nSU;   a8  UR                  SS5      u  pSU;   a  UR                  SS5      u  pSU
;  a  XpeU	nUR                  SS5      u  pSU;   a;  SU;   a5  SnUR                  S5      u  pUR                  SS5      u  p2[        U5      nOSU;   a  SnUR                  S5      u  p?OUn[        UR                  UUUUUUUS9$ )	NFNN@r#   :/T)scheme	bucket_idkey_idporthostordinary_calling_format	access_idaccess_secret)
smart_openutilssafe_urlsplitrV   SCHEMESDEFAULT_PORTDEFAULT_HOSTnetlocpathsplitintdict)uri_as_string	split_urirY   rZ   r[   r\   r]   uri
maybe_authrestmaybe_idmaybe_secretheadrX   	host_portrW   s                   r   	parse_urirr      sG      ..}=Iw&&&DD#  *I 

Y^^
+C cz99S!,
*%/%5%5c1%="H("+3=99S!$LD
d{sd{"&#zz#	__S!,
4y	"&**S/i	 7#	 	r!   c                  ^ [        T5      mU4S jnTR                  S5      nUb:  U S   (       d
  U S   (       a&  [        R                  S5        U R	                  SSS9  O1U S   (       a'  U S   (       a  U" U S   U S   S9  U R	                  SSS9  Ub5  U S	   [
        :w  a(  [        R                  S
5        U R	                  SS9  U T4$ U S	   [
        :w  a)  U S   S:X  a  SnOSnU" USU -  -   S9  U R	                  SS9  U T4$ )aD  Consolidates the parsed Uri with the additional parameters.

This is necessary because the user can pass some of the parameters can in
two different ways:

1) Via the URI itself
2) Via the transport parameters

These are not mutually exclusive, but we have to pick one over the other
in a sensible way in order to proceed.

c                    >  TS   n US   nUR                  " S0 U D6  g ! [          a
    0 =nTS'    N,f = f! [          a	    0 =o!S'    N<f = f)Nclient_kwargs	S3.Clientr=   )KeyErrorupdate)rF   ru   init_kwargstransport_paramss      r   inject#_consolidate_params.<locals>.inject   su    	C,_=M	:'4K 	$V$  	C@BBM,_=	C
  	:799K4	:s   ! 8 55A
ArE   Nr\   r]   zignoring credentials parsed from URL because they conflict with transport_params["client"]. Set transport_params["client"] to None to suppress this warning.)r\   r]   )aws_access_key_idaws_secret_access_keyrZ   zignoring endpoint_url parsed from URL because they conflict with transport_params["client"]. Set transport_params["client"] to None to suppress this warning.)rZ   rV   r   httphttpsz://%(host)s:%(port)d)endpoint_url)rh   r/   r(   warningrx   rc   )rk   rz   r{   rE   rV   s    `   r   _consolidate_paramsr      s)    ,-% !!(+Fs;/33G(	

 	

T
6
k
s?3!+."%o"6	
 	

T
6c&k\9(	

 	


     
V	$x=E!FFF%;c%AAB


   r!   c                @   SnU Vs/ s H  oDU;   d  M
  UPM     nnU(       a(  SnSU< SU< S3n[         R                  " U[        5        [        U 5      n[	        X5      u  p[
        R                  R                  [        U5      n	[        US   US   U40 U	D6$ s  snf )N)multipart_upload_kwargsobject_kwargsresourceresource_kwargssessionsinglepart_upload_kwargszVhttps://github.com/piskvorky/smart_open/blob/develop/MIGRATING_FROM_OLDER_VERSIONS.rstz8ignoring the following deprecated transport parameters: z. See <z> for detailsrW   rX   )	warningswarnUserWarningrr   r   r^   r_   check_kwargsopen)
rk   moderz   
deprecatedkdetecteddoc_urlmessage
parsed_urirF   s
             r   open_urir     s    J &?:a.>)>:H?0 	 '/9 	 	g{+3J#6z#T J**41ABF
;'H)=tNvNN- @s
   	BBFc           
        [         R                  S[        5       5        U[        R                  ;  a!  [        SU< S[        R                  < 35      eU[        R                  :X  a  Ub  [        S5      eU[        R                  :X  a  [        U UUUUUU	S9nOBU[        R                  :X  a$  U(       a  [        U UUU	U
US9nO[        U UUU	U
S9nO
 SU-  5       eXl        U$ )	ak
  Open an S3 object for reading or writing.

Parameters
----------
bucket_id: str
    The name of the bucket this object resides in.
key_id: str
    The name of the key within the bucket.
mode: str
    The mode for opening the object.  Must be either "rb" or "wb".
buffer_size: int, optional
    The buffer size to use when performing I/O.
min_part_size: int, optional
    The minimum part size for multipart uploads, in bytes.
    When the writebuffer contains this many bytes, smart_open will upload
    the bytes to S3 as a single part of a multi-part upload, freeing the
    buffer either partially or entirely.  When you close the writer, it
    will assemble the parts together.
    The value determines the upper limit for the writebuffer.  If buffer
    space is short (e.g. you are buffering to memory), then use a smaller
    value for min_part_size, or consider buffering to disk instead (see
    the writebuffer option).
    The value must be between 5MB and 5GB.  If you specify a value outside
    of this range, smart_open will adjust it for you, because otherwise the
    upload _will_ fail.
    For writing only.  Does not apply if you set multipart_upload=False.
multipart_upload: bool, optional
    Default: `True`
    If set to `True`, will use multipart upload for writing to S3. If set
    to `False`, S3 upload will use the S3 Single-Part Upload API, which
    is more ideal for small file sizes.
    For writing only.
version_id: str, optional
    Version of the object, used when reading object.
    If None, will fetch the most recent version.
defer_seek: boolean, optional
    Default: `False`
    If set to `True` on a file opened for reading, GetObject will not be
    called until the first seek() or read().
    Avoids redundant API queries when seeking before reading.
client: object, optional
    The S3 client to use when working with boto3.
    If you don't specify this, then smart_open will create a new client for you.
client_kwargs: dict, optional
    Additional parameters to pass to the relevant functions of the client.
    The keys are fully qualified method names, e.g. `S3.Client.create_multipart_upload`.
    The values are kwargs to pass to that method each time it is called.
writebuffer: IO[bytes], optional
    By default, this module will buffer data in memory using io.BytesIO
    when writing. Pass another binary IO instance here to use it instead.
    For example, you may pass a file object to buffer to local disk instead
    of in RAM. Use this to keep RAM usage low at the expense of additional
    disk IO. If you pass in an open file, then you are responsible for
    cleaning it up after writing completes.
z%rz
bad mode: z expected one of z$version_id must be None when writing)
version_idbuffer_size
defer_seekrE   ru   )rE   ru   writebuffer	part_size)rE   ru   r   zunexpected mode: %r)r(   debuglocalsr   BINARY_MODESNotImplementedErrorWRITE_BINARY
ValueErrorREAD_BINARYReaderMultipartWriterSinglepartWritername)rW   rX   r   r   r   min_part_sizemultipart_uploadr   rE   ru   r   fileobjs               r   r   r   7  s    H LLvx 9)))!tYMcMc"dee	&&&Z-C?@@y$$$!#!'
 
''	'%+''G '+'G 	3+d22uLNr!   c                     [        XS9nU(       a  X5S'   U(       a  XES'   U R                  " S0 UD6$ ! [        R                  R                   a(  n[        SU< SU< SU< SU< 35      nXgl        XveS nAff = f)	NBucketKey	VersionIdRangezunable to access bucket: z key: z
 version: z error: r=   )rh   
get_objectr   rE   r-   r0   backend_error)rE   bucketkeyversionrange_stringparamserrorwrapped_errors           r   _getr     sz    'V-");*7O  *6**??&& 'We

 ',#&'s   03 A9#A44A9c                b     U R                   R                  S   $ ! [        [        4 a     gf = f)zDGiven an IOError from _get, return the 'Error' dictionary from boto.r$   N)r   r.   AttributeErrorrw   )ioes    r   _unwrap_ioerrorr     s5      ))'22H% s    ..c                  ^    \ rS rSrSr S
S jr\R                  4S jrSS jr	SS jr
S rS	rg)_SeekableRawReaderi  z@Read an S3 object.

This class is internal to the S3 submodule.
Nc                ^    Xl         X l        X0l        X@l        S U l        SU l        S U l        g )Nr   )_client_bucket_key_version_id_content_length	_position_body)r   rE   r   r   r   s        r   r   _SeekableRawReader.__init__  s/     	%#
r!   c                   U[         R                  ;  a  [        S[         R                  -  5      eU R                  b  U R                  R	                  5         SU l        SnSnU[         R
                  :X  a  [        SU5      nO:U[         R                  :X  a  [        SXR                  -   5      nO[        SU* 5      nU R                  c  SnO Ub  X0R                  :  a  SnOUS:X  a  SnOSnU(       a7  [        R                  " 5       U l        U R                  U l        U R                  $ U R                  X45        U R                  $ )zSeek to the specified position.

:param int offset: The offset in bytes.
:param int whence: Where the offset is from.

:returns: the position after seeking.
:rtype: int
z"invalid whence, expected one of %rNr   FT)r   WHENCE_CHOICESr   r   closeWHENCE_STARTmaxWHENCE_CURRENTr   r   ioBytesIO
_open_body)r   offsetwhencestartstopreached_eofs         r   seek_SeekableRawReader.seek  s    111AID\D\\]] ::!JJ
Y+++6NEy///6NN23Eq6'?D 'K5,@,@#@KQYKKDJ!11DN ~~ OOE(~~r!   c           	        Uc  Uc  U R                   n[        R                  R                  X5      n [	        U R
                  U R                  U R                  U R                  U5      n[        R                  SU US   S   5        US   S   nU[        R                  R                  :X  a,  [        R                  R                  US   5      u  pap'Xl         O#U[        R                  R                  :X  a  US   nWU l        US   U l        g! [$         a  n['        U5      n	U	b  U	R)                  S	5      [*        :w  a  e  [-        U	S
   5      =U l         U l        [.        R0                  " 5       U l         SnAg! [2         aY    [	        U R
                  U R                  U R                  U R                  S5      nUS   =U l         U l        US   U l          SnAgf = fSnAff = f)a  Open a connection to download the specified range of bytes. Store
the open file handle in self._body.

If no range is specified, start defaults to self._position.
start and stop follow the semantics of the http range header,
so a stop without a start will read bytes beginning at stop.

As a side effect, set self._content_length. Set self._position
to self._content_length if start is past end of file.
Nz%s: RetryAttempts: %dResponseMetadataRetryAttemptsHTTPStatusCodeContentRangeContentLengthBodyr%   ActualObjectSize)r   r^   r_   make_range_stringr   r   r   r   r   r(   r   r   
HTTPStatusPARTIAL_CONTENTparse_content_rangeOKr   r   r0   r   r/   _OUT_OF_RANGErg   r   r   rw   )
r   r   r   r   r.   status_code_lengthr   error_responses
             r   r   _SeekableRawReader._open_body  s    =T\NNE!''99%F2	*		  H@ LL'+,_= ##567GHKdoo===)3)9)9)M)MhWeNf)g&$!& 2 22!/2#)D !&)DJS  	.,S1N%););F)C})T.8;NK]<^8__!5ZZ\

 	.LLLLII$$ 9A8QQ!5%f-

	.	.s6   7D 
G%(G 4E::AGG GG  G%c           	        U R                   c  U R                  5         U R                  U R                  :  a  g[	        / SQS5       Hb  u  p# US:X  a  U R                   R                  5       nOU R                   R                  U5      nU =R                  [        U5      -  sl        Us  $    [#        SXW4-  5      e! [        [        R                  R                  [        R                  R                  4 aJ  n[        R                  SU UUU5        [        R                   " U5        U R                  5          SnAM  SnAff = f)z9Read from the continuous connection with the remote peer.Nr!   )r#               r#   z?%s: caught %r while reading %d bytes, sleeping %ds before retryz-%s: failed to read %d bytes after %d attempts)r   r   r   r   	enumeratereadlenConnectionResetErrorr   r   BotoCoreErrorurllib3	HTTPErrorr(   r   r+   r,   r0   )r   sizer3   secondsbinaryr4   s         r   r   _SeekableRawReader.readK  s   ::OO>>T111 !**:A >G2:!ZZ__.F!ZZ__T2F  #f+-- !?0 EU\H]]^^# %##11"",, "
 U 

7#!!"s   <B===E:?D??Ec                @    SU R                   < SU R                  < S3$ )Nzsmart_open.s3._SeekableReader(, )r   r   r   s    r   __str___SeekableRawReader.__str__z      :>,,		RRr!   )r   r   r   r   r   r   r   rC   rR   r   )r8   r9   r:   r;   rP   r   r   r   r   r   r   r  r<   r=   r!   r   r   r     s6       #,"8"8 0dA*F-_^Sr!   r   c                    Uc  0 nUc(  UR                  S0 5      n[        R                  " S0 UD6nU(       d   e[        X5      U l        X0l        X@l        g)zzCreated the required objects for accessing S3.  Ideally, they have
been already created for us and we can just reuse them.Nrv   )r
   )r/   boto3rE   r@   r   r   r   )rwrE   ru   r   r   ry   s         r   _initialize_boto3r
  ~  sT     ~#''R82k2M66BJJGr!   c                      \ rS rSrSrS\\R                  SSS4S jrS r	S r
SS jrSS	 jrS
 rSS jrS r\R                   4S jrS rSS jrS rS rS rSS jrSS jrS rS rSrg)r   i  zYReads bytes from S3.

Implements the io.BufferedIOBase interface of the standard library.NFc	                B   X0l         X@l        [        XXU5        [        U R                  UUU R                   5      U l        SU l        [        R                  R                  U5      U l
        SU l        XPl        SU l        S U l        U(       d  U R                  S5        g g )Nr   F)r   _buffer_sizer
  r   r   _raw_reader_current_posr^   
bytebuffer
ByteBuffer_buffer_eof_line_terminator_seek_initializedrawr   )	r   r   r   r   r   line_terminatorr   rE   ru   s	            r   r   Reader.__init__  s     &'$sC-LL	
 !,,77D	 /!&
 IIaL r!   c                .    [         R                  S5        g)zFlush and close this stream.close: calledN)r(   r   r   s    r   r   Reader.close  s    _%r!   c                    g)z+Return True if the stream can be read from.Tr=   r   s    r   readableReader.readable      r!   c                   US:X  a  gUS:  aH  U R                  5       U R                  R                  5       -   nU R                  R                  U l        U$ [        U R                  5      U:  a  U R                  U5      $ U R                  (       a  U R                  5       $ U R                  U5        U R                  U5      $ )z6Read up to size bytes from the object and return them.r   r!   )	_read_from_bufferr  r   r   r  r   r  r  _fill_buffer)r   r   outs      r   r   Reader.read  s    19AX((*T-=-=-B-B-DDC $ 0 0 @ @DJ
 t||$))$//
 99))++$%%d++r!   c                     U R                  US9$ )zThis is the same as read().)r   )r   r   r   s     r   read1Reader.read1  s    yydy##r!   c                v    U R                  [        U5      5      nU(       d  gX!S[        U5      & [        U5      $ )zDRead up to len(b) bytes into b, and return the number of bytes
read.r   N)r   r   )r   bdatas      r   readintoReader.readinto  s3     yyQ *3t94yr!   c                H   US:w  a  [        S5      e[        R                  " 5       nU R                  (       a  [	        U R
                  5      S:X  d  U R
                  R                  U R                  5      nUR                  U5        U =R                  [	        U5      -  sl	        UR                  U R                  5      (       a   UR                  5       $ U R                  5         U R                  (       d  M  [	        U R
                  5      S:X  d  M  UR                  5       $ )zCRead up to and including the next newline.  Returns the bytes read.r   z(limits other than -1 not implemented yetr   )r   r   r   r  r   r  readliner  writer  endswithr"  getvalue)r   limitline	line_parts       r   r/  Reader.readline  s    B;%&PQQ
 zz|99T\\!2a!7--d.C.CDIJJy!Y/!!$"7"788 }} !!# 999T\\!2a!7 }}r!   c                    g)zqIf False, seek(), tell() and truncate() will raise IOError.

We offer only seek support, and no truncate support.Tr=   r   s    r   seekableReader.seekable       r!   c                   U[         R                  :X  a  [         R                  nXR                  -  nU R                  (       a#  U[         R                  :X  a  XR                  :X  d:  U R
                  R                  X5      U l        U R                  R                  5         U R                  U R
                  R                  :H  U l
        SU l        U R                  $ )zSeek to the specified position.

:param int offset: The offset in bytes.
:param int whence: Where the offset is from.

Returns the position after seeking.T)r   r   r   r  r  r  r   r  emptyr   r  r   r   r   s      r   r   Reader.seek  s     Y---++F'''F%%i,,,;L;L1L $ 0 0 5 5f EDLL %%)9)9)I)II	!%   r!   c                    U R                   $ )z,Return the current position within the file.)r  r   s    r   tellReader.tell         r!   c                "    [         R                  ezUnsupported.r   UnsupportedOperationr&  s     r   truncateReader.truncate"      %%%r!   c                "    [         R                  erD  rE  r   s    r   detachReader.detach&  rI  r!   c                    g)zDo nothing.Nr=   r   s    r   	terminateReader.terminate*  s    r!   c                    U(       d   S5       eUR                  U R                  U R                  5      nU R                  b  UR	                  U R                  5      $ U$ zCreate an **independent** `boto3.s3.Object` instance that points to
the same S3 object as this instance.
Changes to the returned object will not affect the current instance.
z*resource must be a boto3.resource instance)Objectr   r   r   Version)r   r   objs      r   to_boto3Reader.to_boto3.  sP    
 EEExoodllDII6';;t//00Jr!   c                    US:  a  UO[        U R                  5      nU R                  R                  U5      nU =R                  [        U5      -  sl        U$ )z:Remove at most size bytes from our buffer and return them.r   )r   r  r   r  )r   r   parts      r   r!  Reader._read_from_buffer=  sF    qytc$,,&7||  &SY&r!   c                   [        XR                  R                  5      n[        U R                  5      U:  a  U R                  (       dv  U R                  R                  U R                  5      nUS:X  a  [        R                  SU 5        SU l        [        U R                  5      U:  a  U R                  (       d  Mt  g g g g )Nr   z$%s: reached EOF while filling bufferT)	r   r  _chunk_sizer   r  fillr  r(   r   )r   r   
bytes_reads      r   r"  Reader._fill_bufferD  s    4112$,,$&tyy**4+;+;<JQCTJ 		 $,,$&tyyy&y&r!   c                @    SU R                   < SU R                  < S3$ )Nzsmart_open.s3.Reader(r   r  r  r   s    r   r  Reader.__str__L  s    15tyyIIr!   c                    SU R                   < SU R                  < SU R                  < SU R                  < SU R                  < S3$ )Nzsmart_open.s3.Reader(bucket=, key=z, version_id=z, buffer_size=z, line_terminator=r  )r   r   r   r  r  r   s    r   __repr__Reader.__repr__O  s7     LLII!!
 	
r!   )	r  r  r  r  r  r  r  r   r  r  rC   )r8   r9   r:   r;   rP   DEFAULT_BUFFER_SIZEr   BINARY_NEWLINEr   r   r  r   r'  r,  r/  r8  r   r   r@  rG  rK  rN  rU  r!  r"  r  rc  r<   r=   r!   r   r   r     s    K '!00"P
,2$* #,"8"8 !2!&&
!J
r!   r   c                      \ rS rSrSrSr\SSS4 SS jjrS rS r	\
S 5       rS rS	 r\R                  4S
 jrSS jrS rS rSS jrS rS rSS jrS rS rS rS rSrg)r   i`  zqWrites bytes to S3 using the multi part API.

Implements the io.BufferedIOBase interface of the standard library.Nc                R   [         R                  R                  U[        [        5      nX7:w  a  [
        R                  SU SU 35        UnX0l        [        XXQU5        U   U   U    [        R                  " U R                  R                  UUS9n[        R                  U5      S   U l        Uc  [(        R*                  " 5       U l        OX`l        SU l        SU l        / U l        S U l        g ! [         R"                  R$                   a  n	['        SU< SU	< S35      U	eS n	A	ff = f)	Nzadjusting part_size from z to r   UploadIdzthe bucket z- does not exist, or is forbidden for access (r  r   )r^   r_   clampMIN_PART_SIZEMAX_PART_SIZEr(   r   
_part_sizer
  rJ   rK   r   create_multipart_uploadRETRYr6   
_upload_idr   rE   r-   r   r   r   _buf_total_bytes_total_parts_partsr  )
r   r   r   r   rE   ru   r   adjusted_psrK   r   s
             r   r   MultipartWriter.__init__f  s    !&&,,Y}U#NN6ykk]ST#I#$sC	''44G
 $ii0<DO 

DI#I/1
 ' ** 	E 		s   &AC. .D&D!!D&c                    g rC   r=   r   s    r   flushMultipartWriter.flush      r!   c                P   [         R                  S5        U R                  (       a  g U R                  R	                  5       (       a  U R                  5         [         R                  SU 5        U R                  (       a  U R                  (       a  [        R                  " U R                  R                  U R                  U R                  U R                  SU R                  0S9n[        R!                  U5        [         R                  SU 5        OU R                  (       a~  U R                  R#                  U R                  U R                  U R                  S9  U R                  R%                  U R                  U R                  SS9  [         R                  S	U 5        S U l        g )
Nr  z%s: completing multipart uploadParts)r   r   ri  MultipartUploadz%s: completed multipart uploadr   r   ri  r!   r   r   r   z-%s: wrote 0 bytes to imitate multipart upload)r(   r   closedrq  r@  _upload_next_partrr  rp  rJ   rK   r   complete_multipart_uploadr   r   rt  ro  r6   abort_multipart_upload
put_object)r   rK   s     r   r   MultipartWriter.close  s%   _%;;99>>""$6=''66||II!($++ 6G IIgLL94@__ LL//||II 0 
 LL##||II $ 
 LLH$Or!   c                    U R                   S L $ rC   )rp  r   s    r   r  MultipartWriter.closed  s    $&&r!   c                    g)z+Return True if the stream supports writing.Tr=   r   s    r   writableMultipartWriter.writable  r  r!   c                    g)zyIf False, seek(), tell() and truncate() will raise IOError.

We offer only tell support, and no seek or truncate support.Tr=   r   s    r   r8  MultipartWriter.seekable  r:  r!   c                "    [         R                  erD  rE  r=  s      r   r   MultipartWriter.seek  rI  r!   c                "    [         R                  erD  rE  r&  s     r   rG  MultipartWriter.truncate  rI  r!   c                    U R                   $ )z#Return the current stream position.)rr  r   s    r   r@  MultipartWriter.tell  rB  r!   c                .    [         R                  " S5      e)Nzdetach() not supportedrE  r   s    r   rK  MultipartWriter.detach  s    %%&>??r!   c                   Sn[        U5      nU =R                  [        U5      -  sl        U[        U5      :  a  UnX R                  -   U R                  R                  5       -
  nU R                  R                  X4U 5        U R                  R                  5       U R                  :  a  U[        U5      :  d   e[        U5      $ U R                  5         UnU[        U5      :  a  M  [        U5      $ )a*  Write the given buffer (bytes, bytearray, memoryview or any buffer
interface implementation) to the S3 file.

For more information about buffers, see https://docs.python.org/3/c-api/buffer.html

There's buffering happening under the covers, so this may not actually
do any HTTP transfer right away.r   )
memoryviewrr  r   rm  rq  r@  r0  r  )r   r*  r   mvr   ends         r   r0  MultipartWriter.write  s     ]SW$ s2wE??*TYY^^-==CIIOOBSM*yy~~$//1
 c"g~%~2w""$F s2w 2wr!   c                    U R                   (       a  g[        R                  SU 5        U R                  R	                  U R
                  U R                  U R                  S9  SU l        [        R                  SU 5        g)z'Cancel the underlying multipart upload.Nz %s: terminating multipart uploadr~  z%s: terminated multipart upload)r  r(   r   r   r  r   r   rp  r   s    r   rN  MultipartWriter.terminate  s_    ;;7>++<<		__ 	, 	

 6=r!   c                j    U(       d   S5       eUR                  U R                  U R                  5      $ rQ  )rR  r   r   )r   r   s     r   rU  MultipartWriter.to_boto3  s+    
 EEExt||TYY77r!   c                   U R                   S-   n[        R                  SU UU R                  R	                  5       U R
                  S-  5        U R                  R                  S5        [        R                  [        R                  " U R                  R                  U R                  U R                  U R                  UU R                  S95      nU R                   R#                  US   US.5        [        R%                  SX5        U =R                   S-  sl         U R                  R                  S5        U R                  R'                  S5        g )	Nr#   z3%s: uploading part_num: %i, %i bytes (total %.3fGB)g      Ar   )r   r   ri  
PartNumberr   ETag)r  r  z#%s: upload of part_num #%i finished)rs  r(   inforq  r@  rr  r   ro  r6   rJ   rK   r   upload_partr   r   rp  rt  appendr   rG  )r   part_numuploads      r   r  !MultipartWriter._upload_next_part  s    $$q(AIINN+	
 			q ((||II#YY	
 	F6N(KL:DKQ		q		1r!   c                    U $ rC   r=   r   s    r   	__enter__MultipartWriter.__enter__A      r!   c                L    Ub  U R                  5         g U R                  5         g rC   rN  r   r   exc_typeexc_valexc_tbs       r   __exit__MultipartWriter.__exit__D      NNJJLr!   c                @    SU R                   < SU R                  < S3$ )Nzsmart_open.s3.MultipartWriter(r   r  r  r   s    r   r  MultipartWriter.__str__J  r  r!   c                \    SU R                   < SU R                  < SU R                  < S3$ )Nz%smart_open.s3.MultipartWriter(bucket=rb  z, part_size=r  )r   r   rm  r   s    r   rc  MultipartWriter.__repr__M  s!    LLIIOO
 	
r!   )rq  rm  rt  rr  rs  rp  r  )r   zio.BytesIO | NonerC   )r*  r	   returnrg   )r  None)r8   r9   r:   r;   rP   rp  DEFAULT_PART_SIZEr   rx  r   propertyr  r  r8  r   r   r   rG  r@  rK  r0  rN  rU  r  r  r  r  rc  r<   r=   r!   r   r   r   `  s    K J $)-. '.`&P ' ' #,"8"8 &&!@B>8"HS
r!   r   c                      \ rS rSrSrSr   SS jrS rS r\	S 5       r
S rS	 rS
 r\R                  4S jrSS jrS rS rSS jrSS jrS rS rS rS rS rSrg)r   iU  a
  Writes bytes to S3 using the single part API.

Implements the io.BufferedIOBase interface of the standard library.

This class buffers all of its input in memory until its `close` method is called. Only then will
the data be written to S3 and the buffer is released.Nc                    [        XXAU5        Uc  [        R                  " 5       U l        g UR	                  5       (       d  [        S5      eXPl        g )Nz writebuffer needs to be seekable)r
  r   r   rq  r8  r   )r   r   r   rE   ru   r   s         r   r   SinglepartWriter.__init__^  sC     	$sC

DI%%''?@@#Ir!   c                    g rC   r=   r   s    r   rx  SinglepartWriter.flusho  rz  r!   c                   [         R                  S5        U R                  (       a  g U R                  S5         U R                  R                  U R                  U R                  U R                  S9  [         R                  SU 5        U R                  R                  5         g ! [        R                  R                   a  n[        SU R                  -  5      UeS nAff = f)Nr  r   r  z8the bucket %r does not exist, or is forbidden for accessz%s: direct upload finished)r(   r   r  r   r   r  r   r   rq  r   rE   r-   r   r   )r   es     r   r   SinglepartWriter.closeu  s    _%;;		!	bLL##||IIYY $  	148		 ** 	bJT\\Y[`ab	bs   9B$ $C CC c                X    U R                   S L =(       d    U R                   R                  $ rC   )rq  r  r   s    r   r  SinglepartWriter.closed  s     yyD 4DII$4$44r!   c                6    U R                   R                  5       $ z
Propagate.)rq  r  r   s    r   r  SinglepartWriter.readable      yy!!##r!   c                6    U R                   R                  5       $ r  )rq  r  r   s    r   r  SinglepartWriter.writable  r  r!   c                6    U R                   R                  5       $ r  )rq  r8  r   s    r   r8  SinglepartWriter.seekable  r  r!   c                8    U R                   R                  X5      $ r  )rq  r   r=  s      r   r   SinglepartWriter.seek  s    yy~~f--r!   c                8    U R                   R                  U5      $ r  )rq  rG  r&  s     r   rG  SinglepartWriter.truncate  s    yy!!$''r!   c                6    U R                   R                  5       $ r  )rq  r@  r   s    r   r@  SinglepartWriter.tell  s    yy~~r!   c                8    U R                   R                  U5      $ )a  Write the given buffer (bytes, bytearray, memoryview or any buffer
interface implementation) into the buffer. Content of the buffer will be
written to S3 on close as a single-part upload.

For more information about buffers, see https://docs.python.org/3/c-api/buffer.html)rq  r0  )r   r*  s     r   r0  SinglepartWriter.write  s     yyq!!r!   c                8    U R                   R                  U5      $ r  )rq  r   r&  s     r   r   SinglepartWriter.read  s    yy~~d##r!   c                8    U R                   R                  U5      $ r  )rq  r'  r&  s     r   r'  SinglepartWriter.read1  s    yyt$$r!   c                d    U R                   R                  5         [        R                  SU 5        g)zClose buffer and skip upload.z %s: terminated singlepart uploadN)rq  r   r(   r   r   s    r   rN  SinglepartWriter.terminate  s    		7>r!   c                    U $ rC   r=   r   s    r   r  SinglepartWriter.__enter__  r  r!   c                L    Ub  U R                  5         g U R                  5         g rC   r  r  s       r   r  SinglepartWriter.__exit__  r  r!   c                @    SU R                   < SU R                  < S3$ )Nzsmart_open.s3.SinglepartWriter(r   r  r  r   s    r   r  SinglepartWriter.__str__  s    ;?<<SSr!   c                @    SU R                   < SU R                  < S3$ )Nz&smart_open.s3.SinglepartWriter(bucket=rb  r  r  r   s    r   rc  SinglepartWriter.__repr__  s    FJllTXT]T]^^r!   )rq  )NNNrC   r  )r8   r9   r:   r;   rP   rq  r   rx  r   r  r  r  r  r8  r   r   r   rG  r@  r0  r   r'  rN  r  r  r  rc  r<   r=   r!   r   r   r   U  s    = D $"( 5 5$$$ #,"8"8 .( "$%?T_r!   r   c                    gNTr=   )r   s    r   _accept_allr    s    r!    c           	   +  .  #    Uc  [         n U R                  n Su  px[        U 4UUS.UD6n	[        R
                  " [        4U US.UD6n
[        R                  R                  US9 nUR                  X5      nSn  UR                  5       u  pUS-  S:X  a$  [        R                  SX[        U5      US	-  5        X4v   U[        U5      -  nUb
  US
-   U:  a  OUS
-  nMg  SSS5        [        R                  SUS
-   U4-  5        g! [         a     Nf = f! [        R                   R"                   a>  nSUR$                  ;   a"  UR$                  S   R'                  S5      S:X  d  Ue SnANSnAf[(         a     M  f = f! , (       d  f       N= f7f)a  
Iterate and download all S3 objects under `s3://bucket_name/prefix`.

Parameters
----------
bucket_name: str
    The name of the bucket.
prefix: str, optional
    Limits the iteration to keys starting with the prefix.
accept_key: callable, optional
    This is a function that accepts a key name (unicode string) and
    returns True/False, signalling whether the given key should be downloaded.
    The default behavior is to accept all keys.
key_limit: int, optional
    If specified, the iterator will stop after yielding this many results.
workers: int, optional
    The number of subprocesses to use.
retries: int, optional
    The number of time to retry a failed download.
session_kwargs: dict, optional
    Keyword arguments to pass when creating a new session.
    For a list of available names and values, see:
    https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html#boto3.session.Session


Yields
------
str
    The full key name (does not include the bucket name).
bytes
    The full contents of the key.

Notes
-----
The keys are processed in parallel, using `workers` processes (default: 16),
to speed up downloads greatly. If multiprocessing is not available, thus
_MULTIPROCESSING is False, this parameter will be ignored.

Examples
--------

  >>> # get all JSON files under "mybucket/foo/"
  >>> for key, content in iter_bucket(
  ...         bucket_name, prefix='foo/',
  ...         accept_key=lambda key: key.endswith('.json')):
  ...     print key, len(content)

  >>> # limit to 10k files, using 32 parallel workers (default is 16)
  >>> for key, content in iter_bucket(bucket_name, key_limit=10000, workers=32):
  ...     print key, len(content)
N)r   r   )prefix
accept_key)bucket_nameretries)	processesr   i  z,yielding key #%i: %s, size %i (total %.1fMB)g      0Ar#   r$   r%   404z processed %i keys, total size %i)r  r   r   _list_bucketrJ   rK   _download_keyr^   concurrencycreate_poolimap_unordered__next__r(   r  r   r   r   r-   r.   r/   StopIteration)r  r  r  	key_limitworkersr  session_kwargs
total_sizekey_nokey_iteratordownload_keypoolresult_iteratorr   contentr4   s                   r   iter_bucketr     s    v  
!&& J 	L
 $$ 	L 
			+	+g	+	>$--lI!0!9!9!;D=A%KKFS\:3K l"c'l*
(VaZ9-D aKF/  
?6 KK2fqj*5MMNU  > &&22   3<</CLL4I4M4Mf4UY^4^I 5_  1 
?	>sw   
FD	 AF'F=ADF#&F	
DFDFF74E0+F0F=F FF
FFc                    gr  r=   )r   s    r   <lambda>r  E  s    Tr!   c              +  r  #    [         R                  R                  " S0 UD6nUR                  S5      nS n U(       a  [	        XUS9nO	[	        XS9nUR
                  " S0 UD6n US   n	U	 H  n
U
S   nU" U5      (       d  M  Uv   M     UR                  SS 5      nU(       d  g Mp  ! [         a     N(f = f7f)Nr
   )r   PrefixContinuationToken)r   r  Contentsr   NextContinuationTokenr=   )r  r   SessionrE   rh   list_objects_v2rw   r/   )r  r  r  r  r   rE   ctokenrF   r.   r  cr   s               r   r  r  B  s     
 mm##5n5G^^D!FF
 vVF<F))3F3	z*G hc??I  5t<'   		s0   A"B7%B' *B7$B7'
B41B73B44B7c                B   Uc  [        S5      e[        R                  R                  " S0 UD6nUR	                  S5      nUR                  U5      n[        US-   5       H  n [        X`5      nX4s  $    g ! [        R                  R                   a
    Xr:X  a  e  M>  f = f)Nzbucket_name may not be Noner
   r#   r=   )r   r  r   r  r   r   r&   _download_fileobjr   rE   r-   )	key_namer  r  r  r   r
   r   xcontent_bytess	            r   r  r  a  s    677
 mm##5n5G			$	BYY{#F
 7Q;		+-f?M **   ** 	|	s   #A66$BBc                n    [         R                  " 5       nU R                  X5        UR                  5       $ rC   )r   r   download_fileobjr2  )r   r  bufs      r   r  r  |  s)    
 **,C
H*<<>r!   )r  NNr      )Nr  )@rP   
__future__r   r   r   rJ   loggingr+   r   typingr   r   r   r  botocore.clientr   botocore.exceptionsurllib3.exceptionsr   ImportErrorr>   smart_open.bytebufferr^   smart_open.concurrencysmart_open.utilsr   mypy_boto3_s3.clientr   typing_extensionsr	   	getLoggerr8   r(   rk  r  rl  ra   rb   rc   re  URI_EXAMPLESr   r   r   ro  r@   rr   r   r   r   r   r   objectr   r
  BufferedIOBaser   r   r   r  r   r  r  r  r=   r!   r   <module>r%     s   K "  	           -(			8	$  /   T /
%!    Q  QN !GE3 3(DN:!zOL ##k\'&{S {S| O
R O
dr
b'' r
ju_r(( u_p nOf !>+6*  Ls   D5 5E E