Administrator/AWS-DDNS
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Inital Commit

Browse Source
This commit is contained in:
Brett Woodruff
2024-06-10 12:24:37 -04:00
commit 106024bcb4
2223 changed files with 241071 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
Function Source Files/boto3/s3/__init__.py Normal file
View File

@@ -0,0 +1,12 @@
# Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
# https://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.

BIN
Function Source Files/boto3/s3/__pycache__/__init__.cpython-310.pyc Normal file
View File

Binary file not shown.

BIN
Function Source Files/boto3/s3/__pycache__/constants.cpython-310.pyc Normal file
View File

Binary file not shown.

BIN
Function Source Files/boto3/s3/__pycache__/inject.cpython-310.pyc Normal file
View File

Binary file not shown.

BIN
Function Source Files/boto3/s3/__pycache__/transfer.cpython-310.pyc Normal file
View File

Binary file not shown.

17
Function Source Files/boto3/s3/constants.py Normal file
View File

@@ -0,0 +1,17 @@
# Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
# https://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.
# TransferConfig preferred_transfer_client settings
CLASSIC_TRANSFER_CLIENT = "classic"
AUTO_RESOLVE_TRANSFER_CLIENT = "auto"

897
Function Source Files/boto3/s3/inject.py Normal file
View File

@@ -0,0 +1,897 @@
# Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
# https://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.
import copy as python_copy
from botocore.exceptions import ClientError
from boto3 import utils
from boto3.s3.transfer import (
ProgressCallbackInvoker,
S3Transfer,
TransferConfig,
create_transfer_manager,
)
def inject_s3_transfer_methods(class_attributes, **kwargs):
utils.inject_attribute(class_attributes, 'upload_file', upload_file)
utils.inject_attribute(class_attributes, 'download_file', download_file)
utils.inject_attribute(class_attributes, 'copy', copy)
utils.inject_attribute(class_attributes, 'upload_fileobj', upload_fileobj)
utils.inject_attribute(
class_attributes, 'download_fileobj', download_fileobj
)
def inject_bucket_methods(class_attributes, **kwargs):
utils.inject_attribute(class_attributes, 'load', bucket_load)
utils.inject_attribute(class_attributes, 'upload_file', bucket_upload_file)
utils.inject_attribute(
class_attributes, 'download_file', bucket_download_file
)
utils.inject_attribute(class_attributes, 'copy', bucket_copy)
utils.inject_attribute(
class_attributes, 'upload_fileobj', bucket_upload_fileobj
)
utils.inject_attribute(
class_attributes, 'download_fileobj', bucket_download_fileobj
)
def inject_object_methods(class_attributes, **kwargs):
utils.inject_attribute(class_attributes, 'upload_file', object_upload_file)
utils.inject_attribute(
class_attributes, 'download_file', object_download_file
)
utils.inject_attribute(class_attributes, 'copy', object_copy)
utils.inject_attribute(
class_attributes, 'upload_fileobj', object_upload_fileobj
)
utils.inject_attribute(
class_attributes, 'download_fileobj', object_download_fileobj
)
def inject_object_summary_methods(class_attributes, **kwargs):
utils.inject_attribute(class_attributes, 'load', object_summary_load)
def bucket_load(self, *args, **kwargs):
"""
Calls s3.Client.list_buckets() to update the attributes of the Bucket
resource.
"""
# The docstring above is phrased this way to match what the autogenerated
# docs produce.
# We can't actually get the bucket's attributes from a HeadBucket,
# so we need to use a ListBuckets and search for our bucket.
# However, we may fail if we lack permissions to ListBuckets
# or the bucket is in another account. In which case, creation_date
# will be None.
self.meta.data = {}
try:
response = self.meta.client.list_buckets()
for bucket_data in response['Buckets']:
if bucket_data['Name'] == self.name:
self.meta.data = bucket_data
break
except ClientError as e:
if not e.response.get('Error', {}).get('Code') == 'AccessDenied':
raise
def object_summary_load(self, *args, **kwargs):
"""
Calls s3.Client.head_object to update the attributes of the ObjectSummary
resource.
"""
response = self.meta.client.head_object(
Bucket=self.bucket_name, Key=self.key
)
if 'ContentLength' in response:
response['Size'] = response.pop('ContentLength')
self.meta.data = response
def upload_file(
self, Filename, Bucket, Key, ExtraArgs=None, Callback=None, Config=None
):
"""Upload a file to an S3 object.
Usage::
import boto3
s3 = boto3.client('s3')
s3.upload_file('/tmp/hello.txt', 'mybucket', 'hello.txt')
Similar behavior as S3Transfer's upload_file() method, except that
argument names are capitalized. Detailed examples can be found at
:ref:`S3Transfer's Usage <ref_s3transfer_usage>`.
:type Filename: str
:param Filename: The path to the file to upload.
:type Bucket: str
:param Bucket: The name of the bucket to upload to.
:type Key: str
:param Key: The name of the key to upload to.
:type ExtraArgs: dict
:param ExtraArgs: Extra arguments that may be passed to the
client operation. For allowed upload arguments see
boto3.s3.transfer.S3Transfer.ALLOWED_UPLOAD_ARGS.
:type Callback: function
:param Callback: A method which takes a number of bytes transferred to
be periodically called during the upload.
:type Config: boto3.s3.transfer.TransferConfig
:param Config: The transfer configuration to be used when performing the
transfer.
"""
with S3Transfer(self, Config) as transfer:
return transfer.upload_file(
filename=Filename,
bucket=Bucket,
key=Key,
extra_args=ExtraArgs,
callback=Callback,
)
def download_file(
self, Bucket, Key, Filename, ExtraArgs=None, Callback=None, Config=None
):
"""Download an S3 object to a file.
Usage::
import boto3
s3 = boto3.resource('s3')
s3.meta.client.download_file('mybucket', 'hello.txt', '/tmp/hello.txt')
Similar behavior as S3Transfer's download_file() method,
except that parameters are capitalized. Detailed examples can be found at
:ref:`S3Transfer's Usage <ref_s3transfer_usage>`.
:type Bucket: str
:param Bucket: The name of the bucket to download from.
:type Key: str
:param Key: The name of the key to download from.
:type Filename: str
:param Filename: The path to the file to download to.
:type ExtraArgs: dict
:param ExtraArgs: Extra arguments that may be passed to the
client operation. For allowed download arguments see
boto3.s3.transfer.S3Transfer.ALLOWED_DOWNLOAD_ARGS.
:type Callback: function
:param Callback: A method which takes a number of bytes transferred to
be periodically called during the download.
:type Config: boto3.s3.transfer.TransferConfig
:param Config: The transfer configuration to be used when performing the
transfer.
"""
with S3Transfer(self, Config) as transfer:
return transfer.download_file(
bucket=Bucket,
key=Key,
filename=Filename,
extra_args=ExtraArgs,
callback=Callback,
)
def bucket_upload_file(
self, Filename, Key, ExtraArgs=None, Callback=None, Config=None
):
"""Upload a file to an S3 object.
Usage::
import boto3
s3 = boto3.resource('s3')
s3.Bucket('mybucket').upload_file('/tmp/hello.txt', 'hello.txt')
Similar behavior as S3Transfer's upload_file() method,
except that parameters are capitalized. Detailed examples can be found at
:ref:`S3Transfer's Usage <ref_s3transfer_usage>`.
:type Filename: str
:param Filename: The path to the file to upload.
:type Key: str
:param Key: The name of the key to upload to.
:type ExtraArgs: dict
:param ExtraArgs: Extra arguments that may be passed to the
client operation. For allowed upload arguments see
boto3.s3.transfer.S3Transfer.ALLOWED_UPLOAD_ARGS.
:type Callback: function
:param Callback: A method which takes a number of bytes transferred to
be periodically called during the upload.
:type Config: boto3.s3.transfer.TransferConfig
:param Config: The transfer configuration to be used when performing the
transfer.
"""
return self.meta.client.upload_file(
Filename=Filename,
Bucket=self.name,
Key=Key,
ExtraArgs=ExtraArgs,
Callback=Callback,
Config=Config,
)
def bucket_download_file(
self, Key, Filename, ExtraArgs=None, Callback=None, Config=None
):
"""Download an S3 object to a file.
Usage::
import boto3
s3 = boto3.resource('s3')
s3.Bucket('mybucket').download_file('hello.txt', '/tmp/hello.txt')
Similar behavior as S3Transfer's download_file() method,
except that parameters are capitalized. Detailed examples can be found at
:ref:`S3Transfer's Usage <ref_s3transfer_usage>`.
:type Key: str
:param Key: The name of the key to download from.
:type Filename: str
:param Filename: The path to the file to download to.
:type ExtraArgs: dict
:param ExtraArgs: Extra arguments that may be passed to the
client operation. For allowed download arguments see
boto3.s3.transfer.S3Transfer.ALLOWED_DOWNLOAD_ARGS.
:type Callback: function
:param Callback: A method which takes a number of bytes transferred to
be periodically called during the download.
:type Config: boto3.s3.transfer.TransferConfig
:param Config: The transfer configuration to be used when performing the
transfer.
"""
return self.meta.client.download_file(
Bucket=self.name,
Key=Key,
Filename=Filename,
ExtraArgs=ExtraArgs,
Callback=Callback,
Config=Config,
)
def object_upload_file(
self, Filename, ExtraArgs=None, Callback=None, Config=None
):
"""Upload a file to an S3 object.
Usage::
import boto3
s3 = boto3.resource('s3')
s3.Object('mybucket', 'hello.txt').upload_file('/tmp/hello.txt')
Similar behavior as S3Transfer's upload_file() method,
except that parameters are capitalized. Detailed examples can be found at
:ref:`S3Transfer's Usage <ref_s3transfer_usage>`.
:type Filename: str
:param Filename: The path to the file to upload.
:type ExtraArgs: dict
:param ExtraArgs: Extra arguments that may be passed to the
client operation. For allowed upload arguments see
boto3.s3.transfer.S3Transfer.ALLOWED_UPLOAD_ARGS.
:type Callback: function
:param Callback: A method which takes a number of bytes transferred to
be periodically called during the upload.
:type Config: boto3.s3.transfer.TransferConfig
:param Config: The transfer configuration to be used when performing the
transfer.
"""
return self.meta.client.upload_file(
Filename=Filename,
Bucket=self.bucket_name,
Key=self.key,
ExtraArgs=ExtraArgs,
Callback=Callback,
Config=Config,
)
def object_download_file(
self, Filename, ExtraArgs=None, Callback=None, Config=None
):
"""Download an S3 object to a file.
Usage::
import boto3
s3 = boto3.resource('s3')
s3.Object('mybucket', 'hello.txt').download_file('/tmp/hello.txt')
Similar behavior as S3Transfer's download_file() method,
except that parameters are capitalized. Detailed examples can be found at
:ref:`S3Transfer's Usage <ref_s3transfer_usage>`.
:type Filename: str
:param Filename: The path to the file to download to.
:type ExtraArgs: dict
:param ExtraArgs: Extra arguments that may be passed to the
client operation. For allowed download arguments see
boto3.s3.transfer.S3Transfer.ALLOWED_DOWNLOAD_ARGS.
:type Callback: function
:param Callback: A method which takes a number of bytes transferred to
be periodically called during the download.
:type Config: boto3.s3.transfer.TransferConfig
:param Config: The transfer configuration to be used when performing the
transfer.
"""
return self.meta.client.download_file(
Bucket=self.bucket_name,
Key=self.key,
Filename=Filename,
ExtraArgs=ExtraArgs,
Callback=Callback,
Config=Config,
)
def copy(
self,
CopySource,
Bucket,
Key,
ExtraArgs=None,
Callback=None,
SourceClient=None,
Config=None,
):
"""Copy an object from one S3 location to another.
This is a managed transfer which will perform a multipart copy in
multiple threads if necessary.
Usage::
import boto3
s3 = boto3.resource('s3')
copy_source = {
'Bucket': 'mybucket',
'Key': 'mykey'
}
s3.meta.client.copy(copy_source, 'otherbucket', 'otherkey')
:type CopySource: dict
:param CopySource: The name of the source bucket, key name of the
source object, and optional version ID of the source object. The
dictionary format is:
``{'Bucket': 'bucket', 'Key': 'key', 'VersionId': 'id'}``. Note
that the ``VersionId`` key is optional and may be omitted.
:type Bucket: str
:param Bucket: The name of the bucket to copy to
:type Key: str
:param Key: The name of the key to copy to
:type ExtraArgs: dict
:param ExtraArgs: Extra arguments that may be passed to the
client operation. For allowed download arguments see
boto3.s3.transfer.S3Transfer.ALLOWED_DOWNLOAD_ARGS.
:type Callback: function
:param Callback: A method which takes a number of bytes transferred to
be periodically called during the copy.
:type SourceClient: botocore or boto3 Client
:param SourceClient: The client to be used for operation that
may happen at the source object. For example, this client is
used for the head_object that determines the size of the copy.
If no client is provided, the current client is used as the client
for the source object.
:type Config: boto3.s3.transfer.TransferConfig
:param Config: The transfer configuration to be used when performing the
copy.
"""
subscribers = None
if Callback is not None:
subscribers = [ProgressCallbackInvoker(Callback)]
config = Config
if config is None:
config = TransferConfig()
# copy is not supported in the CRT
new_config = python_copy.copy(config)
new_config.preferred_transfer_client = "classic"
with create_transfer_manager(self, new_config) as manager:
future = manager.copy(
copy_source=CopySource,
bucket=Bucket,
key=Key,
extra_args=ExtraArgs,
subscribers=subscribers,
source_client=SourceClient,
)
return future.result()
def bucket_copy(
self,
CopySource,
Key,
ExtraArgs=None,
Callback=None,
SourceClient=None,
Config=None,
):
"""Copy an object from one S3 location to an object in this bucket.
This is a managed transfer which will perform a multipart copy in
multiple threads if necessary.
Usage::
import boto3
s3 = boto3.resource('s3')
copy_source = {
'Bucket': 'mybucket',
'Key': 'mykey'
}
bucket = s3.Bucket('otherbucket')
bucket.copy(copy_source, 'otherkey')
:type CopySource: dict
:param CopySource: The name of the source bucket, key name of the
source object, and optional version ID of the source object. The
dictionary format is:
``{'Bucket': 'bucket', 'Key': 'key', 'VersionId': 'id'}``. Note
that the ``VersionId`` key is optional and may be omitted.
:type Key: str
:param Key: The name of the key to copy to
:type ExtraArgs: dict
:param ExtraArgs: Extra arguments that may be passed to the
client operation. For allowed download arguments see
boto3.s3.transfer.S3Transfer.ALLOWED_DOWNLOAD_ARGS.
:type Callback: function
:param Callback: A method which takes a number of bytes transferred to
be periodically called during the copy.
:type SourceClient: botocore or boto3 Client
:param SourceClient: The client to be used for operation that
may happen at the source object. For example, this client is
used for the head_object that determines the size of the copy.
If no client is provided, the current client is used as the client
for the source object.
:type Config: boto3.s3.transfer.TransferConfig
:param Config: The transfer configuration to be used when performing the
copy.
"""
return self.meta.client.copy(
CopySource=CopySource,
Bucket=self.name,
Key=Key,
ExtraArgs=ExtraArgs,
Callback=Callback,
SourceClient=SourceClient,
Config=Config,
)
def object_copy(
self,
CopySource,
ExtraArgs=None,
Callback=None,
SourceClient=None,
Config=None,
):
"""Copy an object from one S3 location to this object.
This is a managed transfer which will perform a multipart copy in
multiple threads if necessary.
Usage::
import boto3
s3 = boto3.resource('s3')
copy_source = {
'Bucket': 'mybucket',
'Key': 'mykey'
}
bucket = s3.Bucket('otherbucket')
obj = bucket.Object('otherkey')
obj.copy(copy_source)
:type CopySource: dict
:param CopySource: The name of the source bucket, key name of the
source object, and optional version ID of the source object. The
dictionary format is:
``{'Bucket': 'bucket', 'Key': 'key', 'VersionId': 'id'}``. Note
that the ``VersionId`` key is optional and may be omitted.
:type ExtraArgs: dict
:param ExtraArgs: Extra arguments that may be passed to the
client operation. For allowed download arguments see
boto3.s3.transfer.S3Transfer.ALLOWED_DOWNLOAD_ARGS.
:type Callback: function
:param Callback: A method which takes a number of bytes transferred to
be periodically called during the copy.
:type SourceClient: botocore or boto3 Client
:param SourceClient: The client to be used for operation that
may happen at the source object. For example, this client is
used for the head_object that determines the size of the copy.
If no client is provided, the current client is used as the client
for the source object.
:type Config: boto3.s3.transfer.TransferConfig
:param Config: The transfer configuration to be used when performing the
copy.
"""
return self.meta.client.copy(
CopySource=CopySource,
Bucket=self.bucket_name,
Key=self.key,
ExtraArgs=ExtraArgs,
Callback=Callback,
SourceClient=SourceClient,
Config=Config,
)
def upload_fileobj(
self, Fileobj, Bucket, Key, ExtraArgs=None, Callback=None, Config=None
):
"""Upload a file-like object to S3.
The file-like object must be in binary mode.
This is a managed transfer which will perform a multipart upload in
multiple threads if necessary.
Usage::
import boto3
s3 = boto3.client('s3')
with open('filename', 'rb') as data:
s3.upload_fileobj(data, 'mybucket', 'mykey')
:type Fileobj: a file-like object
:param Fileobj: A file-like object to upload. At a minimum, it must
implement the `read` method, and must return bytes.
:type Bucket: str
:param Bucket: The name of the bucket to upload to.
:type Key: str
:param Key: The name of the key to upload to.
:type ExtraArgs: dict
:param ExtraArgs: Extra arguments that may be passed to the
client operation. For allowed upload arguments see
boto3.s3.transfer.S3Transfer.ALLOWED_UPLOAD_ARGS.
:type Callback: function
:param Callback: A method which takes a number of bytes transferred to
be periodically called during the upload.
:type Config: boto3.s3.transfer.TransferConfig
:param Config: The transfer configuration to be used when performing the
upload.
"""
if not hasattr(Fileobj, 'read'):
raise ValueError('Fileobj must implement read')
subscribers = None
if Callback is not None:
subscribers = [ProgressCallbackInvoker(Callback)]
config = Config
if config is None:
config = TransferConfig()
with create_transfer_manager(self, config) as manager:
future = manager.upload(
fileobj=Fileobj,
bucket=Bucket,
key=Key,
extra_args=ExtraArgs,
subscribers=subscribers,
)
return future.result()
def bucket_upload_fileobj(
self, Fileobj, Key, ExtraArgs=None, Callback=None, Config=None
):
"""Upload a file-like object to this bucket.
The file-like object must be in binary mode.
This is a managed transfer which will perform a multipart upload in
multiple threads if necessary.
Usage::
import boto3
s3 = boto3.resource('s3')
bucket = s3.Bucket('mybucket')
with open('filename', 'rb') as data:
bucket.upload_fileobj(data, 'mykey')
:type Fileobj: a file-like object
:param Fileobj: A file-like object to upload. At a minimum, it must
implement the `read` method, and must return bytes.
:type Key: str
:param Key: The name of the key to upload to.
:type ExtraArgs: dict
:param ExtraArgs: Extra arguments that may be passed to the
client operation. For allowed upload arguments see
boto3.s3.transfer.S3Transfer.ALLOWED_UPLOAD_ARGS.
:type Callback: function
:param Callback: A method which takes a number of bytes transferred to
be periodically called during the upload.
:type Config: boto3.s3.transfer.TransferConfig
:param Config: The transfer configuration to be used when performing the
upload.
"""
return self.meta.client.upload_fileobj(
Fileobj=Fileobj,
Bucket=self.name,
Key=Key,
ExtraArgs=ExtraArgs,
Callback=Callback,
Config=Config,
)
def object_upload_fileobj(
self, Fileobj, ExtraArgs=None, Callback=None, Config=None
):
"""Upload a file-like object to this object.
The file-like object must be in binary mode.
This is a managed transfer which will perform a multipart upload in
multiple threads if necessary.
Usage::
import boto3
s3 = boto3.resource('s3')
bucket = s3.Bucket('mybucket')
obj = bucket.Object('mykey')
with open('filename', 'rb') as data:
obj.upload_fileobj(data)
:type Fileobj: a file-like object
:param Fileobj: A file-like object to upload. At a minimum, it must
implement the `read` method, and must return bytes.
:type ExtraArgs: dict
:param ExtraArgs: Extra arguments that may be passed to the
client operation. For allowed upload arguments see
boto3.s3.transfer.S3Transfer.ALLOWED_UPLOAD_ARGS.
:type Callback: function
:param Callback: A method which takes a number of bytes transferred to
be periodically called during the upload.
:type Config: boto3.s3.transfer.TransferConfig
:param Config: The transfer configuration to be used when performing the
upload.
"""
return self.meta.client.upload_fileobj(
Fileobj=Fileobj,
Bucket=self.bucket_name,
Key=self.key,
ExtraArgs=ExtraArgs,
Callback=Callback,
Config=Config,
)
def download_fileobj(
self, Bucket, Key, Fileobj, ExtraArgs=None, Callback=None, Config=None
):
"""Download an object from S3 to a file-like object.
The file-like object must be in binary mode.
This is a managed transfer which will perform a multipart download in
multiple threads if necessary.
Usage::
import boto3
s3 = boto3.client('s3')
with open('filename', 'wb') as data:
s3.download_fileobj('mybucket', 'mykey', data)
:type Bucket: str
:param Bucket: The name of the bucket to download from.
:type Key: str
:param Key: The name of the key to download from.
:type Fileobj: a file-like object
:param Fileobj: A file-like object to download into. At a minimum, it must
implement the `write` method and must accept bytes.
:type ExtraArgs: dict
:param ExtraArgs: Extra arguments that may be passed to the
client operation. For allowed download arguments see
boto3.s3.transfer.S3Transfer.ALLOWED_DOWNLOAD_ARGS.
:type Callback: function
:param Callback: A method which takes a number of bytes transferred to
be periodically called during the download.
:type Config: boto3.s3.transfer.TransferConfig
:param Config: The transfer configuration to be used when performing the
download.
"""
if not hasattr(Fileobj, 'write'):
raise ValueError('Fileobj must implement write')
subscribers = None
if Callback is not None:
subscribers = [ProgressCallbackInvoker(Callback)]
config = Config
if config is None:
config = TransferConfig()
with create_transfer_manager(self, config) as manager:
future = manager.download(
bucket=Bucket,
key=Key,
fileobj=Fileobj,
extra_args=ExtraArgs,
subscribers=subscribers,
)
return future.result()
def bucket_download_fileobj(
self, Key, Fileobj, ExtraArgs=None, Callback=None, Config=None
):
"""Download an object from this bucket to a file-like-object.
The file-like object must be in binary mode.
This is a managed transfer which will perform a multipart download in
multiple threads if necessary.
Usage::
import boto3
s3 = boto3.resource('s3')
bucket = s3.Bucket('mybucket')
with open('filename', 'wb') as data:
bucket.download_fileobj('mykey', data)
:type Fileobj: a file-like object
:param Fileobj: A file-like object to download into. At a minimum, it must
implement the `write` method and must accept bytes.
:type Key: str
:param Key: The name of the key to download from.
:type ExtraArgs: dict
:param ExtraArgs: Extra arguments that may be passed to the
client operation. For allowed download arguments see
boto3.s3.transfer.S3Transfer.ALLOWED_DOWNLOAD_ARGS.
:type Callback: function
:param Callback: A method which takes a number of bytes transferred to
be periodically called during the download.
:type Config: boto3.s3.transfer.TransferConfig
:param Config: The transfer configuration to be used when performing the
download.
"""
return self.meta.client.download_fileobj(
Bucket=self.name,
Key=Key,
Fileobj=Fileobj,
ExtraArgs=ExtraArgs,
Callback=Callback,
Config=Config,
)
def object_download_fileobj(
self, Fileobj, ExtraArgs=None, Callback=None, Config=None
):
"""Download this object from S3 to a file-like object.
The file-like object must be in binary mode.
This is a managed transfer which will perform a multipart download in
multiple threads if necessary.
Usage::
import boto3
s3 = boto3.resource('s3')
bucket = s3.Bucket('mybucket')
obj = bucket.Object('mykey')
with open('filename', 'wb') as data:
obj.download_fileobj(data)
:type Fileobj: a file-like object
:param Fileobj: A file-like object to download into. At a minimum, it must
implement the `write` method and must accept bytes.
:type ExtraArgs: dict
:param ExtraArgs: Extra arguments that may be passed to the
client operation. For allowed download arguments see
boto3.s3.transfer.S3Transfer.ALLOWED_DOWNLOAD_ARGS.
:type Callback: function
:param Callback: A method which takes a number of bytes transferred to
be periodically called during the download.
:type Config: boto3.s3.transfer.TransferConfig
:param Config: The transfer configuration to be used when performing the
download.
"""
return self.meta.client.download_fileobj(
Bucket=self.bucket_name,
Key=self.key,
Fileobj=Fileobj,
ExtraArgs=ExtraArgs,
Callback=Callback,
Config=Config,
)

437
Function Source Files/boto3/s3/transfer.py Normal file
View File

@@ -0,0 +1,437 @@
# Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
# https://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.
"""Abstractions over S3's upload/download operations.
This module provides high level abstractions for efficient
uploads/downloads. It handles several things for the user:
* Automatically switching to multipart transfers when
a file is over a specific size threshold
* Uploading/downloading a file in parallel
* Progress callbacks to monitor transfers
* Retries. While botocore handles retries for streaming uploads,
it is not possible for it to handle retries for streaming
downloads. This module handles retries for both cases so
you don't need to implement any retry logic yourself.
This module has a reasonable set of defaults. It also allows you
to configure many aspects of the transfer process including:
* Multipart threshold size
* Max parallel downloads
* Socket timeouts
* Retry amounts
There is no support for s3->s3 multipart copies at this
time.
.. _ref_s3transfer_usage:
Usage
=====
The simplest way to use this module is:
.. code-block:: python
client = boto3.client('s3', 'us-west-2')
transfer = S3Transfer(client)
# Upload /tmp/myfile to s3://bucket/key
transfer.upload_file('/tmp/myfile', 'bucket', 'key')
# Download s3://bucket/key to /tmp/myfile
transfer.download_file('bucket', 'key', '/tmp/myfile')
The ``upload_file`` and ``download_file`` methods also accept
``**kwargs``, which will be forwarded through to the corresponding
client operation. Here are a few examples using ``upload_file``::
# Making the object public
transfer.upload_file('/tmp/myfile', 'bucket', 'key',
extra_args={'ACL': 'public-read'})
# Setting metadata
transfer.upload_file('/tmp/myfile', 'bucket', 'key',
extra_args={'Metadata': {'a': 'b', 'c': 'd'}})
# Setting content type
transfer.upload_file('/tmp/myfile.json', 'bucket', 'key',
extra_args={'ContentType': "application/json"})
The ``S3Transfer`` class also supports progress callbacks so you can
provide transfer progress to users. Both the ``upload_file`` and
``download_file`` methods take an optional ``callback`` parameter.
Here's an example of how to print a simple progress percentage
to the user:
.. code-block:: python
class ProgressPercentage(object):
def __init__(self, filename):
self._filename = filename
self._size = float(os.path.getsize(filename))
self._seen_so_far = 0
self._lock = threading.Lock()
def __call__(self, bytes_amount):
# To simplify we'll assume this is hooked up
# to a single filename.
with self._lock:
self._seen_so_far += bytes_amount
percentage = (self._seen_so_far / self._size) * 100
sys.stdout.write(
"\r%s %s / %s (%.2f%%)" % (
self._filename, self._seen_so_far, self._size,
percentage))
sys.stdout.flush()
transfer = S3Transfer(boto3.client('s3', 'us-west-2'))
# Upload /tmp/myfile to s3://bucket/key and print upload progress.
transfer.upload_file('/tmp/myfile', 'bucket', 'key',
callback=ProgressPercentage('/tmp/myfile'))
You can also provide a TransferConfig object to the S3Transfer
object that gives you more fine grained control over the
transfer. For example:
.. code-block:: python
client = boto3.client('s3', 'us-west-2')
config = TransferConfig(
multipart_threshold=8 * 1024 * 1024,
max_concurrency=10,
num_download_attempts=10,
)
transfer = S3Transfer(client, config)
transfer.upload_file('/tmp/foo', 'bucket', 'key')
"""
import logging
import threading
from os import PathLike, fspath, getpid
from botocore.compat import HAS_CRT
from botocore.exceptions import ClientError
from s3transfer.exceptions import (
RetriesExceededError as S3TransferRetriesExceededError,
)
from s3transfer.futures import NonThreadedExecutor
from s3transfer.manager import TransferConfig as S3TransferConfig
from s3transfer.manager import TransferManager
from s3transfer.subscribers import BaseSubscriber
from s3transfer.utils import OSUtils
import boto3.s3.constants as constants
from boto3.exceptions import RetriesExceededError, S3UploadFailedError
if HAS_CRT:
import awscrt.s3
from boto3.crt import create_crt_transfer_manager
KB = 1024
MB = KB * KB
logger = logging.getLogger(__name__)
def create_transfer_manager(client, config, osutil=None):
"""Creates a transfer manager based on configuration
:type client: boto3.client
:param client: The S3 client to use
:type config: boto3.s3.transfer.TransferConfig
:param config: The transfer config to use
:type osutil: s3transfer.utils.OSUtils
:param osutil: The os utility to use
:rtype: s3transfer.manager.TransferManager
:returns: A transfer manager based on parameters provided
"""
if _should_use_crt(config):
crt_transfer_manager = create_crt_transfer_manager(client, config)
if crt_transfer_manager is not None:
logger.debug(
f"Using CRT client. pid: {getpid()}, thread: {threading.get_ident()}"
)
return crt_transfer_manager
# If we don't resolve something above, fallback to the default.
logger.debug(
f"Using default client. pid: {getpid()}, thread: {threading.get_ident()}"
)
return _create_default_transfer_manager(client, config, osutil)
def _should_use_crt(config):
# This feature requires awscrt>=0.19.18
if HAS_CRT and has_minimum_crt_version((0, 19, 18)):
is_optimized_instance = awscrt.s3.is_optimized_for_system()
else:
is_optimized_instance = False
pref_transfer_client = config.preferred_transfer_client.lower()
if (
is_optimized_instance
and pref_transfer_client == constants.AUTO_RESOLVE_TRANSFER_CLIENT
):
logger.debug(
"Attempting to use CRTTransferManager. Config settings may be ignored."
)
return True
logger.debug(
"Opting out of CRT Transfer Manager. Preferred client: "
f"{pref_transfer_client}, CRT available: {HAS_CRT}, "
f"Instance Optimized: {is_optimized_instance}."
)
return False
def has_minimum_crt_version(minimum_version):
"""Not intended for use outside boto3."""
if not HAS_CRT:
return False
crt_version_str = awscrt.__version__
try:
crt_version_ints = map(int, crt_version_str.split("."))
crt_version_tuple = tuple(crt_version_ints)
except (TypeError, ValueError):
return False
return crt_version_tuple >= minimum_version
def _create_default_transfer_manager(client, config, osutil):
"""Create the default TransferManager implementation for s3transfer."""
executor_cls = None
if not config.use_threads:
executor_cls = NonThreadedExecutor
return TransferManager(client, config, osutil, executor_cls)
class TransferConfig(S3TransferConfig):
ALIAS = {
'max_concurrency': 'max_request_concurrency',
'max_io_queue': 'max_io_queue_size',
}
def __init__(
self,
multipart_threshold=8 * MB,
max_concurrency=10,
multipart_chunksize=8 * MB,
num_download_attempts=5,
max_io_queue=100,
io_chunksize=256 * KB,
use_threads=True,
max_bandwidth=None,
preferred_transfer_client=constants.AUTO_RESOLVE_TRANSFER_CLIENT,
):
"""Configuration object for managed S3 transfers
:param multipart_threshold: The transfer size threshold for which
multipart uploads, downloads, and copies will automatically be
triggered.
:param max_concurrency: The maximum number of threads that will be
making requests to perform a transfer. If ``use_threads`` is
set to ``False``, the value provided is ignored as the transfer
will only ever use the main thread.
:param multipart_chunksize: The partition size of each part for a
multipart transfer.
:param num_download_attempts: The number of download attempts that
will be retried upon errors with downloading an object in S3.
Note that these retries account for errors that occur when
streaming down the data from s3 (i.e. socket errors and read
timeouts that occur after receiving an OK response from s3).
Other retryable exceptions such as throttling errors and 5xx
errors are already retried by botocore (this default is 5). This
does not take into account the number of exceptions retried by
botocore.
:param max_io_queue: The maximum amount of read parts that can be
queued in memory to be written for a download. The size of each
of these read parts is at most the size of ``io_chunksize``.
:param io_chunksize: The max size of each chunk in the io queue.
Currently, this is size used when ``read`` is called on the
downloaded stream as well.
:param use_threads: If True, threads will be used when performing
S3 transfers. If False, no threads will be used in
performing transfers; all logic will be run in the main thread.
:param max_bandwidth: The maximum bandwidth that will be consumed
in uploading and downloading file content. The value is an integer
in terms of bytes per second.
:param preferred_transfer_client: String specifying preferred transfer
client for transfer operations.
Current supported settings are:
* auto (default) - Use the CRTTransferManager when calls
are made with supported environment and settings.
* classic - Only use the origin S3TransferManager with
requests. Disables possible CRT upgrade on requests.
"""
super().__init__(
multipart_threshold=multipart_threshold,
max_request_concurrency=max_concurrency,
multipart_chunksize=multipart_chunksize,
num_download_attempts=num_download_attempts,
max_io_queue_size=max_io_queue,
io_chunksize=io_chunksize,
max_bandwidth=max_bandwidth,
)
# Some of the argument names are not the same as the inherited
# S3TransferConfig so we add aliases so you can still access the
# old version of the names.
for alias in self.ALIAS:
setattr(self, alias, getattr(self, self.ALIAS[alias]))
self.use_threads = use_threads
self.preferred_transfer_client = preferred_transfer_client
def __setattr__(self, name, value):
# If the alias name is used, make sure we set the name that it points
# to as that is what actually is used in governing the TransferManager.
if name in self.ALIAS:
super().__setattr__(self.ALIAS[name], value)
# Always set the value of the actual name provided.
super().__setattr__(name, value)
class S3Transfer:
ALLOWED_DOWNLOAD_ARGS = TransferManager.ALLOWED_DOWNLOAD_ARGS
ALLOWED_UPLOAD_ARGS = TransferManager.ALLOWED_UPLOAD_ARGS
def __init__(self, client=None, config=None, osutil=None, manager=None):
if not client and not manager:
raise ValueError(
'Either a boto3.Client or s3transfer.manager.TransferManager '
'must be provided'
)
if manager and any([client, config, osutil]):
raise ValueError(
'Manager cannot be provided with client, config, '
'nor osutil. These parameters are mutually exclusive.'
)
if config is None:
config = TransferConfig()
if osutil is None:
osutil = OSUtils()
if manager:
self._manager = manager
else:
self._manager = create_transfer_manager(client, config, osutil)
def upload_file(
self, filename, bucket, key, callback=None, extra_args=None
):
"""Upload a file to an S3 object.
Variants have also been injected into S3 client, Bucket and Object.
You don't have to use S3Transfer.upload_file() directly.
.. seealso::
:py:meth:`S3.Client.upload_file`
:py:meth:`S3.Client.upload_fileobj`
"""
if isinstance(filename, PathLike):
filename = fspath(filename)
if not isinstance(filename, str):
raise ValueError('Filename must be a string or a path-like object')
subscribers = self._get_subscribers(callback)
future = self._manager.upload(
filename, bucket, key, extra_args, subscribers
)
try:
future.result()
# If a client error was raised, add the backwards compatibility layer
# that raises a S3UploadFailedError. These specific errors were only
# ever thrown for upload_parts but now can be thrown for any related
# client error.
except ClientError as e:
raise S3UploadFailedError(
"Failed to upload {} to {}: {}".format(
filename, '/'.join([bucket, key]), e
)
)
def download_file(
self, bucket, key, filename, extra_args=None, callback=None
):
"""Download an S3 object to a file.
Variants have also been injected into S3 client, Bucket and Object.
You don't have to use S3Transfer.download_file() directly.
.. seealso::
:py:meth:`S3.Client.download_file`
:py:meth:`S3.Client.download_fileobj`
"""
if isinstance(filename, PathLike):
filename = fspath(filename)
if not isinstance(filename, str):
raise ValueError('Filename must be a string or a path-like object')
subscribers = self._get_subscribers(callback)
future = self._manager.download(
bucket, key, filename, extra_args, subscribers
)
try:
future.result()
# This is for backwards compatibility where when retries are
# exceeded we need to throw the same error from boto3 instead of
# s3transfer's built in RetriesExceededError as current users are
# catching the boto3 one instead of the s3transfer exception to do
# their own retries.
except S3TransferRetriesExceededError as e:
raise RetriesExceededError(e.last_exception)
def _get_subscribers(self, callback):
if not callback:
return None
return [ProgressCallbackInvoker(callback)]
def __enter__(self):
return self
def __exit__(self, *args):
self._manager.__exit__(*args)
class ProgressCallbackInvoker(BaseSubscriber):
"""A back-compat wrapper to invoke a provided callback via a subscriber
:param callback: A callable that takes a single positional argument for
how many bytes were transferred.
"""
def __init__(self, callback):
self._callback = callback
def on_progress(self, bytes_transferred, **kwargs):
self._callback(bytes_transferred)