Exasol BucketFS
Uploading and downloading to/from the BucketFS¶
With these functions you can upload and download data to and from the BucketFS. The upload and download functions can have different sources or targets, such as files, file-objects, strings or Python objects. For more details, please refer to our
Upload API
Download API
Uploading and downloading data from and to files¶
In their simplest form the download and upload functions simply upload a local file to the BucketFS or download the data from the BucketFS into a local file.
Example:
from pathlib import Path
from exasol_bucketfs_utils_python import upload, download
from exasol_bucketfs_utils_python.bucket_config import BucketConfig
from exasol_bucketfs_utils_python.bucketfs_config import BucketFSConfig
from exasol_bucketfs_utils_python.bucketfs_connection_config import BucketFSConnectionConfig
connection_config = BucketFSConnectionConfig(
host="localhost", port=6666,
user="w", pwd="write",
is_https=False)
bucketfs_config = BucketFSConfig(
connection_config=connection_config,
bucketfs_name="bfsdefault")
bucket_config = BucketConfig(
bucket_name="default",
bucketfs_config=bucketfs_config)
local_input_file_path = Path("local_input_file.txt")
path_in_bucket = "path/in/bucket/file.txt"
upload.upload_file_to_bucketfs(
bucket_config=bucket_config,
bucket_file_path=path_in_bucket,
local_file_path=local_input_file_path)
local_output_file_path = Path("local_output_file.txt")
download.download_from_bucketfs_to_file(
bucket_config=bucket_config,
bucket_file_path=path_in_bucket,
local_file_path=local_output_file_path)
Uploading and downloading data from and to file-objects¶
A more sophisticated version of the previous function allows you
to upload from or download into a
file-object.
A file-object can be a local file you opened with the open() function or
it could be also an in-memory stream such as io.BytesIO.
Other modules may provide additional ways to create file-objects. See
socket.socket.makefile() for example.
Example:
from pathlib import Path
from tempfile import NamedTemporaryFile
from exasol_bucketfs_utils_python import upload, download
from exasol_bucketfs_utils_python.bucket_config import BucketConfig
from exasol_bucketfs_utils_python.bucketfs_config import BucketFSConfig
from exasol_bucketfs_utils_python.bucketfs_connection_config import BucketFSConnectionConfig
connection_config = BucketFSConnectionConfig(
host="localhost", port=6666,
user="w", pwd="write",
is_https=False)
bucketfs_config = BucketFSConfig(
connection_config=connection_config,
bucketfs_name="bfsdefault")
bucket_config = BucketConfig(
bucket_name="default",
bucketfs_config=bucketfs_config)
with NamedTemporaryFile() as input_temp_file:
test_byte_string = b"test_byte_string"
input_temp_file.write(test_byte_string)
input_temp_file.flush()
input_temp_file.seek(0)
path_in_bucket = "path/in/bucket/file.txt"
upload.upload_fileobj_to_bucketfs(
bucket_config=bucket_config,
bucket_file_path=path_in_bucket,
fileobj=input_temp_file)
with NamedTemporaryFile() as output_temp_file:
download.download_from_bucketfs_to_fileobj(
bucket_config=bucket_config,
bucket_file_path=path_in_bucket,
fileobj=output_temp_file)
output_temp_file.flush()
output_temp_file.seek(0)
output_test_byte_string = output_temp_file.read()
Uploading and downloading data from and to a string¶
This library also provides functions to simply upload a Python string or download into a Python string. This for example, can be useful to upload or download configuration in json or yaml format from the BucketFS.
Example:
from exasol_bucketfs_utils_python import upload, download
from exasol_bucketfs_utils_python.bucket_config import BucketConfig
from exasol_bucketfs_utils_python.bucketfs_config import BucketFSConfig
from exasol_bucketfs_utils_python.bucketfs_connection_config import BucketFSConnectionConfig
connection_config = BucketFSConnectionConfig(
host="localhost", port=6666,
user="w", pwd="write",
is_https=False)
bucketfs_config = BucketFSConfig(
connection_config=connection_config,
bucketfs_name="bfsdefault")
bucket_config = BucketConfig(
bucket_name="default",
bucketfs_config=bucketfs_config)
test_string = "test_string"
path_in_bucket = "path/in/bucket/file.txt"
upload.upload_string_to_bucketfs(
bucket_config=bucket_config,
bucket_file_path=path_in_bucket,
string=test_string)
output_test_string = \
download.download_from_bucketfs_to_string(
bucket_config=bucket_config,
bucket_file_path=path_in_bucket)
Serialize and deserialize a Python object directly into or from BucketFS¶
Sometimes it can be useful to directly upload a serialized Python object to the BucketFS and later download it and deserialize it. For example, Machine Learning Models are often stored as serialized objects. We use joblib.dump and joblib.load for serialization and deserialization, because these functions are well suited also for Python objects containing large data and it supports compression for the output.
Example:
from exasol_bucketfs_utils_python import upload, download
from exasol_bucketfs_utils_python.bucket_config import BucketConfig
from exasol_bucketfs_utils_python.bucketfs_config import BucketFSConfig
from exasol_bucketfs_utils_python.bucketfs_connection_config import BucketFSConnectionConfig
connection_config = BucketFSConnectionConfig(
host="localhost", port=6666,
user="w", pwd="write",
is_https=False)
bucketfs_config = BucketFSConfig(
connection_config=connection_config,
bucketfs_name="bfsdefault")
bucket_config = BucketConfig(
bucket_name="default",
bucketfs_config=bucketfs_config)
class TestClass:
def __init__(self, attribute: str):
self.attribute = attribute
def __eq__(self, other: "TestClass"):
return isinstance(other, TestClass) and self.attribute == other.attribute
test_python_object = TestClass("test_string")
path_in_bucket = "path/in/bucket/file.txt"
upload.upload_object_to_bucketfs_via_joblib(
bucket_config=bucket_config,
bucket_file_path=path_in_bucket,
object=test_python_object)
output_test_python_object = \
download.download_object_from_bucketfs_via_joblib(
bucket_config=bucket_config,
bucket_file_path=path_in_bucket)