CoreRasurae
124 lines
3.6 KiB
Markdown
124 lines
3.6 KiB
Markdown
# CleverCloud Storage Framework
|
|
|
|
A unified framework for interacting with various AWS storage services including S3, EFS, and Glacier. This library provides a consistent interface for common storage operations across different storage providers.
|
|
|
|
## Features
|
|
|
|
- Unified API for multiple storage providers (S3, EFS, Glacier, Local)
|
|
- Streaming support for large files
|
|
- Automatic provider detection based on path format
|
|
- Chunked reading and writing for efficient memory usage
|
|
- Comprehensive error handling
|
|
- Support for SAS tokens (for S3)
|
|
|
|
## Installation
|
|
|
|
```bash
|
|
pip install clevercloud-storage-framework
|
|
```
|
|
|
|
## Quick Start
|
|
|
|
```python
|
|
from clevercloud_storage_framework.factory import StorageClientFactory
|
|
from clevercloud_storage_framework.client import StorageClient
|
|
|
|
# Create a client for S3
|
|
s3_client = StorageClientFactory.create_client("s3", region_name="us-west-2")
|
|
|
|
# List files in an S3 bucket
|
|
files = s3_client.list_files("s3://my-bucket/my-prefix/")
|
|
|
|
# Read a file from S3
|
|
content = s3_client.read_file("s3://my-bucket/my-file.txt")
|
|
|
|
# Write a file to S3
|
|
s3_client.write_file("s3://my-bucket/new-file.txt", "Hello, World!")
|
|
|
|
# Copy a file between providers
|
|
s3_client.copy("s3://my-bucket/source.txt", "efs://my-filesystem/destination.txt")
|
|
```
|
|
|
|
## Working with Different Storage Providers
|
|
|
|
### S3
|
|
|
|
```python
|
|
# Create an S3 client
|
|
s3_client = StorageClientFactory.create_client("s3",
|
|
region_name="us-west-2",
|
|
aws_access_key_id="YOUR_ACCESS_KEY",
|
|
aws_secret_access_key="YOUR_SECRET_KEY")
|
|
|
|
# Generate a presigned URL
|
|
from clevercloud_storage_framework.providers.s3_provider import S3StorageProvider
|
|
s3_provider = s3_client._provider
|
|
presigned_url = s3_provider.generate_presigned_url("s3://my-bucket/my-file.txt", expiration=3600)
|
|
```
|
|
|
|
### EFS
|
|
|
|
```python
|
|
# Create an EFS client
|
|
efs_client = StorageClientFactory.create_client("efs",
|
|
region_name="us-west-2",
|
|
mount_point="/mnt/efs")
|
|
|
|
# Create a directory
|
|
efs_client.create_directory("efs://fs-12345/my-directory")
|
|
|
|
# List files recursively
|
|
files = efs_client.list_files("efs://fs-12345/", recursive=True)
|
|
```
|
|
|
|
### Glacier
|
|
|
|
```python
|
|
# Create a Glacier client
|
|
glacier_client = StorageClientFactory.create_client("glacier", region_name="us-west-2")
|
|
|
|
# Initiate an archive retrieval
|
|
from clevercloud_storage_framework.providers.glacier_provider import GlacierStorageProvider
|
|
glacier_provider = glacier_client._provider
|
|
job_id = glacier_provider.initiate_archive_retrieval("my-vault", "archive-id")
|
|
|
|
# Check job status
|
|
status = glacier_provider.check_job_status("my-vault", job_id)
|
|
|
|
# Get job output when complete
|
|
if status == "Succeeded":
|
|
content = b"".join(list(glacier_provider.get_job_output("my-vault", job_id)))
|
|
```
|
|
|
|
### Local Storage
|
|
|
|
```python
|
|
# Create a local storage client
|
|
local_client = StorageClientFactory.create_client("local")
|
|
|
|
# Work with local files
|
|
local_client.write_file("/path/to/local/file.txt", "Hello, World!")
|
|
content = local_client.read_file("/path/to/local/file.txt")
|
|
```
|
|
|
|
## Error Handling
|
|
|
|
```python
|
|
from clevercloud_storage_framework.exceptions import StorageError, ResourceNotFoundError
|
|
|
|
try:
|
|
content = client.read_file("s3://non-existent-bucket/file.txt")
|
|
except ResourceNotFoundError as e:
|
|
print(f"Resource not found: {e.resource}")
|
|
except StorageError as e:
|
|
print(f"Storage error: {e.message}")
|
|
```
|
|
|
|
## Contributing
|
|
|
|
Contributions are welcome! Please feel free to submit a Pull Request.
|
|
|
|
## License
|
|
|
|
This project is licensed under the Apache License 2.0 - see the LICENSE file for details.
|