48 lines
1.2 KiB
YAML
48 lines
1.2 KiB
YAML
services:
|
|
cleverdatasets:
|
|
build:
|
|
context: .
|
|
dockerfile: Dockerfile
|
|
image: cleverdatasets:latest
|
|
container_name: cleverdatasets
|
|
|
|
# Mount volumes for data persistence
|
|
volumes:
|
|
# Data directory for downloads and conversions
|
|
- ./dataset_processing:/data/dataset_processing
|
|
# HuggingFace cache for faster subsequent runs
|
|
- huggingface_cache:/data/.cache/huggingface
|
|
# Optional: mount config for custom dataset configurations
|
|
- ./scripts/dataset_config.json:/app/scripts/dataset_config.json:ro
|
|
|
|
# Environment variables
|
|
environment:
|
|
# HuggingFace authentication token (required for uploads)
|
|
- HF_TOKEN=${HF_TOKEN:-}
|
|
# Optional: customize base directory
|
|
- BASE_DIR=/data/dataset_processing
|
|
# Python configuration
|
|
- PYTHONUNBUFFERED=1
|
|
|
|
# Resource limits (adjust based on workload)
|
|
deploy:
|
|
resources:
|
|
limits:
|
|
memory: 16G
|
|
cpus: '4'
|
|
reservations:
|
|
memory: 4G
|
|
cpus: '2'
|
|
|
|
# Restart policy
|
|
restart: on-failure
|
|
|
|
# Network mode
|
|
network_mode: bridge
|
|
|
|
volumes:
|
|
# Named volume for HuggingFace cache persistence
|
|
huggingface_cache:
|
|
driver: local
|
|
|