55 lines
1.5 KiB
Python
55 lines
1.5 KiB
Python
"""Dataset validation utilities for HuggingFace uploads."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataset_registry import get_dataset_config
|
|
from huggingface_hub import HfApi
|
|
from huggingface_hub.utils import HfHubHTTPError
|
|
|
|
|
|
def resolve_organization(organization: str | None = None) -> str:
|
|
"""Resolve the HuggingFace organization name."""
|
|
if organization:
|
|
return organization
|
|
|
|
config = get_dataset_config()
|
|
return config.get("organization", "CleverThis")
|
|
|
|
|
|
def build_repo_id(dataset_id: str, organization: str | None = None) -> str:
|
|
"""Build the full HuggingFace repo ID for a dataset."""
|
|
org_name = resolve_organization(organization)
|
|
return f"{org_name}/{dataset_id}"
|
|
|
|
|
|
def dataset_exists_on_hf(
|
|
dataset_id: str,
|
|
organization: str | None = None,
|
|
api: HfApi | None = None,
|
|
) -> bool:
|
|
"""Check whether a dataset exists on HuggingFace Hub.
|
|
|
|
Args:
|
|
dataset_id: Dataset identifier (without organization prefix).
|
|
organization: HuggingFace organization name.
|
|
api: Optional HfApi instance for dependency injection.
|
|
|
|
Returns:
|
|
True if dataset exists, False if not found.
|
|
|
|
Raises:
|
|
HfHubHTTPError: For non-404 errors.
|
|
"""
|
|
repo_id = build_repo_id(dataset_id, organization)
|
|
client = api or HfApi()
|
|
|
|
try:
|
|
client.dataset_info(repo_id=repo_id)
|
|
except HfHubHTTPError as exc:
|
|
status_code = exc.response.status_code if exc.response else None
|
|
if status_code == 404:
|
|
return False
|
|
raise
|
|
|
|
return True
|