#!/bin/bash
set -e

if [ -z "$DATASET_NAME" ]; then
    echo "ERROR: DATASET_NAME not set" >&2
    exit 1
fi

if [ -z "$HF_TOKEN" ]; then
    echo "ERROR: HF_TOKEN not set" >&2
    exit 1
fi

REPO_NAME="CleverThis/${DATASET_NAME}-v1"
CHECKPOINT_FILE="${DATASET_NAME}_checkpoint.json"

python scripts/rdf_to_hf_incremental.py "$DATASET_NAME" "$REPO_NAME" --checkpoint "$CHECKPOINT_FILE" --token "$HF_TOKEN" --rows-per-shard 5000000 --record-batch-size 500000


# Note: For same dataset,you need to restart the same container (to not loose the checkpoint file) again until you see the upload complete logs of the script.