20 lines
618 B
Bash
20 lines
618 B
Bash
#!/bin/bash
|
|
set -e
|
|
|
|
if [ -z "$DATASET_NAME" ]; then
|
|
echo "ERROR: DATASET_NAME not set" >&2
|
|
exit 1
|
|
fi
|
|
|
|
if [ -z "$HF_TOKEN" ]; then
|
|
echo "ERROR: HF_TOKEN not set" >&2
|
|
exit 1
|
|
fi
|
|
|
|
REPO_NAME="CleverThis/${DATASET_NAME}-v1"
|
|
CHECKPOINT_FILE="${DATASET_NAME}_checkpoint.json"
|
|
|
|
python scripts/rdf_to_hf_incremental.py "$DATASET_NAME" "$REPO_NAME" --checkpoint "$CHECKPOINT_FILE" --token "$HF_TOKEN" --rows-per-shard 5000000 --record-batch-size 500000
|
|
|
|
|
|
# Note: For same dataset,you need to restart the same container (to not loose the checkpoint file) again until you see the upload complete logs of the script. |