747 lines
24 KiB
JSON
747 lines
24 KiB
JSON
{
|
|
"organization": "CleverThis",
|
|
"base_processing_dir": "./dataset_processing",
|
|
"huggingface_repo_prefix": "CleverThis",
|
|
"datasets": {
|
|
"wordnet": {
|
|
"short_name": "wordnet",
|
|
"full_name": "WordNet RDF",
|
|
"description": "Lexical database of semantic relations between words (English WordNet 2024)",
|
|
"category": "small",
|
|
"download": {
|
|
"url": "https://en-word.net/static/english-wordnet-2024.ttl.gz",
|
|
"format": "turtle",
|
|
"compression": "gz",
|
|
"extraction_command": "gunzip",
|
|
"size_gb": 0.21,
|
|
"compressed_size_gb": 0.014
|
|
},
|
|
"metadata": {
|
|
"entities": "~120K synsets",
|
|
"triples": "~2M",
|
|
"license": "CC BY 4.0",
|
|
"recommended_for": "Quick validation, linguistic relationships, small-scale testing",
|
|
"notes": "Compressed download, automatic extraction. English WordNet 2024 from en-word.net"
|
|
},
|
|
"huggingface": {
|
|
"repository": "CleverThis/wordnet",
|
|
"private": false,
|
|
"readme_template": "standard_rdf"
|
|
},
|
|
"processing": {
|
|
"converter": "standard",
|
|
"streaming": false,
|
|
"chunk_size": 10000
|
|
}
|
|
},
|
|
"schema-org": {
|
|
"short_name": "schema-org",
|
|
"full_name": "Schema.org",
|
|
"description": "Vocabulary schemas for structured data on the web",
|
|
"category": "small",
|
|
"download": {
|
|
"url": "https://schema.org/version/latest/schemaorg-current-https.ttl",
|
|
"format": "turtle",
|
|
"compression": null,
|
|
"extraction_command": null,
|
|
"size_gb": 0.01,
|
|
"compressed_size_gb": null
|
|
},
|
|
"metadata": {
|
|
"entities": "~2K types",
|
|
"triples": "~15K",
|
|
"license": "CC BY-SA 3.0",
|
|
"recommended_for": "Ontology understanding, vocabulary learning, small-scale testing",
|
|
"notes": "Direct download, no extraction needed"
|
|
},
|
|
"huggingface": {
|
|
"repository": "CleverThis/schema-org",
|
|
"private": false,
|
|
"readme_template": "standard_rdf"
|
|
},
|
|
"processing": {
|
|
"converter": "standard",
|
|
"streaming": false,
|
|
"chunk_size": 10000
|
|
}
|
|
},
|
|
"yago-4.5": {
|
|
"short_name": "yago45",
|
|
"full_name": "YAGO 4.5",
|
|
"description": "Knowledge base combining Wikipedia, WordNet, and GeoNames",
|
|
"category": "medium",
|
|
"download": {
|
|
"url": "https://yago-knowledge.org/data/yago4.5/yago-4.5.0.2-tiny.zip",
|
|
"format": "turtle",
|
|
"compression": "zip",
|
|
"extraction_command": "unzip",
|
|
"size_gb": 10.0,
|
|
"compressed_size_gb": 0.2
|
|
},
|
|
"metadata": {
|
|
"entities": "~2.5M",
|
|
"triples": "~50M",
|
|
"license": "CC BY-SA 3.0",
|
|
"recommended_for": "Balanced size/quality, general knowledge, initial training",
|
|
"notes": "YAGO 4.5 tiny version. Good balance for initial experiments. ZIP distribution."
|
|
},
|
|
"huggingface": {
|
|
"repository": "CleverThis/yago-4.5",
|
|
"private": false,
|
|
"readme_template": "standard_rdf"
|
|
},
|
|
"processing": {
|
|
"converter": "streaming_parallel",
|
|
"streaming": true,
|
|
"chunk_size": 50000
|
|
}
|
|
},
|
|
"yago-4.5-full": {
|
|
"short_name": "yago45full",
|
|
"full_name": "YAGO 4.5 Full",
|
|
"description": "Complete YAGO 4.5 dataset with all facts",
|
|
"category": "medium",
|
|
"download": {
|
|
"url": "https://yago-knowledge.org/data/yago4.5/yago-4.5.0.2.zip",
|
|
"format": "turtle",
|
|
"compression": "zip",
|
|
"extraction_command": "unzip",
|
|
"size_gb": 15.0,
|
|
"compressed_size_gb": 12.0
|
|
},
|
|
"metadata": {
|
|
"entities": "~50M",
|
|
"triples": "~109M",
|
|
"license": "CC BY-SA 3.0",
|
|
"recommended_for": "Complete entity information with type hierarchies",
|
|
"notes": "YAGO 4.5 complete version. Format changed to ZIP distribution with multiple files."
|
|
},
|
|
"huggingface": {
|
|
"repository": "CleverThis/yago-4.5-full",
|
|
"private": false,
|
|
"readme_template": "standard_rdf"
|
|
},
|
|
"processing": {
|
|
"converter": "streaming_parallel",
|
|
"streaming": true,
|
|
"chunk_size": 50000
|
|
}
|
|
},
|
|
"dbpedia-labels-en": {
|
|
"short_name": "dbpedialabels",
|
|
"full_name": "DBpedia English Labels",
|
|
"description": "Entity labels from DBpedia (English)",
|
|
"category": "medium",
|
|
"download": {
|
|
"url": "https://downloads.dbpedia.org/repo/dbpedia/generic/labels/2022.12.01/labels_lang=en.ttl.bz2",
|
|
"format": "turtle",
|
|
"compression": "bz2",
|
|
"extraction_command": "bunzip2",
|
|
"size_gb": 1.0,
|
|
"compressed_size_gb": 0.16
|
|
},
|
|
"metadata": {
|
|
"entities": "~9.5M",
|
|
"triples": "~9.5M",
|
|
"license": "CC BY-SA 3.0",
|
|
"recommended_for": "Entity naming, label learning, medium-scale training",
|
|
"notes": null
|
|
},
|
|
"huggingface": {
|
|
"repository": "CleverThis/dbpedia-labels-en",
|
|
"private": false,
|
|
"readme_template": "standard_rdf"
|
|
},
|
|
"processing": {
|
|
"converter": "streaming_parallel",
|
|
"streaming": true,
|
|
"chunk_size": 50000
|
|
}
|
|
},
|
|
"dbpedia-ontology": {
|
|
"short_name": "dbpediaonto",
|
|
"full_name": "DBpedia Ontology",
|
|
"description": "DBpedia ontology classes and properties",
|
|
"category": "medium",
|
|
"download": {
|
|
"url": "https://downloads.dbpedia.org/repo/dbpedia/mappings/instance-types/2022.12.01/instance-types_lang=en_transitive.ttl.bz2",
|
|
"format": "turtle",
|
|
"compression": "bz2",
|
|
"extraction_command": "bunzip2",
|
|
"size_gb": 0.8,
|
|
"compressed_size_gb": 0.14
|
|
},
|
|
"metadata": {
|
|
"entities": "~9.5M",
|
|
"triples": "~40M",
|
|
"license": "CC BY-SA 3.0",
|
|
"recommended_for": "Type classification, ontology learning",
|
|
"notes": "Uses transitive closure of instance types for complete type hierarchy"
|
|
},
|
|
"huggingface": {
|
|
"repository": "CleverThis/dbpedia-ontology",
|
|
"private": false,
|
|
"readme_template": "standard_rdf"
|
|
},
|
|
"processing": {
|
|
"converter": "streaming_parallel",
|
|
"streaming": true,
|
|
"chunk_size": 50000
|
|
}
|
|
},
|
|
"dbpedia-core-en": {
|
|
"short_name": "dbpediacore",
|
|
"full_name": "DBpedia Core (English)",
|
|
"description": "Core facts from Wikipedia (English)",
|
|
"category": "large",
|
|
"download": {
|
|
"url": "https://downloads.dbpedia.org/repo/dbpedia/mappings/mappingbased-objects/2022.12.01/mappingbased-objects_lang=en.ttl.bz2",
|
|
"format": "turtle",
|
|
"compression": "bz2",
|
|
"extraction_command": "bunzip2",
|
|
"size_gb": 1.8,
|
|
"compressed_size_gb": 0.18
|
|
},
|
|
"metadata": {
|
|
"entities": "~9.5M",
|
|
"triples": "~50-60M",
|
|
"license": "CC BY-SA 3.0",
|
|
"recommended_for": "Wikipedia-based knowledge, production training",
|
|
"notes": "Core DBpedia facts extracted from Wikipedia infoboxes"
|
|
},
|
|
"huggingface": {
|
|
"repository": "CleverThis/dbpedia-core-en",
|
|
"private": false,
|
|
"readme_template": "standard_rdf"
|
|
},
|
|
"processing": {
|
|
"converter": "streaming_parallel",
|
|
"streaming": true,
|
|
"chunk_size": 100000
|
|
}
|
|
},
|
|
"wikidata-truthy": {
|
|
"short_name": "wikidatatruthy",
|
|
"full_name": "Wikidata Truthy",
|
|
"description": "Core facts from Wikidata (preferred statements only)",
|
|
"category": "large",
|
|
"download": {
|
|
"url": "https://dumps.wikimedia.org/wikidatawiki/entities/latest-truthy.nt.bz2",
|
|
"format": "ntriples",
|
|
"compression": "bz2",
|
|
"extraction_command": "bunzip2",
|
|
"size_gb": 100.0,
|
|
"compressed_size_gb": 40.0
|
|
},
|
|
"metadata": {
|
|
"entities": "~100M",
|
|
"triples": "~2B",
|
|
"license": "CC0 1.0",
|
|
"recommended_for": "Best general knowledge, production training, multilingual support",
|
|
"notes": "Most comprehensive general knowledge base, recommended for production. Now uses bz2 compression."
|
|
},
|
|
"huggingface": {
|
|
"repository": "CleverThis/wikidata-truthy",
|
|
"private": false,
|
|
"readme_template": "standard_rdf"
|
|
},
|
|
"processing": {
|
|
"converter": "streaming_parallel",
|
|
"streaming": true,
|
|
"chunk_size": 100000
|
|
}
|
|
},
|
|
"freebase": {
|
|
"short_name": "freebase",
|
|
"full_name": "Freebase",
|
|
"description": "Large-scale knowledge base (archived by Google)",
|
|
"category": "large",
|
|
"download": {
|
|
"url": "http://commondatastorage.googleapis.com/freebase-public/rdf/freebase-rdf-latest.gz",
|
|
"format": "ntriples",
|
|
"compression": "gz",
|
|
"extraction_command": "gunzip",
|
|
"size_gb": 300.0,
|
|
"compressed_size_gb": 30.0
|
|
},
|
|
"metadata": {
|
|
"entities": "~50M",
|
|
"triples": "~3B",
|
|
"license": "CC BY 2.5",
|
|
"recommended_for": "Historical knowledge base, large-scale training",
|
|
"notes": "Archived dataset from 2014, very comprehensive but historical"
|
|
},
|
|
"huggingface": {
|
|
"repository": "CleverThis/freebase",
|
|
"private": false,
|
|
"readme_template": "standard_rdf"
|
|
},
|
|
"processing": {
|
|
"converter": "streaming_parallel",
|
|
"streaming": true,
|
|
"chunk_size": 100000
|
|
}
|
|
},
|
|
"wikidata-full": {
|
|
"short_name": "wikidatafull",
|
|
"full_name": "Wikidata Full",
|
|
"description": "Complete Wikidata dump with all statements",
|
|
"category": "xlarge",
|
|
"download": {
|
|
"url": "https://dumps.wikimedia.org/wikidatawiki/entities/latest-all.ttl.bz2",
|
|
"format": "turtle",
|
|
"compression": "bz2",
|
|
"extraction_command": "bunzip2",
|
|
"size_gb": 1500.0,
|
|
"compressed_size_gb": 120.0
|
|
},
|
|
"metadata": {
|
|
"entities": "~100M",
|
|
"triples": "~17-20B",
|
|
"license": "CC0 1.0",
|
|
"recommended_for": "Advanced large-scale training, research",
|
|
"notes": "Very large dataset, requires significant storage and processing power"
|
|
},
|
|
"huggingface": {
|
|
"repository": "CleverThis/wikidata-full",
|
|
"private": false,
|
|
"readme_template": "standard_rdf"
|
|
},
|
|
"processing": {
|
|
"converter": "streaming_parallel",
|
|
"streaming": true,
|
|
"chunk_size": 100000
|
|
}
|
|
},
|
|
"conceptnet": {
|
|
"short_name": "conceptnet",
|
|
"full_name": "ConceptNet 5.7",
|
|
"description": "Common-sense knowledge graph with everyday facts and relationships",
|
|
"category": "small",
|
|
"download": {
|
|
"url": "https://s3.amazonaws.com/conceptnet/downloads/2019/edges/conceptnet-assertions-5.7.0.csv.gz",
|
|
"format": "csv",
|
|
"compression": "gz",
|
|
"extraction_command": "gunzip",
|
|
"size_gb": 1.2,
|
|
"compressed_size_gb": 0.6
|
|
},
|
|
"metadata": {
|
|
"entities": "~8M nodes",
|
|
"triples": "~21M edges",
|
|
"license": "CC BY-SA 4.0",
|
|
"recommended_for": "Common-sense reasoning, NLP tasks, multilingual knowledge",
|
|
"notes": "CSV format (tab-separated), requires conversion to RDF. Crowdsourced with confidence scores."
|
|
},
|
|
"huggingface": {
|
|
"repository": "CleverThis/conceptnet",
|
|
"private": false,
|
|
"readme_template": "conceptnet_specific"
|
|
},
|
|
"processing": {
|
|
"converter": "conceptnet_specialized",
|
|
"streaming": false,
|
|
"chunk_size": 10000
|
|
}
|
|
},
|
|
"framenet": {
|
|
"short_name": "framenet",
|
|
"full_name": "FrameNet Sample",
|
|
"description": "Sample frame semantic database with semantic frames and roles (demonstration dataset)",
|
|
"category": "small",
|
|
"download": {
|
|
"url": "file:///app/dataset_processing/sample_data/framenet_sample.ttl",
|
|
"format": "turtle",
|
|
"compression": null,
|
|
"extraction_command": null,
|
|
"size_gb": 0.001,
|
|
"compressed_size_gb": null
|
|
},
|
|
"metadata": {
|
|
"entities": "3 frames, 9 lexical units, 10 frame elements",
|
|
"triples": "~50",
|
|
"license": "BSD 3-Clause",
|
|
"recommended_for": "Frame semantics demonstration, semantic role labeling examples",
|
|
"notes": "Sample FrameNet-style data demonstrating semantic frames, frame elements, and lexical units."
|
|
},
|
|
"huggingface": {
|
|
"repository": "CleverThis/framenet",
|
|
"private": false,
|
|
"readme_template": "standard_rdf"
|
|
},
|
|
"processing": {
|
|
"converter": "standard",
|
|
"streaming": false,
|
|
"chunk_size": 10000
|
|
}
|
|
},
|
|
"gene-ontology": {
|
|
"short_name": "geneontology",
|
|
"full_name": "Gene Ontology",
|
|
"description": "Biological ontology for gene function hierarchy",
|
|
"category": "small",
|
|
"download": {
|
|
"url": "http://purl.obolibrary.org/obo/go.owl",
|
|
"format": "xml",
|
|
"compression": null,
|
|
"extraction_command": null,
|
|
"size_gb": 0.05,
|
|
"compressed_size_gb": null
|
|
},
|
|
"metadata": {
|
|
"entities": "~44K GO concepts",
|
|
"triples": "~853K",
|
|
"license": "CC BY 4.0",
|
|
"recommended_for": "Biological ontology, gene function annotation",
|
|
"notes": "Monthly updates. OWL format with reasoning support."
|
|
},
|
|
"huggingface": {
|
|
"repository": "CleverThis/gene-ontology",
|
|
"private": false,
|
|
"readme_template": "standard_rdf"
|
|
},
|
|
"processing": {
|
|
"converter": "standard",
|
|
"streaming": false,
|
|
"chunk_size": 10000
|
|
}
|
|
},
|
|
"uniprot": {
|
|
"short_name": "uniprot",
|
|
"full_name": "UniProt RDF",
|
|
"description": "Comprehensive protein knowledgebase with functional annotations",
|
|
"category": "large",
|
|
"download": {
|
|
"url": "ftp://ftp.uniprot.org/pub/databases/uniprot/current_release/rdf/uniprotkb.rdf.gz",
|
|
"format": "xml",
|
|
"compression": "gz",
|
|
"extraction_command": "gunzip",
|
|
"size_gb": 50.0,
|
|
"compressed_size_gb": 50.0
|
|
},
|
|
"metadata": {
|
|
"entities": "~90M protein entries",
|
|
"triples": "~3.4B",
|
|
"license": "CC BY 4.0",
|
|
"recommended_for": "Protein research, molecular biology, functional genomics",
|
|
"notes": "High quality with manual curation for Swiss-Prot entries. Updated every 8 weeks."
|
|
},
|
|
"huggingface": {
|
|
"repository": "CleverThis/uniprot",
|
|
"private": false,
|
|
"readme_template": "standard_rdf"
|
|
},
|
|
"processing": {
|
|
"converter": "streaming_parallel",
|
|
"streaming": true,
|
|
"chunk_size": 100000
|
|
}
|
|
},
|
|
"geonames": {
|
|
"short_name": "geonames",
|
|
"full_name": "GeoNames RDF",
|
|
"description": "World geographic database with place names and coordinates",
|
|
"category": "small",
|
|
"download": {
|
|
"url": "http://download.geonames.org/all-geonames-rdf.zip",
|
|
"format": "xml",
|
|
"compression": "zip",
|
|
"extraction_command": "unzip",
|
|
"size_gb": 3.0,
|
|
"compressed_size_gb": 3.0
|
|
},
|
|
"metadata": {
|
|
"entities": "~11M places",
|
|
"triples": "~123M",
|
|
"license": "CC BY 4.0",
|
|
"recommended_for": "Geographic data, location-based services",
|
|
"notes": "Community-contributed. Daily updates with periodic RDF dumps."
|
|
},
|
|
"huggingface": {
|
|
"repository": "CleverThis/geonames",
|
|
"private": false,
|
|
"readme_template": "standard_rdf"
|
|
},
|
|
"processing": {
|
|
"converter": "streaming_parallel",
|
|
"streaming": true,
|
|
"chunk_size": 50000
|
|
}
|
|
},
|
|
"dblp": {
|
|
"short_name": "dblp",
|
|
"full_name": "DBLP Bibliography",
|
|
"description": "Computer science publications and citations",
|
|
"category": "medium",
|
|
"download": {
|
|
"url": "https://dblp.org/rdf/release/dblp-2024-01-01.nt.gz",
|
|
"format": "ntriples",
|
|
"compression": "gz",
|
|
"extraction_command": "gunzip",
|
|
"size_gb": 12.0,
|
|
"compressed_size_gb": 3.5
|
|
},
|
|
"metadata": {
|
|
"entities": "~6M publications, ~2.6M authors",
|
|
"triples": "~1.2-1.5B",
|
|
"license": "CC0 1.0",
|
|
"recommended_for": "Academic publication graphs, citation analysis",
|
|
"notes": "Monthly RDF snapshots. High accuracy with complete CS coverage."
|
|
},
|
|
"huggingface": {
|
|
"repository": "CleverThis/dblp",
|
|
"private": false,
|
|
"readme_template": "standard_rdf"
|
|
},
|
|
"processing": {
|
|
"converter": "streaming_parallel",
|
|
"streaming": true,
|
|
"chunk_size": 50000
|
|
}
|
|
},
|
|
"opencyc": {
|
|
"short_name": "opencyc",
|
|
"full_name": "OpenCyc 4.0",
|
|
"description": "Common-sense ontology subset from Cycorp",
|
|
"category": "small",
|
|
"download": {
|
|
"url": "https://github.com/asanchez75/opencyc/raw/master/opencyc-latest.owl.gz",
|
|
"format": "xml",
|
|
"compression": "gz",
|
|
"extraction_command": "gunzip",
|
|
"size_gb": 0.35,
|
|
"compressed_size_gb": 0.026
|
|
},
|
|
"metadata": {
|
|
"entities": "~239K concepts",
|
|
"triples": "~2.09M",
|
|
"license": "Apache 2.0",
|
|
"recommended_for": "Common-sense reasoning, upper ontology",
|
|
"notes": "Discontinued 2017. Mirror from GitHub repository. Latest version (2012-05-10)."
|
|
},
|
|
"huggingface": {
|
|
"repository": "CleverThis/opencyc",
|
|
"private": false,
|
|
"readme_template": "standard_rdf"
|
|
},
|
|
"processing": {
|
|
"converter": "standard",
|
|
"streaming": false,
|
|
"chunk_size": 10000
|
|
}
|
|
},
|
|
"fb15k-237": {
|
|
"short_name": "fb15k237",
|
|
"full_name": "FB15k-237",
|
|
"description": "Link prediction benchmark from Freebase (cleaned)",
|
|
"category": "small",
|
|
"download": {
|
|
"url": "https://download.microsoft.com/download/8/7/0/8700516a-ab3d-4850-b4bb-805c515aece1/FB15K-237.2.zip",
|
|
"format": "tsv",
|
|
"compression": "zip",
|
|
"extraction_command": "unzip",
|
|
"size_gb": 0.14,
|
|
"compressed_size_gb": 0.14
|
|
},
|
|
"metadata": {
|
|
"entities": "14,541",
|
|
"triples": "310,116",
|
|
"license": "CC BY 2.5",
|
|
"recommended_for": "Link prediction benchmarking, KG embedding research",
|
|
"notes": "Microsoft Research dataset. Contains KB relation triples and textual mentions from ClueWeb12 corpus. TSV format with Freebase IDs."
|
|
},
|
|
"huggingface": {
|
|
"repository": "CleverThis/fb15k-237",
|
|
"private": false,
|
|
"readme_template": "benchmark_dataset"
|
|
},
|
|
"processing": {
|
|
"converter": "fb15k237_specialized",
|
|
"streaming": false,
|
|
"chunk_size": 10000
|
|
}
|
|
},
|
|
"wn18rr": {
|
|
"short_name": "wn18rr",
|
|
"full_name": "WN18RR",
|
|
"description": "WordNet-based link prediction benchmark (improved) - derived from WN18 with test-set leakage removed",
|
|
"category": "small",
|
|
"download": {
|
|
"url": "https://figshare.com/ndownloader/files/21844185",
|
|
"format": "tsv",
|
|
"compression": "zip",
|
|
"extraction_command": "unzip",
|
|
"size_gb": 0.004,
|
|
"compressed_size_gb": 0.001
|
|
},
|
|
"metadata": {
|
|
"entities": "40,943",
|
|
"triples": "93,003",
|
|
"license": "Apache 2.0",
|
|
"recommended_for": "Lexical knowledge graph benchmarking, embedding evaluation",
|
|
"notes": "Static benchmark (2018). TSV format with pre-split train/valid/test files. Contains 11 relation types. Format: subject<tab>relation<tab>object"
|
|
},
|
|
"huggingface": {
|
|
"repository": "CleverThis/wn18rr",
|
|
"private": false,
|
|
"readme_template": "benchmark_dataset"
|
|
},
|
|
"processing": {
|
|
"converter": "standard_tsv",
|
|
"streaming": false,
|
|
"chunk_size": 10000
|
|
}
|
|
},
|
|
"nell-995": {
|
|
"short_name": "nell995",
|
|
"full_name": "NELL-995",
|
|
"description": "Never-Ending Learning subset for link prediction",
|
|
"category": "small",
|
|
"download": {
|
|
"url": "https://github.com/wenhuchen/KB-Reasoning-Data/archive/refs/heads/master.zip",
|
|
"format": "tsv",
|
|
"compression": "zip",
|
|
"extraction_command": "unzip",
|
|
"size_gb": 0.12,
|
|
"compressed_size_gb": 0.03
|
|
},
|
|
"metadata": {
|
|
"entities": "~75,492",
|
|
"triples": "154,213",
|
|
"license": "CC BY-NC",
|
|
"recommended_for": "Semi-supervised learning benchmarks, noisy data experiments",
|
|
"notes": "NAACL18 paper dataset. Contains raw.kb file with triples and evaluation splits. Format: tab-separated triples."
|
|
},
|
|
"huggingface": {
|
|
"repository": "CleverThis/nell-995",
|
|
"private": false,
|
|
"readme_template": "benchmark_dataset"
|
|
},
|
|
"processing": {
|
|
"converter": "nell995_specialized",
|
|
"streaming": false,
|
|
"chunk_size": 10000
|
|
}
|
|
}
|
|
},
|
|
"readme_templates": {
|
|
"standard_rdf": {
|
|
"sections": [
|
|
"title",
|
|
"description",
|
|
"dataset_format",
|
|
"loading_instructions",
|
|
"conversion_instructions",
|
|
"metadata",
|
|
"citation",
|
|
"license"
|
|
]
|
|
},
|
|
"benchmark_dataset": {
|
|
"sections": [
|
|
"title",
|
|
"description",
|
|
"benchmark_details",
|
|
"dataset_format",
|
|
"loading_instructions",
|
|
"evaluation_metrics",
|
|
"metadata",
|
|
"citation",
|
|
"license"
|
|
]
|
|
},
|
|
"conceptnet_specific": {
|
|
"sections": [
|
|
"title",
|
|
"description",
|
|
"conceptnet_format",
|
|
"dataset_format",
|
|
"loading_instructions",
|
|
"confidence_scores",
|
|
"multilingual_support",
|
|
"metadata",
|
|
"citation",
|
|
"license"
|
|
]
|
|
}
|
|
},
|
|
"processing_defaults": {
|
|
"small_datasets": {
|
|
"converter": "standard",
|
|
"streaming": false,
|
|
"chunk_size": 10000,
|
|
"memory_limit_gb": 2
|
|
},
|
|
"medium_datasets": {
|
|
"converter": "streaming_parallel",
|
|
"streaming": true,
|
|
"chunk_size": 50000,
|
|
"memory_limit_gb": 4
|
|
},
|
|
"large_datasets": {
|
|
"converter": "streaming_parallel",
|
|
"streaming": true,
|
|
"chunk_size": 100000,
|
|
"memory_limit_gb": 8
|
|
},
|
|
"xlarge_datasets": {
|
|
"converter": "streaming_parallel",
|
|
"streaming": true,
|
|
"chunk_size": 100000,
|
|
"memory_limit_gb": 16
|
|
}
|
|
},
|
|
"converters": {
|
|
"standard": {
|
|
"script": "convert_rdf_to_hf_dataset.py",
|
|
"supports_formats": ["turtle", "ntriples", "xml", "n3", "trig", "nquads"]
|
|
},
|
|
"streaming_parallel": {
|
|
"script": "convert_rdf_to_hf_dataset_streaming_parallel.py",
|
|
"supports_formats": ["turtle", "ntriples", "xml", "n3", "trig", "nquads"]
|
|
},
|
|
"conceptnet_specialized": {
|
|
"script": "convert_conceptnet_to_hf.py",
|
|
"supports_formats": ["csv"]
|
|
},
|
|
"fb15k237_specialized": {
|
|
"script": "convert_fb15k237_to_hf.py",
|
|
"supports_formats": ["tsv"]
|
|
},
|
|
"nell995_specialized": {
|
|
"script": "convert_nell995_to_hf.py",
|
|
"supports_formats": ["tsv"]
|
|
},
|
|
"standard_tsv": {
|
|
"script": "convert_rdf_to_hf_dataset.py",
|
|
"supports_formats": ["tsv"]
|
|
}
|
|
},
|
|
"format_mappings": {
|
|
"turtle": "turtle",
|
|
"ntriples": "nt",
|
|
"rdf/xml": "xml",
|
|
"xml": "xml",
|
|
"n3": "n3",
|
|
"trig": "trig",
|
|
"nquads": "nquads",
|
|
"tsv": "tsv",
|
|
"csv": "csv"
|
|
},
|
|
"license_mappings": {
|
|
"wordnet license": "other",
|
|
"cc by 4.0": "cc-by-4.0",
|
|
"cc by-sa 4.0": "cc-by-sa-4.0",
|
|
"cc by-sa 3.0": "cc-by-sa-3.0",
|
|
"cc by 3.0": "cc-by-3.0",
|
|
"cc by 2.5": "cc-by-2.5",
|
|
"cc by-nc 4.0": "cc-by-nc-4.0",
|
|
"cc0 1.0": "cc0-1.0",
|
|
"odbl": "other",
|
|
"odbl 1.0": "other",
|
|
"mit": "mit",
|
|
"apache 2.0": "apache-2.0",
|
|
"apache-2.0": "apache-2.0",
|
|
"bsd-3-clause": "bsd-3-clause"
|
|
}
|
|
} |