{
  "organization": "CleverThis",
  "base_processing_dir": "./dataset_processing",
  "huggingface_repo_prefix": "CleverThis",
  "datasets": {
    "wordnet": {
      "short_name": "wordnet",
      "full_name": "WordNet RDF",
      "description": "Lexical database of semantic relations between words (English WordNet 2024)",
      "category": "small",
      "download": {
        "url": "https://en-word.net/static/english-wordnet-2024.ttl.gz",
        "format": "turtle",
        "compression": "gz",
        "extraction_command": "gunzip",
        "size_gb": 0.21,
        "compressed_size_gb": 0.014
      },
      "metadata": {
        "entities": "~120K synsets",
        "triples": "~2M",
        "license": "CC BY 4.0",
        "recommended_for": "Quick validation, linguistic relationships, small-scale testing",
        "notes": "Compressed download, automatic extraction. English WordNet 2024 from en-word.net"
      },
      "huggingface": {
        "repository": "CleverThis/wordnet",
        "private": false,
        "readme_template": "standard_rdf"
      },
      "processing": {
        "converter": "standard",
        "streaming": false,
        "chunk_size": 10000
      }
    },
    "schema-org": {
      "short_name": "schema-org",
      "full_name": "Schema.org",
      "description": "Vocabulary schemas for structured data on the web",
      "category": "small",
      "download": {
        "url": "https://schema.org/version/latest/schemaorg-current-https.ttl",
        "format": "turtle",
        "compression": null,
        "extraction_command": null,
        "size_gb": 0.01,
        "compressed_size_gb": null
      },
      "metadata": {
        "entities": "~2K types",
        "triples": "~15K",
        "license": "CC BY-SA 3.0",
        "recommended_for": "Ontology understanding, vocabulary learning, small-scale testing",
        "notes": "Direct download, no extraction needed"
      },
      "huggingface": {
        "repository": "CleverThis/schema-org",
        "private": false,
        "readme_template": "standard_rdf"
      },
      "processing": {
        "converter": "standard",
        "streaming": false,
        "chunk_size": 10000
      }
    },
    "yago-4.5": {
      "short_name": "yago45",
      "full_name": "YAGO 4.5",
      "description": "Knowledge base combining Wikipedia, WordNet, and GeoNames",
      "category": "medium",
      "download": {
        "url": "https://yago-knowledge.org/data/yago4.5/yago-4.5.0.2-tiny.zip",
        "format": "turtle",
        "compression": "zip",
        "extraction_command": "unzip",
        "size_gb": 10.0,
        "compressed_size_gb": 0.2
      },
      "metadata": {
        "entities": "~2.5M",
        "triples": "~50M",
        "license": "CC BY-SA 3.0",
        "recommended_for": "Balanced size/quality, general knowledge, initial training",
        "notes": "YAGO 4.5 tiny version. Good balance for initial experiments. ZIP distribution."
      },
      "huggingface": {
        "repository": "CleverThis/yago-4.5",
        "private": false,
        "readme_template": "standard_rdf"
      },
      "processing": {
        "converter": "streaming_parallel",
        "streaming": true,
        "chunk_size": 50000
      }
    },
    "yago-4.5-full": {
      "short_name": "yago45full",
      "full_name": "YAGO 4.5 Full",
      "description": "Complete YAGO 4.5 dataset with all facts",
      "category": "medium",
      "download": {
        "url": "https://yago-knowledge.org/data/yago4.5/yago-4.5.0.2.zip",
        "format": "turtle",
        "compression": "zip",
        "extraction_command": "unzip",
        "size_gb": 15.0,
        "compressed_size_gb": 12.0
      },
      "metadata": {
        "entities": "~50M",
        "triples": "~109M",
        "license": "CC BY-SA 3.0",
        "recommended_for": "Complete entity information with type hierarchies",
        "notes": "YAGO 4.5 complete version. Format changed to ZIP distribution with multiple files."
      },
      "huggingface": {
        "repository": "CleverThis/yago-4.5-full",
        "private": false,
        "readme_template": "standard_rdf"
      },
      "processing": {
        "converter": "streaming_parallel",
        "streaming": true,
        "chunk_size": 50000
      }
    },
    "dbpedia-labels-en": {
      "short_name": "dbpedialabels",
      "full_name": "DBpedia English Labels",
      "description": "Entity labels from DBpedia (English)",
      "category": "medium",
      "download": {
        "url": "https://downloads.dbpedia.org/repo/dbpedia/generic/labels/2022.12.01/labels_lang=en.ttl.bz2",
        "format": "turtle",
        "compression": "bz2",
        "extraction_command": "bunzip2",
        "size_gb": 1.0,
        "compressed_size_gb": 0.16
      },
      "metadata": {
        "entities": "~9.5M",
        "triples": "~9.5M",
        "license": "CC BY-SA 3.0",
        "recommended_for": "Entity naming, label learning, medium-scale training",
        "notes": null
      },
      "huggingface": {
        "repository": "CleverThis/dbpedia-labels-en",
        "private": false,
        "readme_template": "standard_rdf"
      },
      "processing": {
        "converter": "streaming_parallel",
        "streaming": true,
        "chunk_size": 50000
      }
    },
    "dbpedia-ontology": {
      "short_name": "dbpediaonto",
      "full_name": "DBpedia Ontology",
      "description": "DBpedia ontology classes and properties",
      "category": "medium",
      "download": {
        "url": "https://downloads.dbpedia.org/repo/dbpedia/mappings/instance-types/2022.12.01/instance-types_lang=en_transitive.ttl.bz2",
        "format": "turtle",
        "compression": "bz2",
        "extraction_command": "bunzip2",
        "size_gb": 0.8,
        "compressed_size_gb": 0.14
      },
      "metadata": {
        "entities": "~9.5M",
        "triples": "~40M",
        "license": "CC BY-SA 3.0",
        "recommended_for": "Type classification, ontology learning",
        "notes": "Uses transitive closure of instance types for complete type hierarchy"
      },
      "huggingface": {
        "repository": "CleverThis/dbpedia-ontology",
        "private": false,
        "readme_template": "standard_rdf"
      },
      "processing": {
        "converter": "streaming_parallel",
        "streaming": true,
        "chunk_size": 50000
      }
    },
    "dbpedia-core-en": {
      "short_name": "dbpediacore",
      "full_name": "DBpedia Core (English)",
      "description": "Core facts from Wikipedia (English)",
      "category": "large",
      "download": {
        "url": "https://downloads.dbpedia.org/repo/dbpedia/mappings/mappingbased-objects/2022.12.01/mappingbased-objects_lang=en.ttl.bz2",
        "format": "turtle",
        "compression": "bz2",
        "extraction_command": "bunzip2",
        "size_gb": 1.8,
        "compressed_size_gb": 0.18
      },
      "metadata": {
        "entities": "~9.5M",
        "triples": "~50-60M",
        "license": "CC BY-SA 3.0",
        "recommended_for": "Wikipedia-based knowledge, production training",
        "notes": "Core DBpedia facts extracted from Wikipedia infoboxes"
      },
      "huggingface": {
        "repository": "CleverThis/dbpedia-core-en",
        "private": false,
        "readme_template": "standard_rdf"
      },
      "processing": {
        "converter": "streaming_parallel",
        "streaming": true,
        "chunk_size": 100000
      }
    },
    "wikidata-truthy": {
      "short_name": "wikidatatruthy",
      "full_name": "Wikidata Truthy",
      "description": "Core facts from Wikidata (preferred statements only)",
      "category": "large",
      "download": {
        "url": "https://dumps.wikimedia.org/wikidatawiki/entities/latest-truthy.nt.bz2",
        "format": "ntriples",
        "compression": "bz2",
        "extraction_command": "bunzip2",
        "size_gb": 100.0,
        "compressed_size_gb": 40.0
      },
      "metadata": {
        "entities": "~100M",
        "triples": "~2B",
        "license": "CC0 1.0",
        "recommended_for": "Best general knowledge, production training, multilingual support",
        "notes": "Most comprehensive general knowledge base, recommended for production. Now uses bz2 compression."
      },
      "huggingface": {
        "repository": "CleverThis/wikidata-truthy",
        "private": false,
        "readme_template": "standard_rdf"
      },
      "processing": {
        "converter": "streaming_parallel",
        "streaming": true,
        "chunk_size": 100000
      }
    },
    "freebase": {
      "short_name": "freebase",
      "full_name": "Freebase",
      "description": "Large-scale knowledge base (archived by Google)",
      "category": "large",
      "download": {
        "url": "http://commondatastorage.googleapis.com/freebase-public/rdf/freebase-rdf-latest.gz",
        "format": "ntriples",
        "compression": "gz",
        "extraction_command": "gunzip",
        "size_gb": 300.0,
        "compressed_size_gb": 30.0
      },
      "metadata": {
        "entities": "~50M",
        "triples": "~3B",
        "license": "CC BY 2.5",
        "recommended_for": "Historical knowledge base, large-scale training",
        "notes": "Archived dataset from 2014, very comprehensive but historical"
      },
      "huggingface": {
        "repository": "CleverThis/freebase",
        "private": false,
        "readme_template": "standard_rdf"
      },
      "processing": {
        "converter": "streaming_parallel",
        "streaming": true,
        "chunk_size": 100000
      }
    },
    "wikidata-full": {
      "short_name": "wikidatafull",
      "full_name": "Wikidata Full",
      "description": "Complete Wikidata dump with all statements",
      "category": "xlarge",
      "download": {
        "url": "https://dumps.wikimedia.org/wikidatawiki/entities/latest-all.ttl.bz2",
        "format": "turtle",
        "compression": "bz2",
        "extraction_command": "bunzip2",
        "size_gb": 1500.0,
        "compressed_size_gb": 120.0
      },
      "metadata": {
        "entities": "~100M",
        "triples": "~17-20B",
        "license": "CC0 1.0",
        "recommended_for": "Advanced large-scale training, research",
        "notes": "Very large dataset, requires significant storage and processing power"
      },
      "huggingface": {
        "repository": "CleverThis/wikidata-full",
        "private": false,
        "readme_template": "standard_rdf"
      },
      "processing": {
        "converter": "streaming_parallel",
        "streaming": true,
        "chunk_size": 100000
      }
    },
    "conceptnet": {
      "short_name": "conceptnet",
      "full_name": "ConceptNet 5.7",
      "description": "Common-sense knowledge graph with everyday facts and relationships",
      "category": "small",
      "download": {
        "url": "https://s3.amazonaws.com/conceptnet/downloads/2019/edges/conceptnet-assertions-5.7.0.csv.gz",
        "format": "csv",
        "compression": "gz",
        "extraction_command": "gunzip",
        "size_gb": 1.2,
        "compressed_size_gb": 0.6
      },
      "metadata": {
        "entities": "~8M nodes",
        "triples": "~21M edges",
        "license": "CC BY-SA 4.0",
        "recommended_for": "Common-sense reasoning, NLP tasks, multilingual knowledge",
        "notes": "CSV format (tab-separated), requires conversion to RDF. Crowdsourced with confidence scores."
      },
      "huggingface": {
        "repository": "CleverThis/conceptnet",
        "private": false,
        "readme_template": "conceptnet_specific"
      },
      "processing": {
        "converter": "conceptnet_specialized",
        "streaming": false,
        "chunk_size": 10000
      }
    },
    "framenet": {
      "short_name": "framenet",
      "full_name": "FrameNet Sample",
      "description": "Sample frame semantic database with semantic frames and roles (demonstration dataset)",
      "category": "small",
      "download": {
        "url": "file:///app/dataset_processing/sample_data/framenet_sample.ttl",
        "format": "turtle",
        "compression": null,
        "extraction_command": null,
        "size_gb": 0.001,
        "compressed_size_gb": null
      },
      "metadata": {
        "entities": "3 frames, 9 lexical units, 10 frame elements",
        "triples": "~50",
        "license": "BSD 3-Clause",
        "recommended_for": "Frame semantics demonstration, semantic role labeling examples",
        "notes": "Sample FrameNet-style data demonstrating semantic frames, frame elements, and lexical units."
      },
      "huggingface": {
        "repository": "CleverThis/framenet",
        "private": false,
        "readme_template": "standard_rdf"
      },
      "processing": {
        "converter": "standard",
        "streaming": false,
        "chunk_size": 10000
      }
    },
    "gene-ontology": {
      "short_name": "geneontology",
      "full_name": "Gene Ontology",
      "description": "Biological ontology for gene function hierarchy",
      "category": "small",
      "download": {
        "url": "http://purl.obolibrary.org/obo/go.owl",
        "format": "xml",
        "compression": null,
        "extraction_command": null,
        "size_gb": 0.05,
        "compressed_size_gb": null
      },
      "metadata": {
        "entities": "~44K GO concepts",
        "triples": "~853K",
        "license": "CC BY 4.0",
        "recommended_for": "Biological ontology, gene function annotation",
        "notes": "Monthly updates. OWL format with reasoning support."
      },
      "huggingface": {
        "repository": "CleverThis/gene-ontology",
        "private": false,
        "readme_template": "standard_rdf"
      },
      "processing": {
        "converter": "standard",
        "streaming": false,
        "chunk_size": 10000
      }
    },
    "uniprot": {
      "short_name": "uniprot",
      "full_name": "UniProt RDF",
      "description": "Comprehensive protein knowledgebase with functional annotations",
      "category": "large",
      "download": {
        "url": "ftp://ftp.uniprot.org/pub/databases/uniprot/current_release/rdf/uniprotkb.rdf.gz",
        "format": "xml",
        "compression": "gz",
        "extraction_command": "gunzip",
        "size_gb": 50.0,
        "compressed_size_gb": 50.0
      },
      "metadata": {
        "entities": "~90M protein entries",
        "triples": "~3.4B",
        "license": "CC BY 4.0",
        "recommended_for": "Protein research, molecular biology, functional genomics",
        "notes": "High quality with manual curation for Swiss-Prot entries. Updated every 8 weeks."
      },
      "huggingface": {
        "repository": "CleverThis/uniprot",
        "private": false,
        "readme_template": "standard_rdf"
      },
      "processing": {
        "converter": "streaming_parallel",
        "streaming": true,
        "chunk_size": 100000
      }
    },
    "geonames": {
      "short_name": "geonames",
      "full_name": "GeoNames RDF",
      "description": "World geographic database with place names and coordinates",
      "category": "small",
      "download": {
        "url": "http://download.geonames.org/all-geonames-rdf.zip",
        "format": "xml",
        "compression": "zip",
        "extraction_command": "unzip",
        "size_gb": 3.0,
        "compressed_size_gb": 3.0
      },
      "metadata": {
        "entities": "~11M places",
        "triples": "~123M",
        "license": "CC BY 4.0",
        "recommended_for": "Geographic data, location-based services",
        "notes": "Community-contributed. Daily updates with periodic RDF dumps."
      },
      "huggingface": {
        "repository": "CleverThis/geonames",
        "private": false,
        "readme_template": "standard_rdf"
      },
      "processing": {
        "converter": "streaming_parallel",
        "streaming": true,
        "chunk_size": 50000
      }
    },
    "dblp": {
      "short_name": "dblp",
      "full_name": "DBLP Bibliography",
      "description": "Computer science publications and citations",
      "category": "medium",
      "download": {
        "url": "https://dblp.org/rdf/release/dblp-2024-01-01.nt.gz",
        "format": "ntriples",
        "compression": "gz",
        "extraction_command": "gunzip",
        "size_gb": 12.0,
        "compressed_size_gb": 3.5
      },
      "metadata": {
        "entities": "~6M publications, ~2.6M authors",
        "triples": "~1.2-1.5B",
        "license": "CC0 1.0",
        "recommended_for": "Academic publication graphs, citation analysis",
        "notes": "Monthly RDF snapshots. High accuracy with complete CS coverage."
      },
      "huggingface": {
        "repository": "CleverThis/dblp",
        "private": false,
        "readme_template": "standard_rdf"
      },
      "processing": {
        "converter": "streaming_parallel",
        "streaming": true,
        "chunk_size": 50000
      }
    },
    "opencyc": {
      "short_name": "opencyc",
      "full_name": "OpenCyc 4.0",
      "description": "Common-sense ontology subset from Cycorp",
      "category": "small",
      "download": {
        "url": "https://github.com/asanchez75/opencyc/raw/master/opencyc-latest.owl.gz",
        "format": "xml",
        "compression": "gz",
        "extraction_command": "gunzip",
        "size_gb": 0.35,
        "compressed_size_gb": 0.026
      },
      "metadata": {
        "entities": "~239K concepts",
        "triples": "~2.09M",
        "license": "Apache 2.0",
        "recommended_for": "Common-sense reasoning, upper ontology",
        "notes": "Discontinued 2017. Mirror from GitHub repository. Latest version (2012-05-10)."
      },
      "huggingface": {
        "repository": "CleverThis/opencyc",
        "private": false,
        "readme_template": "standard_rdf"
      },
      "processing": {
        "converter": "standard",
        "streaming": false,
        "chunk_size": 10000
      }
    },
    "fb15k-237": {
      "short_name": "fb15k237",
      "full_name": "FB15k-237",
      "description": "Link prediction benchmark from Freebase (cleaned)",
      "category": "small",
      "download": {
        "url": "https://download.microsoft.com/download/8/7/0/8700516a-ab3d-4850-b4bb-805c515aece1/FB15K-237.2.zip",
        "format": "tsv",
        "compression": "zip",
        "extraction_command": "unzip",
        "size_gb": 0.14,
        "compressed_size_gb": 0.14
      },
      "metadata": {
        "entities": "14,541",
        "triples": "310,116",
        "license": "CC BY 2.5",
        "recommended_for": "Link prediction benchmarking, KG embedding research",
        "notes": "Microsoft Research dataset. Contains KB relation triples and textual mentions from ClueWeb12 corpus. TSV format with Freebase IDs."
      },
      "huggingface": {
        "repository": "CleverThis/fb15k-237",
        "private": false,
        "readme_template": "benchmark_dataset"
      },
      "processing": {
        "converter": "fb15k237_specialized",
        "streaming": false,
        "chunk_size": 10000
      }
    },
    "wn18rr": {
      "short_name": "wn18rr",
      "full_name": "WN18RR",
      "description": "WordNet-based link prediction benchmark (improved) - derived from WN18 with test-set leakage removed",
      "category": "small",
      "download": {
        "url": "https://figshare.com/ndownloader/files/21844185",
        "format": "tsv",
        "compression": "zip",
        "extraction_command": "unzip",
        "size_gb": 0.004,
        "compressed_size_gb": 0.001
      },
      "metadata": {
        "entities": "40,943",
        "triples": "93,003",
        "license": "Apache 2.0",
        "recommended_for": "Lexical knowledge graph benchmarking, embedding evaluation",
        "notes": "Static benchmark (2018). TSV format with pre-split train/valid/test files. Contains 11 relation types. Format: subject<tab>relation<tab>object"
      },
      "huggingface": {
        "repository": "CleverThis/wn18rr",
        "private": false,
        "readme_template": "benchmark_dataset"
      },
      "processing": {
        "converter": "standard_tsv",
        "streaming": false,
        "chunk_size": 10000
      }
    },
    "nell-995": {
      "short_name": "nell995",
      "full_name": "NELL-995",
      "description": "Never-Ending Learning subset for link prediction",
      "category": "small",
      "download": {
        "url": "https://github.com/wenhuchen/KB-Reasoning-Data/archive/refs/heads/master.zip",
        "format": "tsv",
        "compression": "zip",
        "extraction_command": "unzip",
        "size_gb": 0.12,
        "compressed_size_gb": 0.03
      },
      "metadata": {
        "entities": "~75,492",
        "triples": "154,213",
        "license": "CC BY-NC",
        "recommended_for": "Semi-supervised learning benchmarks, noisy data experiments",
        "notes": "NAACL18 paper dataset. Contains raw.kb file with triples and evaluation splits. Format: tab-separated triples."
      },
      "huggingface": {
        "repository": "CleverThis/nell-995",
        "private": false,
        "readme_template": "benchmark_dataset"
      },
      "processing": {
        "converter": "nell995_specialized",
        "streaming": false,
        "chunk_size": 10000
      }
    }
  },
  "readme_templates": {
    "standard_rdf": {
      "sections": [
        "title",
        "description",
        "dataset_format",
        "loading_instructions",
        "conversion_instructions",
        "metadata",
        "citation",
        "license"
      ]
    },
    "benchmark_dataset": {
      "sections": [
        "title",
        "description",
        "benchmark_details",
        "dataset_format",
        "loading_instructions",
        "evaluation_metrics",
        "metadata",
        "citation",
        "license"
      ]
    },
    "conceptnet_specific": {
      "sections": [
        "title",
        "description",
        "conceptnet_format",
        "dataset_format",
        "loading_instructions",
        "confidence_scores",
        "multilingual_support",
        "metadata",
        "citation",
        "license"
      ]
    }
  },
  "processing_defaults": {
    "small_datasets": {
      "converter": "standard",
      "streaming": false,
      "chunk_size": 10000,
      "memory_limit_gb": 2
    },
    "medium_datasets": {
      "converter": "streaming_parallel",
      "streaming": true,
      "chunk_size": 50000,
      "memory_limit_gb": 4
    },
    "large_datasets": {
      "converter": "streaming_parallel",
      "streaming": true,
      "chunk_size": 100000,
      "memory_limit_gb": 8
    },
    "xlarge_datasets": {
      "converter": "streaming_parallel",
      "streaming": true,
      "chunk_size": 100000,
      "memory_limit_gb": 16
    }
  },
  "converters": {
    "standard": {
      "script": "convert_rdf_to_hf_dataset.py",
      "supports_formats": ["turtle", "ntriples", "xml", "n3", "trig", "nquads"]
    },
    "streaming_parallel": {
      "script": "convert_rdf_to_hf_dataset_streaming_parallel.py",
      "supports_formats": ["turtle", "ntriples", "xml", "n3", "trig", "nquads"]
    },
    "conceptnet_specialized": {
      "script": "convert_conceptnet_to_hf.py",
      "supports_formats": ["csv"]
    },
    "fb15k237_specialized": {
      "script": "convert_fb15k237_to_hf.py",
      "supports_formats": ["tsv"]
    },
    "nell995_specialized": {
      "script": "convert_nell995_to_hf.py",
      "supports_formats": ["tsv"]
    },
    "standard_tsv": {
      "script": "convert_rdf_to_hf_dataset.py",
      "supports_formats": ["tsv"]
    }
  },
  "format_mappings": {
    "turtle": "turtle",
    "ntriples": "nt",
    "rdf/xml": "xml",
    "xml": "xml",
    "n3": "n3",
    "trig": "trig",
    "nquads": "nquads",
    "tsv": "tsv",
    "csv": "csv"
  },
  "license_mappings": {
    "wordnet license": "other",
    "cc by 4.0": "cc-by-4.0",
    "cc by-sa 4.0": "cc-by-sa-4.0",
    "cc by-sa 3.0": "cc-by-sa-3.0",
    "cc by 3.0": "cc-by-3.0",
    "cc by 2.5": "cc-by-2.5",
    "cc by-nc 4.0": "cc-by-nc-4.0",
    "cc0 1.0": "cc0-1.0",
    "odbl": "other",
    "odbl 1.0": "other",
    "mit": "mit",
    "apache 2.0": "apache-2.0",
    "apache-2.0": "apache-2.0",
    "bsd-3-clause": "bsd-3-clause"
  }
}