Project Files

schemas

dataset.schema.json

sampling.schema.json

src

strategies

length.ts

priority.ts

relevance.ts

dataset.ts

index.ts

sampler.ts

types.ts

INSTRUCTIONS.md

manifest.json

package-lock.json

package.json

tsconfig.json

schemas / dataset.schema.json

{
  "$schema": "http://json-schema.org/draft-07/schema#",
  "$id": "https://primary-data.dev/schemas/data-sampler/dataset.json",
  "title": "HuggingFace Dataset Loader Options",
  "description": "Describes the load_dataset tool parameters for the data-sampler plugin. Loads sample rows from HuggingFace datasets via the HF Datasets Server REST API.",
  "type": "object",
  "properties": {
    "dataset": {
      "type": "string",
      "description": "HuggingFace dataset name, e.g. 'squad', 'imdb', 'ibm/duorc', or 'bigcode/the-stack-dedup'.",
      "minLength": 1
    },
    "split": {
      "type": "string",
      "description": "Dataset split to load from.",
      "enum": ["train", "test", "validation"],
      "default": "train"
    },
    "config": {
      "type": "string",
      "description": "Dataset config/subset name. Required when a dataset has multiple configs (e.g. 'wikitext-103-raw-v1' for wikitext)."
    },
    "max_samples": {
      "type": "integer",
      "description": "Number of rows to return.",
      "minimum": 1,
      "maximum": 100,
      "default": 5
    },
    "query": {
      "type": "string",
      "description": "Optional BM25 full-text search query to filter rows. When provided, the server returns the most relevant rows."
    },
    "format": {
      "type": "string",
      "description": "Output format: 'text' (key:value lines), 'json' (raw JSON objects), or 'prompt' (=== Sample N === format).",
      "enum": ["text", "json", "prompt"],
      "default": "text"
    },
    "template": {
      "type": "string",
      "description": "Custom template with {text} placeholder for each sample. Only used when format is 'text'."
    }
  },
  "required": ["dataset"],
  "additionalProperties": false
}