Project Files

schemas

dataset.schema.json

sampling.schema.json

src

strategies

length.ts

priority.ts

relevance.ts

dataset.ts

index.ts

sampler.ts

types.ts

INSTRUCTIONS.md

manifest.json

package-lock.json

package.json

tsconfig.json

schemas / sampling.schema.json

{
  "$schema": "http://json-schema.org/draft-07/schema#",
  "$id": "https://primary-data.dev/schemas/data-sampler/sampling.json",
  "title": "Data Sampler Options",
  "description": "Describes the sampling options accepted by the data-sampler sample tool. Supports three strategies: priority (keyword frequency + position scoring), relevance (BM25-style query scoring), and length (fixed-size chunking with optional overlap).",
  "type": "object",
  "properties": {
    "text": {
      "type": "string",
      "description": "The input text to sample into optimised chunks.",
      "minLength": 1
    },
    "strategy": {
      "type": "string",
      "description": "Sampling strategy: 'priority' scores by keyword frequency and position, 'relevance' requires a query string for BM25-style scoring, 'length' produces fixed-size chunks with optional overlap.",
      "enum": ["priority", "relevance", "length"],
      "default": "priority"
    },
    "chunkSize": {
      "type": "integer",
      "description": "Target character size per chunk.",
      "minimum": 1,
      "default": 2000
    },
    "overlap": {
      "type": "integer",
      "description": "Number of overlapping characters between consecutive chunks. Only applies when strategy is 'length'.",
      "minimum": 0,
      "default": 0
    },
    "maxChunks": {
      "type": "integer",
      "description": "Maximum number of chunks to return. Default is unlimited (all chunks returned).",
      "minimum": 1
    },
    "query": {
      "type": "string",
      "description": "Query string for the 'relevance' strategy (BM25-style scoring). Ignored for other strategies."
    }
  },
  "required": ["text"],
  "additionalProperties": false
}