src / extractors / jsonLdExtractor.ts
src / extractors / jsonLdExtractor.ts
import { Logger } from "../utils/Logger";
export interface ExtractedJsonLd {
headline?: string;
articleBody?: string;
description?: string;
author?: string;
datePublished?: string;
dateModified?: string;
publisher?: string;
}
/**
* Extracts author string from a JSON-LD author property value (string, object, or array).
*/
function extractAuthorString(authorVal: unknown): string | undefined {
if (!authorVal) return undefined;
if (typeof authorVal === "string" && authorVal.trim()) {
return authorVal.trim();
}
if (Array.isArray(authorVal)) {
const authors = authorVal
.map((a) => extractAuthorString(a))
.filter((a): a is string => Boolean(a));
return authors.length > 0 ? authors.join(", ") : undefined;
}
if (typeof authorVal === "object" && authorVal !== null) {
const obj = authorVal as Record<string, unknown>;
if (typeof obj.name === "string" && obj.name.trim()) {
return obj.name.trim();
}
}
return undefined;
}
/**
* Extracts publisher string from a JSON-LD publisher property value (string or object).
*/
function extractPublisherString(publisherVal: unknown): string | undefined {
if (!publisherVal) return undefined;
if (typeof publisherVal === "string" && publisherVal.trim()) {
return publisherVal.trim();
}
if (typeof publisherVal === "object" && publisherVal !== null) {
const obj = publisherVal as Record<string, unknown>;
if (typeof obj.name === "string" && obj.name.trim()) {
return obj.name.trim();
}
}
return undefined;
}
/**
* Normalizes raw JSON-LD script text by stripping HTML comment and CDATA wrappers.
* Does not perform entity decoding or structural JSON repair.
*/
export function normalizeJsonLdText(rawText: string): string {
let text = rawText.trim();
// Strip HTML comment wrapper: <!-- ... -->
if (text.startsWith("<!--") && text.endsWith("-->")) {
text = text.slice(4, -3).trim();
}
// Strip CDATA wrappers (single-line comments, multi-line comments, or bare tags)
text = text
.replace(/^\s*\/\/\s*<!\[CDATA\[\s*/i, "")
.replace(/\s*\/\/\s*\]\]>\s*$/i, "")
.replace(/^\s*\/\*\s*<!\[CDATA\[\s*\*\/\s*/i, "")
.replace(/\s*\/\*\s*\]\]>\s*\*\/\s*$/i, "")
.replace(/^\s*<!\[CDATA\[\s*/i, "")
.replace(/\s*\]\]>\s*$/i, "")
.trim();
return text;
}
/**
* Parses all <script type="application/ld+json"> elements in the document
* and extracts relevant structured metadata fields.
*/
export function extractJsonLd(doc: Document): ExtractedJsonLd {
const result: ExtractedJsonLd = {};
const scripts = doc.querySelectorAll('script[type="application/ld+json"]');
if (!scripts || scripts.length === 0) {
return result;
}
const articleTypes = new Set([
"article",
"newsarticle",
"blogposting",
"techarticle",
"webpage",
"report",
"scholarlyarticle",
"socialmediaposting",
]);
for (const script of Array.from(scripts)) {
const rawText = normalizeJsonLdText(script.textContent || "");
if (!rawText) continue;
try {
const parsed = JSON.parse(rawText);
const itemsToProcess: Record<string, unknown>[] = [];
const collectItems = (node: unknown) => {
if (!node || typeof node !== "object") return;
if (Array.isArray(node)) {
node.forEach(collectItems);
} else {
const obj = node as Record<string, unknown>;
itemsToProcess.push(obj);
if (Array.isArray(obj["@graph"])) {
obj["@graph"].forEach(collectItems);
}
}
};
collectItems(parsed);
for (const item of itemsToProcess) {
const rawType = item["@type"];
const types = Array.isArray(rawType)
? rawType.map((t) => String(t).toLowerCase())
: typeof rawType === "string"
? [rawType.toLowerCase()]
: [];
const isArticle = types.some((t) => articleTypes.has(t)) || Boolean(item.articleBody);
if (isArticle || (!result.headline && !result.articleBody)) {
if (typeof item.headline === "string" && item.headline.trim()) {
result.headline = result.headline || item.headline.trim();
} else if (typeof item.name === "string" && item.name.trim()) {
result.headline = result.headline || item.name.trim();
}
if (typeof item.articleBody === "string" && item.articleBody.trim()) {
result.articleBody = result.articleBody || item.articleBody.trim();
}
if (typeof item.description === "string" && item.description.trim()) {
result.description = result.description || item.description.trim();
}
const authorStr = extractAuthorString(item.author);
if (authorStr) {
result.author = result.author || authorStr;
}
if (typeof item.datePublished === "string" && item.datePublished.trim()) {
result.datePublished = result.datePublished || item.datePublished.trim();
} else if (typeof item.dateCreated === "string" && item.dateCreated.trim()) {
result.datePublished = result.datePublished || item.dateCreated.trim();
}
if (typeof item.dateModified === "string" && item.dateModified.trim()) {
result.dateModified = result.dateModified || item.dateModified.trim();
}
const publisherStr = extractPublisherString(item.publisher);
if (publisherStr) {
result.publisher = result.publisher || publisherStr;
}
}
}
} catch (err) {
const errMsg = err instanceof Error ? err.message : String(err);
Logger.debug("[HTMLExtractor]", `Failed to parse JSON-LD script block: ${errMsg}`);
}
}
return result;
}
import { Logger } from "../utils/Logger";
export interface ExtractedJsonLd {
headline?: string;
articleBody?: string;
description?: string;
author?: string;
datePublished?: string;
dateModified?: string;
publisher?: string;
}
/**
* Extracts author string from a JSON-LD author property value (string, object, or array).
*/
function extractAuthorString(authorVal: unknown): string | undefined {
if (!authorVal) return undefined;
if (typeof authorVal === "string" && authorVal.trim()) {
return authorVal.trim();
}
if (Array.isArray(authorVal)) {
const authors = authorVal
.map((a) => extractAuthorString(a))
.filter((a): a is string => Boolean(a));
return authors.length > 0 ? authors.join(", ") : undefined;
}
if (typeof authorVal === "object" && authorVal !== null) {
const obj = authorVal as Record<string, unknown>;
if (typeof obj.name === "string" && obj.name.trim()) {
return obj.name.trim();
}
}
return undefined;
}
/**
* Extracts publisher string from a JSON-LD publisher property value (string or object).
*/
function extractPublisherString(publisherVal: unknown): string | undefined {
if (!publisherVal) return undefined;
if (typeof publisherVal === "string" && publisherVal.trim()) {
return publisherVal.trim();
}
if (typeof publisherVal === "object" && publisherVal !== null) {
const obj = publisherVal as Record<string, unknown>;
if (typeof obj.name === "string" && obj.name.trim()) {
return obj.name.trim();
}
}
return undefined;
}
/**
* Normalizes raw JSON-LD script text by stripping HTML comment and CDATA wrappers.
* Does not perform entity decoding or structural JSON repair.
*/
export function normalizeJsonLdText(rawText: string): string {
let text = rawText.trim();
// Strip HTML comment wrapper: <!-- ... -->
if (text.startsWith("<!--") && text.endsWith("-->")) {
text = text.slice(4, -3).trim();
}
// Strip CDATA wrappers (single-line comments, multi-line comments, or bare tags)
text = text
.replace(/^\s*\/\/\s*<!\[CDATA\[\s*/i, "")
.replace(/\s*\/\/\s*\]\]>\s*$/i, "")
.replace(/^\s*\/\*\s*<!\[CDATA\[\s*\*\/\s*/i, "")
.replace(/\s*\/\*\s*\]\]>\s*\*\/\s*$/i, "")
.replace(/^\s*<!\[CDATA\[\s*/i, "")
.replace(/\s*\]\]>\s*$/i, "")
.trim();
return text;
}
/**
* Parses all <script type="application/ld+json"> elements in the document
* and extracts relevant structured metadata fields.
*/
export function extractJsonLd(doc: Document): ExtractedJsonLd {
const result: ExtractedJsonLd = {};
const scripts = doc.querySelectorAll('script[type="application/ld+json"]');
if (!scripts || scripts.length === 0) {
return result;
}
const articleTypes = new Set([
"article",
"newsarticle",
"blogposting",
"techarticle",
"webpage",
"report",
"scholarlyarticle",
"socialmediaposting",
]);
for (const script of Array.from(scripts)) {
const rawText = normalizeJsonLdText(script.textContent || "");
if (!rawText) continue;
try {
const parsed = JSON.parse(rawText);
const itemsToProcess: Record<string, unknown>[] = [];
const collectItems = (node: unknown) => {
if (!node || typeof node !== "object") return;
if (Array.isArray(node)) {
node.forEach(collectItems);
} else {
const obj = node as Record<string, unknown>;
itemsToProcess.push(obj);
if (Array.isArray(obj["@graph"])) {
obj["@graph"].forEach(collectItems);
}
}
};
collectItems(parsed);
for (const item of itemsToProcess) {
const rawType = item["@type"];
const types = Array.isArray(rawType)
? rawType.map((t) => String(t).toLowerCase())
: typeof rawType === "string"
? [rawType.toLowerCase()]
: [];
const isArticle = types.some((t) => articleTypes.has(t)) || Boolean(item.articleBody);
if (isArticle || (!result.headline && !result.articleBody)) {
if (typeof item.headline === "string" && item.headline.trim()) {
result.headline = result.headline || item.headline.trim();
} else if (typeof item.name === "string" && item.name.trim()) {
result.headline = result.headline || item.name.trim();
}
if (typeof item.articleBody === "string" && item.articleBody.trim()) {
result.articleBody = result.articleBody || item.articleBody.trim();
}
if (typeof item.description === "string" && item.description.trim()) {
result.description = result.description || item.description.trim();
}
const authorStr = extractAuthorString(item.author);
if (authorStr) {
result.author = result.author || authorStr;
}
if (typeof item.datePublished === "string" && item.datePublished.trim()) {
result.datePublished = result.datePublished || item.datePublished.trim();
} else if (typeof item.dateCreated === "string" && item.dateCreated.trim()) {
result.datePublished = result.datePublished || item.dateCreated.trim();
}
if (typeof item.dateModified === "string" && item.dateModified.trim()) {
result.dateModified = result.dateModified || item.dateModified.trim();
}
const publisherStr = extractPublisherString(item.publisher);
if (publisherStr) {
result.publisher = result.publisher || publisherStr;
}
}
}
} catch (err) {
const errMsg = err instanceof Error ? err.message : String(err);
Logger.debug("[HTMLExtractor]", `Failed to parse JSON-LD script block: ${errMsg}`);
}
}
return result;
}