Clean up record management
This commit is contained in:
parent
5b90a8e1b3
commit
ffaeb4841e
5 changed files with 139 additions and 148 deletions
|
@ -1,15 +1,15 @@
|
||||||
import path, { dirname } from "node:path";
|
import path, { dirname } from "node:path";
|
||||||
import { z } from "zod";
|
import { z } from "zod";
|
||||||
|
import type { UnknownRecord } from "#core/records";
|
||||||
import { MuseError, MuseFileNotFoundError } from "#errors";
|
import { MuseError, MuseFileNotFoundError } from "#errors";
|
||||||
import { resolveFilePath } from "#util";
|
import { resolveFilePath } from "#util";
|
||||||
|
import { type MusePlugin, loadPlugin } from "./plugins";
|
||||||
import {
|
import {
|
||||||
type MuseEntry,
|
type MuseEntry,
|
||||||
type MuseSource,
|
type MuseSource,
|
||||||
loadFromSource,
|
loadFromSource,
|
||||||
parseMuseFile,
|
parseMuseFile,
|
||||||
} from "./sources";
|
} from "./sources";
|
||||||
import { type MusePlugin, loadPlugin } from "./plugins";
|
|
||||||
import type { UnknownRecord } from "./types";
|
|
||||||
|
|
||||||
const SourceSchema = z.union([
|
const SourceSchema = z.union([
|
||||||
z.object({
|
z.object({
|
||||||
|
|
107
src/core/records.ts
Normal file
107
src/core/records.ts
Normal file
|
@ -0,0 +1,107 @@
|
||||||
|
import EMDY from "@endeavorance/emdy";
|
||||||
|
import TOML from "smol-toml";
|
||||||
|
import YAML from "yaml";
|
||||||
|
import type { LoadedFile } from "./files";
|
||||||
|
|
||||||
|
export type UnknownRecord = Record<string, unknown>;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Given an unknown shape, ensure it is an object or array of objects,
|
||||||
|
* then return it as an array of objects.
|
||||||
|
*
|
||||||
|
* @param val - The unknown value to format
|
||||||
|
* @returns - An array of objects
|
||||||
|
*/
|
||||||
|
function formatEntries(val: unknown): UnknownRecord[] {
|
||||||
|
if (Array.isArray(val) && val.every((el) => typeof el === "object")) {
|
||||||
|
return val;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (typeof val === "object" && val !== null) {
|
||||||
|
return [val as UnknownRecord];
|
||||||
|
}
|
||||||
|
|
||||||
|
throw new Error(
|
||||||
|
`Invalid data format. Entry files must define an object or array of objects, but found "${val}"`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Parse one or more YAML documents from a string
|
||||||
|
*
|
||||||
|
* @param text - The YAML string to parse
|
||||||
|
* @returns - An array of parsed objects
|
||||||
|
*/
|
||||||
|
export function parseYAMLEntries(text: string): UnknownRecord[] {
|
||||||
|
const parsedDocs = YAML.parseAllDocuments(text);
|
||||||
|
|
||||||
|
if (parsedDocs.some((doc) => doc.toJS() === null)) {
|
||||||
|
throw new Error("Encountered NULL resource");
|
||||||
|
}
|
||||||
|
|
||||||
|
const errors = parsedDocs.flatMap((doc) => doc.errors);
|
||||||
|
|
||||||
|
if (errors.length > 0) {
|
||||||
|
throw new Error(
|
||||||
|
`Error parsing YAML resource: ${errors.map((e) => e.message).join(", ")}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const collection: UnknownRecord[] = parsedDocs.map((doc) => doc.toJS());
|
||||||
|
return collection;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Parse one or more JSON documents from a string
|
||||||
|
*
|
||||||
|
* @param text - The JSON string to parse
|
||||||
|
* @returns - An array of parsed objects
|
||||||
|
*/
|
||||||
|
export function parseJSONEntries(text: string): UnknownRecord[] {
|
||||||
|
return formatEntries(JSON.parse(text));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Parse one or more TOML documents from a string
|
||||||
|
*
|
||||||
|
* @param text - The TOML string to parse
|
||||||
|
* @returns - An array of parsed objects
|
||||||
|
*/
|
||||||
|
export function parseTOMLEntries(text: string): UnknownRecord[] {
|
||||||
|
return formatEntries(TOML.parse(text));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Parse one or more Markdown documents from a string
|
||||||
|
*
|
||||||
|
* @param text - The Markdown string to parse
|
||||||
|
* @returns - An array of parsed objects
|
||||||
|
*/
|
||||||
|
export function parseMarkdownEntry(
|
||||||
|
text: string,
|
||||||
|
contentKey: string,
|
||||||
|
): UnknownRecord[] {
|
||||||
|
return formatEntries(EMDY.parse(text, contentKey));
|
||||||
|
}
|
||||||
|
|
||||||
|
export function autoParseEntry(loadedFile: LoadedFile, contentKey: string) {
|
||||||
|
const { fileType, content } = loadedFile;
|
||||||
|
|
||||||
|
if (fileType === "md") {
|
||||||
|
return parseMarkdownEntry(content, contentKey);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (fileType === "yaml") {
|
||||||
|
return parseYAMLEntries(content);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (fileType === "json") {
|
||||||
|
return parseJSONEntries(content);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (fileType === "toml") {
|
||||||
|
return parseTOMLEntries(content);
|
||||||
|
}
|
||||||
|
|
||||||
|
throw new Error(`Unsupported file type: ${fileType}`);
|
||||||
|
}
|
|
@ -1,12 +1,10 @@
|
||||||
import EMDY from "@endeavorance/emdy";
|
|
||||||
import TOML from "smol-toml";
|
|
||||||
import YAML from "yaml";
|
|
||||||
import { loadFileContent } from "./files";
|
|
||||||
import type { UnknownRecord } from "./types";
|
|
||||||
import { Glob } from "bun";
|
import { Glob } from "bun";
|
||||||
|
import { type LoadedFile, loadFileContent } from "./files";
|
||||||
|
import { type UnknownRecord, autoParseEntry } from "./records";
|
||||||
|
|
||||||
export type SourceType = "file" | "url";
|
export type SourceType = "file" | "url";
|
||||||
|
|
||||||
|
/** A descriptor of a location to load into a Muse binding */
|
||||||
export interface MuseSource {
|
export interface MuseSource {
|
||||||
location: string;
|
location: string;
|
||||||
type: SourceType;
|
type: SourceType;
|
||||||
|
@ -18,6 +16,7 @@ interface SourceOptions {
|
||||||
ignore: string[];
|
ignore: string[];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** A single loaded entry from a Muse source */
|
||||||
export interface MuseEntry<MetaShape = UnknownRecord> {
|
export interface MuseEntry<MetaShape = UnknownRecord> {
|
||||||
_raw: string;
|
_raw: string;
|
||||||
location: string;
|
location: string;
|
||||||
|
@ -26,96 +25,27 @@ export interface MuseEntry<MetaShape = UnknownRecord> {
|
||||||
source: MuseSource;
|
source: MuseSource;
|
||||||
}
|
}
|
||||||
|
|
||||||
function formatEntries(val: unknown): UnknownRecord[] {
|
|
||||||
if (Array.isArray(val) && val.every((el) => typeof el === "object")) {
|
|
||||||
return val;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (typeof val === "object" && val !== null) {
|
|
||||||
return [val as UnknownRecord];
|
|
||||||
}
|
|
||||||
|
|
||||||
throw new Error(
|
|
||||||
`Invalid data format. Entry files must define an object or array of objects, but found "${val}"`,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
function parseYAMLEntries(text: string): UnknownRecord[] {
|
|
||||||
const parsedDocs = YAML.parseAllDocuments(text);
|
|
||||||
|
|
||||||
if (parsedDocs.some((doc) => doc.toJS() === null)) {
|
|
||||||
throw new Error("Encountered NULL resource");
|
|
||||||
}
|
|
||||||
|
|
||||||
const errors = parsedDocs.flatMap((doc) => doc.errors);
|
|
||||||
|
|
||||||
if (errors.length > 0) {
|
|
||||||
throw new Error(
|
|
||||||
`Error parsing YAML resource: ${errors.map((e) => e.message).join(", ")}`,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
const collection: UnknownRecord[] = parsedDocs.map((doc) => doc.toJS());
|
|
||||||
return collection;
|
|
||||||
}
|
|
||||||
|
|
||||||
function parseJSONEntries(text: string): UnknownRecord[] {
|
|
||||||
return formatEntries(JSON.parse(text));
|
|
||||||
}
|
|
||||||
|
|
||||||
function parseTOMLEntries(text: string): UnknownRecord[] {
|
|
||||||
return formatEntries(TOML.parse(text));
|
|
||||||
}
|
|
||||||
|
|
||||||
function parseMarkdownEntry(text: string, contentKey: string): UnknownRecord[] {
|
|
||||||
return formatEntries(EMDY.parse(text, contentKey));
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function parseMuseFile(
|
export async function parseMuseFile(
|
||||||
rawFilePath: string,
|
rawFilePath: string,
|
||||||
{ contentKey = "content" }: SourceOptions,
|
{ contentKey = "content" }: SourceOptions,
|
||||||
): Promise<MuseEntry[]> {
|
): Promise<MuseEntry[]> {
|
||||||
const { content, filePath, fileType } = await loadFileContent(rawFilePath);
|
const file = await loadFileContent(rawFilePath);
|
||||||
|
const entries = autoParseEntry(file, contentKey);
|
||||||
|
|
||||||
const partial = {
|
const partial = {
|
||||||
_raw: content,
|
_raw: file.content,
|
||||||
source: {
|
source: {
|
||||||
location: filePath,
|
location: file.filePath,
|
||||||
type: "file" as SourceType,
|
type: "file" as SourceType,
|
||||||
},
|
},
|
||||||
location: filePath,
|
location: file.filePath,
|
||||||
meta: {},
|
meta: {},
|
||||||
};
|
};
|
||||||
|
|
||||||
if (fileType === "md") {
|
return entries.map((data) => ({
|
||||||
return parseMarkdownEntry(content, contentKey).map((data) => ({
|
...partial,
|
||||||
...partial,
|
data,
|
||||||
data,
|
}));
|
||||||
}));
|
|
||||||
}
|
|
||||||
|
|
||||||
if (fileType === "yaml") {
|
|
||||||
return parseYAMLEntries(content).map((data) => ({
|
|
||||||
...partial,
|
|
||||||
data,
|
|
||||||
}));
|
|
||||||
}
|
|
||||||
|
|
||||||
if (fileType === "json") {
|
|
||||||
return parseJSONEntries(content).map((data) => ({
|
|
||||||
...partial,
|
|
||||||
data,
|
|
||||||
}));
|
|
||||||
}
|
|
||||||
|
|
||||||
if (fileType === "toml") {
|
|
||||||
return parseTOMLEntries(content).map((data) => ({
|
|
||||||
...partial,
|
|
||||||
data,
|
|
||||||
}));
|
|
||||||
}
|
|
||||||
|
|
||||||
throw new Error(`Unsupported file type: ${fileType}`);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async function loadFromFileSource(
|
async function loadFromFileSource(
|
||||||
|
@ -152,23 +82,6 @@ function getFileExtensionFromURL(url: string): string | null {
|
||||||
return extension === filename ? null : extension;
|
return extension === filename ? null : extension;
|
||||||
}
|
}
|
||||||
|
|
||||||
function mimeTypeToFileType(mimeType: string): string | null {
|
|
||||||
switch (mimeType) {
|
|
||||||
case "text/markdown":
|
|
||||||
return "md";
|
|
||||||
case "text/yaml":
|
|
||||||
return "yaml";
|
|
||||||
case "application/x-yaml":
|
|
||||||
return "yaml";
|
|
||||||
case "application/json":
|
|
||||||
return "json";
|
|
||||||
case "application/toml":
|
|
||||||
return "toml";
|
|
||||||
default:
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function loadFromURLSource(
|
async function loadFromURLSource(
|
||||||
source: MuseSource,
|
source: MuseSource,
|
||||||
options: SourceOptions,
|
options: SourceOptions,
|
||||||
|
@ -182,10 +95,18 @@ async function loadFromURLSource(
|
||||||
const content = await response.text();
|
const content = await response.text();
|
||||||
const mimeType = response.headers.get("Content-Type") || "unknown";
|
const mimeType = response.headers.get("Content-Type") || "unknown";
|
||||||
|
|
||||||
const parseType =
|
const parseType = getFileExtensionFromURL(source.location) ?? "unknown";
|
||||||
mimeTypeToFileType(mimeType) ??
|
|
||||||
getFileExtensionFromURL(source.location) ??
|
const loadedFile: LoadedFile = {
|
||||||
"unknown";
|
content,
|
||||||
|
filePath: source.location,
|
||||||
|
fileType: parseType,
|
||||||
|
dirname: "",
|
||||||
|
basename: "",
|
||||||
|
mimeType,
|
||||||
|
};
|
||||||
|
|
||||||
|
const entries = autoParseEntry(loadedFile, contentKey);
|
||||||
|
|
||||||
const partial = {
|
const partial = {
|
||||||
_raw: content,
|
_raw: content,
|
||||||
|
@ -197,46 +118,10 @@ async function loadFromURLSource(
|
||||||
meta: {},
|
meta: {},
|
||||||
};
|
};
|
||||||
|
|
||||||
if (parseType === "md") {
|
return entries.map((data) => ({
|
||||||
return parseMarkdownEntry(content, contentKey).map((data) => ({
|
...partial,
|
||||||
...partial,
|
data,
|
||||||
data,
|
}));
|
||||||
}));
|
|
||||||
}
|
|
||||||
|
|
||||||
if (parseType === "yaml") {
|
|
||||||
return parseYAMLEntries(content).map((data) => ({
|
|
||||||
...partial,
|
|
||||||
data,
|
|
||||||
}));
|
|
||||||
}
|
|
||||||
|
|
||||||
if (parseType === "json") {
|
|
||||||
return parseJSONEntries(content).map((data) => ({
|
|
||||||
...partial,
|
|
||||||
data,
|
|
||||||
}));
|
|
||||||
}
|
|
||||||
|
|
||||||
if (parseType === "toml") {
|
|
||||||
return parseTOMLEntries(content).map((data) => ({
|
|
||||||
...partial,
|
|
||||||
data,
|
|
||||||
}));
|
|
||||||
}
|
|
||||||
|
|
||||||
// If it doesnt match one of these, try brute force parsing
|
|
||||||
// and return the result if any of them work
|
|
||||||
try {
|
|
||||||
return parseMarkdownEntry(content, contentKey).map((data) => ({
|
|
||||||
...partial,
|
|
||||||
data,
|
|
||||||
}));
|
|
||||||
} catch {
|
|
||||||
// noop
|
|
||||||
}
|
|
||||||
|
|
||||||
throw new Error(`Unsupported MIME type from URL source: ${mimeType}`);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function loadFromSource(
|
export async function loadFromSource(
|
||||||
|
|
|
@ -1 +0,0 @@
|
||||||
export type UnknownRecord = Record<string, unknown>;
|
|
|
@ -1,4 +1,4 @@
|
||||||
import { loadBinding, type Binding } from "#core/binding";
|
import type { Binding } from "#core/binding";
|
||||||
|
|
||||||
export class Muse {
|
export class Muse {
|
||||||
bindingLoaded = false;
|
bindingLoaded = false;
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue