Clean up record management
This commit is contained in:
parent
5b90a8e1b3
commit
ffaeb4841e
5 changed files with 139 additions and 148 deletions
|
|
@ -1,15 +1,15 @@
|
|||
import path, { dirname } from "node:path";
|
||||
import { z } from "zod";
|
||||
import type { UnknownRecord } from "#core/records";
|
||||
import { MuseError, MuseFileNotFoundError } from "#errors";
|
||||
import { resolveFilePath } from "#util";
|
||||
import { type MusePlugin, loadPlugin } from "./plugins";
|
||||
import {
|
||||
type MuseEntry,
|
||||
type MuseSource,
|
||||
loadFromSource,
|
||||
parseMuseFile,
|
||||
} from "./sources";
|
||||
import { type MusePlugin, loadPlugin } from "./plugins";
|
||||
import type { UnknownRecord } from "./types";
|
||||
|
||||
const SourceSchema = z.union([
|
||||
z.object({
|
||||
|
|
|
|||
107
src/core/records.ts
Normal file
107
src/core/records.ts
Normal file
|
|
@ -0,0 +1,107 @@
|
|||
import EMDY from "@endeavorance/emdy";
|
||||
import TOML from "smol-toml";
|
||||
import YAML from "yaml";
|
||||
import type { LoadedFile } from "./files";
|
||||
|
||||
export type UnknownRecord = Record<string, unknown>;
|
||||
|
||||
/**
|
||||
* Given an unknown shape, ensure it is an object or array of objects,
|
||||
* then return it as an array of objects.
|
||||
*
|
||||
* @param val - The unknown value to format
|
||||
* @returns - An array of objects
|
||||
*/
|
||||
function formatEntries(val: unknown): UnknownRecord[] {
|
||||
if (Array.isArray(val) && val.every((el) => typeof el === "object")) {
|
||||
return val;
|
||||
}
|
||||
|
||||
if (typeof val === "object" && val !== null) {
|
||||
return [val as UnknownRecord];
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
`Invalid data format. Entry files must define an object or array of objects, but found "${val}"`,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse one or more YAML documents from a string
|
||||
*
|
||||
* @param text - The YAML string to parse
|
||||
* @returns - An array of parsed objects
|
||||
*/
|
||||
export function parseYAMLEntries(text: string): UnknownRecord[] {
|
||||
const parsedDocs = YAML.parseAllDocuments(text);
|
||||
|
||||
if (parsedDocs.some((doc) => doc.toJS() === null)) {
|
||||
throw new Error("Encountered NULL resource");
|
||||
}
|
||||
|
||||
const errors = parsedDocs.flatMap((doc) => doc.errors);
|
||||
|
||||
if (errors.length > 0) {
|
||||
throw new Error(
|
||||
`Error parsing YAML resource: ${errors.map((e) => e.message).join(", ")}`,
|
||||
);
|
||||
}
|
||||
|
||||
const collection: UnknownRecord[] = parsedDocs.map((doc) => doc.toJS());
|
||||
return collection;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse one or more JSON documents from a string
|
||||
*
|
||||
* @param text - The JSON string to parse
|
||||
* @returns - An array of parsed objects
|
||||
*/
|
||||
export function parseJSONEntries(text: string): UnknownRecord[] {
|
||||
return formatEntries(JSON.parse(text));
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse one or more TOML documents from a string
|
||||
*
|
||||
* @param text - The TOML string to parse
|
||||
* @returns - An array of parsed objects
|
||||
*/
|
||||
export function parseTOMLEntries(text: string): UnknownRecord[] {
|
||||
return formatEntries(TOML.parse(text));
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse one or more Markdown documents from a string
|
||||
*
|
||||
* @param text - The Markdown string to parse
|
||||
* @returns - An array of parsed objects
|
||||
*/
|
||||
export function parseMarkdownEntry(
|
||||
text: string,
|
||||
contentKey: string,
|
||||
): UnknownRecord[] {
|
||||
return formatEntries(EMDY.parse(text, contentKey));
|
||||
}
|
||||
|
||||
export function autoParseEntry(loadedFile: LoadedFile, contentKey: string) {
|
||||
const { fileType, content } = loadedFile;
|
||||
|
||||
if (fileType === "md") {
|
||||
return parseMarkdownEntry(content, contentKey);
|
||||
}
|
||||
|
||||
if (fileType === "yaml") {
|
||||
return parseYAMLEntries(content);
|
||||
}
|
||||
|
||||
if (fileType === "json") {
|
||||
return parseJSONEntries(content);
|
||||
}
|
||||
|
||||
if (fileType === "toml") {
|
||||
return parseTOMLEntries(content);
|
||||
}
|
||||
|
||||
throw new Error(`Unsupported file type: ${fileType}`);
|
||||
}
|
||||
|
|
@ -1,12 +1,10 @@
|
|||
import EMDY from "@endeavorance/emdy";
|
||||
import TOML from "smol-toml";
|
||||
import YAML from "yaml";
|
||||
import { loadFileContent } from "./files";
|
||||
import type { UnknownRecord } from "./types";
|
||||
import { Glob } from "bun";
|
||||
import { type LoadedFile, loadFileContent } from "./files";
|
||||
import { type UnknownRecord, autoParseEntry } from "./records";
|
||||
|
||||
export type SourceType = "file" | "url";
|
||||
|
||||
/** A descriptor of a location to load into a Muse binding */
|
||||
export interface MuseSource {
|
||||
location: string;
|
||||
type: SourceType;
|
||||
|
|
@ -18,6 +16,7 @@ interface SourceOptions {
|
|||
ignore: string[];
|
||||
}
|
||||
|
||||
/** A single loaded entry from a Muse source */
|
||||
export interface MuseEntry<MetaShape = UnknownRecord> {
|
||||
_raw: string;
|
||||
location: string;
|
||||
|
|
@ -26,96 +25,27 @@ export interface MuseEntry<MetaShape = UnknownRecord> {
|
|||
source: MuseSource;
|
||||
}
|
||||
|
||||
function formatEntries(val: unknown): UnknownRecord[] {
|
||||
if (Array.isArray(val) && val.every((el) => typeof el === "object")) {
|
||||
return val;
|
||||
}
|
||||
|
||||
if (typeof val === "object" && val !== null) {
|
||||
return [val as UnknownRecord];
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
`Invalid data format. Entry files must define an object or array of objects, but found "${val}"`,
|
||||
);
|
||||
}
|
||||
|
||||
function parseYAMLEntries(text: string): UnknownRecord[] {
|
||||
const parsedDocs = YAML.parseAllDocuments(text);
|
||||
|
||||
if (parsedDocs.some((doc) => doc.toJS() === null)) {
|
||||
throw new Error("Encountered NULL resource");
|
||||
}
|
||||
|
||||
const errors = parsedDocs.flatMap((doc) => doc.errors);
|
||||
|
||||
if (errors.length > 0) {
|
||||
throw new Error(
|
||||
`Error parsing YAML resource: ${errors.map((e) => e.message).join(", ")}`,
|
||||
);
|
||||
}
|
||||
|
||||
const collection: UnknownRecord[] = parsedDocs.map((doc) => doc.toJS());
|
||||
return collection;
|
||||
}
|
||||
|
||||
function parseJSONEntries(text: string): UnknownRecord[] {
|
||||
return formatEntries(JSON.parse(text));
|
||||
}
|
||||
|
||||
function parseTOMLEntries(text: string): UnknownRecord[] {
|
||||
return formatEntries(TOML.parse(text));
|
||||
}
|
||||
|
||||
function parseMarkdownEntry(text: string, contentKey: string): UnknownRecord[] {
|
||||
return formatEntries(EMDY.parse(text, contentKey));
|
||||
}
|
||||
|
||||
export async function parseMuseFile(
|
||||
rawFilePath: string,
|
||||
{ contentKey = "content" }: SourceOptions,
|
||||
): Promise<MuseEntry[]> {
|
||||
const { content, filePath, fileType } = await loadFileContent(rawFilePath);
|
||||
const file = await loadFileContent(rawFilePath);
|
||||
const entries = autoParseEntry(file, contentKey);
|
||||
|
||||
const partial = {
|
||||
_raw: content,
|
||||
_raw: file.content,
|
||||
source: {
|
||||
location: filePath,
|
||||
location: file.filePath,
|
||||
type: "file" as SourceType,
|
||||
},
|
||||
location: filePath,
|
||||
location: file.filePath,
|
||||
meta: {},
|
||||
};
|
||||
|
||||
if (fileType === "md") {
|
||||
return parseMarkdownEntry(content, contentKey).map((data) => ({
|
||||
...partial,
|
||||
data,
|
||||
}));
|
||||
}
|
||||
|
||||
if (fileType === "yaml") {
|
||||
return parseYAMLEntries(content).map((data) => ({
|
||||
...partial,
|
||||
data,
|
||||
}));
|
||||
}
|
||||
|
||||
if (fileType === "json") {
|
||||
return parseJSONEntries(content).map((data) => ({
|
||||
...partial,
|
||||
data,
|
||||
}));
|
||||
}
|
||||
|
||||
if (fileType === "toml") {
|
||||
return parseTOMLEntries(content).map((data) => ({
|
||||
...partial,
|
||||
data,
|
||||
}));
|
||||
}
|
||||
|
||||
throw new Error(`Unsupported file type: ${fileType}`);
|
||||
return entries.map((data) => ({
|
||||
...partial,
|
||||
data,
|
||||
}));
|
||||
}
|
||||
|
||||
async function loadFromFileSource(
|
||||
|
|
@ -152,23 +82,6 @@ function getFileExtensionFromURL(url: string): string | null {
|
|||
return extension === filename ? null : extension;
|
||||
}
|
||||
|
||||
function mimeTypeToFileType(mimeType: string): string | null {
|
||||
switch (mimeType) {
|
||||
case "text/markdown":
|
||||
return "md";
|
||||
case "text/yaml":
|
||||
return "yaml";
|
||||
case "application/x-yaml":
|
||||
return "yaml";
|
||||
case "application/json":
|
||||
return "json";
|
||||
case "application/toml":
|
||||
return "toml";
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function loadFromURLSource(
|
||||
source: MuseSource,
|
||||
options: SourceOptions,
|
||||
|
|
@ -182,10 +95,18 @@ async function loadFromURLSource(
|
|||
const content = await response.text();
|
||||
const mimeType = response.headers.get("Content-Type") || "unknown";
|
||||
|
||||
const parseType =
|
||||
mimeTypeToFileType(mimeType) ??
|
||||
getFileExtensionFromURL(source.location) ??
|
||||
"unknown";
|
||||
const parseType = getFileExtensionFromURL(source.location) ?? "unknown";
|
||||
|
||||
const loadedFile: LoadedFile = {
|
||||
content,
|
||||
filePath: source.location,
|
||||
fileType: parseType,
|
||||
dirname: "",
|
||||
basename: "",
|
||||
mimeType,
|
||||
};
|
||||
|
||||
const entries = autoParseEntry(loadedFile, contentKey);
|
||||
|
||||
const partial = {
|
||||
_raw: content,
|
||||
|
|
@ -197,46 +118,10 @@ async function loadFromURLSource(
|
|||
meta: {},
|
||||
};
|
||||
|
||||
if (parseType === "md") {
|
||||
return parseMarkdownEntry(content, contentKey).map((data) => ({
|
||||
...partial,
|
||||
data,
|
||||
}));
|
||||
}
|
||||
|
||||
if (parseType === "yaml") {
|
||||
return parseYAMLEntries(content).map((data) => ({
|
||||
...partial,
|
||||
data,
|
||||
}));
|
||||
}
|
||||
|
||||
if (parseType === "json") {
|
||||
return parseJSONEntries(content).map((data) => ({
|
||||
...partial,
|
||||
data,
|
||||
}));
|
||||
}
|
||||
|
||||
if (parseType === "toml") {
|
||||
return parseTOMLEntries(content).map((data) => ({
|
||||
...partial,
|
||||
data,
|
||||
}));
|
||||
}
|
||||
|
||||
// If it doesnt match one of these, try brute force parsing
|
||||
// and return the result if any of them work
|
||||
try {
|
||||
return parseMarkdownEntry(content, contentKey).map((data) => ({
|
||||
...partial,
|
||||
data,
|
||||
}));
|
||||
} catch {
|
||||
// noop
|
||||
}
|
||||
|
||||
throw new Error(`Unsupported MIME type from URL source: ${mimeType}`);
|
||||
return entries.map((data) => ({
|
||||
...partial,
|
||||
data,
|
||||
}));
|
||||
}
|
||||
|
||||
export async function loadFromSource(
|
||||
|
|
|
|||
|
|
@ -1 +0,0 @@
|
|||
export type UnknownRecord = Record<string, unknown>;
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
import { loadBinding, type Binding } from "#core/binding";
|
||||
import type { Binding } from "#core/binding";
|
||||
|
||||
export class Muse {
|
||||
bindingLoaded = false;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue