From 5b90a8e1b32e8f7e11644e7cca36350e80a51dd5 Mon Sep 17 00:00:00 2001 From: Endeavorance Date: Fri, 4 Apr 2025 13:50:43 -0400 Subject: [PATCH] Support for url based resources --- demo/main/binding.md | 3 + src/core/binding.ts | 94 ++++++++++------ src/core/files.ts | 2 + src/core/parse.ts | 104 ------------------ src/core/sources.ts | 254 +++++++++++++++++++++++++++++++++++++++++++ src/muse.ts | 20 ++++ 6 files changed, 342 insertions(+), 135 deletions(-) delete mode 100644 src/core/parse.ts create mode 100644 src/core/sources.ts create mode 100644 src/muse.ts diff --git a/demo/main/binding.md b/demo/main/binding.md index b660a51..cb097e7 100644 --- a/demo/main/binding.md +++ b/demo/main/binding.md @@ -3,6 +3,9 @@ markdownKey: description options: dunks: key: dunkaroos +sources: + - file: ./stats/*.toml + - url: https://git.astral.camp/endeavorance/emdy/raw/branch/main/package.json processors: - ./processors/scream.js - ./processors/announce-slam-dunks.js diff --git a/src/core/binding.ts b/src/core/binding.ts index b78292d..7db7c5e 100644 --- a/src/core/binding.ts +++ b/src/core/binding.ts @@ -1,21 +1,37 @@ import path, { dirname } from "node:path"; -import { Glob } from "bun"; import { z } from "zod"; import { MuseError, MuseFileNotFoundError } from "#errors"; import { resolveFilePath } from "#util"; -import { type MuseEntry, parseMuseFile } from "./parse"; +import { + type MuseEntry, + type MuseSource, + loadFromSource, + parseMuseFile, +} from "./sources"; import { type MusePlugin, loadPlugin } from "./plugins"; import type { UnknownRecord } from "./types"; +const SourceSchema = z.union([ + z.object({ + file: z.string(), + }), + z.object({ + url: z.string().url(), + }), +]); + +type SourceShorthand = z.infer; + // Function to parse an unknown object into parts of a binding -export const BindingSchema = z.object({ +const BindingSchema = z.object({ contentKey: z.string().default("content"), - sources: z - .array(z.string()) - .default(["**/*.yaml", "**/*.md", "**/*.toml", "**/*.json"]), + sources: z.array(SourceSchema).default([ + { + file: "./**/*.{json,yaml,toml,md}", + }, + ]), options: z.record(z.string(), z.any()).default({}), processors: z.array(z.string()).default([]), - renderer: z.string().optional(), }); /** @@ -27,6 +43,9 @@ export interface Binding { /** The raw data read from the binding file */ readonly _raw: z.infer; + /** Information about the sources used to load entries */ + readonly sources: MuseSource[]; + /** The full file path to the binding file */ readonly bindingPath: string; @@ -46,6 +65,24 @@ export interface Binding { readonly options: UnknownRecord; } +function sourceShorthandToSource(shorthand: SourceShorthand): MuseSource { + if ("file" in shorthand) { + return { + location: shorthand.file, + type: "file", + }; + } + + if ("url" in shorthand) { + return { + location: shorthand.url, + type: "url", + }; + } + + throw new MuseError("Invalid source shorthand"); +} + /** * Given a path, find the binding file * If the path is to a directory, check for a binding inside @@ -93,7 +130,11 @@ export async function loadBinding(initialPath: string): Promise { // Resolve the file location if possible const bindingPath = await resolveBindingPath(initialPath); const bindingDirname = dirname(bindingPath); - const loadedBindingData = await parseMuseFile(bindingPath); + const loadedBindingData = await parseMuseFile(bindingPath, { + contentKey: "content", + cwd: bindingDirname, + ignore: [], + }); if (loadedBindingData.length === 0) { throw new MuseError("No entries found in binding file"); @@ -102,30 +143,20 @@ export async function loadBinding(initialPath: string): Promise { // Load and parse the Binding YAML content const parsedBinding = BindingSchema.parse(loadedBindingData[0].data); - // Aggregate file paths to import - const includedFilePaths = parsedBinding.sources - .flatMap((thisGlob) => { - return Array.from( - new Glob(thisGlob).scanSync({ - cwd: bindingDirname, - absolute: true, - followSymlinks: true, - onlyFiles: true, - }), - ); - }) - .filter((filePath) => filePath !== bindingPath); + const sourceOptions = { + cwd: bindingDirname, + contentKey: parsedBinding.contentKey, + ignore: [bindingPath], + }; - // Load files - const entries: MuseEntry[] = ( - await Promise.all( - includedFilePaths.map((filePath) => { - return parseMuseFile(filePath, { - contentKey: parsedBinding.contentKey, - }); - }), - ) - ).flat(); + const sources = parsedBinding.sources.map(sourceShorthandToSource); + + const entries: MuseEntry[] = []; + + for (const source of sources) { + const entriesFromSource = await loadFromSource(source, sourceOptions); + entries.push(...entriesFromSource); + } // Load and check plugins const plugins: MusePlugin[] = await Promise.all( @@ -134,6 +165,7 @@ export async function loadBinding(initialPath: string): Promise { return { _raw: parsedBinding, + sources, bindingPath, contentKey: parsedBinding.contentKey, entries, diff --git a/src/core/files.ts b/src/core/files.ts index a32b877..89b2817 100644 --- a/src/core/files.ts +++ b/src/core/files.ts @@ -7,6 +7,7 @@ export interface LoadedFile { fileType: string; dirname: string; basename: string; + mimeType: string; } export async function loadFileContent(filePath: string): Promise { @@ -28,5 +29,6 @@ export async function loadFileContent(filePath: string): Promise { fileType: extension.slice(1), dirname: dirname(normalizedPath), basename: basename(normalizedPath, extension), + mimeType: file.type, }; } diff --git a/src/core/parse.ts b/src/core/parse.ts deleted file mode 100644 index 6617db2..0000000 --- a/src/core/parse.ts +++ /dev/null @@ -1,104 +0,0 @@ -import EMDY from "@endeavorance/emdy"; -import TOML from "smol-toml"; -import YAML from "yaml"; -import { loadFileContent } from "./files"; -import type { UnknownRecord } from "./types"; - -export interface MuseEntry { - _raw: string; - filePath: string; - data: Record; - meta: MetaShape; -} - -function formatEntries(val: unknown): UnknownRecord[] { - if (Array.isArray(val) && val.every((el) => typeof el === "object")) { - return val; - } - - if (typeof val === "object" && val !== null) { - return [val as UnknownRecord]; - } - - throw new Error( - `Invalid data format. Entry files must define an object or array of objects, but found "${val}"`, - ); -} - -function parseYAMLEntries(text: string): UnknownRecord[] { - const parsedDocs = YAML.parseAllDocuments(text); - - if (parsedDocs.some((doc) => doc.toJS() === null)) { - throw new Error("Encountered NULL resource"); - } - - const errors = parsedDocs.flatMap((doc) => doc.errors); - - if (errors.length > 0) { - throw new Error( - `Error parsing YAML resource: ${errors.map((e) => e.message).join(", ")}`, - ); - } - - const collection: UnknownRecord[] = parsedDocs.map((doc) => doc.toJS()); - return collection; -} - -function parseJSONEntries(text: string): UnknownRecord[] { - return formatEntries(JSON.parse(text)); -} - -function parseTOMLEntries(text: string): UnknownRecord[] { - return formatEntries(TOML.parse(text)); -} - -function parseMarkdownEntry(text: string, contentKey: string): UnknownRecord[] { - return formatEntries(EMDY.parse(text, contentKey)); -} - -interface ParseMuseFileOptions { - contentKey?: string; -} - -export async function parseMuseFile( - rawFilePath: string, - { contentKey = "content" }: ParseMuseFileOptions = {}, -): Promise { - const { content, filePath, fileType } = await loadFileContent(rawFilePath); - - const partial = { - _raw: content, - filePath, - meta: {}, - }; - - if (fileType === "md") { - return parseMarkdownEntry(content, contentKey).map((data) => ({ - ...partial, - data, - })); - } - - if (fileType === "yaml") { - return parseYAMLEntries(content).map((data) => ({ - ...partial, - data, - })); - } - - if (fileType === "json") { - return parseJSONEntries(content).map((data) => ({ - ...partial, - data, - })); - } - - if (fileType === "toml") { - return parseTOMLEntries(content).map((data) => ({ - ...partial, - data, - })); - } - - throw new Error(`Unsupported file type: ${fileType}`); -} diff --git a/src/core/sources.ts b/src/core/sources.ts new file mode 100644 index 0000000..8b57103 --- /dev/null +++ b/src/core/sources.ts @@ -0,0 +1,254 @@ +import EMDY from "@endeavorance/emdy"; +import TOML from "smol-toml"; +import YAML from "yaml"; +import { loadFileContent } from "./files"; +import type { UnknownRecord } from "./types"; +import { Glob } from "bun"; + +export type SourceType = "file" | "url"; + +export interface MuseSource { + location: string; + type: SourceType; +} + +interface SourceOptions { + cwd: string; + contentKey: string; + ignore: string[]; +} + +export interface MuseEntry { + _raw: string; + location: string; + data: Record; + meta: MetaShape; + source: MuseSource; +} + +function formatEntries(val: unknown): UnknownRecord[] { + if (Array.isArray(val) && val.every((el) => typeof el === "object")) { + return val; + } + + if (typeof val === "object" && val !== null) { + return [val as UnknownRecord]; + } + + throw new Error( + `Invalid data format. Entry files must define an object or array of objects, but found "${val}"`, + ); +} + +function parseYAMLEntries(text: string): UnknownRecord[] { + const parsedDocs = YAML.parseAllDocuments(text); + + if (parsedDocs.some((doc) => doc.toJS() === null)) { + throw new Error("Encountered NULL resource"); + } + + const errors = parsedDocs.flatMap((doc) => doc.errors); + + if (errors.length > 0) { + throw new Error( + `Error parsing YAML resource: ${errors.map((e) => e.message).join(", ")}`, + ); + } + + const collection: UnknownRecord[] = parsedDocs.map((doc) => doc.toJS()); + return collection; +} + +function parseJSONEntries(text: string): UnknownRecord[] { + return formatEntries(JSON.parse(text)); +} + +function parseTOMLEntries(text: string): UnknownRecord[] { + return formatEntries(TOML.parse(text)); +} + +function parseMarkdownEntry(text: string, contentKey: string): UnknownRecord[] { + return formatEntries(EMDY.parse(text, contentKey)); +} + +export async function parseMuseFile( + rawFilePath: string, + { contentKey = "content" }: SourceOptions, +): Promise { + const { content, filePath, fileType } = await loadFileContent(rawFilePath); + + const partial = { + _raw: content, + source: { + location: filePath, + type: "file" as SourceType, + }, + location: filePath, + meta: {}, + }; + + if (fileType === "md") { + return parseMarkdownEntry(content, contentKey).map((data) => ({ + ...partial, + data, + })); + } + + if (fileType === "yaml") { + return parseYAMLEntries(content).map((data) => ({ + ...partial, + data, + })); + } + + if (fileType === "json") { + return parseJSONEntries(content).map((data) => ({ + ...partial, + data, + })); + } + + if (fileType === "toml") { + return parseTOMLEntries(content).map((data) => ({ + ...partial, + data, + })); + } + + throw new Error(`Unsupported file type: ${fileType}`); +} + +async function loadFromFileSource( + source: MuseSource, + options: SourceOptions, +): Promise { + const paths = Array.from( + new Glob(source.location).scanSync({ + cwd: options.cwd, + absolute: true, + followSymlinks: true, + onlyFiles: true, + }), + ); + + const filteredPaths = paths.filter((path) => !options.ignore.includes(path)); + + const entries: MuseEntry[] = []; + for (const filePath of filteredPaths) { + const fileEntries = await parseMuseFile(filePath, options); + entries.push(...fileEntries); + } + + return entries; +} + +function getFileExtensionFromURL(url: string): string | null { + const parsedUrl = new URL(url); + + const pathname = parsedUrl.pathname; + const filename = pathname.substring(pathname.lastIndexOf("/") + 1); + const extension = filename.substring(filename.lastIndexOf(".") + 1); + + return extension === filename ? null : extension; +} + +function mimeTypeToFileType(mimeType: string): string | null { + switch (mimeType) { + case "text/markdown": + return "md"; + case "text/yaml": + return "yaml"; + case "application/x-yaml": + return "yaml"; + case "application/json": + return "json"; + case "application/toml": + return "toml"; + default: + return null; + } +} + +async function loadFromURLSource( + source: MuseSource, + options: SourceOptions, +): Promise { + const { contentKey = "content" } = options; + const response = await fetch(source.location); + + if (!response.ok) { + throw new Error(`Failed to fetch URL: ${source.location}`); + } + const content = await response.text(); + const mimeType = response.headers.get("Content-Type") || "unknown"; + + const parseType = + mimeTypeToFileType(mimeType) ?? + getFileExtensionFromURL(source.location) ?? + "unknown"; + + const partial = { + _raw: content, + source: { + location: source.location, + type: "url" as SourceType, + }, + location: source.location, + meta: {}, + }; + + if (parseType === "md") { + return parseMarkdownEntry(content, contentKey).map((data) => ({ + ...partial, + data, + })); + } + + if (parseType === "yaml") { + return parseYAMLEntries(content).map((data) => ({ + ...partial, + data, + })); + } + + if (parseType === "json") { + return parseJSONEntries(content).map((data) => ({ + ...partial, + data, + })); + } + + if (parseType === "toml") { + return parseTOMLEntries(content).map((data) => ({ + ...partial, + data, + })); + } + + // If it doesnt match one of these, try brute force parsing + // and return the result if any of them work + try { + return parseMarkdownEntry(content, contentKey).map((data) => ({ + ...partial, + data, + })); + } catch { + // noop + } + + throw new Error(`Unsupported MIME type from URL source: ${mimeType}`); +} + +export async function loadFromSource( + source: MuseSource, + options: SourceOptions, +): Promise { + switch (source.type) { + case "file": + return loadFromFileSource(source, options); + case "url": + return loadFromURLSource(source, options); + default: + throw new Error(`Unsupported source type: ${source.type}`); + } +} diff --git a/src/muse.ts b/src/muse.ts new file mode 100644 index 0000000..d2b61f9 --- /dev/null +++ b/src/muse.ts @@ -0,0 +1,20 @@ +import { loadBinding, type Binding } from "#core/binding"; + +export class Muse { + bindingLoaded = false; + preliminaryBinding: Partial = {}; + additionalSources: string[] = []; + additionalPluginPaths: string[] = []; + + constructor(bindingConfig: Partial) { + this.preliminaryBinding = bindingConfig; + } + + source(newSource: string) { + this.additionalSources.push(newSource); + } + + plugin(pluginPath: string) { + this.additionalPluginPaths.push(pluginPath); + } +}