diff --git a/README.md b/README.md index df86874..d122169 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,17 @@ # Muse -A CLI for wrangling directories of source files. - -Like a static generator, but for whatever. +_Bind data into anything_ ## Overview -**Muse** is a CLI and toolchain for operating on directories of -json, yaml, toml, and markdown files. Each file can specify any shape -of data. Muse scans included files, parses them into data, and streams -the loaded data through processors and plugins. +**Muse** is a CLI and toolchain for operating on data collected from +json, yaml, toml, and markdown files. -Each processor can modify the data at compile time, as well as enact -side effects such as writing files to disk. +Muse scans included files, parses them into data, and streams +the loaded data through plugins. -Muse does not edit source files, it only reads them in. +Each plugin can modify the data as well as enact side effects +such as writing files to disk. ## Usage @@ -32,10 +29,9 @@ Options: Each Muse project should specify a `binding` file with the following shape: ```yaml -include: - - ../another-project # Optional -files: # Optional - - "**/*.yaml" +sources: # Optional + - file: "**/*.yaml" + - web: "https://example.com/data.json" contentKey: "content" # Optional options: # Optional someOption: someValue @@ -44,9 +40,9 @@ processors: - second-processor ``` -The `binding` file can be any of the supported file types for Muse. If -the CLI is invoked with a directory instead of a file, Muse will look -for a binding file in the directory with the following order: +The `binding` file can be any of the supported file types for Muse and can +be named anything. If the CLI is invoked with a directory instead of a file, +Muse will look for a binding file in the directory with the following order: 1. `binding.json` 2. `binding.yaml` @@ -68,3 +64,34 @@ Your markdown content here When loading markdown files, Muse will load the content of the file into a key called `content` by default. This can be changed in the binding configuration by setting a `contentKey` value as an override. + +## Plugin API + +Muse plugins are written in TypeScript/JavaScript and expose information +describing the plugin, as well as the functions to operate with: + +```typescript +export const name = "Plugin Name"; +export const description = "Plugin Description"; +export async function step(binding: Binding): Promise {} +``` + +The main function of a plugin is `step`, which takes a `Binding` object +and returns a modified `Binding` object. + +When returning a new Binding object, it is best practice to make immutable +changes: + +```typescript +export async function step(binding: Binding): Promise { + const newBinding = { ...binding }; + newBinding.options.someOption = "newValue"; + return { + ...binding, + meta: { + ...binding.meta, + customValue: true, + } + }; +} +``` diff --git a/biome.json b/biome.json index 1814905..7988bf6 100644 --- a/biome.json +++ b/biome.json @@ -7,10 +7,7 @@ }, "files": { "ignoreUnknown": false, - "ignore": [ - "*.d.ts", - "*.json" - ], + "ignore": ["*.d.ts", "*.json"], "include": [ "./src/**/*.ts", "./src/**/*.tsx", diff --git a/bunfig.toml b/bunfig.toml new file mode 100644 index 0000000..a7461c6 --- /dev/null +++ b/bunfig.toml @@ -0,0 +1 @@ +preload = ["./src/preload/http-plugin.js"] diff --git a/demo/main/binding.md b/demo/main/binding.md index b660a51..cb097e7 100644 --- a/demo/main/binding.md +++ b/demo/main/binding.md @@ -3,6 +3,9 @@ markdownKey: description options: dunks: key: dunkaroos +sources: + - file: ./stats/*.toml + - url: https://git.astral.camp/endeavorance/emdy/raw/branch/main/package.json processors: - ./processors/scream.js - ./processors/announce-slam-dunks.js diff --git a/demo/main/processors/announce-slam-dunks.js b/demo/main/processors/announce-slam-dunks.js index 24cdb96..e1889a3 100644 --- a/demo/main/processors/announce-slam-dunks.js +++ b/demo/main/processors/announce-slam-dunks.js @@ -1,13 +1,19 @@ export const name = "Announce Slam Dunks"; +export const description = "Get hype when a file has slam dunks"; -export const process = (binding, { dunks }) => { +export const step = ({ entries, options, ...rest }) => { + const { dunks } = options; const slamDunkKey = dunks?.key ?? "slamDunks"; - for (const entry of binding.entries) { + for (const entry of entries) { if (slamDunkKey in entry.data) { console.log(`Slam dunk!`); } } - return binding; + return { + entries, + options, + ...rest, + }; }; diff --git a/demo/main/processors/scream.js b/demo/main/processors/scream.js index 11f177d..4a8a629 100644 --- a/demo/main/processors/scream.js +++ b/demo/main/processors/scream.js @@ -1,5 +1,4 @@ -export const name = "Scream"; -export function process(binding) { +export function step(binding) { const modifiedEntries = binding.entries.map(entry => { if (binding.markdownKey in entry.data) { entry.data = { diff --git a/package.json b/package.json index e9fae55..e7e1428 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { - "name": "@proscenium/muse", - "version": "0.1.0", + "name": "@endeavorance/muse", + "version": "0.3.0", "module": "dist/index.js", "exports": { ".": { @@ -31,9 +31,10 @@ "fmt": "bunx --bun biome check --fix", "clean": "rm -rf dist", "types": "dts-bundle-generator -o dist/muse.d.ts --project ./tsconfig.json ./src/exports.ts", - "compile": "bun build ./src/cli.ts --outfile ./dist/muse --compile", - "build": "bun run clean && bun run compile && bun run types", - "build:install": "bun run build && mv muse ~/.local/bin/muse", - "demo": "bun run ./src/cli.ts -- ./demo/main" + "transpile": "bun build ./src/exports.ts --outfile ./dist/muse.js", + "compile": "bun run clean && bun build ./src/cli/index.ts --outfile ./dist/muse --compile", + "compile:install": "bun run compile && mv ./dist/muse ~/.local/bin/muse", + "build": "bun run clean && bun run transpile && bun run types", + "demo": "bun run ./src/cli/index.ts -- ./demo/main" } } diff --git a/src/binding.ts b/src/binding.ts deleted file mode 100644 index 9499757..0000000 --- a/src/binding.ts +++ /dev/null @@ -1,160 +0,0 @@ -import path, { dirname } from "node:path"; -import { Glob } from "bun"; -import { FileNotFoundError, MuseError } from "./errors"; -import { parseMuseFile } from "./parse"; -import { - type Binding, - BindingSchema, - type MuseEntry, - type MuseProcessor, -} from "./types"; -import { resolveFilePath } from "./util"; - -async function loadProcessor( - bindingDirname: string, - processorPath: string, -): Promise { - const resolvedProcessorPath = path.resolve(bindingDirname, processorPath); - const { process, name, description } = await import(resolvedProcessorPath); - - if (!process || typeof process !== "function") { - throw new MuseError( - `Processor at ${processorPath} does not export a process() function`, - ); - } - - const processorName = String(name ?? "Unnamed Processor"); - const processorDescription = String(description ?? "No description provided"); - - return { - filePath: resolvedProcessorPath, - process: process as MuseProcessor["process"], - name: processorName, - description: processorDescription, - }; -} -/** - * Given a path, find the binding file - * If the path is to a directory, check for a binding inside - * Check order: binding.json, binding.yaml, binding.toml, binding.md - * Otherwise throw an error - * - * @param bindingPath - The path to the binding file - * @returns The absolute resolved path to the binding file - */ -export async function resolveBindingPath(bindingPath: string): Promise { - // If the path does not specify a filename, use binding.yaml - const inputLocation = Bun.file(bindingPath); - - // If it is a directory, try for the four main supported types - const stat = await inputLocation.stat(); - if (stat.isDirectory()) { - const jsonPath = await resolveFilePath( - path.resolve(bindingPath, "binding.json"), - ); - if (jsonPath) { - return jsonPath; - } - - const yamlPath = await resolveFilePath( - path.resolve(bindingPath, "binding.yaml"), - ); - if (yamlPath) { - return yamlPath; - } - - const tomlPath = await resolveFilePath( - path.resolve(bindingPath, "binding.toml"), - ); - if (tomlPath) { - return tomlPath; - } - - const mdPath = await resolveFilePath( - path.resolve(bindingPath, "binding.md"), - ); - if (mdPath) { - return mdPath; - } - - throw new FileNotFoundError(bindingPath); - } - - const exists = await inputLocation.exists(); - - if (!exists) { - throw new FileNotFoundError(bindingPath); - } - - // If it is a file, return the path - return path.resolve(bindingPath); -} - -export async function loadBinding(initialPath: string): Promise { - // Resolve the file location if possible - const bindingPath = await resolveBindingPath(initialPath); - const bindingDirname = dirname(bindingPath); - const loadedBindingData = await parseMuseFile(bindingPath); - - if (loadedBindingData.length === 0) { - throw new MuseError("No entries found in binding file"); - } - - // Load and parse the Binding YAML content - const parsedBinding = BindingSchema.parse(loadedBindingData[0].data); - - // Prepare collections to load into - const extendsFrom: Binding[] = []; - const entries: MuseEntry[] = []; - - // Process imported bindings and bring in their entries - for (const includePath of parsedBinding.include) { - const pathFromHere = path.resolve(bindingDirname, includePath); - const extendBindingPath = await resolveBindingPath(pathFromHere); - const loadedBinding = await loadBinding(extendBindingPath); - extendsFrom.push(loadedBinding); - entries.push(...loadedBinding.entries); - } - - // Aggregate file paths to import - const includedFilePaths = parsedBinding.files - .flatMap((thisGlob) => { - return Array.from( - new Glob(thisGlob).scanSync({ - cwd: bindingDirname, - absolute: true, - followSymlinks: true, - onlyFiles: true, - }), - ); - }) - .filter((filePath) => filePath !== bindingPath); - - // Load files - const loadedEntries = await Promise.all( - includedFilePaths.map((filePath) => { - return parseMuseFile(filePath, { - contentKey: parsedBinding.contentKey, - }); - }), - ); - - // Add loaded entries to main list - entries.push(...loadedEntries.flat()); - - // Load and check processors - const processors: MuseProcessor[] = await Promise.all( - parsedBinding.processors.map(loadProcessor.bind(null, bindingDirname)), - ); - - return { - _raw: parsedBinding, - bindingPath, - contentKey: parsedBinding.contentKey, - entries, - options: parsedBinding.options, - processors: processors, - imports: extendsFrom, - meta: {}, - }; -} diff --git a/src/cli.ts b/src/cli/index.ts similarity index 61% rename from src/cli.ts rename to src/cli/index.ts index 762d30e..edb3d9d 100644 --- a/src/cli.ts +++ b/src/cli/index.ts @@ -1,9 +1,15 @@ import { parseArgs } from "node:util"; import chalk from "chalk"; -import { MuseError } from "./errors"; -import { loadBinding } from "./binding"; -import { version } from "../package.json"; -import type { Binding, CLIArguments } from "./types"; +import { loadBinding } from "#core/binding"; +import type { MusePlugin } from "#core/plugins"; +import { MuseError } from "#errors"; +import { version } from "../../package.json"; + +interface CLIFlags { + help: boolean; + verbose: boolean; + stdout: boolean; +} interface Loggers { log: (msg: string) => void; @@ -36,7 +42,7 @@ Options: * * @throws {CLIError} if the arguments are invalid */ -export function parseCLIArguments(argv: string[]): CLIArguments { +function parseCLIArguments(argv: string[]): [string, CLIFlags] { const { values: options, positionals: args } = parseArgs({ args: argv, options: { @@ -60,59 +66,55 @@ export function parseCLIArguments(argv: string[]): CLIArguments { allowPositionals: true, }); - return { - inputFilePath: args[0] ?? "./binding.yaml", - flags: { - help: options.help, - verbose: options.verbose, - stdout: options.stdout, - }, - }; + return [args[0] ?? "./", options]; } + +function getStepLogLine(plugin: MusePlugin): string { + const { name, description } = plugin; + let processorLogLine = chalk.bold(`↪ ${name}`); + + if (description && description.length > 0) { + processorLogLine += chalk.dim(` (${description})`); + } + + return processorLogLine; +} + async function processBinding( - { inputFilePath, flags }: CLIArguments, + inputFilePath: string, + flags: CLIFlags, { log, verbose }: Loggers, ) { // Load the binding - const binding = await loadBinding(inputFilePath); - + let binding = await loadBinding(inputFilePath); verbose(`Binding ${binding.bindingPath}`); - const stepWord = binding.processors.length === 1 ? "step" : "steps"; - log( - `Processing ${binding.entries.length} entries with ${binding.processors.length} ${stepWord}`, - ); + const entryCount = binding.entries.length; + const stepCount = binding.plugins.length; + const stepWord = stepCount === 1 ? "step" : "steps"; + log(`Processing ${entryCount} entries with ${stepCount} ${stepWord}`); const processStart = performance.now(); - // Run the data through all processors - const processedSteps: Binding[] = [binding]; - for (const processor of binding.processors) { - const lastStep = processedSteps[processedSteps.length - 1]; - - const { process, name } = processor; - - log(chalk.bold(`↪ ${name}`) + chalk.dim(` (${processor.description})`)); - const thisStep = await process(lastStep, binding.options, flags); - processedSteps.push(thisStep); + // Run the data through relevant plugins + for (const plugin of binding.plugins) { + log(getStepLogLine(plugin)); + const { step } = plugin; + binding = await step(binding); } const processEnd = performance.now(); const processTime = ((processEnd - processStart) / 1000).toFixed(2); verbose(`Processing completed in ${processTime}s`); - const finalState = processedSteps[processedSteps.length - 1]; - const serialized = JSON.stringify(finalState.entries, null, 2); - if (flags.stdout) { - console.log(serialized); + console.log(JSON.stringify(binding, null, 2)); } return ExitCode.Success; } async function main(): Promise { - const cliArguments = parseCLIArguments(Bun.argv.slice(2)); - const { flags } = cliArguments; + const [inputFilePath, flags] = parseCLIArguments(Bun.argv.slice(2)); // If --help is specified, print usage and exit if (flags.help) { @@ -127,7 +129,7 @@ async function main(): Promise { } : () => {}; - const lastProcessResult = await processBinding(cliArguments, { + const lastProcessResult = await processBinding(inputFilePath, flags, { log: logFn, verbose: verboseFn, }); diff --git a/src/core/binding.ts b/src/core/binding.ts new file mode 100644 index 0000000..7652c0a --- /dev/null +++ b/src/core/binding.ts @@ -0,0 +1,145 @@ +import path, { dirname } from "node:path"; +import { z } from "zod"; +import type { UnknownRecord } from "#core/records"; +import { MuseError, MuseFileNotFoundError } from "#errors"; +import { resolveFilePath } from "#util"; +import { type MusePlugin, loadPlugin } from "./plugins"; +import { + type MuseEntry, + type MuseSource, + SourceSchema, + loadFromSource, + parseMuseFile, + sourceShorthandToSource, +} from "./sources"; + +// Function to parse an unknown object into parts of a binding +const BindingSchema = z.object({ + contentKey: z.string().default("content"), + sources: z.array(SourceSchema).default([ + { + file: "./**/*.{json,yaml,toml,md}", + }, + ]), + options: z.record(z.string(), z.any()).default({}), + plugins: z.array(z.string()).default([]), +}); + +/** + * The core object of a Muse binding + * Contains information about the binding file, the configuration of + * the project, and a list of entries to be processed + */ +export interface Binding { + /** Information about the sources used to load entries */ + readonly sources: MuseSource[]; + + /** The full file path to the binding file */ + readonly bindingPath: string; + + /** The key used to for default content in unstructured files */ + readonly contentKey: string; + + /** All entries loaded from the binding file */ + readonly entries: MuseEntry[]; + + /** A list of processors to be applied to the entries */ + readonly plugins: MusePlugin[]; + + /** Arbitrary metadata for processors to use to cache project data */ + readonly meta: MetaShape; + + /** Configuration options for Muse and all processors */ + readonly options: UnknownRecord; +} + +/** + * Given a path, find the binding file + * If the path is to a directory, check for a binding inside + * Check order: binding.json, binding.yaml, binding.toml, binding.md + * Otherwise throw an error + * + * @param bindingPath - The path to the binding file + * @returns The absolute resolved path to the binding file + */ +async function resolveBindingPath(bindingPath: string): Promise { + const DEFAULT_BINDING_FILES = [ + "binding.json", + "binding.yaml", + "binding.toml", + "binding.md", + ] as const; + const inputLocation = Bun.file(bindingPath); + + // If it is a directory, try for the four main supported types + const stat = await inputLocation.stat(); + if (stat.isDirectory()) { + for (const bindingFile of DEFAULT_BINDING_FILES) { + const filePath = path.resolve(bindingPath, bindingFile); + const resolvedPath = await resolveFilePath(filePath); + if (resolvedPath) { + return resolvedPath; + } + } + + throw new MuseFileNotFoundError(bindingPath); + } + + // If not a directory, check if its a file that exists + const exists = await inputLocation.exists(); + + if (!exists) { + throw new MuseFileNotFoundError(bindingPath); + } + + // If it is a file, return the path + return path.resolve(bindingPath); +} + +export async function loadBinding(initialPath: string): Promise { + // Resolve the file location if possible + const bindingPath = await resolveBindingPath(initialPath); + const bindingDirname = dirname(bindingPath); + const loadedBindingData = await parseMuseFile(bindingPath, { + contentKey: "content", + cwd: bindingDirname, + ignore: [], + }); + + if (loadedBindingData.length === 0) { + throw new MuseError("No entries found in binding file"); + } + + // Load and parse the Binding YAML content + const parsedBinding = BindingSchema.parse(loadedBindingData[0].data); + + const sourceOptions = { + cwd: bindingDirname, + contentKey: parsedBinding.contentKey, + ignore: [bindingPath], + }; + + const sources = parsedBinding.sources.map(sourceShorthandToSource); + + const entries: MuseEntry[] = []; + + for (const source of sources) { + const entriesFromSource = await loadFromSource(source, sourceOptions); + entries.push(...entriesFromSource); + } + + // Load and check plugins + const plugins: MusePlugin[] = await Promise.all( + parsedBinding.plugins.map(loadPlugin.bind(null, bindingDirname)), + ); + + return { + sources, + bindingPath, + contentKey: parsedBinding.contentKey, + entries, + options: parsedBinding.options, + plugins, + meta: {}, + }; +} diff --git a/src/core/files.ts b/src/core/files.ts new file mode 100644 index 0000000..89b2817 --- /dev/null +++ b/src/core/files.ts @@ -0,0 +1,34 @@ +import { basename, dirname, extname, normalize } from "node:path"; +import { MuseFileNotFoundError } from "#errors"; + +export interface LoadedFile { + content: string; + filePath: string; + fileType: string; + dirname: string; + basename: string; + mimeType: string; +} + +export async function loadFileContent(filePath: string): Promise { + const normalizedPath = normalize(filePath); + const file = Bun.file(normalizedPath); + + const exists = await file.exists(); + if (!exists) { + throw new MuseFileNotFoundError(normalizedPath); + } + + const filecontent = await file.text(); + + const extension = extname(normalizedPath); + + return { + content: filecontent, + filePath: normalizedPath, + fileType: extension.slice(1), + dirname: dirname(normalizedPath), + basename: basename(normalizedPath, extension), + mimeType: file.type, + }; +} diff --git a/src/core/plugins.ts b/src/core/plugins.ts new file mode 100644 index 0000000..3f2ffed --- /dev/null +++ b/src/core/plugins.ts @@ -0,0 +1,52 @@ +import path from "node:path"; +import { MuseError, MusePluginError } from "#errors"; +import type { Binding } from "./binding"; + +export type MuseStepFn = (binding: Binding) => Promise; + +export interface MusePlugin { + readonly filePath: string; + readonly name: string; + readonly description: string; + readonly version: string; + readonly step: MuseStepFn; +} + +export async function loadPlugin( + bindingDirname: string, + pluginPath: string, +): Promise { + // Resolve local paths, use URLs as-is + const resolvedProcessorPath = pluginPath.startsWith("http") + ? pluginPath + : path.resolve(bindingDirname, pluginPath); + const { step, name, description, version } = await import( + resolvedProcessorPath + ); + + if (!step || typeof step !== "function") { + throw new MuseError( + `Processor at ${pluginPath} does not export a step() function`, + ); + } + + if (name && typeof name !== "string") { + throw new MusePluginError(pluginPath, "Plugin name is not a string."); + } + + if (description && typeof description !== "string") { + throw new MusePluginError(pluginPath, "Plugin description is not a string"); + } + + if (version && typeof version !== "string") { + throw new MusePluginError(pluginPath, "Plugin version is not a string"); + } + + return { + filePath: resolvedProcessorPath, + step: step as MuseStepFn, + name: String(name ?? pluginPath), + description: String(description ?? ""), + version: String(version ?? "0.0.0"), + }; +} diff --git a/src/core/records.ts b/src/core/records.ts new file mode 100644 index 0000000..891b18e --- /dev/null +++ b/src/core/records.ts @@ -0,0 +1,107 @@ +import EMDY from "@endeavorance/emdy"; +import TOML from "smol-toml"; +import YAML from "yaml"; +import type { LoadedFile } from "./files"; + +export type UnknownRecord = Record; + +/** + * Given an unknown shape, ensure it is an object or array of objects, + * then return it as an array of objects. + * + * @param val - The unknown value to format + * @returns - An array of objects + */ +function formatEntries(val: unknown): UnknownRecord[] { + if (Array.isArray(val) && val.every((el) => typeof el === "object")) { + return val; + } + + if (typeof val === "object" && val !== null) { + return [val as UnknownRecord]; + } + + throw new Error( + `Invalid data format. Entry files must define an object or array of objects, but found "${val}"`, + ); +} + +/** + * Parse one or more YAML documents from a string + * + * @param text - The YAML string to parse + * @returns - An array of parsed objects + */ +export function parseYAMLEntries(text: string): UnknownRecord[] { + const parsedDocs = YAML.parseAllDocuments(text); + + if (parsedDocs.some((doc) => doc.toJS() === null)) { + throw new Error("Encountered NULL resource"); + } + + const errors = parsedDocs.flatMap((doc) => doc.errors); + + if (errors.length > 0) { + throw new Error( + `Error parsing YAML resource: ${errors.map((e) => e.message).join(", ")}`, + ); + } + + const collection: UnknownRecord[] = parsedDocs.map((doc) => doc.toJS()); + return collection; +} + +/** + * Parse one or more JSON documents from a string + * + * @param text - The JSON string to parse + * @returns - An array of parsed objects + */ +export function parseJSONEntries(text: string): UnknownRecord[] { + return formatEntries(JSON.parse(text)); +} + +/** + * Parse one or more TOML documents from a string + * + * @param text - The TOML string to parse + * @returns - An array of parsed objects + */ +export function parseTOMLEntries(text: string): UnknownRecord[] { + return formatEntries(TOML.parse(text)); +} + +/** + * Parse one or more Markdown documents from a string + * + * @param text - The Markdown string to parse + * @returns - An array of parsed objects + */ +export function parseMarkdownEntry( + text: string, + contentKey: string, +): UnknownRecord[] { + return formatEntries(EMDY.parse(text, contentKey)); +} + +export function autoParseEntry(loadedFile: LoadedFile, contentKey: string) { + const { fileType, content } = loadedFile; + + if (fileType === "md") { + return parseMarkdownEntry(content, contentKey); + } + + if (fileType === "yaml") { + return parseYAMLEntries(content); + } + + if (fileType === "json") { + return parseJSONEntries(content); + } + + if (fileType === "toml") { + return parseTOMLEntries(content); + } + + throw new Error(`Unsupported file type: ${fileType}`); +} diff --git a/src/core/sources.ts b/src/core/sources.ts new file mode 100644 index 0000000..5c38e4b --- /dev/null +++ b/src/core/sources.ts @@ -0,0 +1,172 @@ +import { Glob } from "bun"; +import { type LoadedFile, loadFileContent } from "./files"; +import { type UnknownRecord, autoParseEntry } from "./records"; +import { z } from "zod"; +import { MuseError } from "#errors"; + +export const SourceSchema = z.union([ + z.object({ + file: z.string(), + }), + z.object({ + url: z.string().url(), + }), +]); + +export type SourceShorthand = z.infer; + +export function sourceShorthandToSource( + shorthand: SourceShorthand, +): MuseSource { + if ("file" in shorthand) { + return { + location: shorthand.file, + type: "file", + }; + } + + if ("url" in shorthand) { + return { + location: shorthand.url, + type: "url", + }; + } + + throw new MuseError("Invalid source shorthand"); +} + +export type SourceType = "file" | "url"; + +/** A descriptor of a location to load into a Muse binding */ +export interface MuseSource { + location: string; + type: SourceType; +} + +interface SourceOptions { + cwd: string; + contentKey: string; + ignore: string[]; +} + +/** A single loaded entry from a Muse source */ +export interface MuseEntry { + _raw: string; + location: string; + data: Record; + meta: MetaShape; + source: MuseSource; +} + +export async function parseMuseFile( + rawFilePath: string, + { contentKey = "content" }: SourceOptions, +): Promise { + const file = await loadFileContent(rawFilePath); + const entries = autoParseEntry(file, contentKey); + + const partial = { + _raw: file.content, + source: { + location: file.filePath, + type: "file" as SourceType, + }, + location: file.filePath, + meta: {}, + }; + + return entries.map((data) => ({ + ...partial, + data, + })); +} + +async function loadFromFileSource( + source: MuseSource, + options: SourceOptions, +): Promise { + const paths = Array.from( + new Glob(source.location).scanSync({ + cwd: options.cwd, + absolute: true, + followSymlinks: true, + onlyFiles: true, + }), + ); + + const filteredPaths = paths.filter((path) => !options.ignore.includes(path)); + + const entries: MuseEntry[] = []; + for (const filePath of filteredPaths) { + const fileEntries = await parseMuseFile(filePath, options); + entries.push(...fileEntries); + } + + return entries; +} + +function getFileExtensionFromURL(url: string): string | null { + const parsedUrl = new URL(url); + + const pathname = parsedUrl.pathname; + const filename = pathname.substring(pathname.lastIndexOf("/") + 1); + const extension = filename.substring(filename.lastIndexOf(".") + 1); + + return extension === filename ? null : extension; +} + +async function loadFromURLSource( + source: MuseSource, + options: SourceOptions, +): Promise { + const { contentKey = "content" } = options; + const response = await fetch(source.location); + + if (!response.ok) { + throw new Error(`Failed to fetch URL: ${source.location}`); + } + const content = await response.text(); + const mimeType = response.headers.get("Content-Type") || "unknown"; + + const parseType = getFileExtensionFromURL(source.location) ?? "unknown"; + + const loadedFile: LoadedFile = { + content, + filePath: source.location, + fileType: parseType, + dirname: "", + basename: "", + mimeType, + }; + + const entries = autoParseEntry(loadedFile, contentKey); + + const partial = { + _raw: content, + source: { + location: source.location, + type: "url" as SourceType, + }, + location: source.location, + meta: {}, + }; + + return entries.map((data) => ({ + ...partial, + data, + })); +} + +export async function loadFromSource( + source: MuseSource, + options: SourceOptions, +): Promise { + switch (source.type) { + case "file": + return loadFromFileSource(source, options); + case "url": + return loadFromURLSource(source, options); + default: + throw new Error(`Unsupported source type: ${source.type}`); + } +} diff --git a/src/errors.ts b/src/errors.ts index 7c8c014..5f43323 100644 --- a/src/errors.ts +++ b/src/errors.ts @@ -1,28 +1,39 @@ export class MuseError extends Error { fmt(): string { - return this.message; + return `-- ERROR -- \nDetails: ${this.message}`; } } -export class FileError extends MuseError { - filePath: string; - details?: string; +export class MusePluginError extends MuseError { + plugin: string; - constructor(message: string, filePath: string, details?: string) { + constructor(message: string, plugin: string) { super(message); - this.filePath = filePath; - this.details = details; + this.plugin = plugin; } fmt(): string { - return `-- ${this.message} --\nFile Path: ${this.filePath}\nDetails: ${this.details}`; + return `-- PLUGIN ERROR --\Plugin: ${this.plugin}\nDetails: ${this.message}`; } } -export class FileNotFoundError extends FileError { +export class MuseFileError extends MuseError { + filePath: string; + + constructor(message: string, filePath: string) { + super(message); + this.filePath = filePath; + } + + fmt(): string { + return `-- FILE ERROR --\nFile Path: ${this.filePath}\nDetails: ${this.message}`; + } +} + +export class MuseFileNotFoundError extends MuseFileError { constructor(filePath: string) { super("File not found", filePath); - this.message = "File not found"; + this.message = "File does not exist"; this.filePath = filePath; } } diff --git a/src/exports.ts b/src/exports.ts index 8a07b08..451c4f8 100644 --- a/src/exports.ts +++ b/src/exports.ts @@ -1,2 +1,12 @@ -export type * from "./types"; +// Content exports +export * from "#errors"; + +// Constants export const DEFAULT_CONTENT_KEY = "content"; + +// Types +export type * from "#core/binding"; +export type * from "#core/plugins"; +export type * from "#core/sources"; +export type * from "#core/records"; +export type * from "#core/files"; diff --git a/src/parse.ts b/src/parse.ts deleted file mode 100644 index 842f6d2..0000000 --- a/src/parse.ts +++ /dev/null @@ -1,105 +0,0 @@ -import { normalize, extname } from "node:path"; -import YAML from "yaml"; -import TOML from "smol-toml"; -import EMDY from "@endeavorance/emdy"; -import type { MuseEntry, UnknownRecord } from "./types"; - -function parseYAMLEntries(text: string): UnknownRecord[] { - const parsedDocs = YAML.parseAllDocuments(text); - - if (parsedDocs.some((doc) => doc.toJS() === null)) { - throw new Error("Encountered NULL resource"); - } - - const errors = parsedDocs.flatMap((doc) => doc.errors); - - if (errors.length > 0) { - throw new Error( - `Error parsing YAML resource: ${errors.map((e) => e.message).join(", ")}`, - ); - } - - const collection: UnknownRecord[] = parsedDocs.map((doc) => doc.toJS()); - return collection; -} - -function parseJSONEntries(text: string): UnknownRecord[] { - const parsed = JSON.parse(text); - - if (Array.isArray(parsed)) { - return parsed; - } - - if (typeof parsed === "object") { - return [parsed]; - } - - throw new Error("JSON resource must be an object or an array of objects"); -} - -function parseTOMLEntries(text: string): UnknownRecord[] { - const parsed = TOML.parse(text); - - if (Array.isArray(parsed)) { - return parsed; - } - - if (typeof parsed === "object") { - return [parsed]; - } - - throw new Error("TOML resource must be an object or an array of objects"); -} - -interface ParseMuseFileOptions { - contentKey?: string; -} - -export async function parseMuseFile( - rawFilePath: string, - { contentKey = "content" }: ParseMuseFileOptions = {}, -): Promise { - const filePath = normalize(rawFilePath); - const file = Bun.file(filePath); - const fileType = extname(filePath).slice(1); - const rawFileContent = await file.text(); - - const partial = { - _raw: rawFileContent, - filePath, - meta: {}, - }; - - if (fileType === "md") { - const parsed = EMDY.parse(rawFileContent, contentKey); - return [ - { - ...partial, - data: parsed, - }, - ]; - } - - if (fileType === "yaml") { - return parseYAMLEntries(rawFileContent).map((data) => ({ - ...partial, - data, - })); - } - - if (fileType === "json") { - return parseJSONEntries(rawFileContent).map((data) => ({ - ...partial, - data, - })); - } - - if (fileType === "toml") { - return parseTOMLEntries(rawFileContent).map((data) => ({ - ...partial, - data, - })); - } - - throw new Error(`Unsupported file type: ${fileType}`); -} diff --git a/src/preload/http-plugin.js b/src/preload/http-plugin.js new file mode 100644 index 0000000..bb7f732 --- /dev/null +++ b/src/preload/http-plugin.js @@ -0,0 +1,40 @@ +const rx_any = /./; +const rx_http = /^https?:\/\//; +const rx_relative_path = /^\.\.?\//; +const rx_absolute_path = /^\//; + +async function load_http_module(href) { + return fetch(href).then((response) => + response + .text() + .then((text) => + response.ok + ? { contents: text, loader: "js" } + : Promise.reject( + new Error(`Failed to load module '${href}'': ${text}`), + ), + ), + ); +} + +Bun.plugin({ + name: "http_imports", + setup(build) { + build.onResolve({ filter: rx_relative_path }, (args) => { + if (rx_http.test(args.importer)) { + return { path: new URL(args.path, args.importer).href }; + } + }); + build.onResolve({ filter: rx_absolute_path }, (args) => { + if (rx_http.test(args.importer)) { + return { path: new URL(args.path, args.importer).href }; + } + }); + build.onLoad({ filter: rx_any, namespace: "http" }, (args) => + load_http_module(`http:${args.path}`), + ); + build.onLoad({ filter: rx_any, namespace: "https" }, (args) => + load_http_module(`https:${args.path}`), + ); + }, +}); diff --git a/src/types.ts b/src/types.ts deleted file mode 100644 index f9faa6c..0000000 --- a/src/types.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { z } from "zod"; - -export interface CLIFlags { - help: boolean; - verbose: boolean; - stdout: boolean; -} - -export interface CLIArguments { - inputFilePath: string; - flags: CLIFlags; -} - -export const BindingSchema = z.object({ - include: z.array(z.string()).default([]), - contentKey: z.string().default("content"), - files: z - .array(z.string()) - .default(["**/*.yaml", "**/*.md", "**/*.toml", "**/*.json"]), - options: z.record(z.string(), z.any()).default({}), - processors: z.array(z.string()).default([]), -}); - -export interface Binding { - readonly _raw: z.infer; - readonly bindingPath: string; - readonly contentKey: string; - readonly entries: MuseEntry[]; - readonly meta: Record; - readonly options: Record; - readonly processors: MuseProcessor[]; - readonly imports: Binding[]; -} - -export type MuseProcessorFn = ( - binding: Binding, - opts: Record, - flags: CLIFlags, -) => Promise; - -export interface MuseProcessor { - readonly filePath: string; - readonly name: string; - readonly description: string; - readonly process: MuseProcessorFn; -} - -export interface MuseEntry { - _raw: string; - filePath: string; - data: Record; - meta: Record; -} - -export type UnknownRecord = Record; diff --git a/tsconfig.json b/tsconfig.json index f2b0046..a964c9c 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -26,11 +26,22 @@ "noUnusedParameters": false, "noPropertyAccessFromIndexSignature": false, "baseUrl": "./src", - "paths": {}, + "paths": { + "#core/*": [ + "./core/*.ts" + ], + "#util": [ + "./util.ts" + ], + "#errors": [ + "./errors.ts" + ] + }, "outDir": "./dist", }, "include": [ "src/**/*.ts", "src/**/*.tsx", + "src/preload/http-plugin.js", ] }