Update plugin shape again

README.md

@@ -1,20 +1,17 @@
 # Muse
 
-A CLI for wrangling directories of source files.
-
-Like a static generator, but for whatever.
+_Bind data into anything_
 
 ## Overview
 
-**Muse** is a CLI and toolchain for operating on directories of
-json, yaml, toml, and markdown files. Each file can specify any shape
-of data. Muse scans included files, parses them into data, and streams
-the loaded data through processors and plugins.
+**Muse** is a CLI and toolchain for operating on data collected from
+json, yaml, toml, and markdown files.
 
-Each processor can modify the data at compile time, as well as enact
-side effects such as writing files to disk.
+Muse scans included files, parses them into data, and streams
+the loaded data through plugins.
 
-Muse does not edit source files, it only reads them in.
+Each plugin can modify the data as well as enact side effects
+such as writing files to disk.
 
 ## Usage
 
@@ -32,10 +29,9 @@ Options:
 
 Each Muse project should specify a `binding` file with the following shape:
 ```yaml
-include:
-  - ../another-project      # Optional
-files:                      # Optional
-  - "**/*.yaml"
+sources:                    # Optional
+  - file: "**/*.yaml"
+  - web: "https://example.com/data.json"
 contentKey: "content"       # Optional
 options:                    # Optional
   someOption: someValue
@@ -44,9 +40,9 @@ processors:
   - second-processor
 ```
 
-The `binding` file can be any of the supported file types for Muse. If
-the CLI is invoked with a directory instead of a file, Muse will look
-for a binding file in the directory with the following order:
+The `binding` file can be any of the supported file types for Muse and can
+be named anything. If the CLI is invoked with a directory instead of a file,
+Muse will look for a binding file in the directory with the following order:
 
 1. `binding.json`
 2. `binding.yaml`
@@ -68,3 +64,34 @@ Your markdown content here
 When loading markdown files, Muse will load the content of the file
 into a key called `content` by default. This can be changed in the binding
 configuration by setting a `contentKey` value as an override.
+
+## Plugin API
+
+Muse plugins are written in TypeScript/JavaScript and expose information
+describing the plugin, as well as the functions to operate with:
+
+```typescript
+export const name = "Plugin Name";
+export const description = "Plugin Description";
+export async function step(binding: Binding): Promise<Binding> {}
+```
+
+The main function of a plugin is `step`, which takes a `Binding` object
+and returns a modified `Binding` object.
+
+When returning a new Binding object, it is best practice to make immutable
+changes:
+
+```typescript
+export async function step(binding: Binding): Promise<Binding> {
+  const newBinding = { ...binding };
+  newBinding.options.someOption = "newValue";
+  return {
+    ...binding,
+    meta: {
+      ...binding.meta,
+      customValue: true,
+    }
+  };
+}
+```

biome.json

@@ -7,10 +7,7 @@
   },
   "files": {
     "ignoreUnknown": false,
-    "ignore": [
-      "*.d.ts",
-      "*.json"
-    ],
+    "ignore": ["*.d.ts", "*.json"],
     "include": [
       "./src/**/*.ts",
       "./src/**/*.tsx",

bunfig.toml

@@ -0,0 +1 @@
+preload = ["./src/preload/http-plugin.js"]

demo/main/binding.md

@@ -3,6 +3,9 @@ markdownKey: description
 options:
   dunks:
     key: dunkaroos
+sources:
+  - file: ./stats/*.toml
+  - url: https://git.astral.camp/endeavorance/emdy/raw/branch/main/package.json
 processors:
   - ./processors/scream.js
   - ./processors/announce-slam-dunks.js

demo/main/processors/announce-slam-dunks.js

@@ -1,13 +1,19 @@
 export const name = "Announce Slam Dunks";
+export const description = "Get hype when a file has slam dunks";
 
-export const process = (binding, { dunks }) => {
+export const step = ({ entries, options, ...rest }) => {
+  const { dunks } = options;
   const slamDunkKey = dunks?.key ?? "slamDunks";
 
-  for (const entry of binding.entries) {
+  for (const entry of entries) {
     if (slamDunkKey in entry.data) {
       console.log(`Slam dunk!`);
     }
   }
 
-  return binding;
+  return {
+    entries,
+    options,
+    ...rest,
+  };
 };

demo/main/processors/scream.js

@@ -1,5 +1,4 @@
-export const name = "Scream";
-export function process(binding) {
+export function step(binding) {
   const modifiedEntries = binding.entries.map(entry => {
     if (binding.markdownKey in entry.data) {
       entry.data = {

package.json

@@ -1,6 +1,6 @@
 {
-  "name": "@proscenium/muse",
-  "version": "0.1.0",
+  "name": "@endeavorance/muse",
+  "version": "0.3.0",
   "module": "dist/index.js",
   "exports": {
     ".": {
@@ -31,9 +31,10 @@
     "fmt": "bunx --bun biome check --fix",
     "clean": "rm -rf dist",
     "types": "dts-bundle-generator -o dist/muse.d.ts --project ./tsconfig.json ./src/exports.ts",
-    "compile": "bun build ./src/cli.ts --outfile ./dist/muse --compile",
-    "build": "bun run clean && bun run compile && bun run types",
-    "build:install": "bun run build && mv muse ~/.local/bin/muse",
-    "demo": "bun run ./src/cli.ts -- ./demo/main"
+    "transpile": "bun build ./src/exports.ts --outfile ./dist/muse.js",
+    "compile": "bun run clean && bun build ./src/cli/index.ts --outfile ./dist/muse --compile",
+    "compile:install": "bun run compile && mv ./dist/muse ~/.local/bin/muse",
+    "build": "bun run clean && bun run transpile && bun run types",
+    "demo": "bun run ./src/cli/index.ts -- ./demo/main"
   }
 }

src/binding.ts

@@ -1,160 +0,0 @@
-import path, { dirname } from "node:path";
-import { Glob } from "bun";
-import { FileNotFoundError, MuseError } from "./errors";
-import { parseMuseFile } from "./parse";
-import {
-  type Binding,
-  BindingSchema,
-  type MuseEntry,
-  type MuseProcessor,
-} from "./types";
-import { resolveFilePath } from "./util";
-
-async function loadProcessor(
-  bindingDirname: string,
-  processorPath: string,
-): Promise<MuseProcessor> {
-  const resolvedProcessorPath = path.resolve(bindingDirname, processorPath);
-  const { process, name, description } = await import(resolvedProcessorPath);
-
-  if (!process || typeof process !== "function") {
-    throw new MuseError(
-      `Processor at ${processorPath} does not export a process() function`,
-    );
-  }
-
-  const processorName = String(name ?? "Unnamed Processor");
-  const processorDescription = String(description ?? "No description provided");
-
-  return {
-    filePath: resolvedProcessorPath,
-    process: process as MuseProcessor["process"],
-    name: processorName,
-    description: processorDescription,
-  };
-}
-/**
- * Given a path, find the binding file
- * If the path is to a directory, check for a binding inside
- * Check order: binding.json, binding.yaml, binding.toml, binding.md
- * Otherwise throw an error
- *
- * @param bindingPath - The path to the binding file
- * @returns The absolute resolved path to the binding file
- */
-export async function resolveBindingPath(bindingPath: string): Promise<string> {
-  // If the path does not specify a filename, use binding.yaml
-  const inputLocation = Bun.file(bindingPath);
-
-  // If it is a directory, try for the four main supported types
-  const stat = await inputLocation.stat();
-  if (stat.isDirectory()) {
-    const jsonPath = await resolveFilePath(
-      path.resolve(bindingPath, "binding.json"),
-    );
-    if (jsonPath) {
-      return jsonPath;
-    }
-
-    const yamlPath = await resolveFilePath(
-      path.resolve(bindingPath, "binding.yaml"),
-    );
-    if (yamlPath) {
-      return yamlPath;
-    }
-
-    const tomlPath = await resolveFilePath(
-      path.resolve(bindingPath, "binding.toml"),
-    );
-    if (tomlPath) {
-      return tomlPath;
-    }
-
-    const mdPath = await resolveFilePath(
-      path.resolve(bindingPath, "binding.md"),
-    );
-    if (mdPath) {
-      return mdPath;
-    }
-
-    throw new FileNotFoundError(bindingPath);
-  }
-
-  const exists = await inputLocation.exists();
-
-  if (!exists) {
-    throw new FileNotFoundError(bindingPath);
-  }
-
-  // If it is a file, return the path
-  return path.resolve(bindingPath);
-}
-
-export async function loadBinding(initialPath: string): Promise<Binding> {
-  // Resolve the file location if possible
-  const bindingPath = await resolveBindingPath(initialPath);
-  const bindingDirname = dirname(bindingPath);
-  const loadedBindingData = await parseMuseFile(bindingPath);
-
-  if (loadedBindingData.length === 0) {
-    throw new MuseError("No entries found in binding file");
-  }
-
-  // Load and parse the Binding YAML content
-  const parsedBinding = BindingSchema.parse(loadedBindingData[0].data);
-
-  // Prepare collections to load into
-  const extendsFrom: Binding[] = [];
-  const entries: MuseEntry[] = [];
-
-  // Process imported bindings and bring in their entries
-  for (const includePath of parsedBinding.include) {
-    const pathFromHere = path.resolve(bindingDirname, includePath);
-    const extendBindingPath = await resolveBindingPath(pathFromHere);
-    const loadedBinding = await loadBinding(extendBindingPath);
-    extendsFrom.push(loadedBinding);
-    entries.push(...loadedBinding.entries);
-  }
-
-  // Aggregate file paths to import
-  const includedFilePaths = parsedBinding.files
-    .flatMap((thisGlob) => {
-      return Array.from(
-        new Glob(thisGlob).scanSync({
-          cwd: bindingDirname,
-          absolute: true,
-          followSymlinks: true,
-          onlyFiles: true,
-        }),
-      );
-    })
-    .filter((filePath) => filePath !== bindingPath);
-
-  // Load files
-  const loadedEntries = await Promise.all(
-    includedFilePaths.map((filePath) => {
-      return parseMuseFile(filePath, {
-        contentKey: parsedBinding.contentKey,
-      });
-    }),
-  );
-
-  // Add loaded entries to main list
-  entries.push(...loadedEntries.flat());
-
-  // Load and check processors
-  const processors: MuseProcessor[] = await Promise.all(
-    parsedBinding.processors.map(loadProcessor.bind(null, bindingDirname)),
-  );
-
-  return {
-    _raw: parsedBinding,
-    bindingPath,
-    contentKey: parsedBinding.contentKey,
-    entries,
-    options: parsedBinding.options,
-    processors: processors,
-    imports: extendsFrom,
-    meta: {},
-  };
-}

src/cli/index.ts

@@ -1,9 +1,15 @@
 import { parseArgs } from "node:util";
 import chalk from "chalk";
-import { MuseError } from "./errors";
-import { loadBinding } from "./binding";
-import { version } from "../package.json";
-import type { Binding, CLIArguments } from "./types";
+import { loadBinding } from "#core/binding";
+import type { MusePlugin } from "#core/plugins";
+import { MuseError } from "#errors";
+import { version } from "../../package.json";
+
+interface CLIFlags {
+  help: boolean;
+  verbose: boolean;
+  stdout: boolean;
+}
 
 interface Loggers {
   log: (msg: string) => void;
@@ -36,7 +42,7 @@ Options:
  *
  * @throws {CLIError} if the arguments are invalid
  */
-export function parseCLIArguments(argv: string[]): CLIArguments {
+function parseCLIArguments(argv: string[]): [string, CLIFlags] {
   const { values: options, positionals: args } = parseArgs({
     args: argv,
     options: {
@@ -60,59 +66,55 @@ export function parseCLIArguments(argv: string[]): CLIArguments {
     allowPositionals: true,
   });
 
-  return {
-    inputFilePath: args[0] ?? "./binding.yaml",
-    flags: {
-      help: options.help,
-      verbose: options.verbose,
-      stdout: options.stdout,
-    },
-  };
+  return [args[0] ?? "./", options];
 }
+
+function getStepLogLine(plugin: MusePlugin): string {
+  const { name, description } = plugin;
+  let processorLogLine = chalk.bold(`↪ ${name}`);
+
+  if (description && description.length > 0) {
+    processorLogLine += chalk.dim(` (${description})`);
+  }
+
+  return processorLogLine;
+}
+
 async function processBinding(
-  { inputFilePath, flags }: CLIArguments,
+  inputFilePath: string,
+  flags: CLIFlags,
   { log, verbose }: Loggers,
 ) {
   // Load the binding
-  const binding = await loadBinding(inputFilePath);
-
+  let binding = await loadBinding(inputFilePath);
   verbose(`Binding ${binding.bindingPath}`);
 
-  const stepWord = binding.processors.length === 1 ? "step" : "steps";
-  log(
-    `Processing ${binding.entries.length} entries with ${binding.processors.length} ${stepWord}`,
-  );
+  const entryCount = binding.entries.length;
+  const stepCount = binding.plugins.length;
+  const stepWord = stepCount === 1 ? "step" : "steps";
+  log(`Processing ${entryCount} entries with ${stepCount} ${stepWord}`);
 
   const processStart = performance.now();
 
-  // Run the data through all processors
-  const processedSteps: Binding[] = [binding];
-  for (const processor of binding.processors) {
-    const lastStep = processedSteps[processedSteps.length - 1];
-
-    const { process, name } = processor;
-
-    log(chalk.bold(`↪ ${name}`) + chalk.dim(` (${processor.description})`));
-    const thisStep = await process(lastStep, binding.options, flags);
-    processedSteps.push(thisStep);
+  // Run the data through relevant plugins
+  for (const plugin of binding.plugins) {
+    log(getStepLogLine(plugin));
+    const { step } = plugin;
+    binding = await step(binding);
   }
 
   const processEnd = performance.now();
   const processTime = ((processEnd - processStart) / 1000).toFixed(2);
   verbose(`Processing completed in ${processTime}s`);
 
-  const finalState = processedSteps[processedSteps.length - 1];
-  const serialized = JSON.stringify(finalState.entries, null, 2);
-
   if (flags.stdout) {
-    console.log(serialized);
+    console.log(JSON.stringify(binding, null, 2));
   }
   return ExitCode.Success;
 }
 
 async function main(): Promise<number> {
-  const cliArguments = parseCLIArguments(Bun.argv.slice(2));
-  const { flags } = cliArguments;
+  const [inputFilePath, flags] = parseCLIArguments(Bun.argv.slice(2));
 
   // If --help is specified, print usage and exit
   if (flags.help) {
@@ -127,7 +129,7 @@ async function main(): Promise<number> {
       }
     : () => {};
 
-  const lastProcessResult = await processBinding(cliArguments, {
+  const lastProcessResult = await processBinding(inputFilePath, flags, {
     log: logFn,
     verbose: verboseFn,
   });

src/core/binding.ts

@@ -0,0 +1,145 @@
+import path, { dirname } from "node:path";
+import { z } from "zod";
+import type { UnknownRecord } from "#core/records";
+import { MuseError, MuseFileNotFoundError } from "#errors";
+import { resolveFilePath } from "#util";
+import { type MusePlugin, loadPlugin } from "./plugins";
+import {
+  type MuseEntry,
+  type MuseSource,
+  SourceSchema,
+  loadFromSource,
+  parseMuseFile,
+  sourceShorthandToSource,
+} from "./sources";
+
+// Function to parse an unknown object into parts of a binding
+const BindingSchema = z.object({
+  contentKey: z.string().default("content"),
+  sources: z.array(SourceSchema).default([
+    {
+      file: "./**/*.{json,yaml,toml,md}",
+    },
+  ]),
+  options: z.record(z.string(), z.any()).default({}),
+  plugins: z.array(z.string()).default([]),
+});
+
+/**
+ * The core object of a Muse binding
+ * Contains information about the binding file, the configuration of
+ * the project, and a list of entries to be processed
+ */
+export interface Binding<MetaShape = UnknownRecord> {
+  /** Information about the sources used to load entries */
+  readonly sources: MuseSource[];
+
+  /** The full file path to the binding file */
+  readonly bindingPath: string;
+
+  /** The key used to for default content in unstructured files */
+  readonly contentKey: string;
+
+  /** All entries loaded from the binding file */
+  readonly entries: MuseEntry[];
+
+  /** A list of processors to be applied to the entries */
+  readonly plugins: MusePlugin[];
+
+  /** Arbitrary metadata for processors to use to cache project data */
+  readonly meta: MetaShape;
+
+  /** Configuration options for Muse and all processors */
+  readonly options: UnknownRecord;
+}
+
+/**
+ * Given a path, find the binding file
+ * If the path is to a directory, check for a binding inside
+ * Check order: binding.json, binding.yaml, binding.toml, binding.md
+ * Otherwise throw an error
+ *
+ * @param bindingPath - The path to the binding file
+ * @returns The absolute resolved path to the binding file
+ */
+async function resolveBindingPath(bindingPath: string): Promise<string> {
+  const DEFAULT_BINDING_FILES = [
+    "binding.json",
+    "binding.yaml",
+    "binding.toml",
+    "binding.md",
+  ] as const;
+  const inputLocation = Bun.file(bindingPath);
+
+  // If it is a directory, try for the four main supported types
+  const stat = await inputLocation.stat();
+  if (stat.isDirectory()) {
+    for (const bindingFile of DEFAULT_BINDING_FILES) {
+      const filePath = path.resolve(bindingPath, bindingFile);
+      const resolvedPath = await resolveFilePath(filePath);
+      if (resolvedPath) {
+        return resolvedPath;
+      }
+    }
+
+    throw new MuseFileNotFoundError(bindingPath);
+  }
+
+  // If not a directory, check if its a file that exists
+  const exists = await inputLocation.exists();
+
+  if (!exists) {
+    throw new MuseFileNotFoundError(bindingPath);
+  }
+
+  // If it is a file, return the path
+  return path.resolve(bindingPath);
+}
+
+export async function loadBinding(initialPath: string): Promise<Binding> {
+  // Resolve the file location if possible
+  const bindingPath = await resolveBindingPath(initialPath);
+  const bindingDirname = dirname(bindingPath);
+  const loadedBindingData = await parseMuseFile(bindingPath, {
+    contentKey: "content",
+    cwd: bindingDirname,
+    ignore: [],
+  });
+
+  if (loadedBindingData.length === 0) {
+    throw new MuseError("No entries found in binding file");
+  }
+
+  // Load and parse the Binding YAML content
+  const parsedBinding = BindingSchema.parse(loadedBindingData[0].data);
+
+  const sourceOptions = {
+    cwd: bindingDirname,
+    contentKey: parsedBinding.contentKey,
+    ignore: [bindingPath],
+  };
+
+  const sources = parsedBinding.sources.map(sourceShorthandToSource);
+
+  const entries: MuseEntry[] = [];
+
+  for (const source of sources) {
+    const entriesFromSource = await loadFromSource(source, sourceOptions);
+    entries.push(...entriesFromSource);
+  }
+
+  // Load and check plugins
+  const plugins: MusePlugin[] = await Promise.all(
+    parsedBinding.plugins.map(loadPlugin.bind(null, bindingDirname)),
+  );
+
+  return {
+    sources,
+    bindingPath,
+    contentKey: parsedBinding.contentKey,
+    entries,
+    options: parsedBinding.options,
+    plugins,
+    meta: {},
+  };
+}

src/core/files.ts

@@ -0,0 +1,34 @@
+import { basename, dirname, extname, normalize } from "node:path";
+import { MuseFileNotFoundError } from "#errors";
+
+export interface LoadedFile {
+  content: string;
+  filePath: string;
+  fileType: string;
+  dirname: string;
+  basename: string;
+  mimeType: string;
+}
+
+export async function loadFileContent(filePath: string): Promise<LoadedFile> {
+  const normalizedPath = normalize(filePath);
+  const file = Bun.file(normalizedPath);
+
+  const exists = await file.exists();
+  if (!exists) {
+    throw new MuseFileNotFoundError(normalizedPath);
+  }
+
+  const filecontent = await file.text();
+
+  const extension = extname(normalizedPath);
+
+  return {
+    content: filecontent,
+    filePath: normalizedPath,
+    fileType: extension.slice(1),
+    dirname: dirname(normalizedPath),
+    basename: basename(normalizedPath, extension),
+    mimeType: file.type,
+  };
+}

src/core/plugins.ts

@@ -0,0 +1,52 @@
+import path from "node:path";
+import { MuseError, MusePluginError } from "#errors";
+import type { Binding } from "./binding";
+
+export type MuseStepFn = (binding: Binding) => Promise<Binding>;
+
+export interface MusePlugin {
+  readonly filePath: string;
+  readonly name: string;
+  readonly description: string;
+  readonly version: string;
+  readonly step: MuseStepFn;
+}
+
+export async function loadPlugin(
+  bindingDirname: string,
+  pluginPath: string,
+): Promise<MusePlugin> {
+  // Resolve local paths, use URLs as-is
+  const resolvedProcessorPath = pluginPath.startsWith("http")
+    ? pluginPath
+    : path.resolve(bindingDirname, pluginPath);
+  const { step, name, description, version } = await import(
+    resolvedProcessorPath
+  );
+
+  if (!step || typeof step !== "function") {
+    throw new MuseError(
+      `Processor at ${pluginPath} does not export a step() function`,
+    );
+  }
+
+  if (name && typeof name !== "string") {
+    throw new MusePluginError(pluginPath, "Plugin name is not a string.");
+  }
+
+  if (description && typeof description !== "string") {
+    throw new MusePluginError(pluginPath, "Plugin description is not a string");
+  }
+
+  if (version && typeof version !== "string") {
+    throw new MusePluginError(pluginPath, "Plugin version is not a string");
+  }
+
+  return {
+    filePath: resolvedProcessorPath,
+    step: step as MuseStepFn,
+    name: String(name ?? pluginPath),
+    description: String(description ?? ""),
+    version: String(version ?? "0.0.0"),
+  };
+}

src/core/records.ts

@@ -0,0 +1,107 @@
+import EMDY from "@endeavorance/emdy";
+import TOML from "smol-toml";
+import YAML from "yaml";
+import type { LoadedFile } from "./files";
+
+export type UnknownRecord = Record<string, unknown>;
+
+/**
+ * Given an unknown shape, ensure it is an object or array of objects,
+ * then return it as an array of objects.
+ *
+ * @param val - The unknown value to format
+ * @returns - An array of objects
+ */
+function formatEntries(val: unknown): UnknownRecord[] {
+  if (Array.isArray(val) && val.every((el) => typeof el === "object")) {
+    return val;
+  }
+
+  if (typeof val === "object" && val !== null) {
+    return [val as UnknownRecord];
+  }
+
+  throw new Error(
+    `Invalid data format. Entry files must define an object or array of objects, but found "${val}"`,
+  );
+}
+
+/**
+ * Parse one or more YAML documents from a string
+ *
+ * @param text - The YAML string to parse
+ * @returns - An array of parsed objects
+ */
+export function parseYAMLEntries(text: string): UnknownRecord[] {
+  const parsedDocs = YAML.parseAllDocuments(text);
+
+  if (parsedDocs.some((doc) => doc.toJS() === null)) {
+    throw new Error("Encountered NULL resource");
+  }
+
+  const errors = parsedDocs.flatMap((doc) => doc.errors);
+
+  if (errors.length > 0) {
+    throw new Error(
+      `Error parsing YAML resource: ${errors.map((e) => e.message).join(", ")}`,
+    );
+  }
+
+  const collection: UnknownRecord[] = parsedDocs.map((doc) => doc.toJS());
+  return collection;
+}
+
+/**
+ * Parse one or more JSON documents from a string
+ *
+ * @param text - The JSON string to parse
+ * @returns - An array of parsed objects
+ */
+export function parseJSONEntries(text: string): UnknownRecord[] {
+  return formatEntries(JSON.parse(text));
+}
+
+/**
+ * Parse one or more TOML documents from a string
+ *
+ * @param text - The TOML string to parse
+ * @returns - An array of parsed objects
+ */
+export function parseTOMLEntries(text: string): UnknownRecord[] {
+  return formatEntries(TOML.parse(text));
+}
+
+/**
+ * Parse one or more Markdown documents from a string
+ *
+ * @param text - The Markdown string to parse
+ * @returns - An array of parsed objects
+ */
+export function parseMarkdownEntry(
+  text: string,
+  contentKey: string,
+): UnknownRecord[] {
+  return formatEntries(EMDY.parse(text, contentKey));
+}
+
+export function autoParseEntry(loadedFile: LoadedFile, contentKey: string) {
+  const { fileType, content } = loadedFile;
+
+  if (fileType === "md") {
+    return parseMarkdownEntry(content, contentKey);
+  }
+
+  if (fileType === "yaml") {
+    return parseYAMLEntries(content);
+  }
+
+  if (fileType === "json") {
+    return parseJSONEntries(content);
+  }
+
+  if (fileType === "toml") {
+    return parseTOMLEntries(content);
+  }
+
+  throw new Error(`Unsupported file type: ${fileType}`);
+}

src/core/sources.ts

@@ -0,0 +1,172 @@
+import { Glob } from "bun";
+import { type LoadedFile, loadFileContent } from "./files";
+import { type UnknownRecord, autoParseEntry } from "./records";
+import { z } from "zod";
+import { MuseError } from "#errors";
+
+export const SourceSchema = z.union([
+  z.object({
+    file: z.string(),
+  }),
+  z.object({
+    url: z.string().url(),
+  }),
+]);
+
+export type SourceShorthand = z.infer<typeof SourceSchema>;
+
+export function sourceShorthandToSource(
+  shorthand: SourceShorthand,
+): MuseSource {
+  if ("file" in shorthand) {
+    return {
+      location: shorthand.file,
+      type: "file",
+    };
+  }
+
+  if ("url" in shorthand) {
+    return {
+      location: shorthand.url,
+      type: "url",
+    };
+  }
+
+  throw new MuseError("Invalid source shorthand");
+}
+
+export type SourceType = "file" | "url";
+
+/** A descriptor of a location to load into a Muse binding */
+export interface MuseSource {
+  location: string;
+  type: SourceType;
+}
+
+interface SourceOptions {
+  cwd: string;
+  contentKey: string;
+  ignore: string[];
+}
+
+/** A single loaded entry from a Muse source */
+export interface MuseEntry<MetaShape = UnknownRecord> {
+  _raw: string;
+  location: string;
+  data: Record<string, unknown>;
+  meta: MetaShape;
+  source: MuseSource;
+}
+
+export async function parseMuseFile(
+  rawFilePath: string,
+  { contentKey = "content" }: SourceOptions,
+): Promise<MuseEntry[]> {
+  const file = await loadFileContent(rawFilePath);
+  const entries = autoParseEntry(file, contentKey);
+
+  const partial = {
+    _raw: file.content,
+    source: {
+      location: file.filePath,
+      type: "file" as SourceType,
+    },
+    location: file.filePath,
+    meta: {},
+  };
+
+  return entries.map((data) => ({
+    ...partial,
+    data,
+  }));
+}
+
+async function loadFromFileSource(
+  source: MuseSource,
+  options: SourceOptions,
+): Promise<MuseEntry[]> {
+  const paths = Array.from(
+    new Glob(source.location).scanSync({
+      cwd: options.cwd,
+      absolute: true,
+      followSymlinks: true,
+      onlyFiles: true,
+    }),
+  );
+
+  const filteredPaths = paths.filter((path) => !options.ignore.includes(path));
+
+  const entries: MuseEntry[] = [];
+  for (const filePath of filteredPaths) {
+    const fileEntries = await parseMuseFile(filePath, options);
+    entries.push(...fileEntries);
+  }
+
+  return entries;
+}
+
+function getFileExtensionFromURL(url: string): string | null {
+  const parsedUrl = new URL(url);
+
+  const pathname = parsedUrl.pathname;
+  const filename = pathname.substring(pathname.lastIndexOf("/") + 1);
+  const extension = filename.substring(filename.lastIndexOf(".") + 1);
+
+  return extension === filename ? null : extension;
+}
+
+async function loadFromURLSource(
+  source: MuseSource,
+  options: SourceOptions,
+): Promise<MuseEntry[]> {
+  const { contentKey = "content" } = options;
+  const response = await fetch(source.location);
+
+  if (!response.ok) {
+    throw new Error(`Failed to fetch URL: ${source.location}`);
+  }
+  const content = await response.text();
+  const mimeType = response.headers.get("Content-Type") || "unknown";
+
+  const parseType = getFileExtensionFromURL(source.location) ?? "unknown";
+
+  const loadedFile: LoadedFile = {
+    content,
+    filePath: source.location,
+    fileType: parseType,
+    dirname: "",
+    basename: "",
+    mimeType,
+  };
+
+  const entries = autoParseEntry(loadedFile, contentKey);
+
+  const partial = {
+    _raw: content,
+    source: {
+      location: source.location,
+      type: "url" as SourceType,
+    },
+    location: source.location,
+    meta: {},
+  };
+
+  return entries.map((data) => ({
+    ...partial,
+    data,
+  }));
+}
+
+export async function loadFromSource(
+  source: MuseSource,
+  options: SourceOptions,
+): Promise<MuseEntry[]> {
+  switch (source.type) {
+    case "file":
+      return loadFromFileSource(source, options);
+    case "url":
+      return loadFromURLSource(source, options);
+    default:
+      throw new Error(`Unsupported source type: ${source.type}`);
+  }
+}

src/errors.ts

@@ -1,28 +1,39 @@
 export class MuseError extends Error {
   fmt(): string {
-    return this.message;
+    return `-- ERROR -- \nDetails: ${this.message}`;
   }
 }
 
-export class FileError extends MuseError {
-  filePath: string;
-  details?: string;
+export class MusePluginError extends MuseError {
+  plugin: string;
 
-  constructor(message: string, filePath: string, details?: string) {
+  constructor(message: string, plugin: string) {
     super(message);
-    this.filePath = filePath;
-    this.details = details;
+    this.plugin = plugin;
   }
 
   fmt(): string {
-    return `-- ${this.message} --\nFile Path: ${this.filePath}\nDetails: ${this.details}`;
+    return `-- PLUGIN ERROR --\Plugin: ${this.plugin}\nDetails: ${this.message}`;
   }
 }
 
-export class FileNotFoundError extends FileError {
+export class MuseFileError extends MuseError {
+  filePath: string;
+
+  constructor(message: string, filePath: string) {
+    super(message);
+    this.filePath = filePath;
+  }
+
+  fmt(): string {
+    return `-- FILE ERROR --\nFile Path: ${this.filePath}\nDetails: ${this.message}`;
+  }
+}
+
+export class MuseFileNotFoundError extends MuseFileError {
   constructor(filePath: string) {
     super("File not found", filePath);
-    this.message = "File not found";
+    this.message = "File does not exist";
     this.filePath = filePath;
   }
 }

src/exports.ts

@@ -1,2 +1,12 @@
-export type * from "./types";
+// Content exports
+export * from "#errors";
+
+// Constants
 export const DEFAULT_CONTENT_KEY = "content";
+
+// Types
+export type * from "#core/binding";
+export type * from "#core/plugins";
+export type * from "#core/sources";
+export type * from "#core/records";
+export type * from "#core/files";

src/parse.ts

@@ -1,105 +0,0 @@
-import { normalize, extname } from "node:path";
-import YAML from "yaml";
-import TOML from "smol-toml";
-import EMDY from "@endeavorance/emdy";
-import type { MuseEntry, UnknownRecord } from "./types";
-
-function parseYAMLEntries(text: string): UnknownRecord[] {
-  const parsedDocs = YAML.parseAllDocuments(text);
-
-  if (parsedDocs.some((doc) => doc.toJS() === null)) {
-    throw new Error("Encountered NULL resource");
-  }
-
-  const errors = parsedDocs.flatMap((doc) => doc.errors);
-
-  if (errors.length > 0) {
-    throw new Error(
-      `Error parsing YAML resource: ${errors.map((e) => e.message).join(", ")}`,
-    );
-  }
-
-  const collection: UnknownRecord[] = parsedDocs.map((doc) => doc.toJS());
-  return collection;
-}
-
-function parseJSONEntries(text: string): UnknownRecord[] {
-  const parsed = JSON.parse(text);
-
-  if (Array.isArray(parsed)) {
-    return parsed;
-  }
-
-  if (typeof parsed === "object") {
-    return [parsed];
-  }
-
-  throw new Error("JSON resource must be an object or an array of objects");
-}
-
-function parseTOMLEntries(text: string): UnknownRecord[] {
-  const parsed = TOML.parse(text);
-
-  if (Array.isArray(parsed)) {
-    return parsed;
-  }
-
-  if (typeof parsed === "object") {
-    return [parsed];
-  }
-
-  throw new Error("TOML resource must be an object or an array of objects");
-}
-
-interface ParseMuseFileOptions {
-  contentKey?: string;
-}
-
-export async function parseMuseFile(
-  rawFilePath: string,
-  { contentKey = "content" }: ParseMuseFileOptions = {},
-): Promise<MuseEntry[]> {
-  const filePath = normalize(rawFilePath);
-  const file = Bun.file(filePath);
-  const fileType = extname(filePath).slice(1);
-  const rawFileContent = await file.text();
-
-  const partial = {
-    _raw: rawFileContent,
-    filePath,
-    meta: {},
-  };
-
-  if (fileType === "md") {
-    const parsed = EMDY.parse(rawFileContent, contentKey);
-    return [
-      {
-        ...partial,
-        data: parsed,
-      },
-    ];
-  }
-
-  if (fileType === "yaml") {
-    return parseYAMLEntries(rawFileContent).map((data) => ({
-      ...partial,
-      data,
-    }));
-  }
-
-  if (fileType === "json") {
-    return parseJSONEntries(rawFileContent).map((data) => ({
-      ...partial,
-      data,
-    }));
-  }
-
-  if (fileType === "toml") {
-    return parseTOMLEntries(rawFileContent).map((data) => ({
-      ...partial,
-      data,
-    }));
-  }
-
-  throw new Error(`Unsupported file type: ${fileType}`);
-}

src/preload/http-plugin.js

@@ -0,0 +1,40 @@
+const rx_any = /./;
+const rx_http = /^https?:\/\//;
+const rx_relative_path = /^\.\.?\//;
+const rx_absolute_path = /^\//;
+
+async function load_http_module(href) {
+  return fetch(href).then((response) =>
+    response
+      .text()
+      .then((text) =>
+        response.ok
+          ? { contents: text, loader: "js" }
+          : Promise.reject(
+              new Error(`Failed to load module '${href}'': ${text}`),
+            ),
+      ),
+  );
+}
+
+Bun.plugin({
+  name: "http_imports",
+  setup(build) {
+    build.onResolve({ filter: rx_relative_path }, (args) => {
+      if (rx_http.test(args.importer)) {
+        return { path: new URL(args.path, args.importer).href };
+      }
+    });
+    build.onResolve({ filter: rx_absolute_path }, (args) => {
+      if (rx_http.test(args.importer)) {
+        return { path: new URL(args.path, args.importer).href };
+      }
+    });
+    build.onLoad({ filter: rx_any, namespace: "http" }, (args) =>
+      load_http_module(`http:${args.path}`),
+    );
+    build.onLoad({ filter: rx_any, namespace: "https" }, (args) =>
+      load_http_module(`https:${args.path}`),
+    );
+  },
+});

src/types.ts

@@ -1,55 +0,0 @@
-import { z } from "zod";
-
-export interface CLIFlags {
-  help: boolean;
-  verbose: boolean;
-  stdout: boolean;
-}
-
-export interface CLIArguments {
-  inputFilePath: string;
-  flags: CLIFlags;
-}
-
-export const BindingSchema = z.object({
-  include: z.array(z.string()).default([]),
-  contentKey: z.string().default("content"),
-  files: z
-    .array(z.string())
-    .default(["**/*.yaml", "**/*.md", "**/*.toml", "**/*.json"]),
-  options: z.record(z.string(), z.any()).default({}),
-  processors: z.array(z.string()).default([]),
-});
-
-export interface Binding {
-  readonly _raw: z.infer<typeof BindingSchema>;
-  readonly bindingPath: string;
-  readonly contentKey: string;
-  readonly entries: MuseEntry[];
-  readonly meta: Record<string, unknown>;
-  readonly options: Record<string, unknown>;
-  readonly processors: MuseProcessor[];
-  readonly imports: Binding[];
-}
-
-export type MuseProcessorFn = (
-  binding: Binding,
-  opts: Record<string, unknown>,
-  flags: CLIFlags,
-) => Promise<Binding>;
-
-export interface MuseProcessor {
-  readonly filePath: string;
-  readonly name: string;
-  readonly description: string;
-  readonly process: MuseProcessorFn;
-}
-
-export interface MuseEntry {
-  _raw: string;
-  filePath: string;
-  data: Record<string, unknown>;
-  meta: Record<string, unknown>;
-}
-
-export type UnknownRecord = Record<string, unknown>;

tsconfig.json

@@ -26,11 +26,22 @@
     "noUnusedParameters": false,
     "noPropertyAccessFromIndexSignature": false,
     "baseUrl": "./src",
-    "paths": {},
+    "paths": {
+      "#core/*": [
+        "./core/*.ts"
+      ],
+      "#util": [
+        "./util.ts"
+      ],
+      "#errors": [
+        "./errors.ts"
+      ]
+    },
     "outDir": "./dist",
   },
   "include": [
     "src/**/*.ts",
     "src/**/*.tsx",
+    "src/preload/http-plugin.js",
   ]
 }