Support for url based resources

demo/main/binding.md

@@ -3,6 +3,9 @@ markdownKey: description
 options:
   dunks:
     key: dunkaroos
+sources:
+  - file: ./stats/*.toml
+  - url: https://git.astral.camp/endeavorance/emdy/raw/branch/main/package.json
 processors:
   - ./processors/scream.js
   - ./processors/announce-slam-dunks.js

src/core/binding.ts

@@ -1,21 +1,37 @@
 import path, { dirname } from "node:path";
-import { Glob } from "bun";
 import { z } from "zod";
 import { MuseError, MuseFileNotFoundError } from "#errors";
 import { resolveFilePath } from "#util";
-import { type MuseEntry, parseMuseFile } from "./parse";
+import {
+  type MuseEntry,
+  type MuseSource,
+  loadFromSource,
+  parseMuseFile,
+} from "./sources";
 import { type MusePlugin, loadPlugin } from "./plugins";
 import type { UnknownRecord } from "./types";
 
+const SourceSchema = z.union([
+  z.object({
+    file: z.string(),
+  }),
+  z.object({
+    url: z.string().url(),
+  }),
+]);
+
+type SourceShorthand = z.infer<typeof SourceSchema>;
+
 // Function to parse an unknown object into parts of a binding
-export const BindingSchema = z.object({
+const BindingSchema = z.object({
   contentKey: z.string().default("content"),
-  sources: z
+  sources: z.array(SourceSchema).default([
-    .array(z.string())
+    {
-    .default(["**/*.yaml", "**/*.md", "**/*.toml", "**/*.json"]),
+      file: "./**/*.{json,yaml,toml,md}",
+    },
+  ]),
   options: z.record(z.string(), z.any()).default({}),
   processors: z.array(z.string()).default([]),
-  renderer: z.string().optional(),
 });
 
 /**
@@ -27,6 +43,9 @@ export interface Binding<MetaShape = UnknownRecord> {
   /** The raw data read from the binding file */
   readonly _raw: z.infer<typeof BindingSchema>;
 
+  /** Information about the sources used to load entries */
+  readonly sources: MuseSource[];
+
   /** The full file path to the binding file */
   readonly bindingPath: string;
 
@@ -46,6 +65,24 @@ export interface Binding<MetaShape = UnknownRecord> {
   readonly options: UnknownRecord;
 }
 
+function sourceShorthandToSource(shorthand: SourceShorthand): MuseSource {
+  if ("file" in shorthand) {
+    return {
+      location: shorthand.file,
+      type: "file",
+    };
+  }
+
+  if ("url" in shorthand) {
+    return {
+      location: shorthand.url,
+      type: "url",
+    };
+  }
+
+  throw new MuseError("Invalid source shorthand");
+}
+
 /**
  * Given a path, find the binding file
  * If the path is to a directory, check for a binding inside
@@ -93,7 +130,11 @@ export async function loadBinding(initialPath: string): Promise<Binding> {
   // Resolve the file location if possible
   const bindingPath = await resolveBindingPath(initialPath);
   const bindingDirname = dirname(bindingPath);
-  const loadedBindingData = await parseMuseFile(bindingPath);
+  const loadedBindingData = await parseMuseFile(bindingPath, {
+    contentKey: "content",
+    cwd: bindingDirname,
+    ignore: [],
+  });
 
   if (loadedBindingData.length === 0) {
     throw new MuseError("No entries found in binding file");
@@ -102,30 +143,20 @@ export async function loadBinding(initialPath: string): Promise<Binding> {
   // Load and parse the Binding YAML content
   const parsedBinding = BindingSchema.parse(loadedBindingData[0].data);
 
-  // Aggregate file paths to import
+  const sourceOptions = {
-  const includedFilePaths = parsedBinding.sources
-    .flatMap((thisGlob) => {
-      return Array.from(
-        new Glob(thisGlob).scanSync({
     cwd: bindingDirname,
-          absolute: true,
-          followSymlinks: true,
-          onlyFiles: true,
-        }),
-      );
-    })
-    .filter((filePath) => filePath !== bindingPath);
-
-  // Load files
-  const entries: MuseEntry[] = (
-    await Promise.all(
-      includedFilePaths.map((filePath) => {
-        return parseMuseFile(filePath, {
     contentKey: parsedBinding.contentKey,
-        });
+    ignore: [bindingPath],
-      }),
+  };
-    )
+
-  ).flat();
+  const sources = parsedBinding.sources.map(sourceShorthandToSource);
+
+  const entries: MuseEntry[] = [];
+
+  for (const source of sources) {
+    const entriesFromSource = await loadFromSource(source, sourceOptions);
+    entries.push(...entriesFromSource);
+  }
 
   // Load and check plugins
   const plugins: MusePlugin[] = await Promise.all(
@@ -134,6 +165,7 @@ export async function loadBinding(initialPath: string): Promise<Binding> {
 
   return {
     _raw: parsedBinding,
+    sources,
     bindingPath,
     contentKey: parsedBinding.contentKey,
     entries,

src/core/files.ts

@@ -7,6 +7,7 @@ export interface LoadedFile {
   fileType: string;
   dirname: string;
   basename: string;
+  mimeType: string;
 }
 
 export async function loadFileContent(filePath: string): Promise<LoadedFile> {
@@ -28,5 +29,6 @@ export async function loadFileContent(filePath: string): Promise<LoadedFile> {
     fileType: extension.slice(1),
     dirname: dirname(normalizedPath),
     basename: basename(normalizedPath, extension),
+    mimeType: file.type,
   };
 }

src/core/parse.ts

@@ -1,104 +0,0 @@
-import EMDY from "@endeavorance/emdy";
-import TOML from "smol-toml";
-import YAML from "yaml";
-import { loadFileContent } from "./files";
-import type { UnknownRecord } from "./types";
-
-export interface MuseEntry<MetaShape = UnknownRecord> {
-  _raw: string;
-  filePath: string;
-  data: Record<string, unknown>;
-  meta: MetaShape;
-}
-
-function formatEntries(val: unknown): UnknownRecord[] {
-  if (Array.isArray(val) && val.every((el) => typeof el === "object")) {
-    return val;
-  }
-
-  if (typeof val === "object" && val !== null) {
-    return [val as UnknownRecord];
-  }
-
-  throw new Error(
-    `Invalid data format. Entry files must define an object or array of objects, but found "${val}"`,
-  );
-}
-
-function parseYAMLEntries(text: string): UnknownRecord[] {
-  const parsedDocs = YAML.parseAllDocuments(text);
-
-  if (parsedDocs.some((doc) => doc.toJS() === null)) {
-    throw new Error("Encountered NULL resource");
-  }
-
-  const errors = parsedDocs.flatMap((doc) => doc.errors);
-
-  if (errors.length > 0) {
-    throw new Error(
-      `Error parsing YAML resource: ${errors.map((e) => e.message).join(", ")}`,
-    );
-  }
-
-  const collection: UnknownRecord[] = parsedDocs.map((doc) => doc.toJS());
-  return collection;
-}
-
-function parseJSONEntries(text: string): UnknownRecord[] {
-  return formatEntries(JSON.parse(text));
-}
-
-function parseTOMLEntries(text: string): UnknownRecord[] {
-  return formatEntries(TOML.parse(text));
-}
-
-function parseMarkdownEntry(text: string, contentKey: string): UnknownRecord[] {
-  return formatEntries(EMDY.parse(text, contentKey));
-}
-
-interface ParseMuseFileOptions {
-  contentKey?: string;
-}
-
-export async function parseMuseFile(
-  rawFilePath: string,
-  { contentKey = "content" }: ParseMuseFileOptions = {},
-): Promise<MuseEntry[]> {
-  const { content, filePath, fileType } = await loadFileContent(rawFilePath);
-
-  const partial = {
-    _raw: content,
-    filePath,
-    meta: {},
-  };
-
-  if (fileType === "md") {
-    return parseMarkdownEntry(content, contentKey).map((data) => ({
-      ...partial,
-      data,
-    }));
-  }
-
-  if (fileType === "yaml") {
-    return parseYAMLEntries(content).map((data) => ({
-      ...partial,
-      data,
-    }));
-  }
-
-  if (fileType === "json") {
-    return parseJSONEntries(content).map((data) => ({
-      ...partial,
-      data,
-    }));
-  }
-
-  if (fileType === "toml") {
-    return parseTOMLEntries(content).map((data) => ({
-      ...partial,
-      data,
-    }));
-  }
-
-  throw new Error(`Unsupported file type: ${fileType}`);
-}

src/core/sources.ts

@@ -0,0 +1,254 @@
+import EMDY from "@endeavorance/emdy";
+import TOML from "smol-toml";
+import YAML from "yaml";
+import { loadFileContent } from "./files";
+import type { UnknownRecord } from "./types";
+import { Glob } from "bun";
+
+export type SourceType = "file" | "url";
+
+export interface MuseSource {
+  location: string;
+  type: SourceType;
+}
+
+interface SourceOptions {
+  cwd: string;
+  contentKey: string;
+  ignore: string[];
+}
+
+export interface MuseEntry<MetaShape = UnknownRecord> {
+  _raw: string;
+  location: string;
+  data: Record<string, unknown>;
+  meta: MetaShape;
+  source: MuseSource;
+}
+
+function formatEntries(val: unknown): UnknownRecord[] {
+  if (Array.isArray(val) && val.every((el) => typeof el === "object")) {
+    return val;
+  }
+
+  if (typeof val === "object" && val !== null) {
+    return [val as UnknownRecord];
+  }
+
+  throw new Error(
+    `Invalid data format. Entry files must define an object or array of objects, but found "${val}"`,
+  );
+}
+
+function parseYAMLEntries(text: string): UnknownRecord[] {
+  const parsedDocs = YAML.parseAllDocuments(text);
+
+  if (parsedDocs.some((doc) => doc.toJS() === null)) {
+    throw new Error("Encountered NULL resource");
+  }
+
+  const errors = parsedDocs.flatMap((doc) => doc.errors);
+
+  if (errors.length > 0) {
+    throw new Error(
+      `Error parsing YAML resource: ${errors.map((e) => e.message).join(", ")}`,
+    );
+  }
+
+  const collection: UnknownRecord[] = parsedDocs.map((doc) => doc.toJS());
+  return collection;
+}
+
+function parseJSONEntries(text: string): UnknownRecord[] {
+  return formatEntries(JSON.parse(text));
+}
+
+function parseTOMLEntries(text: string): UnknownRecord[] {
+  return formatEntries(TOML.parse(text));
+}
+
+function parseMarkdownEntry(text: string, contentKey: string): UnknownRecord[] {
+  return formatEntries(EMDY.parse(text, contentKey));
+}
+
+export async function parseMuseFile(
+  rawFilePath: string,
+  { contentKey = "content" }: SourceOptions,
+): Promise<MuseEntry[]> {
+  const { content, filePath, fileType } = await loadFileContent(rawFilePath);
+
+  const partial = {
+    _raw: content,
+    source: {
+      location: filePath,
+      type: "file" as SourceType,
+    },
+    location: filePath,
+    meta: {},
+  };
+
+  if (fileType === "md") {
+    return parseMarkdownEntry(content, contentKey).map((data) => ({
+      ...partial,
+      data,
+    }));
+  }
+
+  if (fileType === "yaml") {
+    return parseYAMLEntries(content).map((data) => ({
+      ...partial,
+      data,
+    }));
+  }
+
+  if (fileType === "json") {
+    return parseJSONEntries(content).map((data) => ({
+      ...partial,
+      data,
+    }));
+  }
+
+  if (fileType === "toml") {
+    return parseTOMLEntries(content).map((data) => ({
+      ...partial,
+      data,
+    }));
+  }
+
+  throw new Error(`Unsupported file type: ${fileType}`);
+}
+
+async function loadFromFileSource(
+  source: MuseSource,
+  options: SourceOptions,
+): Promise<MuseEntry[]> {
+  const paths = Array.from(
+    new Glob(source.location).scanSync({
+      cwd: options.cwd,
+      absolute: true,
+      followSymlinks: true,
+      onlyFiles: true,
+    }),
+  );
+
+  const filteredPaths = paths.filter((path) => !options.ignore.includes(path));
+
+  const entries: MuseEntry[] = [];
+  for (const filePath of filteredPaths) {
+    const fileEntries = await parseMuseFile(filePath, options);
+    entries.push(...fileEntries);
+  }
+
+  return entries;
+}
+
+function getFileExtensionFromURL(url: string): string | null {
+  const parsedUrl = new URL(url);
+
+  const pathname = parsedUrl.pathname;
+  const filename = pathname.substring(pathname.lastIndexOf("/") + 1);
+  const extension = filename.substring(filename.lastIndexOf(".") + 1);
+
+  return extension === filename ? null : extension;
+}
+
+function mimeTypeToFileType(mimeType: string): string | null {
+  switch (mimeType) {
+    case "text/markdown":
+      return "md";
+    case "text/yaml":
+      return "yaml";
+    case "application/x-yaml":
+      return "yaml";
+    case "application/json":
+      return "json";
+    case "application/toml":
+      return "toml";
+    default:
+      return null;
+  }
+}
+
+async function loadFromURLSource(
+  source: MuseSource,
+  options: SourceOptions,
+): Promise<MuseEntry[]> {
+  const { contentKey = "content" } = options;
+  const response = await fetch(source.location);
+
+  if (!response.ok) {
+    throw new Error(`Failed to fetch URL: ${source.location}`);
+  }
+  const content = await response.text();
+  const mimeType = response.headers.get("Content-Type") || "unknown";
+
+  const parseType =
+    mimeTypeToFileType(mimeType) ??
+    getFileExtensionFromURL(source.location) ??
+    "unknown";
+
+  const partial = {
+    _raw: content,
+    source: {
+      location: source.location,
+      type: "url" as SourceType,
+    },
+    location: source.location,
+    meta: {},
+  };
+
+  if (parseType === "md") {
+    return parseMarkdownEntry(content, contentKey).map((data) => ({
+      ...partial,
+      data,
+    }));
+  }
+
+  if (parseType === "yaml") {
+    return parseYAMLEntries(content).map((data) => ({
+      ...partial,
+      data,
+    }));
+  }
+
+  if (parseType === "json") {
+    return parseJSONEntries(content).map((data) => ({
+      ...partial,
+      data,
+    }));
+  }
+
+  if (parseType === "toml") {
+    return parseTOMLEntries(content).map((data) => ({
+      ...partial,
+      data,
+    }));
+  }
+
+  // If it doesnt match one of these, try brute force parsing
+  // and return the result if any of them work
+  try {
+    return parseMarkdownEntry(content, contentKey).map((data) => ({
+      ...partial,
+      data,
+    }));
+  } catch {
+    // noop
+  }
+
+  throw new Error(`Unsupported MIME type from URL source: ${mimeType}`);
+}
+
+export async function loadFromSource(
+  source: MuseSource,
+  options: SourceOptions,
+): Promise<MuseEntry[]> {
+  switch (source.type) {
+    case "file":
+      return loadFromFileSource(source, options);
+    case "url":
+      return loadFromURLSource(source, options);
+    default:
+      throw new Error(`Unsupported source type: ${source.type}`);
+  }
+}

src/muse.ts

@@ -0,0 +1,20 @@
+import { loadBinding, type Binding } from "#core/binding";
+
+export class Muse {
+  bindingLoaded = false;
+  preliminaryBinding: Partial<Binding> = {};
+  additionalSources: string[] = [];
+  additionalPluginPaths: string[] = [];
+
+  constructor(bindingConfig: Partial<Binding>) {
+    this.preliminaryBinding = bindingConfig;
+  }
+
+  source(newSource: string) {
+    this.additionalSources.push(newSource);
+  }
+
+  plugin(pluginPath: string) {
+    this.additionalPluginPaths.push(pluginPath);
+  }
+}