muse/src/core/sources.ts

import EMDY from "@endeavorance/emdy";
import TOML from "smol-toml";
import YAML from "yaml";
import { loadFileContent } from "./files";
import type { UnknownRecord } from "./types";
import { Glob } from "bun";

export type SourceType = "file" | "url";

export interface MuseSource {
  location: string;
  type: SourceType;
}

interface SourceOptions {
  cwd: string;
  contentKey: string;
  ignore: string[];
}

export interface MuseEntry<MetaShape = UnknownRecord> {
  _raw: string;
  location: string;
  data: Record<string, unknown>;
  meta: MetaShape;
  source: MuseSource;
}

function formatEntries(val: unknown): UnknownRecord[] {
  if (Array.isArray(val) && val.every((el) => typeof el === "object")) {
    return val;
  }

  if (typeof val === "object" && val !== null) {
    return [val as UnknownRecord];
  }

  throw new Error(
    `Invalid data format. Entry files must define an object or array of objects, but found "${val}"`,
  );
}

function parseYAMLEntries(text: string): UnknownRecord[] {
  const parsedDocs = YAML.parseAllDocuments(text);

  if (parsedDocs.some((doc) => doc.toJS() === null)) {
    throw new Error("Encountered NULL resource");
  }

  const errors = parsedDocs.flatMap((doc) => doc.errors);

  if (errors.length > 0) {
    throw new Error(
      `Error parsing YAML resource: ${errors.map((e) => e.message).join(", ")}`,
    );
  }

  const collection: UnknownRecord[] = parsedDocs.map((doc) => doc.toJS());
  return collection;
}

function parseJSONEntries(text: string): UnknownRecord[] {
  return formatEntries(JSON.parse(text));
}

function parseTOMLEntries(text: string): UnknownRecord[] {
  return formatEntries(TOML.parse(text));
}

function parseMarkdownEntry(text: string, contentKey: string): UnknownRecord[] {
  return formatEntries(EMDY.parse(text, contentKey));
}

export async function parseMuseFile(
  rawFilePath: string,
  { contentKey = "content" }: SourceOptions,
): Promise<MuseEntry[]> {
  const { content, filePath, fileType } = await loadFileContent(rawFilePath);

  const partial = {
    _raw: content,
    source: {
      location: filePath,
      type: "file" as SourceType,
    },
    location: filePath,
    meta: {},
  };

  if (fileType === "md") {
    return parseMarkdownEntry(content, contentKey).map((data) => ({
      ...partial,
      data,
    }));
  }

  if (fileType === "yaml") {
    return parseYAMLEntries(content).map((data) => ({
      ...partial,
      data,
    }));
  }

  if (fileType === "json") {
    return parseJSONEntries(content).map((data) => ({
      ...partial,
      data,
    }));
  }

  if (fileType === "toml") {
    return parseTOMLEntries(content).map((data) => ({
      ...partial,
      data,
    }));
  }

  throw new Error(`Unsupported file type: ${fileType}`);
}

async function loadFromFileSource(
  source: MuseSource,
  options: SourceOptions,
): Promise<MuseEntry[]> {
  const paths = Array.from(
    new Glob(source.location).scanSync({
      cwd: options.cwd,
      absolute: true,
      followSymlinks: true,
      onlyFiles: true,
    }),
  );

  const filteredPaths = paths.filter((path) => !options.ignore.includes(path));

  const entries: MuseEntry[] = [];
  for (const filePath of filteredPaths) {
    const fileEntries = await parseMuseFile(filePath, options);
    entries.push(...fileEntries);
  }

  return entries;
}

function getFileExtensionFromURL(url: string): string | null {
  const parsedUrl = new URL(url);

  const pathname = parsedUrl.pathname;
  const filename = pathname.substring(pathname.lastIndexOf("/") + 1);
  const extension = filename.substring(filename.lastIndexOf(".") + 1);

  return extension === filename ? null : extension;
}

function mimeTypeToFileType(mimeType: string): string | null {
  switch (mimeType) {
    case "text/markdown":
      return "md";
    case "text/yaml":
      return "yaml";
    case "application/x-yaml":
      return "yaml";
    case "application/json":
      return "json";
    case "application/toml":
      return "toml";
    default:
      return null;
  }
}

async function loadFromURLSource(
  source: MuseSource,
  options: SourceOptions,
): Promise<MuseEntry[]> {
  const { contentKey = "content" } = options;
  const response = await fetch(source.location);

  if (!response.ok) {
    throw new Error(`Failed to fetch URL: ${source.location}`);
  }
  const content = await response.text();
  const mimeType = response.headers.get("Content-Type") || "unknown";

  const parseType =
    mimeTypeToFileType(mimeType) ??
    getFileExtensionFromURL(source.location) ??
    "unknown";

  const partial = {
    _raw: content,
    source: {
      location: source.location,
      type: "url" as SourceType,
    },
    location: source.location,
    meta: {},
  };

  if (parseType === "md") {
    return parseMarkdownEntry(content, contentKey).map((data) => ({
      ...partial,
      data,
    }));
  }

  if (parseType === "yaml") {
    return parseYAMLEntries(content).map((data) => ({
      ...partial,
      data,
    }));
  }

  if (parseType === "json") {
    return parseJSONEntries(content).map((data) => ({
      ...partial,
      data,
    }));
  }

  if (parseType === "toml") {
    return parseTOMLEntries(content).map((data) => ({
      ...partial,
      data,
    }));
  }

  // If it doesnt match one of these, try brute force parsing
  // and return the result if any of them work
  try {
    return parseMarkdownEntry(content, contentKey).map((data) => ({
      ...partial,
      data,
    }));
  } catch {
    // noop
  }

  throw new Error(`Unsupported MIME type from URL source: ${mimeType}`);
}

export async function loadFromSource(
  source: MuseSource,
  options: SourceOptions,
): Promise<MuseEntry[]> {
  switch (source.type) {
    case "file":
      return loadFromFileSource(source, options);
    case "url":
      return loadFromURLSource(source, options);
    default:
      throw new Error(`Unsupported source type: ${source.type}`);
  }
}