Support for url based resources
This commit is contained in:
parent
ae1b9e262e
commit
5b90a8e1b3
6 changed files with 342 additions and 135 deletions
|
|
@ -3,6 +3,9 @@ markdownKey: description
|
|||
options:
|
||||
dunks:
|
||||
key: dunkaroos
|
||||
sources:
|
||||
- file: ./stats/*.toml
|
||||
- url: https://git.astral.camp/endeavorance/emdy/raw/branch/main/package.json
|
||||
processors:
|
||||
- ./processors/scream.js
|
||||
- ./processors/announce-slam-dunks.js
|
||||
|
|
|
|||
|
|
@ -1,21 +1,37 @@
|
|||
import path, { dirname } from "node:path";
|
||||
import { Glob } from "bun";
|
||||
import { z } from "zod";
|
||||
import { MuseError, MuseFileNotFoundError } from "#errors";
|
||||
import { resolveFilePath } from "#util";
|
||||
import { type MuseEntry, parseMuseFile } from "./parse";
|
||||
import {
|
||||
type MuseEntry,
|
||||
type MuseSource,
|
||||
loadFromSource,
|
||||
parseMuseFile,
|
||||
} from "./sources";
|
||||
import { type MusePlugin, loadPlugin } from "./plugins";
|
||||
import type { UnknownRecord } from "./types";
|
||||
|
||||
const SourceSchema = z.union([
|
||||
z.object({
|
||||
file: z.string(),
|
||||
}),
|
||||
z.object({
|
||||
url: z.string().url(),
|
||||
}),
|
||||
]);
|
||||
|
||||
type SourceShorthand = z.infer<typeof SourceSchema>;
|
||||
|
||||
// Function to parse an unknown object into parts of a binding
|
||||
export const BindingSchema = z.object({
|
||||
const BindingSchema = z.object({
|
||||
contentKey: z.string().default("content"),
|
||||
sources: z
|
||||
.array(z.string())
|
||||
.default(["**/*.yaml", "**/*.md", "**/*.toml", "**/*.json"]),
|
||||
sources: z.array(SourceSchema).default([
|
||||
{
|
||||
file: "./**/*.{json,yaml,toml,md}",
|
||||
},
|
||||
]),
|
||||
options: z.record(z.string(), z.any()).default({}),
|
||||
processors: z.array(z.string()).default([]),
|
||||
renderer: z.string().optional(),
|
||||
});
|
||||
|
||||
/**
|
||||
|
|
@ -27,6 +43,9 @@ export interface Binding<MetaShape = UnknownRecord> {
|
|||
/** The raw data read from the binding file */
|
||||
readonly _raw: z.infer<typeof BindingSchema>;
|
||||
|
||||
/** Information about the sources used to load entries */
|
||||
readonly sources: MuseSource[];
|
||||
|
||||
/** The full file path to the binding file */
|
||||
readonly bindingPath: string;
|
||||
|
||||
|
|
@ -46,6 +65,24 @@ export interface Binding<MetaShape = UnknownRecord> {
|
|||
readonly options: UnknownRecord;
|
||||
}
|
||||
|
||||
function sourceShorthandToSource(shorthand: SourceShorthand): MuseSource {
|
||||
if ("file" in shorthand) {
|
||||
return {
|
||||
location: shorthand.file,
|
||||
type: "file",
|
||||
};
|
||||
}
|
||||
|
||||
if ("url" in shorthand) {
|
||||
return {
|
||||
location: shorthand.url,
|
||||
type: "url",
|
||||
};
|
||||
}
|
||||
|
||||
throw new MuseError("Invalid source shorthand");
|
||||
}
|
||||
|
||||
/**
|
||||
* Given a path, find the binding file
|
||||
* If the path is to a directory, check for a binding inside
|
||||
|
|
@ -93,7 +130,11 @@ export async function loadBinding(initialPath: string): Promise<Binding> {
|
|||
// Resolve the file location if possible
|
||||
const bindingPath = await resolveBindingPath(initialPath);
|
||||
const bindingDirname = dirname(bindingPath);
|
||||
const loadedBindingData = await parseMuseFile(bindingPath);
|
||||
const loadedBindingData = await parseMuseFile(bindingPath, {
|
||||
contentKey: "content",
|
||||
cwd: bindingDirname,
|
||||
ignore: [],
|
||||
});
|
||||
|
||||
if (loadedBindingData.length === 0) {
|
||||
throw new MuseError("No entries found in binding file");
|
||||
|
|
@ -102,30 +143,20 @@ export async function loadBinding(initialPath: string): Promise<Binding> {
|
|||
// Load and parse the Binding YAML content
|
||||
const parsedBinding = BindingSchema.parse(loadedBindingData[0].data);
|
||||
|
||||
// Aggregate file paths to import
|
||||
const includedFilePaths = parsedBinding.sources
|
||||
.flatMap((thisGlob) => {
|
||||
return Array.from(
|
||||
new Glob(thisGlob).scanSync({
|
||||
cwd: bindingDirname,
|
||||
absolute: true,
|
||||
followSymlinks: true,
|
||||
onlyFiles: true,
|
||||
}),
|
||||
);
|
||||
})
|
||||
.filter((filePath) => filePath !== bindingPath);
|
||||
const sourceOptions = {
|
||||
cwd: bindingDirname,
|
||||
contentKey: parsedBinding.contentKey,
|
||||
ignore: [bindingPath],
|
||||
};
|
||||
|
||||
// Load files
|
||||
const entries: MuseEntry[] = (
|
||||
await Promise.all(
|
||||
includedFilePaths.map((filePath) => {
|
||||
return parseMuseFile(filePath, {
|
||||
contentKey: parsedBinding.contentKey,
|
||||
});
|
||||
}),
|
||||
)
|
||||
).flat();
|
||||
const sources = parsedBinding.sources.map(sourceShorthandToSource);
|
||||
|
||||
const entries: MuseEntry[] = [];
|
||||
|
||||
for (const source of sources) {
|
||||
const entriesFromSource = await loadFromSource(source, sourceOptions);
|
||||
entries.push(...entriesFromSource);
|
||||
}
|
||||
|
||||
// Load and check plugins
|
||||
const plugins: MusePlugin[] = await Promise.all(
|
||||
|
|
@ -134,6 +165,7 @@ export async function loadBinding(initialPath: string): Promise<Binding> {
|
|||
|
||||
return {
|
||||
_raw: parsedBinding,
|
||||
sources,
|
||||
bindingPath,
|
||||
contentKey: parsedBinding.contentKey,
|
||||
entries,
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ export interface LoadedFile {
|
|||
fileType: string;
|
||||
dirname: string;
|
||||
basename: string;
|
||||
mimeType: string;
|
||||
}
|
||||
|
||||
export async function loadFileContent(filePath: string): Promise<LoadedFile> {
|
||||
|
|
@ -28,5 +29,6 @@ export async function loadFileContent(filePath: string): Promise<LoadedFile> {
|
|||
fileType: extension.slice(1),
|
||||
dirname: dirname(normalizedPath),
|
||||
basename: basename(normalizedPath, extension),
|
||||
mimeType: file.type,
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,104 +0,0 @@
|
|||
import EMDY from "@endeavorance/emdy";
|
||||
import TOML from "smol-toml";
|
||||
import YAML from "yaml";
|
||||
import { loadFileContent } from "./files";
|
||||
import type { UnknownRecord } from "./types";
|
||||
|
||||
export interface MuseEntry<MetaShape = UnknownRecord> {
|
||||
_raw: string;
|
||||
filePath: string;
|
||||
data: Record<string, unknown>;
|
||||
meta: MetaShape;
|
||||
}
|
||||
|
||||
function formatEntries(val: unknown): UnknownRecord[] {
|
||||
if (Array.isArray(val) && val.every((el) => typeof el === "object")) {
|
||||
return val;
|
||||
}
|
||||
|
||||
if (typeof val === "object" && val !== null) {
|
||||
return [val as UnknownRecord];
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
`Invalid data format. Entry files must define an object or array of objects, but found "${val}"`,
|
||||
);
|
||||
}
|
||||
|
||||
function parseYAMLEntries(text: string): UnknownRecord[] {
|
||||
const parsedDocs = YAML.parseAllDocuments(text);
|
||||
|
||||
if (parsedDocs.some((doc) => doc.toJS() === null)) {
|
||||
throw new Error("Encountered NULL resource");
|
||||
}
|
||||
|
||||
const errors = parsedDocs.flatMap((doc) => doc.errors);
|
||||
|
||||
if (errors.length > 0) {
|
||||
throw new Error(
|
||||
`Error parsing YAML resource: ${errors.map((e) => e.message).join(", ")}`,
|
||||
);
|
||||
}
|
||||
|
||||
const collection: UnknownRecord[] = parsedDocs.map((doc) => doc.toJS());
|
||||
return collection;
|
||||
}
|
||||
|
||||
function parseJSONEntries(text: string): UnknownRecord[] {
|
||||
return formatEntries(JSON.parse(text));
|
||||
}
|
||||
|
||||
function parseTOMLEntries(text: string): UnknownRecord[] {
|
||||
return formatEntries(TOML.parse(text));
|
||||
}
|
||||
|
||||
function parseMarkdownEntry(text: string, contentKey: string): UnknownRecord[] {
|
||||
return formatEntries(EMDY.parse(text, contentKey));
|
||||
}
|
||||
|
||||
interface ParseMuseFileOptions {
|
||||
contentKey?: string;
|
||||
}
|
||||
|
||||
export async function parseMuseFile(
|
||||
rawFilePath: string,
|
||||
{ contentKey = "content" }: ParseMuseFileOptions = {},
|
||||
): Promise<MuseEntry[]> {
|
||||
const { content, filePath, fileType } = await loadFileContent(rawFilePath);
|
||||
|
||||
const partial = {
|
||||
_raw: content,
|
||||
filePath,
|
||||
meta: {},
|
||||
};
|
||||
|
||||
if (fileType === "md") {
|
||||
return parseMarkdownEntry(content, contentKey).map((data) => ({
|
||||
...partial,
|
||||
data,
|
||||
}));
|
||||
}
|
||||
|
||||
if (fileType === "yaml") {
|
||||
return parseYAMLEntries(content).map((data) => ({
|
||||
...partial,
|
||||
data,
|
||||
}));
|
||||
}
|
||||
|
||||
if (fileType === "json") {
|
||||
return parseJSONEntries(content).map((data) => ({
|
||||
...partial,
|
||||
data,
|
||||
}));
|
||||
}
|
||||
|
||||
if (fileType === "toml") {
|
||||
return parseTOMLEntries(content).map((data) => ({
|
||||
...partial,
|
||||
data,
|
||||
}));
|
||||
}
|
||||
|
||||
throw new Error(`Unsupported file type: ${fileType}`);
|
||||
}
|
||||
254
src/core/sources.ts
Normal file
254
src/core/sources.ts
Normal file
|
|
@ -0,0 +1,254 @@
|
|||
import EMDY from "@endeavorance/emdy";
|
||||
import TOML from "smol-toml";
|
||||
import YAML from "yaml";
|
||||
import { loadFileContent } from "./files";
|
||||
import type { UnknownRecord } from "./types";
|
||||
import { Glob } from "bun";
|
||||
|
||||
export type SourceType = "file" | "url";
|
||||
|
||||
export interface MuseSource {
|
||||
location: string;
|
||||
type: SourceType;
|
||||
}
|
||||
|
||||
interface SourceOptions {
|
||||
cwd: string;
|
||||
contentKey: string;
|
||||
ignore: string[];
|
||||
}
|
||||
|
||||
export interface MuseEntry<MetaShape = UnknownRecord> {
|
||||
_raw: string;
|
||||
location: string;
|
||||
data: Record<string, unknown>;
|
||||
meta: MetaShape;
|
||||
source: MuseSource;
|
||||
}
|
||||
|
||||
function formatEntries(val: unknown): UnknownRecord[] {
|
||||
if (Array.isArray(val) && val.every((el) => typeof el === "object")) {
|
||||
return val;
|
||||
}
|
||||
|
||||
if (typeof val === "object" && val !== null) {
|
||||
return [val as UnknownRecord];
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
`Invalid data format. Entry files must define an object or array of objects, but found "${val}"`,
|
||||
);
|
||||
}
|
||||
|
||||
function parseYAMLEntries(text: string): UnknownRecord[] {
|
||||
const parsedDocs = YAML.parseAllDocuments(text);
|
||||
|
||||
if (parsedDocs.some((doc) => doc.toJS() === null)) {
|
||||
throw new Error("Encountered NULL resource");
|
||||
}
|
||||
|
||||
const errors = parsedDocs.flatMap((doc) => doc.errors);
|
||||
|
||||
if (errors.length > 0) {
|
||||
throw new Error(
|
||||
`Error parsing YAML resource: ${errors.map((e) => e.message).join(", ")}`,
|
||||
);
|
||||
}
|
||||
|
||||
const collection: UnknownRecord[] = parsedDocs.map((doc) => doc.toJS());
|
||||
return collection;
|
||||
}
|
||||
|
||||
function parseJSONEntries(text: string): UnknownRecord[] {
|
||||
return formatEntries(JSON.parse(text));
|
||||
}
|
||||
|
||||
function parseTOMLEntries(text: string): UnknownRecord[] {
|
||||
return formatEntries(TOML.parse(text));
|
||||
}
|
||||
|
||||
function parseMarkdownEntry(text: string, contentKey: string): UnknownRecord[] {
|
||||
return formatEntries(EMDY.parse(text, contentKey));
|
||||
}
|
||||
|
||||
export async function parseMuseFile(
|
||||
rawFilePath: string,
|
||||
{ contentKey = "content" }: SourceOptions,
|
||||
): Promise<MuseEntry[]> {
|
||||
const { content, filePath, fileType } = await loadFileContent(rawFilePath);
|
||||
|
||||
const partial = {
|
||||
_raw: content,
|
||||
source: {
|
||||
location: filePath,
|
||||
type: "file" as SourceType,
|
||||
},
|
||||
location: filePath,
|
||||
meta: {},
|
||||
};
|
||||
|
||||
if (fileType === "md") {
|
||||
return parseMarkdownEntry(content, contentKey).map((data) => ({
|
||||
...partial,
|
||||
data,
|
||||
}));
|
||||
}
|
||||
|
||||
if (fileType === "yaml") {
|
||||
return parseYAMLEntries(content).map((data) => ({
|
||||
...partial,
|
||||
data,
|
||||
}));
|
||||
}
|
||||
|
||||
if (fileType === "json") {
|
||||
return parseJSONEntries(content).map((data) => ({
|
||||
...partial,
|
||||
data,
|
||||
}));
|
||||
}
|
||||
|
||||
if (fileType === "toml") {
|
||||
return parseTOMLEntries(content).map((data) => ({
|
||||
...partial,
|
||||
data,
|
||||
}));
|
||||
}
|
||||
|
||||
throw new Error(`Unsupported file type: ${fileType}`);
|
||||
}
|
||||
|
||||
async function loadFromFileSource(
|
||||
source: MuseSource,
|
||||
options: SourceOptions,
|
||||
): Promise<MuseEntry[]> {
|
||||
const paths = Array.from(
|
||||
new Glob(source.location).scanSync({
|
||||
cwd: options.cwd,
|
||||
absolute: true,
|
||||
followSymlinks: true,
|
||||
onlyFiles: true,
|
||||
}),
|
||||
);
|
||||
|
||||
const filteredPaths = paths.filter((path) => !options.ignore.includes(path));
|
||||
|
||||
const entries: MuseEntry[] = [];
|
||||
for (const filePath of filteredPaths) {
|
||||
const fileEntries = await parseMuseFile(filePath, options);
|
||||
entries.push(...fileEntries);
|
||||
}
|
||||
|
||||
return entries;
|
||||
}
|
||||
|
||||
function getFileExtensionFromURL(url: string): string | null {
|
||||
const parsedUrl = new URL(url);
|
||||
|
||||
const pathname = parsedUrl.pathname;
|
||||
const filename = pathname.substring(pathname.lastIndexOf("/") + 1);
|
||||
const extension = filename.substring(filename.lastIndexOf(".") + 1);
|
||||
|
||||
return extension === filename ? null : extension;
|
||||
}
|
||||
|
||||
function mimeTypeToFileType(mimeType: string): string | null {
|
||||
switch (mimeType) {
|
||||
case "text/markdown":
|
||||
return "md";
|
||||
case "text/yaml":
|
||||
return "yaml";
|
||||
case "application/x-yaml":
|
||||
return "yaml";
|
||||
case "application/json":
|
||||
return "json";
|
||||
case "application/toml":
|
||||
return "toml";
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function loadFromURLSource(
|
||||
source: MuseSource,
|
||||
options: SourceOptions,
|
||||
): Promise<MuseEntry[]> {
|
||||
const { contentKey = "content" } = options;
|
||||
const response = await fetch(source.location);
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch URL: ${source.location}`);
|
||||
}
|
||||
const content = await response.text();
|
||||
const mimeType = response.headers.get("Content-Type") || "unknown";
|
||||
|
||||
const parseType =
|
||||
mimeTypeToFileType(mimeType) ??
|
||||
getFileExtensionFromURL(source.location) ??
|
||||
"unknown";
|
||||
|
||||
const partial = {
|
||||
_raw: content,
|
||||
source: {
|
||||
location: source.location,
|
||||
type: "url" as SourceType,
|
||||
},
|
||||
location: source.location,
|
||||
meta: {},
|
||||
};
|
||||
|
||||
if (parseType === "md") {
|
||||
return parseMarkdownEntry(content, contentKey).map((data) => ({
|
||||
...partial,
|
||||
data,
|
||||
}));
|
||||
}
|
||||
|
||||
if (parseType === "yaml") {
|
||||
return parseYAMLEntries(content).map((data) => ({
|
||||
...partial,
|
||||
data,
|
||||
}));
|
||||
}
|
||||
|
||||
if (parseType === "json") {
|
||||
return parseJSONEntries(content).map((data) => ({
|
||||
...partial,
|
||||
data,
|
||||
}));
|
||||
}
|
||||
|
||||
if (parseType === "toml") {
|
||||
return parseTOMLEntries(content).map((data) => ({
|
||||
...partial,
|
||||
data,
|
||||
}));
|
||||
}
|
||||
|
||||
// If it doesnt match one of these, try brute force parsing
|
||||
// and return the result if any of them work
|
||||
try {
|
||||
return parseMarkdownEntry(content, contentKey).map((data) => ({
|
||||
...partial,
|
||||
data,
|
||||
}));
|
||||
} catch {
|
||||
// noop
|
||||
}
|
||||
|
||||
throw new Error(`Unsupported MIME type from URL source: ${mimeType}`);
|
||||
}
|
||||
|
||||
export async function loadFromSource(
|
||||
source: MuseSource,
|
||||
options: SourceOptions,
|
||||
): Promise<MuseEntry[]> {
|
||||
switch (source.type) {
|
||||
case "file":
|
||||
return loadFromFileSource(source, options);
|
||||
case "url":
|
||||
return loadFromURLSource(source, options);
|
||||
default:
|
||||
throw new Error(`Unsupported source type: ${source.type}`);
|
||||
}
|
||||
}
|
||||
20
src/muse.ts
Normal file
20
src/muse.ts
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
import { loadBinding, type Binding } from "#core/binding";
|
||||
|
||||
export class Muse {
|
||||
bindingLoaded = false;
|
||||
preliminaryBinding: Partial<Binding> = {};
|
||||
additionalSources: string[] = [];
|
||||
additionalPluginPaths: string[] = [];
|
||||
|
||||
constructor(bindingConfig: Partial<Binding>) {
|
||||
this.preliminaryBinding = bindingConfig;
|
||||
}
|
||||
|
||||
source(newSource: string) {
|
||||
this.additionalSources.push(newSource);
|
||||
}
|
||||
|
||||
plugin(pluginPath: string) {
|
||||
this.additionalPluginPaths.push(pluginPath);
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue