From 4acd7281f550aa73b0c4c7f4627f374fa503e200 Mon Sep 17 00:00:00 2001 From: gene9831 Date: Wed, 17 Jun 2026 15:40:16 +0800 Subject: [PATCH 1/2] feat(kit): add skill loader foundation --- packages/kit/package.json | 14 +- .../kit/scripts/download-skill-fixtures.mjs | 128 +++++++++ packages/kit/src/node.ts | 8 + packages/kit/src/skills/loader/browser.ts | 66 +++++ packages/kit/src/skills/loader/definition.ts | 130 +++++++++ packages/kit/src/skills/loader/fs.ts | 58 ++++ packages/kit/src/skills/loader/github.ts | 199 ++++++++++++++ packages/kit/src/skills/loader/index.ts | 45 ++++ packages/kit/src/skills/loader/node.ts | 44 +++ packages/kit/src/skills/loader/type.ts | 108 ++++++++ packages/kit/src/skills/loader/utils.ts | 104 +++++++ packages/kit/src/skills/test/.gitignore | 1 + .../kit/src/skills/test/loaderBrowser.test.ts | 121 +++++++++ .../src/skills/test/loaderDefinition.test.ts | 255 ++++++++++++++++++ .../kit/src/skills/test/loaderNode.test.ts | 131 +++++++++ packages/kit/src/skills/types/index.ts | 112 ++++++++ packages/kit/src/skills/utils.ts | 26 ++ 17 files changed, 1547 insertions(+), 3 deletions(-) create mode 100644 packages/kit/scripts/download-skill-fixtures.mjs create mode 100644 packages/kit/src/node.ts create mode 100644 packages/kit/src/skills/loader/browser.ts create mode 100644 packages/kit/src/skills/loader/definition.ts create mode 100644 packages/kit/src/skills/loader/fs.ts create mode 100644 packages/kit/src/skills/loader/github.ts create mode 100644 packages/kit/src/skills/loader/index.ts create mode 100644 packages/kit/src/skills/loader/node.ts create mode 100644 packages/kit/src/skills/loader/type.ts create mode 100644 packages/kit/src/skills/loader/utils.ts create mode 100644 packages/kit/src/skills/test/.gitignore create mode 100644 packages/kit/src/skills/test/loaderBrowser.test.ts create mode 100644 packages/kit/src/skills/test/loaderDefinition.test.ts create mode 100644 packages/kit/src/skills/test/loaderNode.test.ts create mode 100644 packages/kit/src/skills/types/index.ts create mode 100644 packages/kit/src/skills/utils.ts diff --git a/packages/kit/package.json b/packages/kit/package.json index 2c4ba37a8..313eb3c88 100644 --- a/packages/kit/package.json +++ b/packages/kit/package.json @@ -44,6 +44,11 @@ "types": "./dist/core.d.ts", "import": "./dist/core.mjs", "require": "./dist/core.js" + }, + "./node": { + "types": "./dist/node.d.ts", + "import": "./dist/node.mjs", + "require": "./dist/node.js" } }, "files": [ @@ -51,8 +56,10 @@ ], "sideEffects": false, "scripts": { - "build": "tsup src/index.ts src/core.ts --format cjs,esm --dts --minify", - "dev": "tsup src/index.ts src/core.ts --format cjs,esm --dts --watch", + "build": "tsup src/index.ts src/core.ts src/node.ts --format cjs,esm --dts --minify", + "dev": "tsup src/index.ts src/core.ts src/node.ts --format cjs,esm --dts --watch", + "lint": "eslint src", + "pretest": "node scripts/download-skill-fixtures.mjs", "test": "vitest run", "test:watch": "vitest" }, @@ -69,6 +76,7 @@ "vue": ">=3.0.0" }, "dependencies": { - "idb": "^8.0.3" + "idb": "^8.0.3", + "yaml": "^2.8.3" } } diff --git a/packages/kit/scripts/download-skill-fixtures.mjs b/packages/kit/scripts/download-skill-fixtures.mjs new file mode 100644 index 000000000..f58cb140a --- /dev/null +++ b/packages/kit/scripts/download-skill-fixtures.mjs @@ -0,0 +1,128 @@ +import { mkdir, readFile, rm, writeFile } from 'node:fs/promises' +import { dirname, join } from 'node:path' +import { fileURLToPath } from 'node:url' + +const __dirname = dirname(fileURLToPath(import.meta.url)) +const cacheDirectory = join(__dirname, '../src/skills/test/.cache') + +const fixtures = [ + { + repo: 'openclaw/openclaw', + commit: '58672075219d09495de6489ad0821d276ac84f13', + sourcePath: 'skills/weather', + }, + { + repo: 'vuejs-ai/skills', + commit: 'b9d14d022da6a0a8bdcb824557f40bca6fbc1845', + sourcePath: 'skills/vue-best-practices', + }, +] + +const getFixtureTargetPath = (fixture) => { + const normalizedSourcePath = fixture.sourcePath.split('\\').join('/') + const targetName = normalizedSourcePath.split('/').filter(Boolean).at(-1) + + if (!targetName) { + throw new Error(`Invalid fixture source path: ${fixture.sourcePath}`) + } + + return join(cacheDirectory, targetName) +} + +const fetchJson = async (url) => { + const response = await fetch(url, { + headers: { + accept: 'application/vnd.github+json', + 'user-agent': '@opentiny/tiny-robot-kit skill fixture downloader', + }, + }) + + if (!response.ok) { + throw new Error(`Failed to fetch ${url}: ${response.status} ${response.statusText}`) + } + + return response.json() +} + +const fetchBytes = async (url) => { + const response = await fetch(url, { + headers: { + 'user-agent': '@opentiny/tiny-robot-kit skill fixture downloader', + }, + }) + + if (!response.ok) { + throw new Error(`Failed to download ${url}: ${response.status} ${response.statusText}`) + } + + return new Uint8Array(await response.arrayBuffer()) +} + +const getMarkerPath = (targetPath) => join(targetPath, '.fixture-source.json') + +const hasCurrentFixture = async (fixture) => { + const targetPath = getFixtureTargetPath(fixture) + + try { + const marker = JSON.parse(await readFile(getMarkerPath(targetPath), 'utf8')) + return ( + marker.repo === fixture.repo && + marker.commit === fixture.commit && + marker.sourcePath === fixture.sourcePath + ) + } catch { + return false + } +} + +const downloadDirectory = async (fixture, sourcePath, targetPath) => { + const url = new URL(`https://api.github.com/repos/${fixture.repo}/contents/${sourcePath}`) + url.searchParams.set('ref', fixture.commit) + + const entries = await fetchJson(url) + if (!Array.isArray(entries)) { + throw new Error(`Expected directory listing for ${sourcePath}`) + } + + for (const entry of entries) { + const entryTargetPath = join(targetPath, entry.name) + + if (entry.type === 'dir') { + await downloadDirectory(fixture, entry.path, entryTargetPath) + continue + } + + if (entry.type !== 'file' || !entry.download_url) { + continue + } + + await mkdir(dirname(entryTargetPath), { recursive: true }) + await writeFile(entryTargetPath, await fetchBytes(entry.download_url)) + } +} + +for (const fixture of fixtures) { + const targetPath = getFixtureTargetPath(fixture) + + if (await hasCurrentFixture(fixture)) { + console.log(`Skill fixture already cached: ${fixture.sourcePath}@${fixture.commit}`) + continue + } + + console.log(`Downloading skill fixture: ${fixture.sourcePath}@${fixture.commit}`) + await rm(targetPath, { recursive: true, force: true }) + await mkdir(targetPath, { recursive: true }) + await downloadDirectory(fixture, fixture.sourcePath, targetPath) + await writeFile( + getMarkerPath(targetPath), + `${JSON.stringify( + { + repo: fixture.repo, + commit: fixture.commit, + sourcePath: fixture.sourcePath, + }, + null, + 2, + )}\n`, + ) +} diff --git a/packages/kit/src/node.ts b/packages/kit/src/node.ts new file mode 100644 index 000000000..d72db2267 --- /dev/null +++ b/packages/kit/src/node.ts @@ -0,0 +1,8 @@ +export { loadSkill, loadSkillWithDetails } from './skills/loader/node' +export type { + FsSkillLoadOptions, + GithubSkillLoadOptions, + SkillLoadJob, + SkillLoadOptions, + SkillLoadResult, +} from './skills/loader/node' diff --git a/packages/kit/src/skills/loader/browser.ts b/packages/kit/src/skills/loader/browser.ts new file mode 100644 index 000000000..6ea10d103 --- /dev/null +++ b/packages/kit/src/skills/loader/browser.ts @@ -0,0 +1,66 @@ +import { isTextSkillFilePath, normalizeSkillPath, stripRootDirectory, throwIfSkillLoadCancelled } from './utils' +import type { BrowserSkillLoadOptions, LoadableSkillFile, SkillLoadContext } from './type' + +type FileWithRelativePath = File & { + webkitRelativePath?: string +} + +export async function loadBrowserSkillFiles( + options: BrowserSkillLoadOptions, + context: SkillLoadContext, +): Promise { + if ('fileList' in options && options.fileList) { + return Promise.all( + Array.from(options.fileList) + .filter((file): file is FileWithRelativePath => Boolean(file)) + .map((file) => loadBrowserFile(file, stripRootDirectory(file.webkitRelativePath || file.name), context)), + ) + } + + const result: LoadableSkillFile[] = [] + + const walk = async (directory: FileSystemDirectoryHandle, parentPath = '') => { + throwIfSkillLoadCancelled(context.signal) + const entries = ( + directory as FileSystemDirectoryHandle & { + entries(): AsyncIterable<[string, FileSystemDirectoryHandle | FileSystemFileHandle]> + } + ).entries() + + for await (const [name, handle] of entries) { + const path = parentPath ? `${parentPath}/${name}` : name + + if (handle.kind === 'directory') { + await walk(handle, path) + continue + } + + result.push(await loadBrowserFile(await handle.getFile(), path, context)) + } + } + + await walk(options.directoryHandle) + return result +} + +async function loadBrowserFile(file: File, rawPath: string, context: SkillLoadContext): Promise { + const path = normalizeSkillPath(rawPath) + + if (!path) { + throw new Error(`Invalid skill file path: ${rawPath}`) + } + + const kind = isTextSkillFilePath(path) ? 'text' : 'binary' + const content = kind === 'text' ? await file.text() : new Uint8Array(await file.arrayBuffer()) + + throwIfSkillLoadCancelled(context.signal) + + return { + path, + kind, + content, + mimeType: file.type, + size: file.size, + lastModified: file.lastModified, + } +} diff --git a/packages/kit/src/skills/loader/definition.ts b/packages/kit/src/skills/loader/definition.ts new file mode 100644 index 000000000..16f9f8661 --- /dev/null +++ b/packages/kit/src/skills/loader/definition.ts @@ -0,0 +1,130 @@ +import type { SkillResourceDescriptor } from '../types' +import type { LoadableSkillFile, SkillLoadBaseOptions, SkillLoadResult, SkillLoadWarning } from './type' +import { + getFallbackSkillName, + getRecord, + getString, + isTextSkillFilePath, + normalizeSkillPath, + parseMarkdownFrontmatter, + pushWarning, +} from './utils' + +export function createSkillDefinition(files: LoadableSkillFile[], options: SkillLoadBaseOptions): SkillLoadResult { + const warnings: SkillLoadWarning[] = [] + const entryFile = options.entryFile ?? 'SKILL.md' + const normalizedFiles = normalizeFiles(files, options, warnings) + const skillEntry = normalizedFiles.find((file) => file.path === entryFile) + + if (!skillEntry) { + throw new Error(`Skill entry file "${entryFile}" is missing.`) + } + + if (skillEntry.kind !== 'text') { + throw new Error(`Skill entry file "${entryFile}" must be a text file.`) + } + + const { frontmatter, body } = parseMarkdownFrontmatter(String(skillEntry.content)) + const instructions = body.trim() + + if (!instructions) { + throw new Error(`Skill entry file "${entryFile}" must contain instructions.`) + } + + const resources = normalizedFiles.flatMap((file) => { + if (file.path === entryFile) return [] + if (file.kind === 'text' && !isTextSkillFilePath(file.path)) { + pushWarning(warnings, options, { + code: 'unsupported-text-file-ignored', + message: 'Only markdown, text, and json files are converted to text skill files.', + path: file.path, + }) + return [] + } + + return [toSkillResource(file)] + }) + + return { + skill: { + name: getString(frontmatter.name) || getFallbackSkillName(entryFile), + description: getString(frontmatter.description) || '', + instructions, + resources: resources.length ? resources : undefined, + metadata: { + ...getRecord(frontmatter.metadata), + ...(getString(frontmatter.homepage) ? { homepage: getString(frontmatter.homepage) } : {}), + }, + }, + warnings, + } +} + +function normalizeFiles( + files: T[], + options: SkillLoadBaseOptions, + warnings: SkillLoadWarning[], +) { + const result: T[] = [] + const seenPaths = new Set() + + for (const file of files) { + const path = normalizeSkillPath(file.path) + + if (!path) { + pushWarning(warnings, options, { + code: 'invalid-path', + message: `Invalid skill file path: ${file.path}`, + path: file.path, + }) + continue + } + + if (seenPaths.has(path)) { + pushWarning(warnings, options, { + code: 'duplicate-path', + message: `Duplicate skill file path: ${path}`, + path, + }) + continue + } + + seenPaths.add(path) + result.push({ ...file, path }) + } + + return result.sort((a, b) => a.path.localeCompare(b.path)) +} + +function toSkillResource(file: LoadableSkillFile): SkillResourceDescriptor { + if (file.kind === 'text') { + const text = typeof file.content === 'string' ? file.content : new TextDecoder().decode(file.content) + + return { + path: file.path, + kind: file.kind, + resourceId: file.path, + mimeType: file.mimeType, + size: file.size, + lastModified: file.lastModified, + metadata: file.metadata, + text, + readText: async () => text, + readBinary: async () => new TextEncoder().encode(text), + } + } + + const binary = file.content instanceof Uint8Array ? file.content : new TextEncoder().encode(file.content) + + return { + path: file.path, + kind: file.kind, + resourceId: file.path, + mimeType: file.mimeType, + size: file.size, + lastModified: file.lastModified, + metadata: file.metadata, + binary, + readBinary: async () => binary, + } +} diff --git a/packages/kit/src/skills/loader/fs.ts b/packages/kit/src/skills/loader/fs.ts new file mode 100644 index 000000000..e8b27990d --- /dev/null +++ b/packages/kit/src/skills/loader/fs.ts @@ -0,0 +1,58 @@ +import { readFile, readdir, stat } from 'node:fs/promises' +import { join, relative } from 'node:path' +import type { FsSkillLoadOptions, LoadableSkillFile, SkillLoadContext } from './type' +import { isTextSkillFilePath, normalizeSkillPath, throwIfSkillLoadCancelled } from './utils' + +export async function loadFsSkillFiles( + options: FsSkillLoadOptions, + context: SkillLoadContext, +): Promise { + const ignored = new Set(options.ignoredDirectories ?? ['node_modules']) + const result: LoadableSkillFile[] = [] + + const walk = async (directory: string) => { + throwIfSkillLoadCancelled(context.signal) + const entries = await readdir(directory, { + withFileTypes: true, + }) + + for (const entry of entries) { + const fullPath = join(directory, entry.name) + + if (entry.isDirectory()) { + if (!ignored.has(entry.name) && !entry.name.startsWith('.')) { + await walk(fullPath) + } + continue + } + + if (!entry.isFile()) { + continue + } + + const path = normalizeSkillPath(relative(options.root, fullPath)) + + if (!path) { + continue + } + + const fileStat = await stat(fullPath) + const kind = isTextSkillFilePath(path) ? 'text' : 'binary' + const content = + kind === 'text' + ? await readFile(fullPath, { encoding: 'utf8', signal: context.signal }) + : new Uint8Array(await readFile(fullPath, { signal: context.signal })) + + result.push({ + path, + kind, + content, + size: fileStat.size, + lastModified: fileStat.mtimeMs, + }) + } + } + + await walk(options.root) + return result +} diff --git a/packages/kit/src/skills/loader/github.ts b/packages/kit/src/skills/loader/github.ts new file mode 100644 index 000000000..a04dfd4fc --- /dev/null +++ b/packages/kit/src/skills/loader/github.ts @@ -0,0 +1,199 @@ +import type { GithubSkillLoadOptions, LoadableSkillFile, SkillLoadContext } from './type' +import { isTextSkillFilePath, normalizeSkillPath, throwIfSkillLoadCancelled } from './utils' + +const userAgent = '@opentiny/tiny-robot-kit skill loader' +const maxGithubFetchRetries = 5 +const githubFetchRetryBaseDelay = 200 + +type GithubContentEntry = { + name: string + path: string + type: 'file' | 'dir' | 'symlink' | 'submodule' + size?: number + download_url?: string | null +} + +type GithubRepository = { + default_branch: string +} + +export async function loadGithubSkillFiles( + options: GithubSkillLoadOptions, + context: SkillLoadContext, +): Promise { + const result: LoadableSkillFile[] = [] + const ref = await resolveGithubRef(options, context) + const skillRoot = normalizeRepoPath(options.path) + + const walk = async (sourcePath: string) => { + const url = new URL(`https://api.github.com/repos/${options.repo}/contents/${sourcePath}`) + url.searchParams.set('ref', ref) + + const entries = await fetchGithubJson(url, context) + + if (!Array.isArray(entries)) { + throw new Error(`Expected directory listing for ${sourcePath}`) + } + + for (const entry of entries) { + if (entry.type === 'dir') { + if (entry.name.startsWith('.')) { + continue + } + + await walk(entry.path) + continue + } + + if (entry.type !== 'file' || !entry.download_url) { + continue + } + + const path = toSkillRelativePath(skillRoot, entry.path) + + if (!path) { + continue + } + + const kind = isTextSkillFilePath(path) ? 'text' : 'binary' + const bytes = await fetchGithubBytes(entry.download_url, context) + const content = kind === 'text' ? new TextDecoder().decode(bytes) : bytes + + result.push({ + path, + kind, + content, + size: entry.size, + }) + } + } + + await walk(skillRoot) + return result +} + +async function resolveGithubRef(options: GithubSkillLoadOptions, context: SkillLoadContext) { + if (options.ref) { + return options.ref + } + + const repository = await fetchGithubJson(`https://api.github.com/repos/${options.repo}`, context) + + if (!repository.default_branch) { + throw new Error(`Repository "${options.repo}" does not expose a default branch.`) + } + + return repository.default_branch +} + +async function fetchGithubJson(url: URL | string, context: SkillLoadContext): Promise { + const response = await fetchGithubWithRetry(url, context, { + headers: { + accept: 'application/vnd.github+json', + 'user-agent': userAgent, + }, + }) + + if (!response.ok) { + throw new Error(`Failed to fetch ${response}: ${response.status} ${response.statusText}`) + } + + return response.json() as Promise +} + +async function fetchGithubBytes(url: string, context: SkillLoadContext) { + const response = await fetchGithubWithRetry(url, context, { + headers: { + 'user-agent': userAgent, + }, + }) + + if (!response.ok) { + throw new Error(`Failed to download ${url}: ${response.status} ${response.statusText}`) + } + + const bytes = new Uint8Array(await response.arrayBuffer()) + return bytes +} + +async function fetchGithubWithRetry( + url: URL | string, + context: SkillLoadContext, + init: RequestInit, +): Promise { + let lastError: unknown + + for (let retryCount = 0; retryCount <= maxGithubFetchRetries; retryCount += 1) { + try { + const response = await fetch(url, { + ...init, + signal: context.signal, + }) + + if (response.ok || !shouldRetryGithubResponse(response)) { + return response + } + + lastError = new Error(`GitHub request failed with ${response.status} ${response.statusText}`) + } catch (error) { + if (context.signal.aborted) { + throw error + } + + lastError = error + } + + if (retryCount < maxGithubFetchRetries) { + await waitForGithubFetchRetry(retryCount, context) + } + } + + throw lastError +} + +function shouldRetryGithubResponse(response: Response) { + return response.status === 429 || response.status >= 500 +} + +function waitForGithubFetchRetry(retryCount: number, context: SkillLoadContext) { + throwIfSkillLoadCancelled(context.signal) + const delay = githubFetchRetryBaseDelay * 2 ** retryCount + + return new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + context.signal.removeEventListener('abort', onAbort) + resolve() + }, delay) + + const onAbort = () => { + clearTimeout(timeout) + reject(context.signal.reason) + } + + context.signal.addEventListener('abort', onAbort, { + once: true, + }) + }) +} + +function normalizeRepoPath(path: string) { + return path + .split('\\') + .join('/') + .replace(/^\/+|\/+$/g, '') +} + +function toSkillRelativePath(skillRoot: string, entryPath: string) { + const normalizedEntry = normalizeRepoPath(entryPath) + const prefix = `${skillRoot}/` + + if (normalizedEntry === skillRoot) { + return normalizeSkillPath(normalizedEntry.split('/').at(-1) || normalizedEntry) + } + + if (!normalizedEntry.startsWith(prefix)) { + return normalizeSkillPath(normalizedEntry) + } + + return normalizeSkillPath(normalizedEntry.slice(prefix.length)) +} diff --git a/packages/kit/src/skills/loader/index.ts b/packages/kit/src/skills/loader/index.ts new file mode 100644 index 000000000..3ffb75d43 --- /dev/null +++ b/packages/kit/src/skills/loader/index.ts @@ -0,0 +1,45 @@ +import { loadBrowserSkillFiles } from './browser' +import { createSkillDefinition } from './definition' +import { loadGithubSkillFiles } from './github' +import type { BrowserSkillLoadOptions, GithubSkillLoadOptions, SkillLoadJob, SkillLoadResult } from './type' +import { createSkillLoadJob, throwIfSkillLoadCancelled } from './utils' + +export type SkillLoadOptions = BrowserSkillLoadOptions | GithubSkillLoadOptions + +export function loadSkillWithDetails(options: SkillLoadOptions): SkillLoadJob { + return createSkillLoadJob(async (context) => { + const files = await (async () => { + switch (options.source) { + case 'browser': + return loadBrowserSkillFiles(options, context) + case 'github': + return loadGithubSkillFiles(options, context) + default: + throw new Error(`Unsupported skill source: ${(options as { source?: string }).source}`) + } + })() + + throwIfSkillLoadCancelled(context.signal) + return createSkillDefinition(files, options) + }) +} + +export function loadSkill(options: SkillLoadOptions): SkillLoadJob { + const detailsJob = loadSkillWithDetails(options) + const job = detailsJob.then((result) => result.skill) as SkillLoadJob + + job.cancel = () => { + detailsJob.cancel() + } + + return job +} + +export type { + BrowserSkillLoadOptions, + FsSkillLoadOptions, + GithubSkillLoadOptions, + SkillLoadJob, + SkillLoadProgressEvent, + SkillLoadResult, +} from './type' diff --git a/packages/kit/src/skills/loader/node.ts b/packages/kit/src/skills/loader/node.ts new file mode 100644 index 000000000..cfb059e72 --- /dev/null +++ b/packages/kit/src/skills/loader/node.ts @@ -0,0 +1,44 @@ +import { createSkillDefinition } from './definition' +import { loadFsSkillFiles } from './fs' +import { loadGithubSkillFiles } from './github' +import type { FsSkillLoadOptions, GithubSkillLoadOptions, SkillLoadJob, SkillLoadResult } from './type' +import { createSkillLoadJob, throwIfSkillLoadCancelled } from './utils' + +export type SkillLoadOptions = FsSkillLoadOptions | GithubSkillLoadOptions + +export function loadSkillWithDetails(options: SkillLoadOptions): SkillLoadJob { + return createSkillLoadJob(async (context) => { + const files = await (async () => { + switch (options.source) { + case 'fs': + return loadFsSkillFiles(options, context) + case 'github': + return loadGithubSkillFiles(options, context) + default: + throw new Error(`Unsupported skill source: ${(options as { source?: string }).source}`) + } + })() + + throwIfSkillLoadCancelled(context.signal) + return createSkillDefinition(files, options) + }) +} + +export function loadSkill(options: SkillLoadOptions): SkillLoadJob { + const detailsJob = loadSkillWithDetails(options) + const job = detailsJob.then((result) => result.skill) as SkillLoadJob + + job.cancel = () => { + detailsJob.cancel() + } + + return job +} + +export type { + FsSkillLoadOptions, + GithubSkillLoadOptions, + SkillLoadJob, + SkillLoadProgressEvent, + SkillLoadResult, +} from './type' diff --git a/packages/kit/src/skills/loader/type.ts b/packages/kit/src/skills/loader/type.ts new file mode 100644 index 000000000..101a43a29 --- /dev/null +++ b/packages/kit/src/skills/loader/type.ts @@ -0,0 +1,108 @@ +import type { SkillDefinition, SkillFileKind } from '../types' + +export type SkillLoadWarning = { + code: string + message: string + path?: string +} + +export type SkillLoadResult = { + skill: SkillDefinition + warnings: SkillLoadWarning[] +} + +export type SkillLoadJob = Promise & { + cancel(): void +} + +export type SkillLoadContext = { + signal: AbortSignal +} + +export type SkillLoadProgressPhase = 'discover' | 'read' | 'download' | 'parse' | 'store' | 'complete' + +export interface SkillLoadProgressEvent { + /** + * 当前加载阶段。不同 source 可按自身能力选择上报阶段。 + */ + phase: SkillLoadProgressPhase + /** + * 当前阶段已处理的数量。 + */ + loaded: number + /** + * 当前阶段总数量；目录遍历或远端下载时可能未知。 + */ + total?: number + /** + * 当前处理的 skill 内相对路径。 + */ + path?: string + /** + * 面向调用方展示或记录的补充说明。 + */ + message?: string +} + +export type SkillLoadBaseOptions = { + /** + * skill 入口文件名。 + */ + entryFile?: string + /** + * 启用后，非致命问题会直接抛出为错误。 + */ + strict?: boolean + /** + * @experimental + * + * 加载进度回调预留。当前 loader 暂未实现进度事件上报。 + */ + onProgress?: (event: SkillLoadProgressEvent) => void +} + +export type LoadableSkillFile = { + path: string + kind: SkillFileKind + content: string | Uint8Array + mimeType?: string + size?: number + lastModified?: number + metadata?: Record +} + +export type BrowserSkillLoadOptions = SkillLoadBaseOptions & + ( + | { + source: 'browser' + fileList: ArrayLike + directoryHandle?: never + } + | { + source: 'browser' + directoryHandle: FileSystemDirectoryHandle + fileList?: never + } + ) + +export type FsSkillLoadOptions = SkillLoadBaseOptions & { + source: 'fs' + root: string + ignoredDirectories?: string[] +} + +export type GithubSkillLoadOptions = SkillLoadBaseOptions & { + source: 'github' + /** + * GitHub 仓库，格式为 `owner/repo`。 + */ + repo: string + /** + * 分支、标签或 commit SHA。省略时使用仓库默认分支。 + */ + ref?: string + /** + * 仓库内 skill 根目录（含 SKILL.md），例如 `skills/weather`。 + */ + path: string +} diff --git a/packages/kit/src/skills/loader/utils.ts b/packages/kit/src/skills/loader/utils.ts new file mode 100644 index 000000000..9e095ed7d --- /dev/null +++ b/packages/kit/src/skills/loader/utils.ts @@ -0,0 +1,104 @@ +import { parse as parseYaml } from 'yaml' +import { getExtension, isTextSkillFilePath, normalizeSkillPath } from '../utils' +import type { SkillLoadBaseOptions, SkillLoadContext, SkillLoadJob, SkillLoadWarning } from './type' + +class SkillLoadCancelledError extends Error { + constructor() { + super('Skill load was cancelled.') + this.name = 'SkillLoadCancelledError' + } +} + +export function throwIfSkillLoadCancelled(signal: AbortSignal) { + if (!signal.aborted) { + return + } + + if (signal.reason instanceof Error) { + throw signal.reason + } + + throw new SkillLoadCancelledError() +} + +const normalizeAbortError = (error: unknown): never => { + if (error instanceof DOMException && error.name === 'AbortError') { + throw new SkillLoadCancelledError() + } + + throw error +} + +export function createSkillLoadJob(load: (context: SkillLoadContext) => Promise): SkillLoadJob { + const controller = new AbortController() + + const job = (async () => { + try { + throwIfSkillLoadCancelled(controller.signal) + const result = await load({ signal: controller.signal }) + throwIfSkillLoadCancelled(controller.signal) + return result + } catch (error) { + normalizeAbortError(error) + } + })() as SkillLoadJob + + job.cancel = () => { + controller.abort(new SkillLoadCancelledError()) + } + + return job +} + +export { getExtension, isTextSkillFilePath, normalizeSkillPath } + +export function parseMarkdownFrontmatter(content: string) { + if (!content.startsWith('---')) { + return { + frontmatter: {} as Record, + body: content, + } + } + + const endIndex = content.indexOf('\n---', 3) + + if (endIndex === -1) { + return { + frontmatter: {} as Record, + body: content, + } + } + + const rawFrontmatter = content.slice(3, endIndex).trim() + const body = content.slice(endIndex + 4) + + return { + frontmatter: getRecord(parseYaml(rawFrontmatter)) ?? {}, + body, + } +} + +export function stripRootDirectory(path: string) { + const normalized = path.split('\\').join('/') + const parts = normalized.split('/').filter(Boolean) + return parts.length <= 1 ? normalized : parts.slice(1).join('/') +} + +export function getFallbackSkillName(entryFile: string) { + const filename = entryFile.split('/').at(-1) || entryFile + const ext = getExtension(filename) + return ext ? filename.slice(0, -ext.length) : filename +} + +export function pushWarning(warnings: SkillLoadWarning[], options: SkillLoadBaseOptions, warning: SkillLoadWarning) { + if (options.strict) { + throw new Error(warning.path ? `${warning.path}: ${warning.message}` : warning.message) + } + + warnings.push(warning) +} + +export const getString = (value: unknown) => (typeof value === 'string' ? value : undefined) + +export const getRecord = (value: unknown) => + value && typeof value === 'object' && !Array.isArray(value) ? (value as Record) : undefined diff --git a/packages/kit/src/skills/test/.gitignore b/packages/kit/src/skills/test/.gitignore new file mode 100644 index 000000000..ceddaa37f --- /dev/null +++ b/packages/kit/src/skills/test/.gitignore @@ -0,0 +1 @@ +.cache/ diff --git a/packages/kit/src/skills/test/loaderBrowser.test.ts b/packages/kit/src/skills/test/loaderBrowser.test.ts new file mode 100644 index 000000000..6c14c9e5c --- /dev/null +++ b/packages/kit/src/skills/test/loaderBrowser.test.ts @@ -0,0 +1,121 @@ +import { describe, expect, it } from 'vitest' +import { loadSkill, loadSkillWithDetails } from '../loader' + +type TestFile = File & { + webkitRelativePath?: string +} + +const createTestFile = (path: string, content: string | Uint8Array, type = 'text/plain'): TestFile => { + const fileContent: BlobPart = typeof content === 'string' ? content : new Uint8Array(content) + const file = new File([fileContent], path.split('/').at(-1) ?? path, { + type, + lastModified: 123, + }) as TestFile + + file.webkitRelativePath = path + return file +} + +describe('browser loadSkill', () => { + it('loads fileList skills and strips the root directory from resource paths', async () => { + const loadedSkill = await loadSkill({ + source: 'browser', + fileList: [ + createTestFile( + 'weather/SKILL.md', + ['---', 'name: weather', 'description: Weather skill', '---', '', '# Weather Skill'].join('\n'), + 'text/markdown', + ), + createTestFile('weather/references/usage.md', '# Usage', 'text/markdown'), + ], + }) + + expect(loadedSkill.name).toBe('weather') + expect(loadedSkill.instructions).toContain('# Weather Skill') + expect(loadedSkill.resources?.map((resource) => resource.path)).toEqual(['references/usage.md']) + await expect(loadedSkill.resources?.[0]?.readText?.()).resolves.toBe('# Usage') + }) + + it('loads binary browser resources', async () => { + const image = new Uint8Array([1, 2, 3]) + const loadedSkill = await loadSkill({ + source: 'browser', + fileList: [ + createTestFile( + 'binary-skill/SKILL.md', + ['---', 'name: binary-skill', 'description: Binary skill', '---', '', '# Binary Skill'].join('\n'), + 'text/markdown', + ), + createTestFile('binary-skill/assets/icon.png', image, 'image/png'), + ], + }) + + expect(loadedSkill.resources).toEqual([ + expect.objectContaining({ + path: 'assets/icon.png', + kind: 'binary', + mimeType: 'image/png', + binary: image, + }), + ]) + }) + + it('uses file names when webkitRelativePath is not available', async () => { + const file = createTestFile( + 'SKILL.md', + ['---', 'name: single-file', 'description: Single file skill', '---', '', '# Single'].join('\n'), + 'text/markdown', + ) + file.webkitRelativePath = '' + + const loadedSkill = await loadSkill({ + source: 'browser', + fileList: [file], + }) + + expect(loadedSkill.name).toBe('single-file') + expect(loadedSkill.resources).toBeUndefined() + }) + + it('loads skill details with warnings', async () => { + const loadedSkill = await loadSkillWithDetails({ + source: 'browser', + fileList: [ + createTestFile( + 'weather/SKILL.md', + ['---', 'name: weather', 'description: Weather skill', '---', '', '# Weather Skill'].join('\n'), + 'text/markdown', + ), + ], + }) + + expect(loadedSkill.skill.name).toBe('weather') + expect(loadedSkill.warnings).toEqual([]) + }) + + it('returns a cancellable load job', async () => { + let releaseWait!: () => void + const waitForText = new Promise((resolve) => { + releaseWait = () => resolve('# Cancelled') + }) + const file = createTestFile( + 'cancelled/SKILL.md', + ['---', 'name: cancelled', 'description: Cancelled skill', '---', '', '# Cancelled'].join('\n'), + 'text/markdown', + ) + + file.text = async () => waitForText + + const job = loadSkill({ + source: 'browser', + fileList: [file], + }) + + job.cancel() + releaseWait() + + await expect(job).rejects.toMatchObject({ + name: 'SkillLoadCancelledError', + }) + }) +}) diff --git a/packages/kit/src/skills/test/loaderDefinition.test.ts b/packages/kit/src/skills/test/loaderDefinition.test.ts new file mode 100644 index 000000000..a2f69ba0d --- /dev/null +++ b/packages/kit/src/skills/test/loaderDefinition.test.ts @@ -0,0 +1,255 @@ +import { describe, expect, it } from 'vitest' +import { createSkillDefinition } from '../loader/definition' + +describe('createSkillDefinition', () => { + it('creates a SkillDefinition from skill entry frontmatter and instructions', () => { + const loadedSkill = createSkillDefinition( + [ + { + path: 'SKILL.md', + kind: 'text', + content: [ + '---', + 'name: weather', + 'description: Get weather information', + 'homepage: https://wttr.in/:help', + '---', + '', + '# Weather Skill', + ].join('\n'), + }, + ], + {}, + ) + const { skill } = loadedSkill + + expect(skill.name).toBe('weather') + expect(skill.description).toContain('weather') + expect(skill.instructions).toContain('# Weather Skill') + expect(skill.metadata?.homepage).toBe('https://wttr.in/:help') + expect(loadedSkill.warnings).toEqual([]) + }) + + it('creates resources from multi-file skill references', () => { + const loadedSkill = createSkillDefinition( + [ + { + path: 'SKILL.md', + kind: 'text', + content: [ + '---', + 'name: vue-best-practices', + 'description: Vue.js tasks', + 'metadata:', + ' author: github.com/vuejs-ai', + ' version: 18.0.0', + '---', + '', + '# Vue Best Practices Workflow', + ].join('\n'), + }, + { + path: 'references/reactivity.md', + kind: 'text', + content: '# Reactivity', + }, + { + path: 'references/sfc.md', + kind: 'text', + content: '# SFC', + }, + ], + {}, + ) + const { skill } = loadedSkill + + expect(skill.name).toBe('vue-best-practices') + expect(skill.description).toContain('Vue.js tasks') + expect(skill.instructions).toContain('# Vue Best Practices Workflow') + expect(skill.metadata).toMatchObject({ + author: 'github.com/vuejs-ai', + version: '18.0.0', + }) + expect(skill.resources).toHaveLength(2) + expect(skill.resources?.map((file) => file.path)).toEqual(['references/reactivity.md', 'references/sfc.md']) + expect(skill.resources?.find((file) => file.path === 'references/reactivity.md')).toMatchObject({ + path: 'references/reactivity.md', + kind: 'text', + text: expect.stringContaining('# Reactivity'), + }) + expect(loadedSkill.warnings).toEqual([]) + }) + + it('keeps binary files as skill resources', () => { + const image = new Uint8Array([1, 2, 3]) + const loadedSkill = createSkillDefinition( + [ + { + path: 'SKILL.md', + kind: 'text', + content: [ + '---', + 'name: binary-skill', + 'description: Skill with binary assets', + '---', + '', + '# Binary Skill', + ].join('\n'), + }, + { + path: 'assets/icon.png', + kind: 'binary', + content: image, + mimeType: 'image/png', + size: image.byteLength, + lastModified: 123, + }, + ], + {}, + ) + + expect(loadedSkill.skill.resources).toEqual([ + expect.objectContaining({ + path: 'assets/icon.png', + kind: 'binary', + binary: image, + mimeType: 'image/png', + size: 3, + lastModified: 123, + }), + ]) + expect(loadedSkill.warnings).toEqual([]) + }) + + it('throws when the entry file is missing', () => { + expect(() => createSkillDefinition([], {})).toThrow('Skill entry file "SKILL.md" is missing.') + }) + + it('throws when the entry file is binary', () => { + expect(() => + createSkillDefinition( + [ + { + path: 'SKILL.md', + kind: 'binary', + content: new Uint8Array([1, 2, 3]), + }, + ], + {}, + ), + ).toThrow('Skill entry file "SKILL.md" must be a text file.') + }) + + it('throws when the entry file has no instructions', () => { + expect(() => + createSkillDefinition( + [ + { + path: 'SKILL.md', + kind: 'text', + content: ['---', 'name: empty-skill', 'description: Empty skill', '---', ''].join('\n'), + }, + ], + {}, + ), + ).toThrow('Skill entry file "SKILL.md" must contain instructions.') + }) + + it('reports duplicate and unsupported file warnings', () => { + const loadedSkill = createSkillDefinition( + [ + { + path: 'SKILL.md', + kind: 'text', + content: ['---', 'name: warning-skill', 'description: Warning skill', '---', '', '# Warning'].join('\n'), + }, + { + path: 'notes.md', + kind: 'text', + content: 'first', + }, + { + path: 'notes.md', + kind: 'text', + content: 'second', + }, + { + path: 'script.ts', + kind: 'text', + content: 'export {}', + }, + ], + {}, + ) + + expect(loadedSkill.warnings).toEqual([ + { + code: 'duplicate-path', + message: 'Duplicate skill file path: notes.md', + path: 'notes.md', + }, + { + code: 'unsupported-text-file-ignored', + message: 'Only markdown, text, and json files are converted to text skill files.', + path: 'script.ts', + }, + ]) + expect(loadedSkill.skill.resources?.map((file) => file.path)).toEqual(['notes.md']) + }) + + it('throws warnings as errors in strict mode', () => { + expect(() => + createSkillDefinition( + [ + { + path: 'SKILL.md', + kind: 'text', + content: ['---', 'name: strict-skill', 'description: Strict skill', '---', '', '# Strict'].join('\n'), + }, + { + path: 'notes.md', + kind: 'text', + content: 'first', + }, + { + path: 'notes.md', + kind: 'text', + content: 'second', + }, + ], + { strict: true }, + ), + ).toThrow('notes.md: Duplicate skill file path: notes.md') + }) + + it('keeps json files as regular skill resources', () => { + const loadedSkill = createSkillDefinition( + [ + { + path: 'SKILL.md', + kind: 'text', + content: ['---', 'name: tool-skill', 'description: Tool skill', '---', '', '# Tool'].join('\n'), + }, + { + path: 'references/weather-format.json', + kind: 'text', + content: JSON.stringify({ + type: 'function', + function: { + name: 'run_tool', + description: 'Run tool', + parameters: { + type: 'object', + properties: {}, + }, + }, + }), + }, + ], + {}, + ) + + expect(loadedSkill.skill.resources?.map((file) => file.path)).toEqual(['references/weather-format.json']) + expect(loadedSkill.warnings).toEqual([]) + }) +}) diff --git a/packages/kit/src/skills/test/loaderNode.test.ts b/packages/kit/src/skills/test/loaderNode.test.ts new file mode 100644 index 000000000..f4da63bfc --- /dev/null +++ b/packages/kit/src/skills/test/loaderNode.test.ts @@ -0,0 +1,131 @@ +import { fileURLToPath } from 'node:url' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { loadSkill, loadSkillWithDetails } from '../loader/node' + +const createResponse = ( + status: number, + body: unknown, + statusText = status >= 200 && status < 300 ? 'OK' : 'Server Error', +) => + ({ + ok: status >= 200 && status < 300, + status, + statusText, + json: async () => body, + arrayBuffer: async () => { + const bytes = new TextEncoder().encode(String(body)) + return bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength) + }, + }) as Response + +describe('node loadSkill', () => { + afterEach(() => { + vi.useRealTimers() + vi.unstubAllGlobals() + }) + + it('loads weather skill directory as SkillDefinition', async () => { + const root = fileURLToPath(new URL('./.cache/weather', import.meta.url)) + const loadedSkill = await loadSkill({ source: 'fs', root }) + + expect(loadedSkill.name).toBe('weather') + expect(loadedSkill.description).toContain('weather') + expect(loadedSkill.instructions).toContain('# Weather Skill') + expect(loadedSkill.metadata?.homepage).toBe('https://wttr.in/:help') + }) + + it('loads multi-file skill references as resources', async () => { + const root = fileURLToPath(new URL('./.cache/vue-best-practices', import.meta.url)) + const loadedSkill = await loadSkill({ source: 'fs', root }) + + expect(loadedSkill.name).toBe('vue-best-practices') + expect(loadedSkill.description).toContain('Vue.js tasks') + expect(loadedSkill.instructions).toContain('# Vue Best Practices Workflow') + expect(loadedSkill.metadata).toMatchObject({ + author: 'github.com/vuejs-ai', + version: '18.0.0', + }) + expect(loadedSkill.resources).toBeDefined() + expect(loadedSkill.resources?.map((file) => file.path)).toEqual( + expect.arrayContaining([ + 'references/reactivity.md', + 'references/sfc.md', + 'references/component-data-flow.md', + 'references/composables.md', + ]), + ) + expect(loadedSkill.resources?.find((file) => file.path === 'references/reactivity.md')).toMatchObject({ + path: 'references/reactivity.md', + kind: 'text', + text: expect.stringContaining('# Reactivity'), + }) + }) + + it('loads weather skill from GitHub over the network', async () => { + const expectedRoot = fileURLToPath(new URL('./.cache/weather', import.meta.url)) + const expectedSkill = await loadSkill({ source: 'fs', root: expectedRoot }) + const loadedSkill = await loadSkill({ + source: 'github', + repo: 'openclaw/openclaw', + ref: '58672075219d09495de6489ad0821d276ac84f13', + path: 'skills/weather', + }) + + expect(loadedSkill.name).toBe(expectedSkill.name) + expect(loadedSkill.description).toBe(expectedSkill.description) + expect(loadedSkill.instructions).toBe(expectedSkill.instructions) + expect(loadedSkill.metadata).toEqual(expectedSkill.metadata) + const expectedResources = expectedSkill.resources?.filter((resource) => resource.path !== '.fixture-source.json') + expect(loadedSkill.resources).toEqual(expectedResources?.length ? expectedResources : undefined) + }) + + it('retries transient GitHub fetch failures with exponential backoff', async () => { + vi.useFakeTimers() + const skillMarkdown = ['---', 'name: retry-weather', 'description: Retry weather skill', '---', '', '# Retry'].join( + '\n', + ) + const fetch = vi + .fn() + .mockResolvedValueOnce(createResponse(500, { message: 'server error' })) + .mockResolvedValueOnce( + createResponse(200, [ + { + name: 'SKILL.md', + path: 'skills/weather/SKILL.md', + type: 'file', + size: skillMarkdown.length, + download_url: 'https://raw.githubusercontent.com/openclaw/openclaw/SKILL.md', + }, + ]), + ) + .mockResolvedValueOnce(createResponse(502, 'bad gateway')) + .mockResolvedValueOnce(createResponse(503, 'unavailable')) + .mockResolvedValueOnce(createResponse(200, skillMarkdown)) + + vi.stubGlobal('fetch', fetch) + + const job = loadSkill({ + source: 'github', + repo: 'openclaw/openclaw', + ref: '58672075219d09495de6489ad0821d276ac84f13', + path: 'skills/weather', + }) + + await vi.advanceTimersByTimeAsync(200 + 200 + 400) + + await expect(job).resolves.toMatchObject({ + name: 'retry-weather', + description: 'Retry weather skill', + instructions: expect.stringContaining('# Retry'), + }) + expect(fetch).toHaveBeenCalledTimes(5) + }) + + it('loads skill details with warnings', async () => { + const root = fileURLToPath(new URL('./.cache/weather', import.meta.url)) + const loadedSkill = await loadSkillWithDetails({ source: 'fs', root }) + + expect(loadedSkill.skill.name).toBe('weather') + expect(loadedSkill.warnings).toEqual([]) + }) +}) diff --git a/packages/kit/src/skills/types/index.ts b/packages/kit/src/skills/types/index.ts new file mode 100644 index 000000000..476aae458 --- /dev/null +++ b/packages/kit/src/skills/types/index.ts @@ -0,0 +1,112 @@ +export type SkillFileKind = 'text' | 'binary' + +/** + * skill 文件基础描述。 + */ +interface SkillFileDescriptor { + /** + * skill 内相对路径。 + */ + path: string + /** + * 文件类型。 + */ + kind: SkillFileKind + /** + * 文件 MIME 类型。 + */ + mimeType?: string + /** + * 文件大小（字节）。 + */ + size?: number + /** + * 最后修改时间。 + */ + lastModified?: number + /** + * 自定义元数据。 + */ + metadata?: Record +} + +interface SkillResourceBase extends Omit { + /** + * 文件类型。 + */ + kind: K + /** 资源 ID，用于在 storage 内定位文件内容。 */ + resourceId: string +} + +type SkillTextResourceContent = + | { + /** 已加载的文本内容，适合内存中的完整 skill。 */ + text: string + /** 读取文本内容。 */ + readText?: () => Promise + } + | { + /** 已加载的文本内容，适合内存中的完整 skill。 */ + text?: string + /** 读取文本内容。 */ + readText: () => Promise + } + +type SkillBinaryResourceContent = + | { + /** 已加载的二进制内容，适合内存中的完整 skill。 */ + binary: Uint8Array + /** 读取二进制内容。 */ + readBinary?: () => Promise + } + | { + /** 已加载的二进制内容，适合内存中的完整 skill。 */ + binary?: Uint8Array + /** 读取二进制内容。 */ + readBinary: () => Promise + } + +/** skill 能力定义。 */ +export interface SkillDefinition { + /** + * 唯一 skill 名称。 + */ + name: string + /** + * skill 描述。 + */ + description: string + /** + * 注入模型的 instructions。 + */ + instructions: string + /** + * skill 资源描述。 + */ + resources?: SkillResourceDescriptor[] + /** + * 自定义 metadata。 + */ + metadata?: Record +} + +/** selection 阶段暴露给模型的 skill 候选项。 */ +export type SkillCandidate = Pick + +/** skill 资源文件描述；文本资源至少包含 text/readText 之一，二进制资源至少包含 binary/readBinary 之一。 */ +export type SkillResourceDescriptor = + | (SkillResourceBase<'text'> & + SkillTextResourceContent & { + /** 已加载的二进制内容，适合内存中的完整 skill。 */ + binary?: Uint8Array + /** 读取二进制内容。 */ + readBinary?: () => Promise + }) + | (SkillResourceBase<'binary'> & + SkillBinaryResourceContent & { + /** 已加载的文本内容，适合内存中的完整 skill。 */ + text?: string + /** 读取文本内容。 */ + readText?: () => Promise + }) diff --git a/packages/kit/src/skills/utils.ts b/packages/kit/src/skills/utils.ts new file mode 100644 index 000000000..044b38b0e --- /dev/null +++ b/packages/kit/src/skills/utils.ts @@ -0,0 +1,26 @@ +export const normalizeSkillPath = (path: string) => { + const normalized = path + .split('\\') + .join('/') + .replace(/^\.\/+/, '') + + if (!normalized || normalized.startsWith('/') || normalized.includes('\0')) { + return null + } + + if (normalized.split('/').some((part) => part === '..' || part === '')) { + return null + } + + return normalized +} + +export const isTextSkillFilePath = (path: string) => { + return ['.md', '.txt', '.json'].includes(getExtension(path)) +} + +export const getExtension = (path: string) => { + const filename = path.split('/').at(-1) || path + const index = filename.lastIndexOf('.') + return index === -1 ? '' : filename.slice(index).toLowerCase() +} From 2017713adb677b38d0cdec57007cd43329d11718 Mon Sep 17 00:00:00 2001 From: gene9831 Date: Wed, 17 Jun 2026 16:12:25 +0800 Subject: [PATCH 2/2] feat(kit): add skill storage providers --- packages/kit/package.json | 1 + packages/kit/src/node.ts | 1 + packages/kit/src/skills/storage/fs.ts | 328 +++++++++++++++++ .../kit/src/skills/storage/importSkill.ts | 26 ++ packages/kit/src/skills/storage/index.ts | 18 + packages/kit/src/skills/storage/indexedDB.ts | 341 ++++++++++++++++++ packages/kit/src/skills/storage/memory.ts | 117 ++++++ packages/kit/src/skills/storage/node.ts | 18 + packages/kit/src/skills/storage/types.ts | 48 +++ .../kit/src/skills/test/fsStorage.test.ts | 106 ++++++ .../src/skills/test/indexedDBStorage.test.ts | 180 +++++++++ .../kit/src/skills/test/memoryStorage.test.ts | 130 +++++++ 12 files changed, 1314 insertions(+) create mode 100644 packages/kit/src/skills/storage/fs.ts create mode 100644 packages/kit/src/skills/storage/importSkill.ts create mode 100644 packages/kit/src/skills/storage/index.ts create mode 100644 packages/kit/src/skills/storage/indexedDB.ts create mode 100644 packages/kit/src/skills/storage/memory.ts create mode 100644 packages/kit/src/skills/storage/node.ts create mode 100644 packages/kit/src/skills/storage/types.ts create mode 100644 packages/kit/src/skills/test/fsStorage.test.ts create mode 100644 packages/kit/src/skills/test/indexedDBStorage.test.ts create mode 100644 packages/kit/src/skills/test/memoryStorage.test.ts diff --git a/packages/kit/package.json b/packages/kit/package.json index 313eb3c88..2f54e6b24 100644 --- a/packages/kit/package.json +++ b/packages/kit/package.json @@ -67,6 +67,7 @@ "license": "MIT", "devDependencies": { "@types/node": "^22.13.17", + "fake-indexeddb": "^6.2.5", "openai": "^6.34.0", "tsup": "^8.0.1", "typescript": "^5.8.2", diff --git a/packages/kit/src/node.ts b/packages/kit/src/node.ts index d72db2267..c0245ae46 100644 --- a/packages/kit/src/node.ts +++ b/packages/kit/src/node.ts @@ -6,3 +6,4 @@ export type { SkillLoadOptions, SkillLoadResult, } from './skills/loader/node' +export * from './skills/storage/node' diff --git a/packages/kit/src/skills/storage/fs.ts b/packages/kit/src/skills/storage/fs.ts new file mode 100644 index 000000000..445d6c6dc --- /dev/null +++ b/packages/kit/src/skills/storage/fs.ts @@ -0,0 +1,328 @@ +import { mkdir, readFile, readdir, rm, stat, writeFile } from 'node:fs/promises' +import { dirname, join, relative } from 'node:path' +import { stringify as stringifyYaml } from 'yaml' +import { loadSkillWithDetails } from '../loader/node' +import type { SkillLoadOptions } from '../loader/node' +import { + getRecord, + getString, + isTextSkillFilePath, + normalizeSkillPath, + parseMarkdownFrontmatter, +} from '../loader/utils' +import type { SkillDefinition, SkillResourceDescriptor } from '../types' +import { createImportSkill } from './importSkill' +import type { SkillStorage } from './types' + +/** 一个标准 skill 目录集合的文件系统 storage。 */ +export interface FsSkillStorageOptions { + root: string + /** 只读 storage 不允许 add/import 写入，也不允许 delete。 */ + readonly?: boolean +} + +const entryFile = 'SKILL.md' +const importSkill = createImportSkill(loadSkillWithDetails) + +export class FsSkillStorage implements SkillStorage { + readonly root: string + readonly readonly: boolean + + constructor(options: FsSkillStorageOptions) { + this.root = options.root + this.readonly = options.readonly ?? false + } + + async add(skill: SkillDefinition) { + this.assertWritable() + + const directory = this.getSkillDirectory(skill.name) + await rm(directory, { + recursive: true, + force: true, + }) + await mkdir(directory, { + recursive: true, + }) + await writeFile(join(directory, entryFile), serializeSkillEntry(skill), 'utf8') + + for (const resource of skill.resources ?? []) { + await this.writeResource(directory, resource) + } + + const storedSkill = await this.get(skill.name) + if (!storedSkill) { + throw new Error(`Failed to store skill "${skill.name}".`) + } + + return storedSkill + } + + async get(name: string) { + const directory = this.getSkillDirectory(name) + + try { + return await this.readSkillDirectory(directory) + } catch (error) { + if (isFileNotFoundError(error)) { + return undefined + } + + throw error + } + } + + async has(name: string) { + return Boolean(await this.get(name)) + } + + async delete(name: string) { + this.assertWritable() + + const directory = this.getSkillDirectory(name) + const exists = await this.has(name) + + if (!exists) { + return false + } + + await rm(directory, { + recursive: true, + force: true, + }) + return true + } + + async list() { + const entries = await readdir(this.root, { + withFileTypes: true, + }).catch((error: unknown) => { + if (isFileNotFoundError(error)) { + return [] + } + + throw error + }) + const summaries = await Promise.all( + entries + .filter((entry) => entry.isDirectory() && !entry.name.startsWith('.')) + .map(async (entry) => this.get(entry.name)), + ) + + return summaries + .filter((skill): skill is SkillDefinition => Boolean(skill)) + .map((skill) => ({ + name: skill.name, + description: skill.description, + resourceCount: skill.resources?.length ?? 0, + metadata: skill.metadata, + })) + .sort((a, b) => a.name.localeCompare(b.name)) + } + + import(options: SkillLoadOptions) { + this.assertWritable() + const task = importSkill(options) + + return Object.assign( + task.then(async (result) => { + const skill = await this.add(result.skill) + return { + ...result, + name: skill.name, + skill, + } + }), + { cancel: task.cancel }, + ) + } + + private async readSkillDirectory(directory: string): Promise { + const entryPath = join(directory, entryFile) + const entryContent = await readFile(entryPath, 'utf8') + const { frontmatter, body } = parseMarkdownFrontmatter(entryContent) + const instructions = body.trim() + + if (!instructions) { + throw new Error(`Skill entry file "${entryFile}" must contain instructions.`) + } + + const resources = await this.readResourceDescriptors(directory) + + return { + name: getString(frontmatter.name) || directory.split(/[\\/]/).at(-1) || '', + description: getString(frontmatter.description) || '', + instructions, + resources: resources.length ? resources : undefined, + metadata: { + ...getRecord(frontmatter.metadata), + ...(getString(frontmatter.homepage) ? { homepage: getString(frontmatter.homepage) } : {}), + }, + } + } + + private async readResourceDescriptors(directory: string) { + const resources: SkillResourceDescriptor[] = [] + + const walk = async (currentDirectory: string) => { + const entries = await readdir(currentDirectory, { + withFileTypes: true, + }) + + for (const entry of entries) { + if (entry.name.startsWith('.')) { + continue + } + + const fullPath = join(currentDirectory, entry.name) + + if (entry.isDirectory()) { + await walk(fullPath) + continue + } + + if (!entry.isFile()) { + continue + } + + const path = normalizeSkillPath(relative(directory, fullPath)) + if (!path || path === entryFile) { + continue + } + + const fileStat = await stat(fullPath) + const kind = isTextSkillFilePath(path) ? 'text' : 'binary' + const base = { + path, + kind, + resourceId: path, + size: fileStat.size, + lastModified: fileStat.mtimeMs, + } + + resources.push( + kind === 'text' + ? { + ...base, + kind, + readText: async () => readFile(fullPath, 'utf8'), + readBinary: async () => new Uint8Array(await readFile(fullPath)), + } + : { + ...base, + kind, + readBinary: async () => new Uint8Array(await readFile(fullPath)), + readText: async () => new TextDecoder().decode(await readFile(fullPath)), + }, + ) + } + } + + await walk(directory) + return resources.sort((a, b) => a.path.localeCompare(b.path)) + } + + private async writeResource(directory: string, resource: SkillResourceDescriptor) { + const path = normalizeSkillPath(resource.path) + + if (!path || path === entryFile) { + return + } + + const fullPath = join(directory, path) + await mkdir(dirname(fullPath), { + recursive: true, + }) + + if (resource.kind === 'text') { + await writeFile(fullPath, await getResourceText(resource), 'utf8') + return + } + + await writeFile(fullPath, await getResourceBinary(resource)) + } + + private getSkillDirectory(name: string) { + const directoryName = normalizeSkillPath(name) + + if (!directoryName || directoryName.includes('/')) { + throw new Error(`Invalid skill name for file storage: ${name}`) + } + + return join(this.root, directoryName) + } + + private assertWritable() { + if (this.readonly) { + throw new Error('File system skill storage is readonly.') + } + } +} + +export function createFsSkillStorage(options: FsSkillStorageOptions) { + return new FsSkillStorage(options) +} + +function serializeSkillEntry(skill: SkillDefinition) { + const metadata = { ...skill.metadata } + const homepage = typeof metadata.homepage === 'string' ? metadata.homepage : undefined + delete metadata.homepage + const frontmatter: Record = { + name: skill.name, + description: skill.description, + } + + if (homepage) { + frontmatter.homepage = homepage + } + + if (Object.keys(metadata).length > 0) { + frontmatter.metadata = metadata + } + + return `---\n${stringifyYaml(frontmatter).trimEnd()}\n---\n\n${skill.instructions.trim()}\n` +} + +async function getResourceText(resource: SkillResourceDescriptor) { + if (resource.text !== undefined) { + return resource.text + } + + if (resource.readText) { + return resource.readText() + } + + if (resource.binary) { + return new TextDecoder().decode(resource.binary) + } + + if (resource.readBinary) { + return new TextDecoder().decode(await resource.readBinary()) + } + + throw new Error(`Skill resource "${resource.path}" has no text content.`) +} + +async function getResourceBinary(resource: SkillResourceDescriptor) { + if (resource.binary) { + return resource.binary + } + + if (resource.readBinary) { + return resource.readBinary() + } + + if (resource.text !== undefined) { + return new TextEncoder().encode(resource.text) + } + + if (resource.readText) { + return new TextEncoder().encode(await resource.readText()) + } + + throw new Error(`Skill resource "${resource.path}" has no binary content.`) +} + +function isFileNotFoundError(error: unknown) { + return Boolean(error && typeof error === 'object' && 'code' in error && error.code === 'ENOENT') +} diff --git a/packages/kit/src/skills/storage/importSkill.ts b/packages/kit/src/skills/storage/importSkill.ts new file mode 100644 index 000000000..88a786add --- /dev/null +++ b/packages/kit/src/skills/storage/importSkill.ts @@ -0,0 +1,26 @@ +import type { SkillLoadJob, SkillLoadResult } from '../loader/type' +import type { SkillImporter, SkillImportJob, SkillImportResult } from './types' + +export function createImportSkill( + loadSkill: (options: TImportOptions) => SkillLoadJob, +): SkillImporter { + return (options) => { + const loadJob = loadSkill(options) + + const task = (async (): Promise => { + const { skill, warnings } = await loadJob + + return { + name: skill.name, + skill, + warnings, + } + })() as SkillImportJob + + task.cancel = () => { + loadJob.cancel() + } + + return task + } +} diff --git a/packages/kit/src/skills/storage/index.ts b/packages/kit/src/skills/storage/index.ts new file mode 100644 index 000000000..fb9bcf887 --- /dev/null +++ b/packages/kit/src/skills/storage/index.ts @@ -0,0 +1,18 @@ +import { loadSkillWithDetails } from '../loader' +import type { SkillLoadOptions } from '../loader' +import type { SkillStorage as SkillStorageBase } from './types' +import { createImportSkill } from './importSkill' +import { createMemorySkillStorage as createMemorySkillStorageBase } from './memory' + +export type { SkillImportJob, SkillImportResult } from './types' +export type SkillImportOptions = SkillLoadOptions +export type SkillStorage = SkillStorageBase +export { createIndexedDBSkillStorage, IndexedDBSkillStorage } from './indexedDB' +export type { IndexedDBSkillStorageOptions } from './indexedDB' +export { MemorySkillStorage } from './memory' + +export const importSkill = createImportSkill(loadSkillWithDetails) + +export function createMemorySkillStorage() { + return createMemorySkillStorageBase(importSkill) +} diff --git a/packages/kit/src/skills/storage/indexedDB.ts b/packages/kit/src/skills/storage/indexedDB.ts new file mode 100644 index 000000000..0a9205b59 --- /dev/null +++ b/packages/kit/src/skills/storage/indexedDB.ts @@ -0,0 +1,341 @@ +import { openDB, type DBSchema, type IDBPDatabase, type IDBPTransaction } from 'idb' +import { loadSkillWithDetails } from '../loader' +import type { SkillLoadOptions } from '../loader' +import type { SkillDefinition, SkillResourceDescriptor } from '../types' +import { createImportSkill } from './importSkill' +import type { SkillImportJob, SkillImporter, SkillStorage, SkillSummary } from './types' + +const defaultVersion = 1 +const defaultSkillStoreName = 'skills' +const defaultResourceStoreName = 'resources' + +export interface IndexedDBSkillStorageOptions { + /** + * IndexedDB database name. Tests should pass a unique name to avoid cross-test state. + */ + databaseName: string +} + +interface IndexedDBSkillStorageSkillRecord { + name: string + description: string + instructions: string + metadata?: Record + resources?: IndexedDBSkillStorageResourceMetadata[] +} + +interface IndexedDBSkillStorageResourceMetadata { + path: string + kind: SkillResourceDescriptor['kind'] + resourceId: string + mimeType?: string + size?: number + lastModified?: number + metadata?: Record +} + +interface IndexedDBSkillStorageResourceRecord { + skillName: string + resourceId: string + kind: SkillResourceDescriptor['kind'] + text?: string + binary?: Uint8Array +} + +interface IndexedDBSkillStorageSchema extends DBSchema { + skills: { + key: string + value: IndexedDBSkillStorageSkillRecord + } + resources: { + key: [string, string] + value: IndexedDBSkillStorageResourceRecord + indexes: { + skillName: string + } + } +} + +type IndexedDBSkillStorageTransaction = IDBPTransaction< + IndexedDBSkillStorageSchema, + ['skills', 'resources'], + 'readwrite' +> + +type SkillImportOptions = SkillLoadOptions +const importSkill = createImportSkill(loadSkillWithDetails) + +export class IndexedDBSkillStorage implements SkillStorage { + readonly databaseName: string + readonly skillStoreName = defaultSkillStoreName + readonly resourceStoreName = defaultResourceStoreName + private dbPromise?: Promise> + + constructor( + options: IndexedDBSkillStorageOptions, + private readonly importer: SkillImporter = importSkill as SkillImporter, + ) { + this.databaseName = options.databaseName + } + + private getDB() { + this.dbPromise ??= openDB(this.databaseName, defaultVersion, { + upgrade: (db) => { + if (!db.objectStoreNames.contains(this.skillStoreName)) { + db.createObjectStore(defaultSkillStoreName, { + keyPath: 'name', + }) + } + + if (!db.objectStoreNames.contains(this.resourceStoreName)) { + const resourceStore = db.createObjectStore(defaultResourceStoreName, { + keyPath: ['skillName', 'resourceId'], + }) + resourceStore.createIndex('skillName', 'skillName') + } + }, + }) + + return this.dbPromise + } + + async add(skill: SkillDefinition) { + const db = await this.getDB() + const tx = db.transaction([defaultSkillStoreName, defaultResourceStoreName], 'readwrite') + const skillStore = tx.objectStore(defaultSkillStoreName) + const resourceStore = tx.objectStore(defaultResourceStoreName) + + await this.deleteResourceRecords(tx, skill.name) + await skillStore.put(toSkillRecord(skill)) + + for (const resource of skill.resources ?? []) { + await resourceStore.put(await toResourceRecord(skill.name, resource)) + } + + await tx.done + return this.getStoredSkill(skill.name) + } + + async get(name: string) { + const db = await this.getDB() + const record = await db.get(defaultSkillStoreName, name) + + return record ? this.toSkillDefinition(record) : undefined + } + + async has(name: string) { + const db = await this.getDB() + return (await db.count(defaultSkillStoreName, name)) > 0 + } + + async delete(name: string) { + const db = await this.getDB() + const tx = db.transaction([defaultSkillStoreName, defaultResourceStoreName], 'readwrite') + const skillStore = tx.objectStore(defaultSkillStoreName) + const existed = (await skillStore.count(name)) > 0 + + await skillStore.delete(name) + await this.deleteResourceRecords(tx, name) + await tx.done + + return existed + } + + async list(): Promise { + const db = await this.getDB() + const records = await db.getAll(defaultSkillStoreName) + + return records + .map((record) => ({ + name: record.name, + description: record.description, + resourceCount: record.resources?.length ?? 0, + metadata: record.metadata, + })) + .sort((a, b) => a.name.localeCompare(b.name)) + } + + import(options: TImportOptions): SkillImportJob { + const task = this.importer(options) + + return Object.assign( + task.then(async (result) => { + await this.add(result.skill) + return result + }), + { cancel: task.cancel }, + ) + } + + private async getStoredSkill(name: string) { + const skill = await this.get(name) + + if (!skill) { + throw new Error(`Failed to store skill "${name}".`) + } + + return skill + } + + private toSkillDefinition(record: IndexedDBSkillStorageSkillRecord): SkillDefinition { + return { + name: record.name, + description: record.description, + instructions: record.instructions, + metadata: record.metadata ? { ...record.metadata } : undefined, + resources: record.resources?.map((resource) => this.toSkillResource(record.name, resource)), + } + } + + private toSkillResource(skillName: string, resource: IndexedDBSkillStorageResourceMetadata): SkillResourceDescriptor { + const base = { + path: resource.path, + resourceId: resource.resourceId, + mimeType: resource.mimeType, + size: resource.size, + lastModified: resource.lastModified, + metadata: resource.metadata ? { ...resource.metadata } : undefined, + } + + if (resource.kind === 'text') { + return { + ...base, + kind: resource.kind, + readText: async () => this.readResourceText(skillName, resource.resourceId), + readBinary: async () => this.readResourceBinary(skillName, resource.resourceId), + } + } + + return { + ...base, + kind: resource.kind, + readBinary: async () => this.readResourceBinary(skillName, resource.resourceId), + readText: async () => this.readResourceText(skillName, resource.resourceId), + } + } + + private async readResourceText(skillName: string, resourceId: string) { + const resource = await this.getResourceRecord(skillName, resourceId) + + if (typeof resource.text === 'string') { + return resource.text + } + + if (resource.binary) { + return new TextDecoder().decode(resource.binary) + } + + throw new Error(`Skill resource "${resourceId}" has no text content.`) + } + + private async readResourceBinary(skillName: string, resourceId: string) { + const resource = await this.getResourceRecord(skillName, resourceId) + + if (resource.binary) { + return new Uint8Array(resource.binary) + } + + if (typeof resource.text === 'string') { + return new TextEncoder().encode(resource.text) + } + + throw new Error(`Skill resource "${resourceId}" has no binary content.`) + } + + private async getResourceRecord(skillName: string, resourceId: string) { + const db = await this.getDB() + const resource = await db.get(defaultResourceStoreName, [skillName, resourceId]) + + if (!resource) { + throw new Error(`Skill resource "${resourceId}" was not found.`) + } + + return resource + } + + private async deleteResourceRecords(tx: IndexedDBSkillStorageTransaction, skillName: string) { + const resourceStore = tx.objectStore(defaultResourceStoreName) + const resourceKeys = await resourceStore.index('skillName').getAllKeys(skillName) + + await Promise.all(resourceKeys.map((key) => resourceStore.delete(key))) + } +} + +export function createIndexedDBSkillStorage(options: IndexedDBSkillStorageOptions) { + return new IndexedDBSkillStorage(options) +} + +function toSkillRecord(skill: SkillDefinition): IndexedDBSkillStorageSkillRecord { + return { + name: skill.name, + description: skill.description, + instructions: skill.instructions, + metadata: skill.metadata ? { ...skill.metadata } : undefined, + resources: skill.resources?.map((resource) => ({ + path: resource.path, + kind: resource.kind, + resourceId: resource.resourceId, + mimeType: resource.mimeType, + size: resource.size, + lastModified: resource.lastModified, + metadata: resource.metadata ? { ...resource.metadata } : undefined, + })), + } +} + +async function toResourceRecord( + skillName: string, + resource: SkillResourceDescriptor, +): Promise { + if (resource.kind === 'text') { + const text = resource.text ?? (await readTextContent(resource)) + + if (typeof text !== 'string') { + throw new Error(`Skill resource "${resource.resourceId}" has no text content to store.`) + } + + return { + skillName, + resourceId: resource.resourceId, + kind: resource.kind, + text, + } + } + + const binary = resource.binary ?? (await readBinaryContent(resource)) + + if (!binary) { + throw new Error(`Skill resource "${resource.resourceId}" has no binary content to store.`) + } + + return { + skillName, + resourceId: resource.resourceId, + kind: resource.kind, + binary: new Uint8Array(binary), + } +} + +async function readTextContent(resource: SkillResourceDescriptor) { + if (resource.readText) { + return resource.readText() + } + + if (resource.binary) { + return new TextDecoder().decode(resource.binary) + } + + return undefined +} + +async function readBinaryContent(resource: SkillResourceDescriptor) { + if (resource.readBinary) { + return resource.readBinary() + } + + if (resource.text) { + return new TextEncoder().encode(resource.text) + } + + return undefined +} diff --git a/packages/kit/src/skills/storage/memory.ts b/packages/kit/src/skills/storage/memory.ts new file mode 100644 index 000000000..5ae401226 --- /dev/null +++ b/packages/kit/src/skills/storage/memory.ts @@ -0,0 +1,117 @@ +import type { SkillDefinition, SkillResourceDescriptor } from '../types' +import type { SkillImporter, SkillStorage, SkillSummary } from './types' + +const toSummary = (skill: SkillDefinition): SkillSummary => ({ + name: skill.name, + description: skill.description, + resourceCount: skill.resources?.length ?? 0, + metadata: skill.metadata, +}) + +const cloneSkill = (skill: SkillDefinition): SkillDefinition => ({ + name: skill.name, + description: skill.description, + instructions: skill.instructions, + metadata: skill.metadata ? { ...skill.metadata } : undefined, + resources: skill.resources?.map(cloneResource), +}) + +const cloneResource = (resource: SkillResourceDescriptor): SkillResourceDescriptor => { + const base = { + path: resource.path, + resourceId: resource.resourceId, + mimeType: resource.mimeType, + size: resource.size, + lastModified: resource.lastModified, + metadata: resource.metadata ? { ...resource.metadata } : undefined, + } + + if (resource.kind === 'text') { + const content = { + binary: resource.binary ? new Uint8Array(resource.binary) : undefined, + readBinary: resource.readBinary, + } + + return resource.text !== undefined + ? { + ...base, + ...content, + kind: resource.kind, + text: resource.text, + readText: resource.readText, + } + : { + ...base, + ...content, + kind: resource.kind, + readText: resource.readText!, + } + } + + const content = { + text: resource.text, + readText: resource.readText, + } + + return resource.binary + ? { + ...base, + ...content, + kind: resource.kind, + binary: new Uint8Array(resource.binary), + readBinary: resource.readBinary, + } + : { + ...base, + ...content, + kind: resource.kind, + readBinary: resource.readBinary!, + } +} + +export class MemorySkillStorage implements SkillStorage { + private skills = new Map() + + constructor(private readonly importer: SkillImporter) {} + + async add(skill: SkillDefinition) { + const saved = cloneSkill(skill) + this.skills.set(skill.name, saved) + return cloneSkill(saved) + } + + async get(name: string) { + const skill = this.skills.get(name) + return skill ? cloneSkill(skill) : undefined + } + + async has(name: string) { + return this.skills.has(name) + } + + async delete(name: string) { + return this.skills.delete(name) + } + + async list() { + return Array.from(this.skills.values()) + .map(toSummary) + .sort((a, b) => a.name.localeCompare(b.name)) + } + + import(options: TImportOptions) { + const task = this.importer(options) + + return Object.assign( + task.then(async (result) => { + await this.add(result.skill) + return result + }), + { cancel: task.cancel }, + ) + } +} + +export function createMemorySkillStorage(importer: SkillImporter) { + return new MemorySkillStorage(importer) +} diff --git a/packages/kit/src/skills/storage/node.ts b/packages/kit/src/skills/storage/node.ts new file mode 100644 index 000000000..0555270c1 --- /dev/null +++ b/packages/kit/src/skills/storage/node.ts @@ -0,0 +1,18 @@ +import { loadSkillWithDetails } from '../loader/node' +import type { SkillLoadOptions } from '../loader/node' +import type { SkillStorage as SkillStorageBase } from './types' +import { createImportSkill } from './importSkill' +import { createMemorySkillStorage as createMemorySkillStorageBase } from './memory' + +export type { SkillImportJob, SkillImportResult } from './types' +export type SkillImportOptions = SkillLoadOptions +export type SkillStorage = SkillStorageBase +export { MemorySkillStorage } from './memory' + +export const importSkill = createImportSkill(loadSkillWithDetails) + +export function createMemorySkillStorage() { + return createMemorySkillStorageBase(importSkill) +} +export { createFsSkillStorage, FsSkillStorage } from './fs' +export type { FsSkillStorageOptions } from './fs' diff --git a/packages/kit/src/skills/storage/types.ts b/packages/kit/src/skills/storage/types.ts new file mode 100644 index 000000000..c6a269bf4 --- /dev/null +++ b/packages/kit/src/skills/storage/types.ts @@ -0,0 +1,48 @@ +import type { SkillDefinition } from '../types' +import type { SkillLoadWarning } from '../loader/type' + +/** skill 摘要，用于 list()。 */ +export interface SkillSummary { + name: string + description: string + resourceCount: number + metadata?: Record +} + +/** + * skill 持久化与导入。 + * + * @example + * await storage.add(skill) + * const saved = await storage.get('weather') + * const summaries = await storage.list() + */ +export interface SkillStorage { + add(skill: SkillDefinition): Promise + get(name: string): Promise + has(name: string): Promise + delete(name: string): Promise + list(): Promise + import(options: TImportOptions): SkillImportJob +} + +export interface SkillImportResult { + name: string + skill: SkillDefinition + warnings: SkillLoadWarning[] +} + +/** + * 进行中的导入操作；await 得到 SkillImportResult。 + * + * @example + * const job = storage.import({ source: 'browser', fileList: input.files }) + * job.cancel() + * const { name, warnings } = await job + */ +export type SkillImportJob = Promise & { + /** 中止导入。 */ + cancel(): void +} + +export type SkillImporter = (options: TImportOptions) => SkillImportJob diff --git a/packages/kit/src/skills/test/fsStorage.test.ts b/packages/kit/src/skills/test/fsStorage.test.ts new file mode 100644 index 000000000..718352a06 --- /dev/null +++ b/packages/kit/src/skills/test/fsStorage.test.ts @@ -0,0 +1,106 @@ +import { cp, mkdtemp, readFile, writeFile } from 'node:fs/promises' +import { join } from 'node:path' +import { tmpdir } from 'node:os' +import { fileURLToPath } from 'node:url' +import { describe, expect, it } from 'vitest' +import { createFsSkillStorage } from '../storage/node' + +const createTempRoot = () => mkdtemp(join(tmpdir(), 'tiny-robot-skill-storage-')) + +describe('FsSkillStorage', () => { + it('adds and restores skills in native directory format with lazy resources', async () => { + const root = await createTempRoot() + const storage = createFsSkillStorage({ root }) + + await storage.add({ + name: 'demo', + description: 'Demo skill', + instructions: '# Demo\n\nUse this skill.', + metadata: { + homepage: 'https://example.com/demo', + version: '1.0.0', + }, + resources: [ + { + path: 'references/guide.md', + kind: 'text', + resourceId: 'references/guide.md', + text: '# Guide', + }, + { + path: 'assets/icon.bin', + kind: 'binary', + resourceId: 'assets/icon.bin', + binary: new Uint8Array([1, 2, 3]), + }, + ], + }) + + await expect(readFile(join(root, 'demo', 'SKILL.md'), 'utf8')).resolves.toContain( + 'homepage: https://example.com/demo', + ) + await expect(readFile(join(root, 'demo', 'references', 'guide.md'), 'utf8')).resolves.toBe('# Guide') + + const storedSkill = await storage.get('demo') + const guide = storedSkill?.resources?.find((resource) => resource.path === 'references/guide.md') + + expect(storedSkill).toMatchObject({ + name: 'demo', + description: 'Demo skill', + instructions: '# Demo\n\nUse this skill.', + metadata: { + homepage: 'https://example.com/demo', + version: '1.0.0', + }, + }) + expect(guide).toMatchObject({ + path: 'references/guide.md', + kind: 'text', + }) + expect(guide).not.toHaveProperty('text') + + await writeFile(join(root, 'demo', 'references', 'guide.md'), '# Updated', 'utf8') + await expect(guide?.readText?.()).resolves.toBe('# Updated') + }) + + it('lists existing skill directories, imports another skill, and deletes skills', async () => { + const root = await createTempRoot() + const weatherRoot = fileURLToPath(new URL('./.cache/weather', import.meta.url)) + const vueRoot = fileURLToPath(new URL('./.cache/vue-best-practices', import.meta.url)) + await cp(weatherRoot, join(root, 'weather'), { + recursive: true, + }) + + const storage = createFsSkillStorage({ root }) + + await expect(storage.list()).resolves.toEqual([ + expect.objectContaining({ + name: 'weather', + description: expect.stringContaining('weather'), + }), + ]) + const existingSkill = await storage.get('weather') + expect(existingSkill?.instructions).toContain('# Weather Skill') + + const result = await storage.import({ + source: 'fs', + root: vueRoot, + }) + + expect(result.skill.name).toBe('vue-best-practices') + await expect(storage.list()).resolves.toEqual( + expect.arrayContaining([ + expect.objectContaining({ + name: 'weather', + }), + expect.objectContaining({ + name: 'vue-best-practices', + }), + ]), + ) + expect(await storage.has('weather')).toBe(true) + expect(await storage.delete('weather')).toBe(true) + expect(await storage.get('weather')).toBeUndefined() + expect(await storage.has('vue-best-practices')).toBe(true) + }) +}) diff --git a/packages/kit/src/skills/test/indexedDBStorage.test.ts b/packages/kit/src/skills/test/indexedDBStorage.test.ts new file mode 100644 index 000000000..9e4aa0022 --- /dev/null +++ b/packages/kit/src/skills/test/indexedDBStorage.test.ts @@ -0,0 +1,180 @@ +import 'fake-indexeddb/auto' +import { describe, expect, it } from 'vitest' +import { createIndexedDBSkillStorage } from '../storage' +import type { SkillDefinition } from '../types' + +const databaseName = () => `tiny-robot-skills-test-${crypto.randomUUID()}` + +describe('IndexedDBSkillStorage', () => { + it('adds, gets, lists, checks, and deletes skills', async () => { + const storage = createIndexedDBSkillStorage({ databaseName: databaseName() }) + + const saved = await storage.add({ + name: 'demo', + description: 'Demo skill', + instructions: '# Demo', + metadata: { + homepage: 'https://example.com', + }, + }) + + expect(saved).toMatchObject({ + name: 'demo', + description: 'Demo skill', + instructions: '# Demo', + metadata: { + homepage: 'https://example.com', + }, + }) + expect(await storage.has('demo')).toBe(true) + expect(await storage.list()).toEqual([ + { + name: 'demo', + description: 'Demo skill', + resourceCount: 0, + metadata: { + homepage: 'https://example.com', + }, + }, + ]) + + expect(await storage.delete('demo')).toBe(true) + expect(await storage.delete('demo')).toBe(false) + expect(await storage.has('demo')).toBe(false) + expect(await storage.get('demo')).toBeUndefined() + }) + + it('restores resources as lazy readers without eager content', async () => { + const storage = createIndexedDBSkillStorage({ databaseName: databaseName() }) + + await storage.add({ + name: 'docs', + description: 'Docs skill', + instructions: '# Docs', + resources: [ + { + path: 'references/guide.md', + kind: 'text', + resourceId: 'references/guide.md', + text: '# Guide', + readText: async () => '# Guide', + }, + { + path: 'assets/icon.png', + kind: 'binary', + resourceId: 'assets/icon.png', + binary: new Uint8Array([1, 2, 3]), + readBinary: async () => new Uint8Array([1, 2, 3]), + mimeType: 'image/png', + }, + ], + }) + + const skill = await storage.get('docs') + const textResource = skill?.resources?.find((resource) => resource.path === 'references/guide.md') + const binaryResource = skill?.resources?.find((resource) => resource.path === 'assets/icon.png') + + expect(textResource).toMatchObject({ + path: 'references/guide.md', + kind: 'text', + resourceId: 'references/guide.md', + }) + expect(textResource).not.toHaveProperty('text') + await expect(textResource?.readText?.()).resolves.toBe('# Guide') + await expect(textResource?.readBinary?.()).resolves.toEqual(new TextEncoder().encode('# Guide')) + + expect(binaryResource).toMatchObject({ + path: 'assets/icon.png', + kind: 'binary', + resourceId: 'assets/icon.png', + mimeType: 'image/png', + }) + expect(binaryResource).not.toHaveProperty('binary') + await expect(binaryResource?.readBinary?.()).resolves.toEqual(new Uint8Array([1, 2, 3])) + await expect(binaryResource?.readText?.()).resolves.toBe(new TextDecoder().decode(new Uint8Array([1, 2, 3]))) + }) + + it('overwrites stale resource records when replacing a skill', async () => { + const storage = createIndexedDBSkillStorage({ databaseName: databaseName() }) + + await storage.add({ + name: 'docs', + description: 'Docs skill', + instructions: '# Docs', + resources: [ + { + path: 'old.md', + kind: 'text', + resourceId: 'old.md', + text: 'old', + }, + ], + }) + + await storage.add({ + name: 'docs', + description: 'Updated docs skill', + instructions: '# Updated', + resources: [ + { + path: 'new.md', + kind: 'text', + resourceId: 'new.md', + text: 'new', + }, + ], + }) + + const skill = await storage.get('docs') + + expect(skill?.description).toBe('Updated docs skill') + expect(skill?.resources?.map((resource) => resource.path)).toEqual(['new.md']) + await expect(skill?.resources?.[0]?.readText?.()).resolves.toBe('new') + }) + + it('imports skills from browser sources', async () => { + const storage = createIndexedDBSkillStorage({ databaseName: databaseName() }) + + const file = new File( + [['---', 'name: browser-docs', 'description: Browser docs skill', '---', '', '# Browser Docs'].join('\n')], + 'SKILL.md', + { type: 'text/markdown' }, + ) + + const result = await storage.import({ + source: 'browser', + fileList: [file], + }) + + expect(result.name).toBe('browser-docs') + expect(await storage.get('browser-docs')).toMatchObject({ + name: 'browser-docs', + instructions: '# Browser Docs', + }) + }) + + it('persists resources through lazy readers during add', async () => { + const storage = createIndexedDBSkillStorage({ databaseName: databaseName() }) + const skill: SkillDefinition = { + name: 'lazy', + description: 'Lazy skill', + instructions: '# Lazy', + resources: [ + { + path: 'lazy.md', + kind: 'text', + resourceId: 'lazy.md', + readText: async () => 'lazy text', + }, + ], + } + + await storage.add(skill) + + const storedSkill = await storage.get('lazy') + const resource = storedSkill?.resources?.[0] + + expect(resource).not.toHaveProperty('text') + await expect(resource?.readText?.()).resolves.toBe('lazy text') + }) +}) diff --git a/packages/kit/src/skills/test/memoryStorage.test.ts b/packages/kit/src/skills/test/memoryStorage.test.ts new file mode 100644 index 000000000..6101a2494 --- /dev/null +++ b/packages/kit/src/skills/test/memoryStorage.test.ts @@ -0,0 +1,130 @@ +import { describe, expect, it } from 'vitest' +import { createMemorySkillStorage, importSkill } from '../storage' + +type TestFile = File & { + webkitRelativePath?: string +} + +const createTestFile = (path: string, content: string): TestFile => + ({ + name: path.split('/').at(-1) ?? path, + webkitRelativePath: path, + type: 'text/markdown', + size: content.length, + lastModified: 123, + text: async () => content, + arrayBuffer: async () => { + const bytes = new TextEncoder().encode(content) + return bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength) + }, + }) as TestFile + +describe('MemorySkillStorage', () => { + it('add, get, has, delete, and list', async () => { + const storage = createMemorySkillStorage() + + const saved = await storage.add({ + name: 'demo', + description: 'Demo skill', + instructions: '# Demo', + }) + + expect(saved.name).toBe('demo') + expect(await storage.has('demo')).toBe(true) + expect(await storage.get('demo')).toMatchObject({ + name: 'demo', + description: 'Demo skill', + instructions: '# Demo', + }) + + const summaries = await storage.list() + expect(summaries).toEqual([ + { + name: 'demo', + description: 'Demo skill', + resourceCount: 0, + metadata: undefined, + }, + ]) + + expect(await storage.delete('demo')).toBe(true) + expect(await storage.has('demo')).toBe(false) + expect(await storage.get('demo')).toBeUndefined() + }) + + it('imports skill from browser source', async () => { + const storage = createMemorySkillStorage() + + const { name, skill, warnings } = await storage.import({ + source: 'browser', + fileList: [ + createTestFile( + 'weather/SKILL.md', + ['---', 'name: weather', 'description: Weather skill', '---', '', '# Weather Skill'].join('\n'), + ), + ], + }) + + expect(name).toBe('weather') + expect(skill.name).toBe('weather') + expect(skill.instructions).toContain('# Weather Skill') + expect(warnings).toEqual([]) + + const storedSkill = await storage.get('weather') + expect(storedSkill?.instructions).toContain('# Weather Skill') + expect(storedSkill?.resources?.some((resource) => resource.path === 'SKILL.md')).toBeFalsy() + }) + + it('imports multi-file skill with readable resources', async () => { + const storage = createMemorySkillStorage() + + await storage.import({ + source: 'browser', + fileList: [ + createTestFile( + 'vue-best-practices/SKILL.md', + [ + '---', + 'name: vue-best-practices', + 'description: Vue.js tasks', + '---', + '', + '# Vue Best Practices Workflow', + ].join('\n'), + ), + createTestFile('vue-best-practices/references/reactivity.md', '# Reactivity'), + ], + }) + + const skill = await storage.get('vue-best-practices') + const resource = skill?.resources?.find((item) => item.path === 'references/reactivity.md') + + expect(resource).toBeDefined() + await expect(resource?.readText?.()).resolves.toContain('# Reactivity') + }) + + it('supports cancel on import task', async () => { + let releaseWait!: () => void + const waitForText = new Promise((resolve) => { + releaseWait = () => resolve('# Cancelled') + }) + const task = importSkill({ + source: 'browser', + fileList: [ + { + ...createTestFile( + 'cancelled/SKILL.md', + ['---', 'name: cancelled', 'description: Cancelled skill', '---', '', '# Cancelled'].join('\n'), + ), + text: async () => waitForText, + }, + ], + }) + task.cancel() + releaseWait() + + await expect(task).rejects.toMatchObject({ + name: 'SkillLoadCancelledError', + }) + }) +})