feat(kit): add skill loader foundation

packages/kit/package.json

@@ -44,15 +44,22 @@
       "types": "./dist/core.d.ts",
       "import": "./dist/core.mjs",
       "require": "./dist/core.js"
+    },
+    "./node": {
+      "types": "./dist/node.d.ts",
+      "import": "./dist/node.mjs",
+      "require": "./dist/node.js"
     }
   },
   "files": [
     "dist"
   ],
   "sideEffects": false,
   "scripts": {
-    "build": "tsup src/index.ts src/core.ts --format cjs,esm --dts --minify",
-    "dev": "tsup src/index.ts src/core.ts --format cjs,esm --dts --watch",
+    "build": "tsup src/index.ts src/core.ts src/node.ts --format cjs,esm --dts --minify",
+    "dev": "tsup src/index.ts src/core.ts src/node.ts --format cjs,esm --dts --watch",
+    "lint": "eslint src",
+    "pretest": "node scripts/download-skill-fixtures.mjs",
     "test": "vitest run",
     "test:watch": "vitest"
   },
@@ -69,6 +76,7 @@
     "vue": ">=3.0.0"
   },
   "dependencies": {
-    "idb": "^8.0.3"
+    "idb": "^8.0.3",
+    "yaml": "^2.8.3"
   }
 }

packages/kit/scripts/download-skill-fixtures.mjs

@@ -0,0 +1,128 @@
+import { mkdir, readFile, rm, writeFile } from 'node:fs/promises'
+import { dirname, join } from 'node:path'
+import { fileURLToPath } from 'node:url'
+
+const __dirname = dirname(fileURLToPath(import.meta.url))
+const cacheDirectory = join(__dirname, '../src/skills/test/.cache')
+
+const fixtures = [
+  {
+    repo: 'openclaw/openclaw',
+    commit: '58672075219d09495de6489ad0821d276ac84f13',
+    sourcePath: 'skills/weather',
+  },
+  {
+    repo: 'vuejs-ai/skills',
+    commit: 'b9d14d022da6a0a8bdcb824557f40bca6fbc1845',
+    sourcePath: 'skills/vue-best-practices',
+  },
+]
+
+const getFixtureTargetPath = (fixture) => {
+  const normalizedSourcePath = fixture.sourcePath.split('\\').join('/')
+  const targetName = normalizedSourcePath.split('/').filter(Boolean).at(-1)
+
+  if (!targetName) {
+    throw new Error(`Invalid fixture source path: ${fixture.sourcePath}`)
+  }
+
+  return join(cacheDirectory, targetName)
+}
+
+const fetchJson = async (url) => {
+  const response = await fetch(url, {
+    headers: {
+      accept: 'application/vnd.github+json',
+      'user-agent': '@opentiny/tiny-robot-kit skill fixture downloader',
+    },
+  })
+
+  if (!response.ok) {
+    throw new Error(`Failed to fetch ${url}: ${response.status} ${response.statusText}`)
+  }
+
+  return response.json()
+}
+
+const fetchBytes = async (url) => {
+  const response = await fetch(url, {
+    headers: {
+      'user-agent': '@opentiny/tiny-robot-kit skill fixture downloader',
+    },
+  })
+
+  if (!response.ok) {
+    throw new Error(`Failed to download ${url}: ${response.status} ${response.statusText}`)
+  }
+
+  return new Uint8Array(await response.arrayBuffer())
+}
+
+const getMarkerPath = (targetPath) => join(targetPath, '.fixture-source.json')
+
+const hasCurrentFixture = async (fixture) => {
+  const targetPath = getFixtureTargetPath(fixture)
+
+  try {
+    const marker = JSON.parse(await readFile(getMarkerPath(targetPath), 'utf8'))
+    return (
+      marker.repo === fixture.repo &&
+      marker.commit === fixture.commit &&
+      marker.sourcePath === fixture.sourcePath
+    )
+  } catch {
+    return false
+  }
+}
+
+const downloadDirectory = async (fixture, sourcePath, targetPath) => {
+  const url = new URL(`https://api.github.com/repos/${fixture.repo}/contents/${sourcePath}`)
+  url.searchParams.set('ref', fixture.commit)
+
+  const entries = await fetchJson(url)
+  if (!Array.isArray(entries)) {
+    throw new Error(`Expected directory listing for ${sourcePath}`)
+  }
+
+  for (const entry of entries) {
+    const entryTargetPath = join(targetPath, entry.name)
+
+    if (entry.type === 'dir') {
+      await downloadDirectory(fixture, entry.path, entryTargetPath)
+      continue
+    }
+
+    if (entry.type !== 'file' || !entry.download_url) {
+      continue
+    }
+
+    await mkdir(dirname(entryTargetPath), { recursive: true })
+    await writeFile(entryTargetPath, await fetchBytes(entry.download_url))
+  }
+}
+
+for (const fixture of fixtures) {
+  const targetPath = getFixtureTargetPath(fixture)
+
+  if (await hasCurrentFixture(fixture)) {
+    console.log(`Skill fixture already cached: ${fixture.sourcePath}@${fixture.commit}`)
+    continue
+  }
+
+  console.log(`Downloading skill fixture: ${fixture.sourcePath}@${fixture.commit}`)
+  await rm(targetPath, { recursive: true, force: true })
+  await mkdir(targetPath, { recursive: true })
+  await downloadDirectory(fixture, fixture.sourcePath, targetPath)
+  await writeFile(
+    getMarkerPath(targetPath),
+    `${JSON.stringify(
+      {
+        repo: fixture.repo,
+        commit: fixture.commit,
+        sourcePath: fixture.sourcePath,
+      },
+      null,
+      2,
+    )}\n`,
+  )
+}

packages/kit/src/node.ts

@@ -0,0 +1,8 @@
+export { loadSkill, loadSkillWithDetails } from './skills/loader/node'
+export type {
+  FsSkillLoadOptions,
+  GithubSkillLoadOptions,
+  SkillLoadJob,
+  SkillLoadOptions,
+  SkillLoadResult,
+} from './skills/loader/node'

packages/kit/src/skills/loader/browser.ts

@@ -0,0 +1,66 @@
+import { isTextSkillFilePath, normalizeSkillPath, stripRootDirectory, throwIfSkillLoadCancelled } from './utils'
+import type { BrowserSkillLoadOptions, LoadableSkillFile, SkillLoadContext } from './type'
+
+type FileWithRelativePath = File & {
+  webkitRelativePath?: string
+}
+
+export async function loadBrowserSkillFiles(
+  options: BrowserSkillLoadOptions,
+  context: SkillLoadContext,
+): Promise<LoadableSkillFile[]> {
+  if ('fileList' in options && options.fileList) {
+    return Promise.all(
+      Array.from(options.fileList)
+        .filter((file): file is FileWithRelativePath => Boolean(file))
+        .map((file) => loadBrowserFile(file, stripRootDirectory(file.webkitRelativePath || file.name), context)),
+    )
+  }
+
+  const result: LoadableSkillFile[] = []
+
+  const walk = async (directory: FileSystemDirectoryHandle, parentPath = '') => {
+    throwIfSkillLoadCancelled(context.signal)
+    const entries = (
+      directory as FileSystemDirectoryHandle & {
+        entries(): AsyncIterable<[string, FileSystemDirectoryHandle | FileSystemFileHandle]>
+      }
+    ).entries()
+
+    for await (const [name, handle] of entries) {
+      const path = parentPath ? `${parentPath}/${name}` : name
+
+      if (handle.kind === 'directory') {
+        await walk(handle, path)
+        continue
+      }
+
+      result.push(await loadBrowserFile(await handle.getFile(), path, context))
+    }
+  }
+
+  await walk(options.directoryHandle)
+  return result
+}
+
+async function loadBrowserFile(file: File, rawPath: string, context: SkillLoadContext): Promise<LoadableSkillFile> {
+  const path = normalizeSkillPath(rawPath)
+
+  if (!path) {
+    throw new Error(`Invalid skill file path: ${rawPath}`)
+  }
+
+  const kind = isTextSkillFilePath(path) ? 'text' : 'binary'
+  const content = kind === 'text' ? await file.text() : new Uint8Array(await file.arrayBuffer())
+
+  throwIfSkillLoadCancelled(context.signal)
+
+  return {
+    path,
+    kind,
+    content,
+    mimeType: file.type,
+    size: file.size,
+    lastModified: file.lastModified,
+  }
+}

packages/kit/src/skills/loader/definition.ts

@@ -0,0 +1,130 @@
+import type { SkillResourceDescriptor } from '../types'
+import type { LoadableSkillFile, SkillLoadBaseOptions, SkillLoadResult, SkillLoadWarning } from './type'
+import {
+  getFallbackSkillName,
+  getRecord,
+  getString,
+  isTextSkillFilePath,
+  normalizeSkillPath,
+  parseMarkdownFrontmatter,
+  pushWarning,
+} from './utils'
+
+export function createSkillDefinition(files: LoadableSkillFile[], options: SkillLoadBaseOptions): SkillLoadResult {
+  const warnings: SkillLoadWarning[] = []
+  const entryFile = options.entryFile ?? 'SKILL.md'
+  const normalizedFiles = normalizeFiles(files, options, warnings)
+  const skillEntry = normalizedFiles.find((file) => file.path === entryFile)
+
+  if (!skillEntry) {
+    throw new Error(`Skill entry file "${entryFile}" is missing.`)
+  }
+
+  if (skillEntry.kind !== 'text') {
+    throw new Error(`Skill entry file "${entryFile}" must be a text file.`)
+  }
+
+  const { frontmatter, body } = parseMarkdownFrontmatter(String(skillEntry.content))
+  const instructions = body.trim()
+
+  if (!instructions) {
+    throw new Error(`Skill entry file "${entryFile}" must contain instructions.`)
+  }
+
+  const resources = normalizedFiles.flatMap((file) => {
+    if (file.path === entryFile) return []
+    if (file.kind === 'text' && !isTextSkillFilePath(file.path)) {
+      pushWarning(warnings, options, {
+        code: 'unsupported-text-file-ignored',
+        message: 'Only markdown, text, and json files are converted to text skill files.',
+        path: file.path,
+      })
+      return []
+    }
+
+    return [toSkillResource(file)]
+  })
+
+  return {
+    skill: {
+      name: getString(frontmatter.name) || getFallbackSkillName(entryFile),
+      description: getString(frontmatter.description) || '',
+      instructions,
+      resources: resources.length ? resources : undefined,
+      metadata: {
+        ...getRecord(frontmatter.metadata),
+        ...(getString(frontmatter.homepage) ? { homepage: getString(frontmatter.homepage) } : {}),
+      },
+    },
+    warnings,
+  }
+}
+
+function normalizeFiles<T extends LoadableSkillFile>(
+  files: T[],
+  options: SkillLoadBaseOptions,
+  warnings: SkillLoadWarning[],
+) {
+  const result: T[] = []
+  const seenPaths = new Set<string>()
+
+  for (const file of files) {
+    const path = normalizeSkillPath(file.path)
+
+    if (!path) {
+      pushWarning(warnings, options, {
+        code: 'invalid-path',
+        message: `Invalid skill file path: ${file.path}`,
+        path: file.path,
+      })
+      continue
+    }
+
+    if (seenPaths.has(path)) {
+      pushWarning(warnings, options, {
+        code: 'duplicate-path',
+        message: `Duplicate skill file path: ${path}`,
+        path,
+      })
+      continue
+    }
+
+    seenPaths.add(path)
+    result.push({ ...file, path })
+  }
+
+  return result.sort((a, b) => a.path.localeCompare(b.path))
+}
+
+function toSkillResource(file: LoadableSkillFile): SkillResourceDescriptor {
+  if (file.kind === 'text') {
+    const text = typeof file.content === 'string' ? file.content : new TextDecoder().decode(file.content)
+
+    return {
+      path: file.path,
+      kind: file.kind,
+      resourceId: file.path,
+      mimeType: file.mimeType,
+      size: file.size,
+      lastModified: file.lastModified,
+      metadata: file.metadata,
+      text,
+      readText: async () => text,
+      readBinary: async () => new TextEncoder().encode(text),
+    }
+  }
+
+  const binary = file.content instanceof Uint8Array ? file.content : new TextEncoder().encode(file.content)
+
+  return {
+    path: file.path,
+    kind: file.kind,
+    resourceId: file.path,
+    mimeType: file.mimeType,
+    size: file.size,
+    lastModified: file.lastModified,
+    metadata: file.metadata,
+    binary,
+    readBinary: async () => binary,
+  }
+}