feat(kit): add skill loader foundation

packages/kit/package.json

@@ -44,15 +44,22 @@
       "types": "./dist/core.d.ts",
       "import": "./dist/core.mjs",
       "require": "./dist/core.js"
+    },
+    "./node": {
+      "types": "./dist/node.d.ts",
+      "import": "./dist/node.mjs",
+      "require": "./dist/node.js"
     }
   },
   "files": [
     "dist"
   ],
   "sideEffects": false,
   "scripts": {
-    "build": "tsup src/index.ts src/core.ts --format cjs,esm --dts --minify",
-    "dev": "tsup src/index.ts src/core.ts --format cjs,esm --dts --watch",
+    "build": "tsup src/index.ts src/core.ts src/node.ts --format cjs,esm --dts --minify",
+    "dev": "tsup src/index.ts src/core.ts src/node.ts --format cjs,esm --dts --watch",
+    "lint": "eslint src",
+    "pretest": "node scripts/download-skill-fixtures.mjs",
     "test": "vitest run",
     "test:watch": "vitest"
   },
@@ -69,6 +76,7 @@
     "vue": ">=3.0.0"
   },
   "dependencies": {
-    "idb": "^8.0.3"
+    "idb": "^8.0.3",
+    "yaml": "^2.8.3"
   }
 }

packages/kit/scripts/download-skill-fixtures.mjs

@@ -0,0 +1,178 @@
+import { mkdir, readFile, rm, writeFile } from 'node:fs/promises'
+import { dirname, join } from 'node:path'
+import { fileURLToPath } from 'node:url'
+
+const __dirname = dirname(fileURLToPath(import.meta.url))
+const cacheDirectory = join(__dirname, '../src/skills/test/.cache')
+const maxRetries = 5
+const retryBaseDelay = 200
+const requestTimeout = 30_000
+
+const fixtures = [
+  {
+    repo: 'openclaw/openclaw',
+    commit: '58672075219d09495de6489ad0821d276ac84f13',
+    sourcePath: 'skills/weather',
+  },
+  {
+    repo: 'vuejs-ai/skills',
+    commit: 'b9d14d022da6a0a8bdcb824557f40bca6fbc1845',
+    sourcePath: 'skills/vue-best-practices',
+  },
+]
+
+const getFixtureTargetPath = (fixture) => {
+  const normalizedSourcePath = fixture.sourcePath.split('\\').join('/')
+  const targetName = normalizedSourcePath.split('/').filter(Boolean).at(-1)
+
+  if (!targetName) {
+    throw new Error(`Invalid fixture source path: ${fixture.sourcePath}`)
+  }
+
+  return join(cacheDirectory, targetName)
+}
+
+const waitForRetry = (retryCount) =>
+  new Promise((resolve) => {
+    setTimeout(resolve, retryBaseDelay * 2 ** retryCount)
+  })
+
+const isRetryableStatus = (status) => status === 429 || status >= 500
+
+const fetchWithRetry = async (url, init, errorPrefix) => {
+  let lastError
+
+  for (let retryCount = 0; retryCount < maxRetries; retryCount += 1) {
+    const controller = new AbortController()
+    const timeout = setTimeout(() => {
+      controller.abort(new Error(`Request timeout after ${requestTimeout}ms`))
+    }, requestTimeout)
+
+    try {
+      const response = await fetch(url, {
+        ...init,
+        signal: controller.signal,
+      })
+
+      if (response.ok) {
+        return response
+      }
+
+      const message = `${errorPrefix} ${url}: ${response.status} ${response.statusText}`
+      if (!isRetryableStatus(response.status) || retryCount === maxRetries - 1) {
+        throw new Error(message)
+      }
+
+      lastError = new Error(message)
+    } catch (error) {
+      lastError = error
+      if (retryCount === maxRetries - 1) {
+        throw error
+      }
+    } finally {
+      clearTimeout(timeout)
+    }
+
+    await waitForRetry(retryCount)
+  }
+
+  throw lastError ?? new Error(`${errorPrefix} ${url}: retry budget exhausted`)
+}
+
+const fetchJson = async (url) => {
+  const response = await fetchWithRetry(
+    url,
+    {
+      headers: {
+        accept: 'application/vnd.github+json',
+        'user-agent': '@opentiny/tiny-robot-kit skill fixture downloader',
+      },
+    },
+    'Failed to fetch',
+  )
+
+  return response.json()
+}
+
+const fetchBytes = async (url) => {
+  const response = await fetchWithRetry(
+    url,
+    {
+      headers: {
+        'user-agent': '@opentiny/tiny-robot-kit skill fixture downloader',
+      },
+    },
+    'Failed to download',
+  )
+
+  return new Uint8Array(await response.arrayBuffer())
+}
+
+const getMarkerPath = (targetPath) => join(targetPath, '.fixture-source.json')
+
+const hasCurrentFixture = async (fixture) => {
+  const targetPath = getFixtureTargetPath(fixture)
+
+  try {
+    const marker = JSON.parse(await readFile(getMarkerPath(targetPath), 'utf8'))
+    return (
+      marker.repo === fixture.repo &&
+      marker.commit === fixture.commit &&
+      marker.sourcePath === fixture.sourcePath
+    )
+  } catch {
+    return false
+  }
+}
+
+const downloadDirectory = async (fixture, sourcePath, targetPath) => {
+  const url = new URL(`https://api.github.com/repos/${fixture.repo}/contents/${sourcePath}`)
+  url.searchParams.set('ref', fixture.commit)
+
+  const entries = await fetchJson(url)
+  if (!Array.isArray(entries)) {
+    throw new Error(`Expected directory listing for ${sourcePath}`)
+  }
+
+  for (const entry of entries) {
+    const entryTargetPath = join(targetPath, entry.name)
+
+    if (entry.type === 'dir') {
+      await downloadDirectory(fixture, entry.path, entryTargetPath)
+      continue
+    }
+
+    if (entry.type !== 'file' || !entry.download_url) {
+      continue
+    }
+
+    await mkdir(dirname(entryTargetPath), { recursive: true })
+    await writeFile(entryTargetPath, await fetchBytes(entry.download_url))
+  }
+}
+
+for (const fixture of fixtures) {
+  const targetPath = getFixtureTargetPath(fixture)
+
+  if (await hasCurrentFixture(fixture)) {
+    console.log(`Skill fixture already cached: ${fixture.sourcePath}@${fixture.commit}`)
+    continue
+  }
+
+  console.log(`Downloading skill fixture: ${fixture.sourcePath}@${fixture.commit}`)
+  await rm(targetPath, { recursive: true, force: true })
+  await mkdir(targetPath, { recursive: true })
+  await downloadDirectory(fixture, fixture.sourcePath, targetPath)
+  await writeFile(
+    getMarkerPath(targetPath),
+    `${JSON.stringify(
+      {
+        repo: fixture.repo,
+        commit: fixture.commit,
+        sourcePath: fixture.sourcePath,
+      },
+      null,
+      2,
+    )}\n`,
+  )
+}

packages/kit/src/node.ts

@@ -0,0 +1,8 @@
+export { loadSkill, loadSkillWithDetails } from './skills/loader/node'
+export type {
+  FsSkillLoadOptions,
+  GithubSkillLoadOptions,
+  SkillLoadJob,
+  SkillLoadOptions,
+  SkillLoadResult,
+} from './skills/loader/node'

packages/kit/src/skills/loader/browser.ts

@@ -0,0 +1,76 @@
+import { isTextSkillFilePath, normalizeSkillPath, stripRootDirectory, throwIfSkillLoadCancelled } from './utils'
+import type { BrowserSkillLoadOptions, LoadableSkillFile, SkillLoadContext } from './type'
+
+type FileWithRelativePath = File & {
+  webkitRelativePath?: string
+}
+
+export async function loadBrowserSkillFiles(
+  options: BrowserSkillLoadOptions,
+  context: SkillLoadContext,
+): Promise<LoadableSkillFile[]> {
+  if ('fileList' in options && options.fileList) {
+    const files: LoadableSkillFile[] = []
+
+    for (const file of Array.from(options.fileList)) {
+      if (!file) {
+        continue
+      }
+
+      throwIfSkillLoadCancelled(context.signal)
+      const fileWithRelativePath = file as FileWithRelativePath
+      const path = stripRootDirectory(fileWithRelativePath.webkitRelativePath || file.name)
+
+      files.push(await loadBrowserFile(file, path, context))
+    }
+
+    return files
+  }
+
+  const result: LoadableSkillFile[] = []
+
+  const walk = async (directory: FileSystemDirectoryHandle, parentPath = '') => {
+    throwIfSkillLoadCancelled(context.signal)
+    const entries = (
+      directory as FileSystemDirectoryHandle & {
+        entries(): AsyncIterable<[string, FileSystemDirectoryHandle | FileSystemFileHandle]>
+      }
+    ).entries()
+
+    for await (const [name, handle] of entries) {
+      const path = parentPath ? `${parentPath}/${name}` : name
+
+      if (handle.kind === 'directory') {
+        await walk(handle, path)
+        continue
+      }
+
+      result.push(await loadBrowserFile(await handle.getFile(), path, context))
+    }
+  }
+
+  await walk(options.directoryHandle)
+  return result
+}
+
+async function loadBrowserFile(file: File, rawPath: string, context: SkillLoadContext): Promise<LoadableSkillFile> {
+  const path = normalizeSkillPath(rawPath)
+
+  if (!path) {
+    throw new Error(`Invalid skill file path: ${rawPath}`)
+  }
+
+  const kind = isTextSkillFilePath(path) ? 'text' : 'binary'
+  const content = kind === 'text' ? await file.text() : new Uint8Array(await file.arrayBuffer())
+
+  throwIfSkillLoadCancelled(context.signal)
+
+  return {
+    path,
+    kind,
+    content,
+    mimeType: file.type,
+    size: file.size,
+    lastModified: file.lastModified,
+  }
+}