diff --git a/website/package.json b/website/package.json index 51dd9081dd0..a998731e5d6 100644 --- a/website/package.json +++ b/website/package.json @@ -12,11 +12,12 @@ "clean": "rm -rf node_modules && rm -rf package-lock.json && rm -rf build", "full-test": "./test-links.sh test", "start": "docusaurus start", + "serve": "docusaurus serve", "build": "./node_modules/.bin/docusaurus build && yarn run process-markdown", "build:dev": "./node_modules/.bin/docusaurus build --dev && yarn run process-markdown", "build:preview": "yarn run process-markdown && ./node_modules/.bin/docusaurus build --locale en && yarn run process-markdown", - "process-markdown": "node ./scripts/copy-md-to-static.js", - "meilisearch:build-index": "npm run process-markdown && node scripts/index-meilisearch.mjs", + "process-markdown": "node ./scripts/copy-md-to-static.mjs", + "meilisearch:build-index": "node scripts/index-meilisearch.mjs", "swizzle": "docusaurus swizzle", "docusaurus": "docusaurus" }, @@ -32,6 +33,7 @@ "serve": "^14.2.4" }, "dependencies": { + "@ai-sdk/react": "^3.0.99", "@docsearch/core": "^4.5.0", "@docusaurus/core": "3.9.2", "@docusaurus/plugin-ideal-image": "3.9.2", @@ -42,6 +44,7 @@ "@hot-labs/near-connect": "^0.8.2", "@mermaid-js/layout-elk": "^0.2.0", "@saucelabs/theme-github-codeblock": "^0.3.0", + "ai": "^6.0.97", "axios": "^1.12.0", "clsx": "^1.1.1", "crypto-browserify": "^3.12.1", @@ -70,6 +73,7 @@ "typed.js": "^2.1.0", "url": "^0.11.4", "util": "^0.12.5", - "vm-browserify": "^1.1.2" + "vm-browserify": "^1.1.2", + "zod": "^4.3.6" } } diff --git a/website/scripts/copy-md-to-static.js b/website/scripts/copy-md-to-static.js deleted file mode 100644 index 9e11c26509d..00000000000 --- a/website/scripts/copy-md-to-static.js +++ /dev/null @@ -1,317 +0,0 @@ -#!/usr/bin/env node - -const glob = require('glob'); -const path = require('path'); -const fs = require('fs'); -const axios = require('axios'); -const { URL } = require('url'); - -async function fetchGitHubCode(tag) { - // Replace single quotes with double quotes for consistent parsing - const normalizedTag = tag.replace(/'/g, '"'); - - // Extract URL from the tag - const urlMatch = normalizedTag.match(/url="(.*?)"/); - if (!urlMatch) return null; - - let url = urlMatch[1]; - // Remove hash fragment - url = url.split('#')[0]; - - // Parse URL and extract components - const urlObj = new URL(url); - const pathSegments = urlObj.pathname.slice(1).split('/'); - - if (pathSegments.length < 4) return null; - - const [org, repo, , branch, ...pathSeg] = pathSegments; - const filePath = pathSeg.join('/'); - - // Construct raw GitHub URL - const rawUrl = `https://raw.githubusercontent.com/${org}/${repo}/${branch}/${filePath}`; - - try { - // Fetch the code - const response = await axios.get(rawUrl); - return { - code: response.data, - normalizedTag, - url: rawUrl - }; - } catch (error) { - console.error(`āŒ ${rawUrl}`); - return null; - } -} - -// Extract code lines based on start/end parameters and format as code block -function formatCodeBlock(codeData, language = '') { - const { code, normalizedTag } = codeData; - - // Extract line numbers - const startMatch = normalizedTag.match(/start="(\d*)"/); - const endMatch = normalizedTag.match(/end="(\d*)"/); - - const codeLines = String(code).split('\n'); - const start = startMatch ? Math.max(parseInt(startMatch[1]) - 1, 0) : 0; - const end = endMatch ? parseInt(endMatch[1]) + 1 : codeLines.length; - - const selectedCode = codeLines.slice(start, end).join('\n'); - - // Return formatted code block with optional language syntax highlighting - return `\`\`\`${language}\n${selectedCode}\n\`\`\``; -} - -async function replaceTagsWithCode(content, tagName, includeLanguage = false) { - const tagRegex = new RegExp(`<${tagName}\\s[^>]*?(?:\\/>|>[^<]*<\\/${tagName}>)`, 'g'); - const tags = content.match(tagRegex) || []; - - for (const tag of tags) { - const codeData = await fetchGitHubCode(tag); - - if (codeData) { - let language = ''; - - // Extract language if needed (for File tags) - if (includeLanguage) { - const languageMatch = codeData.normalizedTag.match(/language="(.*?)"/); - language = languageMatch ? languageMatch[1] : ''; - } - - const codeBlock = formatCodeBlock(codeData, language); - content = content.replace(tag, codeBlock); - } - } - - return content; -} - -async function replaceGithubWithCode(content) { - return replaceTagsWithCode(content, 'Github', false); -} - -async function replaceFileWithCode(content) { - return replaceTagsWithCode(content, 'File', true); -} - -// Directories -const DOCS_DIR = path.join(__dirname, '../../docs'); -const BUILD_DIR = path.join(__dirname, '../build'); - -console.log('šŸš€ Starting markdown files post-processing...'); -console.log('This script will copy processed .md files alongside .html files in build/'); - -// Clear md -async function cleanContent(content) { - let cleaned = content; - - // Remove all imports (including JSX components and MDX imports) - cleaned = cleaned.replace(/^import\s+.*?from\s+['"].*?['"];?\s*$/gm, ''); - cleaned = cleaned.replace(/^import\s+\{[^}]*\}\s+from\s+['"].*?['"];?\s*$/gm, ''); - cleaned = cleaned.replace(/^import\s+.*?$/gm, ''); - - cleaned = await replaceGithubWithCode(cleaned); - cleaned = await replaceFileWithCode(cleaned); - - return cleaned; -} - -function generatePath(filePath) { - const content = fs.readFileSync(filePath, 'utf8'); - - const frontmatterMatch = content.match(/^---\s*\n([\s\S]*?)\n---/); - let id = null; - if (frontmatterMatch) { - const frontmatter = frontmatterMatch[1]; - const idMatch = frontmatter.match(/^id:\s*(.+)$/m); - if (idMatch) { - id = idMatch[1].trim().replace(/['"]/g, ''); - } - } - - const relativePath = path.relative(DOCS_DIR, filePath); - - const dirPath = path.dirname(relativePath); - - let newFilename; - if (id) { - newFilename = `${id}.md`; - } else { - newFilename = path.basename(filePath); - } - - return path.join(BUILD_DIR, dirPath, newFilename); -} - -function extractFrontmatter(content) { - const frontmatter = {}; - let body = content; - - if (content.startsWith('---\n')) { - const endIndex = content.indexOf('\n---\n', 4); - if (endIndex !== -1) { - const frontmatterText = content.substring(4, endIndex); - body = content.substring(endIndex + 5); - const lines = frontmatterText.split('\n'); - for (const line of lines) { - const colonIndex = line.indexOf(':'); - if (colonIndex > 0) { - const key = line.substring(0, colonIndex).trim(); - const value = line.substring(colonIndex + 1).trim().replace(/^["']|["']$/g, ''); - frontmatter[key] = value; - } - } - } - } - - return { frontmatter, body }; -} - -function getDescription(content) { - const lines = content.split('\n'); - for (const line of lines) { - const trimmed = line.trim(); - if (trimmed && - trimmed.length > 0) { - return trimmed - } - } - return '' -} - -async function checkLink(url) { - try { - const response = await axios.head(url, { timeout: 5000 }); - return { url, status: response.status, ok: true }; - } catch (error) { - const status = error.response?.status || 'NO RESPONSE'; - console.log(`āŒ ${url} - ${status}`); - return { url, status, ok: false }; - } -} - -const allMarkdownFiles = glob.sync(path.join(DOCS_DIR, '**/*.md')); - -async function processMarkdownFiles() { - const documentationPages = {}; - - await Promise.all( - allMarkdownFiles.map(async (markdownFilePath) => { - let fileContent = fs.readFileSync(markdownFilePath, 'utf8'); - - fileContent = await cleanContent(fileContent); - - const outputFilePath = generatePath(markdownFilePath); - const outputDirectory = path.dirname(outputFilePath); - - if (!fs.existsSync(outputDirectory)) fs.mkdirSync(outputDirectory, { recursive: true }); - - fs.writeFileSync(outputFilePath, fileContent, 'utf8'); - - const relativeFilename = path.relative(DOCS_DIR, markdownFilePath); - - if (relativeFilename === "index.md" || relativeFilename === "help.md") return; - - const pathSegments = relativeFilename.split('/'); - const sectionName = pathSegments[0]; - const fileName = pathSegments.pop(); - const alternativeId = fileName.replace('.md', ''); - - if (!documentationPages[sectionName]) { - documentationPages[sectionName] = []; - } - - const { frontmatter, body } = extractFrontmatter(fileContent); - const pageDescription = getDescription(body); - - if (pageDescription.startsWith('#') || - pageDescription.startsWith('import') || - pageDescription.startsWith(':::') || - pageDescription.startsWith('![')) { - console.warn(`Warning: No valid description found in ${relativeFilename}`); - } - if(!frontmatter.description){ - console.log(`āŒ No description tag found for ${relativeFilename}`); - } - const pageTitle = frontmatter.title || - frontmatter.sidebar_label || - alternativeId.replace(/[-_]/g, ' ').replace(/\b\w/g, letter => letter.toUpperCase()); - - const pageUrl = `${pathSegments.join("/")}/${frontmatter.id ? frontmatter.id + ".md" : fileName}`; - const pageId = frontmatter.id || alternativeId; - - documentationPages[sectionName].push({ - title: pageTitle, - url: pageUrl, - description: frontmatter.description || pageDescription, - id: pageId, - }); - }) - ); - - const documentationSections = { - "protocol": { name: "Core Protocol" }, - "ai": { name: "AI and Agents" }, - "chain-abstraction": { name: "Chain Abstraction" }, - "smart-contracts": { name: "Smart Contracts" }, - "web3-apps": { name: "Web3 Applications" }, - "primitives": { name: "Tokens and Primitives" }, - "tools": { name: "Developer Tools" }, - "tutorials": { name: "Tutorials and Examples" }, - "api": { name: "API Reference" }, - "data-infrastructure": { name: "Data Infrastructure" }, - "integrations": { name: "Integration Examples" }, - "resources": { name: "Resources" } - }; - - let documentationContent = `# NEAR Protocol Documentation - -> NEAR is a layer-1 blockchain built for scale and multichain compatibility, featuring AI-native infrastructure and chain abstraction capabilities. This documentation covers smart contracts, Web3 applications, AI agents, cross-chain development, and the complete NEAR ecosystem. -NEAR Protocol is a proof-of-stake blockchain that enables developers to build decentralized applications with seamless user experiences. Key features include human-readable account names, minimal transaction fees, and built-in developer tools. The platform supports multiple programming languages and provides chain abstraction for cross-blockchain interactions. -This documentation is organized into several main sections: Protocol fundamentals, AI and agent development, chain abstraction features, smart contract development, Web3 application building, and comprehensive API references. Each section includes tutorials, examples, and detailed technical specifications. - -`; - const links =[]; - for (const sectionKey in documentationSections) { - const section = documentationSections[sectionKey]; - const sectionPages = documentationPages[sectionKey] || []; - - let sectionContent = `## ${section.name}\n`; - - const orderedPages = sectionPages.sort((a, b) => { - return a.url.localeCompare(b.url); - }); - - for (const page of orderedPages) { - const cleanDescription = (page.description || page.title) - .replace(/\s*\n\s*/g, ' ') - .trim(); - links.push(`https://docs.near.org/${page.url}`); - - sectionContent += `- [${page.title}](https://docs.near.org/${page.url}): ${cleanDescription}\n`; - } - - documentationContent += sectionContent + '\n'; - } - - const outputFilePath = path.join(BUILD_DIR, 'llms.txt'); - const outputDirectory = BUILD_DIR; - - if (!fs.existsSync(outputDirectory)) { - fs.mkdirSync(outputDirectory, { recursive: true }); - } - - fs.writeFileSync(outputFilePath, documentationContent, 'utf-8'); - - console.log("Checking links..."); - const results = await Promise.all(links.map(checkLink)); - const broken = results.filter(r => !r.ok); - if (broken.length > 0) { - console.log('\nšŸ”“ Broken URLs:'); - broken.forEach(b => console.log(`${b.url} - Status: ${b.status}`)); - } else { - console.log('šŸŸ¢ All links are valid'); - } -} - -processMarkdownFiles(); diff --git a/website/scripts/copy-md-to-static.mjs b/website/scripts/copy-md-to-static.mjs new file mode 100644 index 00000000000..6ffe0dedf6a --- /dev/null +++ b/website/scripts/copy-md-to-static.mjs @@ -0,0 +1,570 @@ +#!/usr/bin/env node + +import { globSync } from 'glob'; +import path from 'path'; +import fs from 'fs'; +import { + DOCS_DIR, + BUILD_DIR, + BASE_URL, + extractFrontmatter, +} from './shared.mjs'; + + +export const DOCUMENTATION_SECTIONS = { + protocol: 'Core Protocol', + ai: 'AI and Agents', + 'chain-abstraction': 'Chain Abstraction', + 'smart-contracts': 'Smart Contracts', + 'web3-apps': 'Web3 Applications', + primitives: 'Tokens and Primitives', + tools: 'Developer Tools', + tutorials: 'Tutorials and Examples', + api: 'API Reference', + 'data-infrastructure': 'Data Infrastructure', + integrations: 'Integration Examples', + aurora: 'Aurora', + quest: 'Learning Quests', +}; + + +const SKIPPED_FILES = new Set(['index.md', 'help.md']); + +const githubCache = new Map(); +let cacheHits = 0; + +const JSX_COMPONENTS = [ + 'TabItem', 'Tabs', 'CodeTabs', + 'Card', 'ConceptCard', + 'SplitLayoutContainer', 'SplitLayoutLeft', 'SplitLayoutRight', + 'Language', 'Block', + 'Quiz', 'Progress','MultipleChoice', 'Option', + 'LantstoolLabel', 'TryOutOnLantstool', + 'MovingForwardSupportSection', 'SigsSupport', 'TryDemo', + 'ExplainCode', 'CodeBlock', + 'LandingHero', 'Faucet', 'AIBadges', + 'CreateTokenForm', 'MintNFT', + 'FeatureList', 'Column', 'Feature', +]; + +const LLMS_TXT_HEADER = `# NEAR Protocol Documentation + +> NEAR is a layer-1 blockchain built for scale and multichain compatibility, +> featuring AI-native infrastructure and chain abstraction capabilities. +> This documentation covers smart contracts, Web3 applications, AI agents, +> cross-chain development, and the complete NEAR ecosystem. + +NEAR Protocol is a proof-of-stake blockchain that enables developers to build +decentralized applications with seamless user experiences. Key features include +human-readable account names, minimal transaction fees, and built-in developer +tools. The platform supports multiple programming languages and provides chain +abstraction for cross-blockchain interactions. + +This documentation is organized into several main sections: Protocol fundamentals, +AI and agent development, chain abstraction features, smart contract development, +Web3 application building, and comprehensive API references. Each section includes +tutorials, examples, and detailed technical specifications. + + +`; + + +function parseGitHubTag(tag) { + const normalized = tag.replace(/'/g, '"'); + + const urlMatch = normalized.match(/url="(.*?)"/); + if (!urlMatch) return null; + + const url = urlMatch[1].split('#')[0]; + const urlObj = new URL(url); + const segments = urlObj.pathname.slice(1).split('/'); + if (segments.length < 4) return null; + + const [org, repo, , branch, ...rest] = segments; + const filePath = rest.join('/'); + + return { + rawUrl: `https://raw.githubusercontent.com/${org}/${repo}/${branch}/${filePath}`, + normalized, + }; +} + +async function fetchGitHubCode(tag) { + const parsed = parseGitHubTag(tag); + if (!parsed) { + console.warn('Invalid GitHub tag format'); + return null; + } + + if (githubCache.has(parsed.rawUrl)) { + cacheHits++; + return githubCache.get(parsed.rawUrl); + } + + try { + const response = await fetch(parsed.rawUrl); + + if (!response.ok) { + throw new Error(`HTTP ${response.status}`); + } + + const code = await response.text(); + const result = { code, normalized: parsed.normalized }; + + githubCache.set(parsed.rawUrl, result); + + return result; + } catch (error) { + console.error(`āŒ Failed to fetch ${parsed.rawUrl}: ${error.message}`); + return null; + } +} + +function extractCodeSlice(code, tagAttrs) { + const startMatch = tagAttrs.match(/start="(\d*)"/); + const endMatch = tagAttrs.match(/end="(\d*)"/); + + const lines = String(code).split('\n'); + const start = startMatch ? Math.max(parseInt(startMatch[1]) - 1, 0) : 0; + const end = endMatch ? parseInt(endMatch[1]) + 1 : lines.length; + + return lines.slice(start, end).join('\n'); +} + +async function replaceTagsWithCode(content, tagName, { includeLanguage = false } = {}) { + const tagRegex = new RegExp(`<${tagName}\\s[^>]*?(?:\\/>|>[^<]*<\\/${tagName}>)`, 'g'); + const tags = content.match(tagRegex) || []; + + for (const tag of tags) { + const codeData = await fetchGitHubCode(tag); + if (!codeData) continue; + + let language = ''; + if (includeLanguage) { + const langMatch = codeData.normalized.match(/language="(.*?)"/); + language = langMatch ? langMatch[1] : ''; + } + + const slice = extractCodeSlice(codeData.code, codeData.normalized); + content = content.replace(tag, `\`\`\`${language}\n${slice}\n\`\`\``); + } + + return content; +} + +function transformOutsideCodeBlocks(content, transformFn) { + const segments = content.split(/(```[\s\S]*?```)/g); + return segments.map((segment, index) => { + if (index % 2 === 1) return segment; // code block, keep as-is + return transformFn(segment); + }).join(''); +} + +function removeImports(content) { + return transformOutsideCodeBlocks(content, (text) => + text + .replace(/^import\s+.*?from\s+['"].*?['"];?\s*$/gm, '') + .replace(/^import\s+\{[^}]*\}\s+from\s+['"].*?['"];?\s*$/gm, '') + .replace(/^import\s+.*?$/gm, '') + ); +} + + +function convertComponentTitles(content) { + const openTag = /^\s*<(?:Card|ConceptCard)\b/; + const lines = content.split('\n'); + const result = []; + let collecting = false; + let collected = ''; + + for (const line of lines) { + if (!collecting && openTag.test(line)) { + collected = line; + collecting = true; + const trimmed = line.trim(); + if (trimmed.endsWith('>') || trimmed.endsWith('/>')) { + const heading = extractHeadingFromAttrs(collected); + result.push(heading !== null ? heading : line); + collecting = false; + collected = ''; + } + continue; + } + + if (collecting) { + collected += '\n' + line; + const trimmed = line.trim(); + if (trimmed === '>' || trimmed === '/>') { + const heading = extractHeadingFromAttrs(collected); + result.push(heading !== null ? heading : collected); + collecting = false; + collected = ''; + } + continue; + } + + result.push(line); + } + + return result.join('\n'); +} + +function extractHeadingFromAttrs(tagText) { + const titleMatch = tagText.match(/title=["']([^"']+)["']/); + if (!titleMatch) return null; + + const title = titleMatch[1]; + const hrefMatch = tagText.match(/href=["']([^"']+)["']/); + return hrefMatch ? `### [${title}](${hrefMatch[1]})` : `### ${title}`; +} + +function stripJsx(content) { + const names = JSX_COMPONENTS.join('|'); + const singleLineSelfClosing = new RegExp(`^\\s*<(?:${names})\\b.*/>\\s*$`); + const singleLineOpening = new RegExp(`^\\s*<(?:${names})\\b.*>\\s*$`); + const closingTag = new RegExp(`^\\s*\\s*$`); + const multiLineStart = new RegExp(`^\\s*<(?:${names})\\b`); + + const lines = content.split('\n'); + const result = []; + let insideMultiLineTag = false; + + for (const line of lines) { + if (insideMultiLineTag) { + const trimmed = line.trim(); + if (trimmed === '/>' || trimmed === '>' || trimmed.endsWith('/>') || trimmed.endsWith('>')) { + insideMultiLineTag = false; + } + continue; + } + + if (singleLineSelfClosing.test(line)) continue; + if (singleLineOpening.test(line)) continue; + if (closingTag.test(line)) continue; + + if (multiLineStart.test(line) && !line.includes('>')) { + insideMultiLineTag = true; + continue; + } + + result.push(line); + } + + content = result.join('\n'); + + return content + .replace(/]*\/?>/g, '---') + .replace(/]*>([\s\S]*?)<\/a>/g, '[$2]($1)') + .replace(/]*\/?>/g, '![$2]($1)') + .replace(/
  • (.*?)<\/li>/gm, '- $1') + .replace(/<\/?(?:ul|ol)>/g, '') + .replace(/]*>/g, '') + .replace(/<\/div>/g, '') + .replace(/]*>/g, '') + .replace(/<\/span>/g, '') + .replace(/<\/?p>/g, '') + .replace(//g, '\n') + .replace(/|\/>)/g, '') + .replace(/^[ \t]*
    [ \t]*$/gm, '') + .replace(/^[ \t]*<\/details>[ \t]*$/gm, '') + .replace(/^[ \t]*\s*(.*?)\s*<\/summary>[ \t]*$/gm, '**$1**') + .replace(/^[ \t]*:::\w+.*$/gm, '') + .replace(/^[ \t]*:::$/gm, ''); +} + +function removeJsxTags(content) { + return transformOutsideCodeBlocks(content, stripJsx) + .replace(/\n{3,}/g, '\n\n'); +} + +function dedentCodeBlocks(content) { + const lines = content.split('\n'); + const result = []; + let insideCodeBlock = false; + let indentSize = 0; + + for (const line of lines) { + if (!insideCodeBlock) { + const openMatch = line.match(/^(\s+)(```\w)/); + if (openMatch) { + indentSize = openMatch[1].length; + result.push(line.slice(indentSize)); + insideCodeBlock = true; + continue; + } + result.push(line); + } else { + const stripped = line.length >= indentSize && line.slice(0, indentSize).trim() === '' + ? line.slice(indentSize) + : line; + result.push(stripped); + if (stripped.trimEnd() === '```') { + insideCodeBlock = false; + } + } + } + + return result.join('\n'); +} + +function ensureCodeBlockSpacing(content) { + content = content.replace(/```([^\n`]+)```/g, '```\n\n$1\n\n```'); + + const lines = content.split('\n'); + const result = []; + + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + const trimmed = line.trimStart(); + const isOpenFence = /^```\w/.test(trimmed); + const isCloseFence = line.trimEnd() === '```'; + + if (isOpenFence && !isCloseFence) { + if (result.length > 0 && result[result.length - 1].trim() !== '') { + result.push(''); + } + result.push(line); + } else if (isCloseFence && !isOpenFence) { + result.push(line); + if (i + 1 < lines.length && lines[i + 1].trim() !== '') { + result.push(''); + } + } else { + result.push(line); + } + } + + return result.join('\n'); +} + +function minifyMarkdown(content) { + return transformOutsideCodeBlocks(content, (text) => + text + .replace(//g, '') + .replace(/[ \t]+$/gm, '') + ) + .replace(/\n{3,}/g, '\n\n') + .trim() + '\n'; +} + +function cleanFrontmatter(content) { + const fmMatch = content.match(/^---\n([\s\S]*?)\n---/); + if (!fmMatch) return content; + + // 'sidebar_label' + const keysToRemove = new Set([ + 'hide_title', 'slug', 'hide_table_of_contents' + ]); + + const fmLines = fmMatch[1].split('\n'); + const cleaned = fmLines.filter(line => { + const keyMatch = line.match(/^(\w[\w_-]*)\s*:/); + return !keyMatch || !keysToRemove.has(keyMatch[1]); + }); + + return content.replace(fmMatch[0], `---\n${cleaned.join('\n')}\n---`); +} + +function fixImagePaths(content) { + return content.replace(/@site\/static\//g, `${BASE_URL}/`); +} + +function resolveRelativeLinks(content, relativeFilePath) { + const fileDir = path.dirname(relativeFilePath); + + return content.replace(/\]\(([^)]+)\)/g, (match, linkPath) => { + if (/^(https?:|#|data:|\/\/)/.test(linkPath)) return match; + + if (linkPath.startsWith('/')) { + return `](${BASE_URL}${linkPath})`; + } + + if (linkPath.startsWith('./') || linkPath.startsWith('../')) { + const [pathPart, ...anchorParts] = linkPath.split('#'); + const anchor = anchorParts.length ? '#' + anchorParts.join('#') : ''; + const resolved = path.normalize(path.join(fileDir, pathPart)); + return `](${BASE_URL}/${resolved}${anchor})`; + } + + return match; + }); +} + +async function cleanContent(content, relativeFilePath) { + let cleaned = removeImports(content); + cleaned = await replaceTagsWithCode(cleaned, 'Github', { includeLanguage: true }); + cleaned = await replaceTagsWithCode(cleaned, 'File', { includeLanguage: true }); + cleaned = convertComponentTitles(cleaned); + cleaned = removeJsxTags(cleaned); + cleaned = fixImagePaths(cleaned); + cleaned = resolveRelativeLinks(cleaned, relativeFilePath); + cleaned = dedentCodeBlocks(cleaned); + cleaned = ensureCodeBlockSpacing(cleaned); + cleaned = cleanFrontmatter(cleaned); + cleaned = minifyMarkdown(cleaned); + return cleaned; +} + + +function getFirstNonEmptyLine(text) { + for (const line of text.split('\n')) { + const trimmed = line.trim(); + if (trimmed.length > 0) return trimmed; + } + return ''; +} + + +function getOutputPath(filePath, frontmatterId) { + const relativePath = path.relative(DOCS_DIR, filePath); + const dirPath = path.dirname(relativePath); + + // border case, for /frontmatterId/frontmatterId gets transformed into /frontmatterId.md instead of /frontmatterId/frontmatterId.md + if (path.basename(dirPath) === frontmatterId) { + return path.join(BUILD_DIR, path.dirname(dirPath), `${frontmatterId}.md`); + } + + return path.join(BUILD_DIR, dirPath, `${frontmatterId}.md`); +} + + +function writeFileSafe(filePath, content) { + const dir = path.dirname(filePath); + if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync(filePath, content, 'utf8'); +} + +function buildPageMetadata(relativeFilename, frontmatter, body) { + const pathSegments = relativeFilename.split('/'); + const section = pathSegments[0]; + const fileName = pathSegments.pop(); + const title = frontmatter.title || frontmatter.sidebar_label; + + if (!title) { + console.warn(`āš ļø Missing title in ${relativeFilename}`); + } + + const pageId = frontmatter.id; + const url = `${pathSegments.join('/')}/${frontmatter.id ? frontmatter.id + '.md' : fileName}`; + const firstLine = getFirstNonEmptyLine(body); + + // Validate description + let description = frontmatter.description; + if (!description) { + console.log(`āŒ No description tag found for ${relativeFilename}`); + + if (firstLine.startsWith('#') || firstLine.startsWith('import') || + firstLine.startsWith(':::') || firstLine.startsWith('![')) { + console.warn(`āš ļø No valid description found in ${relativeFilename}`); + description = title; // Fallback to title + } else { + description = firstLine; + } + } + + return { + section, + title, + url, + description, + id: pageId, + }; +} + + +async function checkLink(url) { + try { + const response = await fetch(url, { method: 'HEAD', signal: AbortSignal.timeout(5000) }); + return { url, status: response.status, ok: response.ok }; + } catch (error) { + const status = 'NO RESPONSE'; + console.log(`āŒ ${url} - ${status}`); + return { url, status, ok: false }; + } +} + +async function reportBrokenLinks(links) { + console.log('Checking links...'); + const results = await Promise.all(links.map(checkLink)); + const broken = results.filter(r => !r.ok); + + if (broken.length > 0) { + console.log('\nšŸ”“ Broken URLs:'); + broken.forEach(b => console.log(`${b.url} - Status: ${b.status}`)); + } else { + console.log('šŸŸ¢ All links are valid'); + } +} + + +function buildLlmsTxt(pagesBySection) { + let content = LLMS_TXT_HEADER; + const links = []; + + for (const [key, section] of Object.entries(DOCUMENTATION_SECTIONS)) { + const pages = (pagesBySection[key] || []).sort((a, b) => a.url.localeCompare(b.url)); + if (pages.length === 0) continue; + + content += `## ${section}\n`; + for (const page of pages) { + const desc = (page.description || page.title).replace(/\s*\n\s*/g, ' ').trim(); + const fullUrl = `${BASE_URL}/${page.url}`; + links.push(fullUrl); + content += `- [${page.title}](${fullUrl}): ${desc}\n`; + } + content += '\n'; + } + + return { content, links }; +} + +async function processMarkdownFiles() { + console.log('šŸš€ Starting markdown files post-processing...'); + console.log('This script will copy processed .md files alongside .html files in build/'); + + const allMarkdownFiles = globSync(path.join(DOCS_DIR, '**/*.md')); + console.log(`šŸ“ Found ${allMarkdownFiles.length} markdown files`); + + const pagesBySection = {}; + let processedCount = 0; + let errorCount = 0; + + await Promise.all( + allMarkdownFiles.map(async (filePath) => { + try { + const relativeFilename = path.relative(DOCS_DIR, filePath); + const rawContent = fs.readFileSync(filePath, 'utf8'); + const cleanedContent = await cleanContent(rawContent, relativeFilename); + const { frontmatter, body } = extractFrontmatter(cleanedContent); + + const outputPath = getOutputPath(filePath, frontmatter.id); + writeFileSafe(outputPath, cleanedContent); + + processedCount++; + if (processedCount % 10 === 0) { + console.log(`āœ… Processed ${processedCount}/${allMarkdownFiles.length} files`); + } + + if (SKIPPED_FILES.has(relativeFilename)) return; + + const page = buildPageMetadata(relativeFilename, frontmatter, body); + + if (!pagesBySection[page.section]) { + pagesBySection[page.section] = []; + } + pagesBySection[page.section].push(page); + } catch (error) { + errorCount++; + console.error(`āŒ Error processing ${path.relative(DOCS_DIR, filePath)}: ${error.message}`); + } + }) + ); + + const { content: llmsTxt, links } = buildLlmsTxt(pagesBySection); + writeFileSafe(path.join(BUILD_DIR, 'llms.txt'), llmsTxt); + console.log(`šŸ“ Generated llms.txt with ${links.length} links\n`); + + await reportBrokenLinks(links); +} + +processMarkdownFiles(); diff --git a/website/scripts/index-meilisearch.mjs b/website/scripts/index-meilisearch.mjs index fd6e94b82db..118c4254c1d 100644 --- a/website/scripts/index-meilisearch.mjs +++ b/website/scripts/index-meilisearch.mjs @@ -3,19 +3,13 @@ import { MeiliSearch } from 'meilisearch'; import fs from 'fs'; import { globSync } from 'glob'; import path from 'path'; -import { fileURLToPath } from 'url'; +import { + BUILD_DIR, + extractFrontmatter, +} from './shared.mjs'; import { createHash } from 'crypto'; -const __dirname = path.dirname(fileURLToPath(import.meta.url)); - -const MEILI_HOST = process.env.MEILI_HOST || 'http://localhost:7700'; -const MEILI_MASTER_KEY = process.env.MEILI_MASTER_KEY || 'masterKey123'; -const MEILI_INDEX_NAME = process.env.MEILI_INDEX_NAME || 'near-docs'; -const DOCS_PATH = path.resolve(__dirname, '../static'); -const BATCH_SIZE = 100; -const TASK_TIMEOUT = 300000; // 5 minutes timeout for tasks with embedders - -const CATEGORY_MAP = { +export const CATEGORY_MAP = { 'protocol': 'Protocol', 'chain-abstraction': 'Multi-Chain', 'ai': 'AI & Agents', @@ -27,17 +21,22 @@ const CATEGORY_MAP = { 'api': 'API', }; +const MEILI_HOST = process.env.MEILI_HOST || 'http://localhost:7700'; +const MEILI_MASTER_KEY = process.env.MEILI_MASTER_KEY || 'masterKey123'; +const MEILI_INDEX_NAME = process.env.MEILI_INDEX_NAME || 'near-docs'; +const BATCH_SIZE = 100; +const TASK_TIMEOUT = 300000; + function getCategoryFromPath(filePath) { - const relativePath = path.relative(DOCS_PATH, filePath); + const relativePath = path.relative(BUILD_DIR, filePath); const firstFolder = relativePath.split(path.sep)[0]; return CATEGORY_MAP[firstFolder] || 'General'; } function getHierarchy(filePath) { - const relativePath = path.relative(DOCS_PATH, filePath); + const relativePath = path.relative(BUILD_DIR, filePath); const parts = relativePath.split(path.sep); - // Remove file name parts.pop(); const hierarchy = { @@ -59,82 +58,82 @@ function getHierarchy(filePath) { .replace(/^\d+\s*/g, '') .replace(/\b\w/g, c => c.toUpperCase()); } - + return hierarchy; } +function getUrlPath(filePath) { + const relativePath = path.relative(BUILD_DIR, filePath); + const pathParts = relativePath.replace(/\\/g, '/').split('/'); + const fileName = pathParts.pop().replace(/\.mdx?$/, ''); + + const docId = fileName.replace(/^\d+-/, ''); + + const cleanPathParts = pathParts.map(part => part.replace(/^\d+-/, '')); + + let urlPath; + + if (docId === 'index') { + urlPath = cleanPathParts.join('/'); + } else { + const parentFolder = cleanPathParts[cleanPathParts.length - 1]; + if (docId === parentFolder) { + urlPath = cleanPathParts.join('/'); + } else { + urlPath = [...cleanPathParts, docId].join('/'); + } + } + + return '/' + urlPath; +} + async function indexDocuments() { - // console.log('Starting MeiliSearch indexation...'); - // console.log(`Host: ${MEILI_HOST}`); - // console.log(`Index: ${MEILI_INDEX_NAME}`); - - // // Initialize client - // const client = new MeiliSearch({ - // host: MEILI_HOST, - // apiKey: MEILI_MASTER_KEY, - // }); - - // // Check connection - // try { - // const health = await client.health(); - // console.log('MeiliSearch status:', health.status); - // } catch (error) { - // console.error('Failed to connect to MeiliSearch:', error.message); - // console.error('Make sure MeiliSearch is running at', MEILI_HOST); - // process.exit(1); - // } - - // // Get index - // let index; - // index = await client.getIndex(MEILI_INDEX_NAME); - // console.log('Using existing index:', MEILI_INDEX_NAME); - - // // Configure index settings - // console.log('Configuring index settings...'); - // await index.updateSettings({ - // searchableAttributes: ['title', 'content', 'section', 'hierarchy_lvl0', 'hierarchy_lvl1', 'hierarchy_lvl2'], - // filterableAttributes: ['category', 'version', 'hierarchy_lvl0'], - // sortableAttributes: ['timestamp'], - // rankingRules: ['words', 'typo', 'proximity', 'attribute', 'sort', 'exactness'], - // distinctAttribute: 'path', - // embedders: { - // default: { - // source: 'huggingFace', - // model: 'sentence-transformers/all-MiniLM-L6-v2', - // documentTemplate: '{{doc.title}} {{doc.content}}', - // }, - // }, - // }); - - // Get all markdown files - let files = globSync('../static/**/*.md', { cwd: __dirname }); + console.log('Starting MeiliSearch indexation...'); + console.log(`Host: ${MEILI_HOST}`); + console.log(`Index: ${MEILI_INDEX_NAME}`); + + const client = new MeiliSearch({ + host: MEILI_HOST, + apiKey: MEILI_MASTER_KEY, + }); + + let index = await client.getIndex(MEILI_INDEX_NAME); + console.log('Using existing index:', MEILI_INDEX_NAME); + + await index.updateSettings({ + searchableAttributes: ['title', 'content', 'section', 'hierarchy_lvl0', 'hierarchy_lvl1', 'hierarchy_lvl2'], + filterableAttributes: ['category', 'version', 'hierarchy_lvl0'], + sortableAttributes: ['timestamp'], + rankingRules: ['words', 'typo', 'proximity', 'attribute', 'sort', 'exactness'], + distinctAttribute: 'path', + embedders: { + default: { + source: 'huggingFace', + model: 'BAAI/bge-small-en-v1.5', + documentTemplate: '{{doc.title}} {{doc.content}}', + }, + } + }); + + let files = globSync(path.join(BUILD_DIR, '**/*.md')); console.log(`Found ${files.length} markdown files`); - console.log(files[0]); - exit(); - - // Process files into documents const documents = []; for (const filePath of files) { try { const content = fs.readFileSync(filePath, 'utf-8'); const { frontmatter, body } = extractFrontmatter(content); - const headings = extractHeadings(body); - const cleanedContent = cleanContent(body); - const urlPath = getUrlPath(filePath, frontmatter); - const title = frontmatter.title || - frontmatter.sidebar_label || - headings[0] || - path.basename(filePath, path.extname(filePath)).replace(/-/g, ' '); + const urlPath = getUrlPath(filePath); + const title = frontmatter.title; const hierarchy = getHierarchy(filePath); const doc = { - id: generateId(urlPath), + id: createHash('md5').update(urlPath).digest('hex').substring(0, 12), title, - content: cleanedContent, // Limit content size + content: body, path: urlPath, section: frontmatter.sidebar_label || title, category: hierarchy.lvl0, diff --git a/website/scripts/shared.mjs b/website/scripts/shared.mjs new file mode 100644 index 00000000000..53faeb4cd84 --- /dev/null +++ b/website/scripts/shared.mjs @@ -0,0 +1,32 @@ +import path from 'path'; +import { fileURLToPath } from 'url'; + + +const SCRIPTS_DIR = path.dirname(fileURLToPath(import.meta.url)); + +export const DOCS_DIR = path.join(SCRIPTS_DIR, '../../docs'); +export const BUILD_DIR = path.join(SCRIPTS_DIR, '../build'); +export const BASE_URL = 'https://docs.near.org'; + + +export function extractFrontmatter(content) { + if (!content.startsWith('---\n')) return { frontmatter: {}, body: content }; + + const endIndex = content.indexOf('\n---\n', 4); + if (endIndex === -1) return { frontmatter: {}, body: content }; + + const frontmatterText = content.substring(4, endIndex); + const body = content.substring(endIndex + 5); + const frontmatter = {}; + + for (const line of frontmatterText.split('\n')) { + const colonIndex = line.indexOf(':'); + if (colonIndex > 0) { + const key = line.substring(0, colonIndex).trim(); + const value = line.substring(colonIndex + 1).trim().replace(/^["']|["']$/g, ''); + frontmatter[key] = value; + } + } + + return { frontmatter, body }; +} \ No newline at end of file diff --git a/website/src/theme/SearchBar/AIChatInSearch.tsx b/website/src/theme/SearchBar/AIChatInSearch.tsx new file mode 100644 index 00000000000..1ecbc7b5ed9 --- /dev/null +++ b/website/src/theme/SearchBar/AIChatInSearch.tsx @@ -0,0 +1,508 @@ +import React, { useState, useRef, useEffect } from 'react'; +import { useChat, type UIMessage } from '@ai-sdk/react'; +import { useColorMode } from '@docusaurus/theme-common'; +import MarkdownRenderer from '../../components/AIChat/MarkdownRenderer'; +import styles from './styles.module.css'; + +interface Source { + title: string; + path: string; +} + +type ChatUIMessage = UIMessage<{ sources?: Source[] }>; + +interface HistoryMessage { + role: 'user' | 'assistant'; + content: string; +} + +interface ParsedStreamChunk { + textDelta?: string; + fullText?: string; + sources?: Source[]; + done?: boolean; +} + +interface AIChatInSearchProps { + initialQuery: string; + onSaveConversation?: (data: SavedConversation) => void; + savedConversation?: SavedConversation | null; +} + +export interface SavedConversation { + messages: ChatUIMessage[]; +} + +const API_URL = 'https://docs-mcp-f18b.onrender.com/api/chat'; + +const SUGGESTIONS = [ + 'How do I create a NEAR account?', + 'What is a smart contract on NEAR?', + 'How do cross-contract calls work?', + 'How to build a dApp on NEAR?', +]; + +function extractMessageText(message: ChatUIMessage): string { + return message.parts + .filter((part): part is { type: 'text'; text: string } => part.type === 'text') + .map((part) => part.text) + .join(''); +} + +function toChatMessage({ + id, + role, + text, + sources, +}: { + id: string; + role: 'user' | 'assistant'; + text: string; + sources?: Source[]; +}): ChatUIMessage { + return { + id, + role, + parts: [{ type: 'text', text }], + metadata: sources ? { sources } : undefined, + }; +} + +function safeParseJson(value: string): unknown | null { + try { + return JSON.parse(value); + } catch { + return null; + } +} + +function parsePayloadChunk(payload: string): ParsedStreamChunk { + const trimmed = payload.trim(); + if (!trimmed) { + return {}; + } + + if (trimmed === '[DONE]') { + return { done: true }; + } + + const parsed = safeParseJson(trimmed); + if (!parsed || typeof parsed !== 'object') { + return { textDelta: payload }; + } + + const data = parsed as Record; + const type = typeof data.type === 'string' ? data.type : undefined; + + if (type === 'done') { + return { done: true }; + } + + if (type === 'text') { + return { + textDelta: typeof data.text === 'string' ? data.text : '', + done: false, + }; + } + + if (type === 'sources') { + const typedSources = Array.isArray(data.sources) + ? (data.sources.filter( + (source): source is Source => + typeof source === 'object' && + source !== null && + typeof (source as Source).title === 'string' && + typeof (source as Source).path === 'string', + ) as Source[]) + : undefined; + + return { + sources: typedSources, + done: false, + }; + } + + const sources = Array.isArray(data.sources) + ? (data.sources.filter( + (source): source is Source => + typeof source === 'object' && + source !== null && + typeof (source as Source).title === 'string' && + typeof (source as Source).path === 'string', + ) as Source[]) + : undefined; + + const fullText = + typeof data.message === 'string' + ? data.message + : typeof data.content === 'string' + ? data.content + : undefined; + + const textDelta = + typeof data.delta === 'string' + ? data.delta + : typeof data.token === 'string' + ? data.token + : typeof data.chunk === 'string' + ? data.chunk + : undefined; + + return { + fullText, + textDelta, + sources, + done: data.done === true || data.finished === true, + }; +} + +export default function AIChatInSearch({ + initialQuery, + onSaveConversation, + savedConversation, +}: AIChatInSearchProps) { + const { colorMode } = useColorMode(); + const isDarkTheme = colorMode === 'dark'; + + const { messages, setMessages, status } = useChat({ + messages: savedConversation?.messages || [], + }); + + const [inputValue, setInputValue] = useState(''); + const [seconds, setSeconds] = useState(1); + + const isLoading = status === 'submitted' || status === 'streaming'; + + const messagesEndRef = useRef(null); + const inputRef = useRef(null); + const hasSentInitial = useRef(false); + + async function sendMessageToApi(text: string) { + const userMsgId = String(Date.now()); + const aiMsgId = String(Date.now() + 1); + const userMessage = toChatMessage({ id: userMsgId, role: 'user', text }); + + setMessages((prev) => [...prev, userMessage, toChatMessage({ id: aiMsgId, role: 'assistant', text: '' })]); + setInputValue(''); + + try { + const history: HistoryMessage[] = [...messages, userMessage] + .filter((msg) => msg.role === 'assistant' || msg.role === 'user') + .map((msg) => ({ + role: msg.role === 'assistant' ? ('assistant' as const) : ('user' as const), + content: extractMessageText(msg), + })); + + const response = await fetch(API_URL, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Accept: 'application/json, text/event-stream, text/plain', + }, + body: JSON.stringify({ message: text, history }), + }); + + if (!response.ok) { + throw new Error(`Request failed with status ${response.status}`); + } + + const contentType = response.headers.get('content-type') || ''; + const updateAiMessage = (nextText: string, nextSources?: Source[]) => { + setMessages((prev) => + prev.map((msg) => + msg.id === aiMsgId + ? { + ...msg, + parts: [{ type: 'text', text: nextText }], + metadata: nextSources ? { ...msg.metadata, sources: nextSources } : msg.metadata, + } + : msg, + ), + ); + }; + + if (!response.body) { + if (contentType.includes('application/json')) { + const data = await response.json(); + const messageText = + typeof data?.message === 'string' ? data.message : 'No response received.'; + const sources = Array.isArray(data?.sources) ? data.sources : undefined; + updateAiMessage(messageText, sources); + } else { + const textResponse = await response.text(); + updateAiMessage(textResponse || 'No response received.'); + } + return; + } + + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + let accumulatedText = ''; + let streamedSources: Source[] | undefined; + let buffer = ''; + const isSse = contentType.includes('text/event-stream'); + + const applyParsedChunk = (chunk: ParsedStreamChunk) => { + if (chunk.fullText !== undefined) { + accumulatedText = chunk.fullText; + } else if (chunk.textDelta !== undefined) { + accumulatedText += chunk.textDelta; + } + + if (chunk.sources && chunk.sources.length > 0) { + streamedSources = chunk.sources; + } + + if (chunk.fullText !== undefined || chunk.textDelta !== undefined || chunk.sources) { + updateAiMessage(accumulatedText, streamedSources); + } + }; + + const processSseBuffer = () => { + let eventBoundary = buffer.indexOf('\n\n'); + + while (eventBoundary !== -1) { + const rawEvent = buffer.slice(0, eventBoundary); + buffer = buffer.slice(eventBoundary + 2); + + const dataPayload = rawEvent + .split('\n') + .filter((line) => line.startsWith('data:')) + .map((line) => line.slice(5).trimStart()) + .join('\n'); + + if (dataPayload) { + const parsedChunk = parsePayloadChunk(dataPayload); + if (parsedChunk.done) { + return true; + } + applyParsedChunk(parsedChunk); + } + + eventBoundary = buffer.indexOf('\n\n'); + } + + return false; + }; + + const processNdjsonBuffer = () => { + const lines = buffer.split('\n'); + buffer = lines.pop() || ''; + + for (const line of lines) { + const trimmedLine = line.trim(); + if (!trimmedLine) { + continue; + } + + const parsedChunk = parsePayloadChunk(trimmedLine); + if (parsedChunk.done) { + return true; + } + applyParsedChunk(parsedChunk); + } + + return false; + }; + + let finished = false; + while (!finished) { + const { value, done } = await reader.read(); + if (done) { + break; + } + + const chunkText = decoder.decode(value, { stream: true }); + buffer += chunkText; + + if (isSse) { + finished = processSseBuffer(); + } else if (contentType.includes('application/json') || contentType.includes('ndjson')) { + finished = processNdjsonBuffer(); + } else { + accumulatedText += chunkText; + updateAiMessage(accumulatedText, streamedSources); + } + } + + const trailingText = decoder.decode(); + if (trailingText) { + if (isSse || contentType.includes('application/json') || contentType.includes('ndjson')) { + buffer += trailingText; + } else { + accumulatedText += trailingText; + updateAiMessage(accumulatedText, streamedSources); + } + } + + if (buffer.trim()) { + const parsedChunk = parsePayloadChunk(buffer.trim()); + applyParsedChunk(parsedChunk); + } + + if (!accumulatedText.trim()) { + updateAiMessage('No response received.', streamedSources); + } + } catch { + setMessages((prev) => + prev.map((msg) => + msg.id === aiMsgId + ? { + ...msg, + parts: [{ type: 'text', text: 'Sorry, something went wrong. Please try again.' }], + metadata: undefined, + } + : msg, + ), + ); + } finally { + inputRef.current?.focus(); + } + } + + useEffect(() => { + if (messages.length > 0 && onSaveConversation) { + onSaveConversation({ messages }); + } + }, [messages, onSaveConversation]); + + useEffect(() => { + let timer: ReturnType; + if (isLoading) { + timer = setInterval(() => setSeconds((s) => s + 1), 1000); + } else { + setSeconds(1); + } + return () => clearInterval(timer); + }, [isLoading]); + + useEffect(() => { + messagesEndRef.current?.scrollIntoView({ behavior: 'smooth' }); + }, [messages]); + + useEffect(() => { + if (initialQuery.trim() && !hasSentInitial.current && !savedConversation?.messages?.length) { + hasSentInitial.current = true; + sendMessageToApi(initialQuery.trim()); + } + }, [initialQuery, savedConversation]); + + const handleSubmit = (e: React.FormEvent) => { + e.preventDefault(); + if (inputValue.trim() && !isLoading) { + sendMessageToApi(inputValue.trim()); + } + }; + + const visibleMessages = messages.filter((msg) => msg.role === 'assistant' || msg.role === 'user'); + + const turns: Array<{ + id: string; + userText: string; + assistantText?: string; + sources?: Source[]; + }> = []; + + for (let index = 0; index < visibleMessages.length; index += 1) { + const current = visibleMessages[index]; + if (current.role !== 'user') { + continue; + } + + const next = visibleMessages[index + 1]; + const hasAssistant = next && next.role === 'assistant'; + + turns.push({ + id: current.id, + userText: extractMessageText(current), + assistantText: hasAssistant ? extractMessageText(next as ChatUIMessage) : undefined, + sources: hasAssistant ? (next as ChatUIMessage).metadata?.sources : undefined, + }); + + if (hasAssistant) { + index += 1; + } + } + + const showSuggestions = turns.length === 0 && !isLoading; + + return ( +
    +
    + {showSuggestions && ( +
    +

    āœØ Ask anything about NEAR

    +
    + {SUGGESTIONS.map((suggestion) => ( + + ))} +
    +
    + )} + + {turns.map((turn) => { + const hasAssistantText = !!turn.assistantText?.trim(); + + return ( +
    +

    {turn.userText}

    + + {hasAssistantText ? ( +
    + +
    + ) : ( +
    Thinking... ({seconds}s)
    + )} + + {turn.sources && turn.sources.length > 0 && ( +
    + SOURCES: + {turn.sources.map((source) => ( + + {source.title} + + ))} +
    + )} + +
    +
    + ); + })} +
    +
    + +
    + setInputValue(e.target.value)} + disabled={isLoading} + /> + +
    +
    + ); +} diff --git a/website/src/theme/SearchBar/index.tsx b/website/src/theme/SearchBar/index.tsx index 022818b1828..cd35d9b5bb9 100644 --- a/website/src/theme/SearchBar/index.tsx +++ b/website/src/theme/SearchBar/index.tsx @@ -2,8 +2,13 @@ import React, { useState, useEffect, useCallback, useRef } from 'react'; import { useHistory } from '@docusaurus/router'; import useDocusaurusContext from '@docusaurus/useDocusaurusContext'; import { MeiliSearch } from 'meilisearch'; -import { trackSearch, trackSearchResultClick, trackSearchNoResults } from '../../utils/searchAnalytics'; +import { + trackSearch, + trackSearchResultClick, + trackSearchNoResults, +} from '../../utils/searchAnalytics'; import { SearchIcon } from '../Icon/Search'; +import AIChatInSearch, { type SavedConversation } from './AIChatInSearch'; import styles from './styles.module.css'; interface SearchHit { @@ -31,15 +36,15 @@ interface SearchResult { const CATEGORIES = [ { id: 'all', label: 'All' }, - { id: 'Protocol', label: 'Protocol' }, - { id: 'Multi-Chain', label: 'Multi-Chain' }, - { id: 'AI & Agents', label: 'AI' }, - { id: 'Smart Contracts', label: 'Contracts' }, - { id: 'Web3 Apps', label: 'Web3 Apps' }, - { id: 'Primitives', label: 'Tokens & Primitives' }, - { id: 'Data Infrastructure', label: 'Data Infrastructure' }, - { id: 'Tools', label: 'Tools' }, - { id: 'API', label: 'API' }, + { id: 'protocol', label: 'Protocol' }, + { id: 'chain-abstraction', label: 'Multi-Chain' }, + { id: 'ai', label: 'AI & Agents' }, + { id: 'smart-contracts', label: 'Smart Contracts' }, + { id: 'web3-apps', label: 'Web3 Apps' }, + { id: 'primitives', label: 'Primitives' }, + { id: 'data-infrastructure', label: 'Data Infrastructure' }, + { id: 'tools', label: 'Tools' }, + { id: 'api', label: 'API' }, ]; export default function SearchBar(): JSX.Element { @@ -53,6 +58,9 @@ export default function SearchBar(): JSX.Element { const [selectedIndex, setSelectedIndex] = useState(0); const [selectedCategory, setSelectedCategory] = useState('all'); const [client, setClient] = useState(null); + const [mode, setMode] = useState<'search' | 'askDocs'>('search'); + const [askDocsQuery, setAskDocsQuery] = useState(''); + const [savedConversation, setSavedConversation] = useState(null); const inputRef = useRef(null); const resultsRef = useRef(null); @@ -73,6 +81,13 @@ export default function SearchBar(): JSX.Element { } }, [siteConfig]); + const closeModal = useCallback(() => { + setIsOpen(false); + setQuery(''); + setMode('search'); + setAskDocsQuery(''); + }, []); + useEffect(() => { const handleKeyDown = (e: KeyboardEvent) => { if ((e.metaKey || e.ctrlKey) && e.key === 'k') { @@ -80,13 +95,13 @@ export default function SearchBar(): JSX.Element { setIsOpen(true); } if (e.key === 'Escape') { - setIsOpen(false); + closeModal(); } }; document.addEventListener('keydown', handleKeyDown); return () => document.removeEventListener('keydown', handleKeyDown); - }, []); + }, [closeModal]); useEffect(() => { if (isOpen && inputRef.current) { @@ -94,47 +109,50 @@ export default function SearchBar(): JSX.Element { } }, [isOpen]); - const search = useCallback(async (searchQuery: string, category: string) => { - if (!client || !searchQuery.trim()) { - setResults([]); - return; - } + const search = useCallback( + async (searchQuery: string, category: string) => { + if (!client || !searchQuery.trim()) { + setResults([]); + return; + } - const config = siteConfig.customFields?.meilisearch as { indexName?: string } | undefined; - const indexName = config?.indexName || 'near-docs'; - - setLoading(true); - try { - const index = client.index(indexName); - const filter = category !== 'all' ? `category = "${category}"` : undefined; - - const searchResult: SearchResult = await index.search(searchQuery, { - limit: 10, - attributesToHighlight: ['title', 'content'], - highlightPreTag: '', - highlightPostTag: '', - filter, - hybrid: { - semanticRatio: 0.5, - embedder: 'default' - }, - }); + const config = siteConfig.customFields?.meilisearch as { indexName?: string } | undefined; + const indexName = config?.indexName || 'near-docs'; + + setLoading(true); + try { + const index = client.index(indexName); + const filter = category !== 'all' ? `category = "${category}"` : undefined; - setResults(searchResult.hits); - setSelectedIndex(0); + const searchResult: SearchResult = await index.search(searchQuery, { + limit: 10, + attributesToHighlight: ['title', 'content'], + highlightPreTag: '', + highlightPostTag: '', + filter, + hybrid: { + semanticRatio: 0.5, + embedder: 'default', + }, + }); - trackSearch(searchQuery, searchResult.hits.length, category); + setResults(searchResult.hits); + setSelectedIndex(0); - if (searchResult.hits.length === 0) { - trackSearchNoResults(searchQuery); + trackSearch(searchQuery, searchResult.hits.length, category); + + if (searchResult.hits.length === 0) { + trackSearchNoResults(searchQuery); + } + } catch (error) { + console.error('Search error:', error); + setResults([]); + } finally { + setLoading(false); } - } catch (error) { - console.error('Search error:', error); - setResults([]); - } finally { - setLoading(false); - } - }, [client, siteConfig]); + }, + [client, siteConfig], + ); useEffect(() => { const timer = setTimeout(() => { @@ -144,13 +162,19 @@ export default function SearchBar(): JSX.Element { return () => clearTimeout(timer); }, [query, selectedCategory, search]); + const switchToAskDocs = () => { + setAskDocsQuery(query.trim()); + setMode('askDocs'); + }; + const handleKeyDown = (e: React.KeyboardEvent) => { + if (mode === 'askDocs') return; if (e.key === 'ArrowDown') { e.preventDefault(); - setSelectedIndex(prev => Math.min(prev + 1, results.length - 1)); + setSelectedIndex((prev) => Math.min(prev + 1, results.length - 1)); } else if (e.key === 'ArrowUp') { e.preventDefault(); - setSelectedIndex(prev => Math.max(prev - 1, 0)); + setSelectedIndex((prev) => Math.max(prev - 1, 0)); } else if (e.key === 'Enter' && results[selectedIndex]) { e.preventDefault(); navigateToResult(results[selectedIndex], selectedIndex); @@ -159,8 +183,7 @@ export default function SearchBar(): JSX.Element { const navigateToResult = (hit: SearchHit, index: number) => { trackSearchResultClick(query, index, hit.path); - setIsOpen(false); - setQuery(''); + closeModal(); history.push(hit.path); }; @@ -178,11 +201,7 @@ export default function SearchBar(): JSX.Element { return ( <> - -
    - -
    - {CATEGORIES.map((cat) => ( + {loading && mode === 'search' &&
    } +
    - ))} -
    - -
    - {results.length === 0 && query && !loading && ( -
    -

    No results found for "{query}"

    -

    - Try different keywords or browse the documentation -

    -
    - )} - - {results.map((hit, index) => ( - ))} +
    +
    - {results.length > 0 && ( -
    -
    - Enter to select - ā†‘ā†“ to navigate - Esc to close + {mode === 'search' && ( + <> + {query.trim() && ( +
    switchToAskDocs()}> + āœØ + Ask about "{query.trim()}" + AI-powered +
    + )} + +
    + {results.length === 0 && query && !loading && ( +
    +

    No results found for "{query}"

    +

    + Try different keywords or browse the documentation +

    +
    + )} + + {results.map((hit, index) => ( + + ))}
    -
    + +
    +
    + {CATEGORIES.map((cat) => ( + + ))} +
    +
    + + )} + + {mode === 'askDocs' && ( + )}
    diff --git a/website/src/theme/SearchBar/styles.module.css b/website/src/theme/SearchBar/styles.module.css index 5424ba1bf87..0187e354e9a 100644 --- a/website/src/theme/SearchBar/styles.module.css +++ b/website/src/theme/SearchBar/styles.module.css @@ -164,8 +164,7 @@ .categoryFilters { display: flex; gap: 0.5rem; - padding: 0.75rem 1rem; - border-bottom: 1px solid var(--ifm-color-emphasis-200); + padding: 0.75rem 0; background: var(--ifm-color-emphasis-50); overflow-x: auto; overflow-y: hidden; @@ -337,3 +336,280 @@ background: rgba(255, 186, 0, 0.2); color: var(--ifm-color-warning-light); } + +/* Mode Toggle */ +.modeToggle { + display: flex; + background: var(--ifm-color-emphasis-200); + border-radius: 8px; + padding: 2px; + flex-shrink: 0; +} + +.modeToggleBtn { + padding: 0.25rem 0.625rem; + font-size: 0.75rem; + font-weight: 500; + border: none; + border-radius: 6px; + background: transparent; + color: var(--ifm-color-emphasis-600); + cursor: pointer; + transition: all 0.15s ease; + white-space: nowrap; +} + +.modeToggleBtn:hover { + color: var(--ifm-font-color-base); +} + +.modeToggleBtnActive { + background: var(--ifm-background-color); + color: var(--ifm-font-color-base); + box-shadow: 0 1px 3px rgba(0, 0, 0, 0.1); +} + +/* Ask Docs Prompt */ +.askDocsPrompt { + display: flex; + align-items: center; + gap: 0.5rem; + padding: 0.625rem 1rem; + border-bottom: 1px solid var(--ifm-color-emphasis-200); + background: var(--ifm-color-emphasis-50); + cursor: pointer; + transition: background 0.15s ease; +} + +.askDocsPrompt:hover { + background: var(--ifm-color-emphasis-100); +} + +.askDocsPromptIcon { + flex-shrink: 0; + font-size: 0.875rem; +} + +.askDocsPromptText { + font-size: 0.8125rem; + color: var(--ifm-color-primary); + font-weight: 500; +} + +.askDocsPromptHint { + margin-left: auto; + font-size: 0.6875rem; + color: var(--ifm-color-emphasis-500); +} + +/* AI Chat Container */ +.aiChatContainer { + display: flex; + flex-direction: column; + max-height: 400px; +} + +.aiChatMessages { + flex: 1; + overflow-y: auto; + padding: 1.1rem; + display: flex; + flex-direction: column; + gap: 1rem; +} + +.aiChatTurn { + display: flex; + flex-direction: column; + gap: 0.5rem; +} + +.aiChatUserQuery { + margin: 0; + font-size: 0.9375rem; + line-height: 1.6; + font-weight: 700; + color: var(--ifm-font-color-base); +} + +.aiChatAnswer { + font-size: 0.9375rem; + line-height: 1.7; +} + +.aiChatAnswer h1, +.aiChatAnswer h2, +.aiChatAnswer h3, +.aiChatAnswer h4, +.aiChatAnswer h5, +.aiChatAnswer h6 { + font-size: inherit; + font-weight: 600; + margin: 0.5rem 0 0.25rem 0; + line-height: 1.4; +} + +.aiChatAnswer h1 { + font-size: 1rem; +} + +.aiChatAnswer h2 { + font-size: 0.95rem; +} + +.aiChatAnswer h3 { + font-size: 0.9rem; +} + +.aiChatDelimiter { + border: 0; + border-top: 1px solid var(--ifm-color-emphasis-300); + margin: 0; +} + +.aiChatThinking { + color: var(--ifm-color-emphasis-600); + font-style: italic; + font-size: 0.8125rem; +} + +.aiChatFeedback { + margin-top: 0.375rem; + padding-top: 0.375rem; + border-top: 1px solid var(--ifm-color-emphasis-300); +} + +/* AI Chat Input Area */ +.aiChatInputArea { + display: flex; + gap: 0.5rem; + padding: 0.75rem 1rem; + border-top: 1px solid var(--ifm-color-emphasis-200); + background: var(--ifm-color-emphasis-50); +} + +.aiChatInput { + flex: 1; + padding: 0.5rem 0.75rem; + font-size: 0.875rem; + border: 1px solid var(--ifm-color-emphasis-300); + border-radius: 8px; + background: var(--ifm-background-color); + color: var(--ifm-font-color-base); + outline: none; + transition: border-color 0.15s ease; +} + +.aiChatInput:focus { + border-color: var(--ifm-color-primary); +} + +.aiChatInput::placeholder { + color: var(--ifm-color-emphasis-500); +} + +.aiChatSendBtn { + padding: 0.5rem 1rem; + font-size: 0.8125rem; + font-weight: 600; + background: var(--ifm-color-primary); + color: white; + border: none; + border-radius: 8px; + cursor: pointer; + transition: background 0.15s ease; +} + +.aiChatSendBtn:hover:not(:disabled) { + background: var(--ifm-color-primary-dark); +} + +.aiChatSendBtn:disabled { + opacity: 0.5; + cursor: not-allowed; +} + +/* Dark mode - AI Chat */ +[data-theme='dark'] .modeToggleBtnActive { + background: var(--ifm-color-emphasis-300); +} + +[data-theme='dark'] .askDocsPrompt { + background: var(--ifm-color-emphasis-100); +} + +[data-theme='dark'] .askDocsPrompt:hover { + background: var(--ifm-color-emphasis-200); +} + +[data-theme='dark'] .aiChatInputArea { + background: var(--ifm-color-emphasis-100); +} + +/* AI Chat Suggestions */ +.aiChatSuggestions { + display: flex; + flex-direction: column; + align-items: center; + padding: 1.5rem 1rem; + gap: 1rem; +} + +.aiChatSuggestionsTitle { + font-size: 0.9375rem; + font-weight: 600; + color: var(--ifm-font-color-base); + margin: 0; +} + +.aiChatSuggestionsList { + display: flex; + flex-wrap: wrap; + gap: 0.5rem; + justify-content: center; +} + +.aiChatSuggestionChip { + padding: 0.375rem 0.75rem; + font-size: 0.8125rem; + background: var(--ifm-color-emphasis-100); + border: 1px solid var(--ifm-color-emphasis-300); + border-radius: 9999px; + color: var(--ifm-color-emphasis-700); + cursor: pointer; + transition: all 0.15s ease; +} + +.aiChatSuggestionChip:hover { + background: var(--ifm-color-primary); + border-color: var(--ifm-color-primary); + color: white; +} + +/* AI Chat Sources */ +.aiChatSources { + display: flex; + align-items: center; + flex-wrap: wrap; + gap: 0.75rem; + align-self: flex-start; + padding: 0.25rem 0; + font-size: 0.75rem; +} + +.aiChatSourcesLabel { + font-weight: 700; + color: var(--ifm-color-emphasis-600); + text-transform: uppercase; + letter-spacing: 0.025em; +} + +.aiChatSourceLink { + color: var(--ifm-color-primary) !important; + text-decoration: none !important; + font-size: 0.75rem; +} + +.aiChatSourceLink:hover { + text-decoration: underline !important; +} diff --git a/website/static/llms.txt b/website/static/llms.txt index 20b12c2e368..74180e9236e 100644 --- a/website/static/llms.txt +++ b/website/static/llms.txt @@ -16,7 +16,7 @@ NEAR is a blockchain platform designed for building scalable, user-friendly dece - **Account Model**: /protocol/account-model NEAR uses human-readable account IDs. Accounts can store smart contracts, hold tokens, and have access keys. -- **Smart Contracts**: /smart-contracts/anatomy/anatomy +- **Smart Contracts**: /smart-contracts/anatomy/basic-anatomy Contracts are small programs stored on accounts. Write in JavaScript, Rust, Python, or Go. Compiled to WebAssembly. - **Access Keys**: /protocol/access-keys