diff --git a/integration/.gitignore b/integration/.gitignore new file mode 100644 index 0000000..a14702c --- /dev/null +++ b/integration/.gitignore @@ -0,0 +1,34 @@ +# dependencies (bun install) +node_modules + +# output +out +dist +*.tgz + +# code coverage +coverage +*.lcov + +# logs +logs +_.log +report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json + +# dotenv environment variable files +.env +.env.development.local +.env.test.local +.env.production.local +.env.local + +# caches +.eslintcache +.cache +*.tsbuildinfo + +# IntelliJ based IDEs +.idea + +# Finder (MacOS) folder config +.DS_Store diff --git a/integration/README.md b/integration/README.md new file mode 100644 index 0000000..3905210 --- /dev/null +++ b/integration/README.md @@ -0,0 +1,15 @@ +# integration + +To install dependencies: + +```bash +bun install +``` + +To run: + +```bash +bun run index.ts +``` + +This project was created using `bun init` in bun v1.2.13. [Bun](https://bun.sh) is a fast all-in-one JavaScript runtime. diff --git a/integration/bun.lock b/integration/bun.lock new file mode 100644 index 0000000..0c03820 --- /dev/null +++ b/integration/bun.lock @@ -0,0 +1,25 @@ +{ + "lockfileVersion": 1, + "workspaces": { + "": { + "name": "integration", + "devDependencies": { + "@types/bun": "latest", + }, + "peerDependencies": { + "typescript": "^5", + }, + }, + }, + "packages": { + "@types/bun": ["@types/bun@1.2.17", "https://registry.npmmirror.com/@types/bun/-/bun-1.2.17.tgz", { "dependencies": { "bun-types": "1.2.17" } }, "sha512-l/BYs/JYt+cXA/0+wUhulYJB6a6p//GTPiJ7nV+QHa8iiId4HZmnu/3J/SowP5g0rTiERY2kfGKXEK5Ehltx4Q=="], + + "@types/node": ["@types/node@24.0.7", "https://registry.npmmirror.com/@types/node/-/node-24.0.7.tgz", { "dependencies": { "undici-types": "~7.8.0" } }, "sha512-YIEUUr4yf8q8oQoXPpSlnvKNVKDQlPMWrmOcgzoduo7kvA2UF0/BwJ/eMKFTiTtkNL17I0M6Xe2tvwFU7be6iw=="], + + "bun-types": ["bun-types@1.2.17", "https://registry.npmmirror.com/bun-types/-/bun-types-1.2.17.tgz", { "dependencies": { "@types/node": "*" } }, "sha512-ElC7ItwT3SCQwYZDYoAH+q6KT4Fxjl8DtZ6qDulUFBmXA8YB4xo+l54J9ZJN+k2pphfn9vk7kfubeSd5QfTVJQ=="], + + "typescript": ["typescript@5.8.3", "https://registry.npmmirror.com/typescript/-/typescript-5.8.3.tgz", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ=="], + + "undici-types": ["undici-types@7.8.0", "https://registry.npmmirror.com/undici-types/-/undici-types-7.8.0.tgz", {}, "sha512-9UJ2xGDvQ43tYyVMpuHlsgApydB8ZKfVYTsLDhXkFL/6gfkp+U8xTGdh8pMJv1SpZna0zxG1DwsKZsreLbXBxw=="], + } +} diff --git a/integration/compose.yml b/integration/compose.yml new file mode 100644 index 0000000..81f614b --- /dev/null +++ b/integration/compose.yml @@ -0,0 +1,69 @@ +version: '3.3' +services: + postgres-container: + image: postgres + restart: always + environment: + POSTGRES_PASSWORD: your_password + ports: + - 5432:5432 + volumes: + - ./init.sql:/docker-entrypoint-initdb.d/init.sql + healthcheck: + test: [ "CMD-SHELL", "pg_isready -U postgres" ] + interval: 5s + retries: 5 + start_period: 30s + # psql -h localhost -p 5432 -U postgres + end: + image: sixwaaaay/end:0.25.5 + restart: always + ports: + - 5151:5151 + environment: + ConnectionStrings__Default: "Host=postgres-container;Username=postgres;Password=your_password" + ConnectionStrings__ReplicationConnection: "Host=postgres-container;Username=postgres;Password=your_password" + Application__Secret: "the secret key|the secret key|the secret key" + ASPNETCORE_URLS: "http://+:5151" + depends_on: + postgres-container: + condition: service_healthy + meilisearch: + image: getmeili/meilisearch:v1.15 + environment: + MEILI_MASTER_KEY: masterKey|masterKey|masterKey|masterKey + ports: + - 7700:7700 + qdrant: + image: qdrant/qdrant:v1.15 + environment: + QDRANT__SERVICE__API_KEY: qdrant|qdrant|qdrant|qdrant|qdrant|qdrant + ports: + - 6333:6333 + valkey: + image: valkey/valkey:8-alpine3.22 # redis alternative + ports: + - 6379:6379 + content: + image: sixwaaaay/content:latest + ports: + - 5000:5000 + environment: + ConnectionStrings__Default: "Host=postgres-container;Username=postgres;Password=your_password" + ConnectionStrings__User: "http://end:5151" + ConnectionStrings__Vote: "http://end:5151" + ConnectionStrings__Search: "http://meilisearch:7700" + ConnectionStrings__Redis: "valkey:6379" + ConnectionStrings__QdrantUrl: "http://qdrant:6334" + ConnectionStrings__QdrantApiKey: "qdrant|qdrant|qdrant|qdrant|qdrant|qdrant" + Secret: "the secret key|the secret key|the secret key" + Token: "masterKey|masterKey|masterKey|masterKey" + bunserver: + image: oven/bun:1 + command: "bun embeddingserver.ts" + ports: + - "8000:8000" + volumes: + - ./embeddingserver.ts:/app/embeddingserver.ts + working_dir: /app + restart: always diff --git a/integration/embeddingserver.ts b/integration/embeddingserver.ts new file mode 100644 index 0000000..1093528 --- /dev/null +++ b/integration/embeddingserver.ts @@ -0,0 +1,112 @@ +// 定义响应结构 +type EmbeddingData = { + object: "embedding"; + embedding: number[]; + index: number; +}; + +type EmbeddingResponse = { + object: "list"; + data: EmbeddingData[]; + model: string; + usage: { + prompt_tokens: number; + total_tokens: number; + }; +}; + +// 定义支持的模型及其维度 +const modelDimensions: Record = { + "text-embedding-ada-002": 1536, + "text-similarity-ada-001": 1024, + "text-search-ada-001": 1024, + "code-search-ada-code-001": 1024, + "simple-vector": 24 +}; + +// 生成随机嵌入向量 +function generateEmbedding(dimensions: number): number[] { + return Array.from({ length: dimensions }, () => Math.random() * 2 - 1); +} + +// 简单的 token 计数器 +function countTokens(text: string | string[]): number { + if (Array.isArray(text)) { + return text.reduce((acc, t) => acc + countTokens(t), 0); + } + return text.split(/\s+/).length; +} + +// 创建 HTTP 服务器 +const server = Bun.serve({ + port: 8000, + fetch(req) { + const url = new URL(req.url); + + // 处理 Embedding API 请求 + if (req.method === "POST" && url.pathname === "/v1/embeddings") { + return handleEmbeddingRequest(req); + } + + // 默认返回 404 + return new Response(JSON.stringify({ error: "Not Found" }), { + status: 404, + headers: { "Content-Type": "application/json" }, + }); + }, +}); + +// 处理嵌入请求 +async function handleEmbeddingRequest(req: Request): Promise { + try { + // 解析请求体 + const { input, model = "text-embedding-ada-002" } = await req.json(); + + // 验证输入 + if (input === undefined) { + return new Response(JSON.stringify({ error: "Missing 'input' parameter" }), { + status: 400, + headers: { "Content-Type": "application/json" }, + }); + } + + // 确保 input 是数组 + const inputList = Array.isArray(input) ? input : [input]; + + // 获取模型维度 + const dimensions = modelDimensions[model] || 1536; + + // 生成嵌入向量 + const data = inputList.map((text, index) => ({ + object: "embedding", + embedding: generateEmbedding(dimensions), + index, + })) as EmbeddingData[]; + + // 计算 token 使用量 + const promptTokens = countTokens(inputList); + + // 构建响应 + const response: EmbeddingResponse = { + object: "list", + data, + model, + usage: { + prompt_tokens: promptTokens, + total_tokens: promptTokens, + }, + }; + + return new Response(JSON.stringify(response), { + headers: { "Content-Type": "application/json" }, + }); + } catch (error) { + console.error("Error handling request:", error); + return new Response(JSON.stringify({ error: "Internal Server Error" }), { + status: 500, + headers: { "Content-Type": "application/json" }, + }); + } +} + +console.log(`Server running at http://localhost:${server.port}`); \ No newline at end of file diff --git a/integration/index.ts b/integration/index.ts new file mode 100644 index 0000000..6a7b3b7 --- /dev/null +++ b/integration/index.ts @@ -0,0 +1 @@ +console.log("the integration via Bun!"); diff --git a/integration/init.sql b/integration/init.sql new file mode 100644 index 0000000..01a9f40 --- /dev/null +++ b/integration/init.sql @@ -0,0 +1,82 @@ +CREATE TABLE users ( + id bigserial, + name character varying(255) NOT NULL, + email character varying(255) NOT NULL, + code character varying(6) NOT NULL DEFAULT ''::character varying, + code_time timestamptz NOT NULL DEFAULT to_timestamp('2020-01-01 00:00:00'::text, 'YYYY-MM-DD HH24:MI:SS'::text), + code_try integer NOT NULL DEFAULT 0, + bio character varying(255) NOT NULL DEFAULT ''::character varying, + avatar character varying(255) NOT NULL DEFAULT ''::character varying, + created_at timestamptz NOT NULL DEFAULT now(), + background character varying(255) NOT NULL DEFAULT ''::character varying, + PRIMARY KEY (id), + UNIQUE (email) +); + + +-- 一个脚本，生成 10 万 条数据 +INSERT INTO users (name, email, code, code_time, code_try, bio, avatar, created_at, background) +SELECT + 'user_' || i::text, + 'user_' || i::text || '@example.com', + '123456', + now(), + 0, + 'This is a bio for user ' || i::text, + 'https://example.com/avatar_' || i::text || '.jpg', + now(), + 'https://example.com/background_' || i::text || '.jpg' +FROM generate_series(1, 100000) AS s(i); + +CREATE TABLE graph ( + id bigserial, + relation integer NOT NULL, + subject_id bigint NOT NULL, + object_id bigint NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (id), + UNIQUE (relation, subject_id, object_id) +); + +-- 一个脚本，生成 10 万 条数据 +INSERT INTO graph (relation, subject_id, object_id) +SELECT-- 从序列获取下一个值作为 id + floor(random() * 3) + 1, -- 生成 1 到 3 之间的随机整数 + floor(random() * 100000) + 1, -- 生成 1 到 100000 之间的随机整数 + floor(random() * 100000) + 1 -- 生成 1 到 100000 之间的随机整数 +FROM generate_series(1, 100000) AS s(i) +ON CONFLICT (relation, subject_id, object_id) DO NOTHING; + + + +CREATE TABLE videos ( + id bigserial, + user_id bigint NOT NULL, + title character varying(255) NOT NULL, + des text NOT NULL, + cover_url character varying(255) DEFAULT ''::character varying NOT NULL, + video_url character varying(255) DEFAULT ''::character varying NOT NULL, + duration integer, + view_count integer DEFAULT 0 NOT NULL, + like_count integer DEFAULT 0 NOT NULL, + created_at timestamptz DEFAULT now() NOT NULL, + updated_at timestamptz DEFAULT now() NOT NULL, + processed integer DEFAULT 0 NOT NULL, + PRIMARY KEY (id) +); + +-- 一个脚本，生成 10 万 条数据 +INSERT INTO videos (user_id, title, des, cover_url, video_url, duration, view_count, like_count, created_at, updated_at, processed) +SELECT + floor(random() * 100000) + 1, -- 生成 1 到 100000 之间的随机整数 + 'Video ' || i::text, + 'This is a description for video ' || i::text, + 'https://example.com/cover_' || i::text || '.jpg', + 'https://example.com/video_' || i::text || '.mp4', + floor(random() * 3600) + 1, -- 生成 1 到 3600 之间的随机整数 + floor(random() * 10000) + 1, -- 生成 1 到 10000 之间的随机整数 + floor(random() * 1000) + 1, -- 生成 1 到 1000 之间的随机整数 + now(), + now(), + floor(random() * 2) -- 生成 0 或 1 +FROM generate_series(1, 100000) AS s(i); \ No newline at end of file diff --git a/integration/init.ts b/integration/init.ts new file mode 100644 index 0000000..13f4bbf --- /dev/null +++ b/integration/init.ts @@ -0,0 +1,160 @@ +/* + the following code is used to initialize meilisearch search engine +*/ +const meilisearchUrl = "http://localhost:7700" +const meilisearchApiKey = "masterKey|masterKey|masterKey|masterKey" + +// 创建 videos 的index +const videosIndex = await fetch(`${meilisearchUrl}/indexes`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${meilisearchApiKey}`, + }, + body: JSON.stringify({ + uid: 'videos', + primaryKey: 'id', + }), +}) + +if (!videosIndex.ok) { + throw new Error(`创建 videos index 失败`) +} + + +// 配置 embedder +const embedder = `{ + "default": { + "source": "rest", + "url": "http://bunserver:8000/v1/embeddings", + "request": { + "model": "simple-vector", + "input": ["{{text}}"] + }, + "response": { + "data": [ + { + "embedding": "{{embedding}}" + } + ] + }, + "documentTemplate": "{{doc.title}}" + } +}` + +const embeddersResponse = await fetch(`${meilisearchUrl}/indexes/videos/settings/embedders`, { + method: 'PATCH', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${meilisearchApiKey}`, + }, + body: embedder, +}) + +if (!embeddersResponse.ok) { + throw new Error(`配置 embedders 失败`) +} + + +// 插入 数据 +function* generator(start: number, count: number) { + for (let i = start; i < start + count; i++) { + yield { id: i, title: `title_${i}` } + } +} + +const insertResponse = await fetch(`${meilisearchUrl}/indexes/videos/documents`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${meilisearchApiKey}`, + }, + body: JSON.stringify(Array.from(generator(1, 100))) +}) + +if (!insertResponse.ok) { + throw new Error(`插入数据失败`) +} + + + + +/* + +the following is used to initialize qdrant vector database + +*/ + +const qdrantUrl = "http://localhost:6333" +const apikey = 'qdrant|qdrant|qdrant|qdrant|qdrant|qdrant' + + +/* +https://api.qdrant.tech/master/api-reference/collections/create-collection +*/ +const collectionName = 'videos' + +const createResp = await fetch(`${qdrantUrl}/collections/${collectionName}`, { + method: 'PUT', + headers: { + 'api-key': apikey, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + vectors: { + size: 1024, + distance: 'Cosine', + } + }), +}) + +if (!createResp.ok) { + throw new Error(`创建 collection 失败`) +} + + +function generateEmbedding(dimensions: number): number[] { + return Array.from({ length: dimensions }, () => Math.random() * 2 - 1); +} + +type Point = { + id: number + payload: { + id: number; + title: string; + } + vector: number[] +} + +function generatePoint(id: number): Point { + return { + id: id, + payload: { + id: id, + title: `title_${id}`, + }, + vector: generateEmbedding(1024), + } +} + + +// id from 1 to 100 +const points = Array.from({ length: 100 }, (_, i) => generatePoint(i + 1)) + +/* +reference: https://api.qdrant.tech/master/api-reference/points/upsert-points +*/ +const resp = await fetch(`${qdrantUrl}/collections/${collectionName}/points`, { + method: 'PUT', + headers: { + 'api-key': apikey, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + points, + }), +}) + +if (!resp.ok) { + throw new Error(`插入 points 失败`) +} \ No newline at end of file diff --git a/integration/package.json b/integration/package.json new file mode 100644 index 0000000..3dc419c --- /dev/null +++ b/integration/package.json @@ -0,0 +1,11 @@ +{ + "name": "integration", + "module": "index.ts", + "type": "module", + "devDependencies": { + "@types/bun": "latest" + }, + "peerDependencies": { + "typescript": "^5" + } +} diff --git a/integration/tsconfig.json b/integration/tsconfig.json new file mode 100644 index 0000000..9c62f74 --- /dev/null +++ b/integration/tsconfig.json @@ -0,0 +1,28 @@ +{ + "compilerOptions": { + // Environment setup & latest features + "lib": ["ESNext"], + "target": "ESNext", + "module": "ESNext", + "moduleDetection": "force", + "jsx": "react-jsx", + "allowJs": true, + + // Bundler mode + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "verbatimModuleSyntax": true, + "noEmit": true, + + // Best practices + "strict": true, + "skipLibCheck": true, + "noFallthroughCasesInSwitch": true, + "noUncheckedIndexedAccess": true, + + // Some stricter flags (disabled by default) + "noUnusedLocals": false, + "noUnusedParameters": false, + "noPropertyAccessFromIndexSignature": false + } +}