From cfc12dcfb9b676822f97629115cf8c79bf8030a8 Mon Sep 17 00:00:00 2001 From: xiaozhou26 Date: Sat, 6 Jun 2026 18:50:57 +0800 Subject: [PATCH 1/4] up --- chatgpt2api | 1 + conversion/requests/chatgpt/convert.go | 147 +++++++++++++++++- initialize/handlers.go | 137 ++++++++++++++--- initialize/router.go | 2 + internal/chatgpt/files.go | 189 +++++++++++++++++++++++ tests/test_file_qa.py | 154 +++++++++++++++++++ typings/chatgpt/request.go | 33 +++- typings/official/request.go | 202 ++++++++++++++++++++++--- 8 files changed, 812 insertions(+), 53 deletions(-) create mode 160000 chatgpt2api create mode 100644 internal/chatgpt/files.go create mode 100644 tests/test_file_qa.py diff --git a/chatgpt2api b/chatgpt2api new file mode 160000 index 000000000..faea4eb23 --- /dev/null +++ b/chatgpt2api @@ -0,0 +1 @@ +Subproject commit faea4eb237118125f206afaae3e239fcb95ff606 diff --git a/conversion/requests/chatgpt/convert.go b/conversion/requests/chatgpt/convert.go index e0aa5c51d..d60d29f37 100644 --- a/conversion/requests/chatgpt/convert.go +++ b/conversion/requests/chatgpt/convert.go @@ -1,9 +1,11 @@ package chatgpt import ( + backendchatgpt "aurora/internal/chatgpt" "aurora/internal/tokens" chatgpt_types "aurora/typings/chatgpt" official_types "aurora/typings/official" + "strings" ) func ConvertAPIRequest(api_request official_types.APIRequest, secret *tokens.Secret, proxy string) chatgpt_types.ChatGPTRequest { @@ -20,11 +22,16 @@ func ConvertAPIRequest(api_request official_types.APIRequest, secret *tokens.Sec chatgpt_request.PluginIDs = api_request.PluginIDs chatgpt_request.Model = "gpt-4-plugins" } - for _, api_message := range api_request.Messages { - if api_message.Role == "system" { - api_message.Role = "critic" + for _, apiMessage := range api_request.Messages { + if apiMessage.Role == "system" { + apiMessage.Role = "critic" } - chatgpt_request.AddMessage(api_message.Role, api_message.Content) + parts, metadata := buildMessageParts(apiMessage) + if len(metadata) > 0 { + chatgpt_request.AddMultimodalMessage(apiMessage.Role, parts, metadata) + continue + } + chatgpt_request.AddMessage(apiMessage.Role, apiMessage.Text()) } return chatgpt_request } @@ -35,3 +42,135 @@ func ConvertTTSAPIRequest(input string) chatgpt_types.ChatGPTRequest { chatgpt_request.AddAssistantMessage(input) return chatgpt_request } + +func buildMessageParts(message official_types.APIMessage) ([]interface{}, map[string]interface{}) { + text := message.Text() + files := enrichFiles(message.Files()) + if len(files) == 0 { + return []interface{}{text}, nil + } + + parts := make([]interface{}, 0, len(files)+1) + attachments := make([]interface{}, 0, len(files)) + for _, file := range files { + fileID := fileID(file) + if fileID == "" { + continue + } + part := map[string]interface{}{ + "content_type": filePartType(file), + "asset_pointer": "file-service://" + fileID, + } + if file.Size > 0 { + part["size_bytes"] = file.Size + } + if file.Width > 0 { + part["width"] = file.Width + } + if file.Height > 0 { + part["height"] = file.Height + } + parts = append(parts, part) + + attachment := map[string]interface{}{ + "id": fileID, + "size": file.Size, + "name": fileName(file), + "mime_type": fileMime(file), + "mimeType": fileMime(file), + "source": "library", + "is_big_paste": false, + } + if file.Width > 0 { + attachment["width"] = file.Width + } + if file.Height > 0 { + attachment["height"] = file.Height + } + if file.LibraryFileID != "" { + attachment["library_file_id"] = file.LibraryFileID + } + attachments = append(attachments, attachment) + } + if text != "" { + parts = append(parts, text) + } + if len(parts) == 0 { + parts = append(parts, text) + } + return parts, map[string]interface{}{ + "attachments": attachments, + "developer_mode_connector_ids": []interface{}{}, + "selected_sources": []interface{}{}, + "selected_github_repos": []interface{}{}, + "selected_all_github_repos": false, + "serialization_metadata": map[string]interface{}{"custom_symbol_offsets": []interface{}{}}, + } +} + +func enrichFiles(files []official_types.FileAttachment) []official_types.FileAttachment { + enriched := make([]official_types.FileAttachment, 0, len(files)) + seen := make(map[string]bool) + for _, file := range files { + id := fileID(file) + if id == "" || seen[id] { + continue + } + if uploaded, ok := backendchatgpt.LookupUploadedFile(id); ok { + if file.ID == "" { + file.ID = uploaded.ID + } + if file.FileID == "" { + file.FileID = uploaded.FileID + } + if file.Name == "" && file.FileName == "" && file.Filename == "" { + file.Name = uploaded.Filename + file.FileName = uploaded.Filename + file.Filename = uploaded.Filename + } + if file.MimeType == "" && file.MIMEType == "" { + file.MimeType = uploaded.MimeType + file.MIMEType = uploaded.MimeType + } + if file.Size == 0 { + file.Size = uploaded.Bytes + } + if file.LibraryFileID == "" { + file.LibraryFileID = uploaded.LibraryFileID + } + } + seen[id] = true + enriched = append(enriched, file) + } + return enriched +} + +func fileID(file official_types.FileAttachment) string { + if strings.TrimSpace(file.FileID) != "" { + return strings.TrimSpace(file.FileID) + } + return strings.TrimSpace(file.ID) +} + +func fileName(file official_types.FileAttachment) string { + for _, value := range []string{file.Name, file.FileName, file.Filename} { + if strings.TrimSpace(value) != "" { + return strings.TrimSpace(value) + } + } + return fileID(file) +} + +func fileMime(file official_types.FileAttachment) string { + if strings.TrimSpace(file.MimeType) != "" { + return strings.TrimSpace(file.MimeType) + } + return strings.TrimSpace(file.MIMEType) +} + +func filePartType(file official_types.FileAttachment) string { + if strings.HasPrefix(strings.ToLower(fileMime(file)), "image/") { + return "image_asset_pointer" + } + return "file_asset_pointer" +} diff --git a/initialize/handlers.go b/initialize/handlers.go index d7b761020..09ab9cb08 100644 --- a/initialize/handlers.go +++ b/initialize/handlers.go @@ -13,6 +13,7 @@ import ( "io" "os" "strings" + "time" "github.com/gin-gonic/gin" "github.com/google/uuid" @@ -170,16 +171,18 @@ func (h *Handler) nightmare(c *gin.Context) { }}) return } - proxyUrl := h.proxy.GetProxyIP() - input_tokens := util.CountToken(original_request.Messages[0].Content) - secret := h.token.GetSecret() - authHeader := c.GetHeader("Authorization") - if authHeader != "" { - customAccessToken := strings.Replace(authHeader, "Bearer ", "", 1) - if strings.HasPrefix(customAccessToken, "eyJhbGciOiJSUzI1NiI") { - secret = h.token.GenerateTempToken(customAccessToken) - } + if len(original_request.Messages) == 0 { + c.JSON(400, gin.H{"error": gin.H{ + "message": "Missing required parameter: messages", + "type": "invalid_request_error", + "param": "messages", + "code": "missing_required_parameter", + }}) + return } + proxyUrl := h.proxy.GetProxyIP() + input_tokens := countMessagesTokens(original_request.Messages) + secret := h.secretFromAuthorization(c.GetHeader("Authorization"), original_requestHasFiles(original_request), false) if secret == nil { c.JSON(400, gin.H{"error": "Not Account Found."}) c.Abort() @@ -288,16 +291,9 @@ func (h *Handler) responses(c *gin.Context) { proxyUrl := h.proxy.GetProxyIP() input_tokens := 0 for _, message := range original_request.Messages { - input_tokens += util.CountToken(message.Content) - } - secret := h.token.GetSecret() - authHeader := c.GetHeader("Authorization") - if authHeader != "" { - customAccessToken := strings.Replace(authHeader, "Bearer ", "", 1) - if strings.HasPrefix(customAccessToken, "eyJhbGciOiJSUzI1NiI") { - secret = h.token.GenerateTempToken(customAccessToken) - } + input_tokens += util.CountToken(message.Text()) } + secret := h.secretFromAuthorization(c.GetHeader("Authorization"), original_requestHasFiles(original_request), false) if secret == nil { c.JSON(400, gin.H{"error": "Not Account Found."}) c.Abort() @@ -509,6 +505,77 @@ func (h *Handler) imageGenerations(c *gin.Context) { c.JSON(200, officialtypes.NewImageGenerationResponse(data)) } +func (h *Handler) files(c *gin.Context) { + secret := h.secretFromAuthorization(c.GetHeader("Authorization"), true, true) + if secret == nil || secret.Token == "" || secret.IsFree { + c.JSON(400, gin.H{"error": gin.H{ + "message": "Files API requires a logged-in ChatGPT access token.", + "type": "invalid_request_error", + "param": nil, + "code": "missing_access_token", + }}) + return + } + + formFile, err := c.FormFile("file") + if err != nil { + c.JSON(400, gin.H{"error": gin.H{ + "message": "Missing required multipart field: file", + "type": "invalid_request_error", + "param": "file", + "code": "missing_required_parameter", + }}) + return + } + file, err := formFile.Open() + if err != nil { + c.JSON(400, gin.H{"error": gin.H{ + "message": err.Error(), + "type": "invalid_request_error", + "param": "file", + "code": "file_open_error", + }}) + return + } + defer file.Close() + data, err := io.ReadAll(file) + if err != nil { + c.JSON(400, gin.H{"error": gin.H{ + "message": err.Error(), + "type": "invalid_request_error", + "param": "file", + "code": "file_read_error", + }}) + return + } + if len(data) == 0 { + c.JSON(400, gin.H{"error": gin.H{ + "message": "Uploaded file is empty", + "type": "invalid_request_error", + "param": "file", + "code": "empty_file", + }}) + return + } + + contentType := formFile.Header.Get("Content-Type") + client := bogdanfinn.NewStdClient() + client.SetCookies("https://chatgpt.com", chatgpt.BasicCookies) + uploaded, status, err := chatgpt.UploadFile(client, secret, h.proxy.GetProxyIP(), formFile.Filename, contentType, c.PostForm("purpose"), data) + if err != nil { + c.JSON(status, gin.H{"error": gin.H{ + "message": err.Error(), + "type": "file_upload_error", + "param": "file", + "code": "file_upload_error", + }}) + return + } + uploaded.CreatedAt = time.Now().Unix() + chatgpt.RegisterUploadedFile(uploaded) + c.JSON(200, uploaded) +} + func (h *Handler) engines(c *gin.Context) { type ResData struct { ID string `json:"id"` @@ -539,6 +606,40 @@ func (h *Handler) engines(c *gin.Context) { }) } +func (h *Handler) secretFromAuthorization(authHeader string, needsPaid bool, allowFallbackPaid bool) *tokens.Secret { + secret := h.token.GetSecret() + if needsPaid || allowFallbackPaid { + secret = h.token.GetPaidSecret() + } + if authHeader != "" { + customAccessToken := strings.TrimSpace(strings.Replace(authHeader, "Bearer ", "", 1)) + if strings.HasPrefix(customAccessToken, "eyJhbGciOiJSUzI1NiI") { + secret = h.token.GenerateTempToken(customAccessToken) + } + } + if needsPaid && (secret == nil || secret.Token == "" || secret.IsFree) && !allowFallbackPaid { + return nil + } + return secret +} + +func countMessagesTokens(messages []officialtypes.APIMessage) int { + total := 0 + for _, message := range messages { + total += util.CountToken(message.Text()) + } + return total +} + +func original_requestHasFiles(request officialtypes.APIRequest) bool { + for _, message := range request.Messages { + if len(message.Files()) > 0 { + return true + } + } + return false +} + var ttsFmtMap = map[string]string{ "mp3": "mp3", "opus": "opus", diff --git a/initialize/router.go b/initialize/router.go index 5c0834f7f..bcfd74edc 100644 --- a/initialize/router.go +++ b/initialize/router.go @@ -35,10 +35,12 @@ func RegisterRouter() *gin.Engine { router.OPTIONS("/v1/chat/completions", optionsHandler) router.OPTIONS("/v1/responses", optionsHandler) router.OPTIONS("/v1/images/generations", optionsHandler) + router.OPTIONS("/v1/files", optionsHandler) authGroup := router.Group("").Use(middlewares.Authorization) authGroup.POST("/v1/chat/completions", handler.nightmare) authGroup.POST("/v1/responses", handler.responses) + authGroup.POST("/v1/files", handler.files) authGroup.GET("/v1/models", handler.engines) authGroup.POST("/backend-api/conversation", handler.chatgptConversation) authGroup.POST("/v1/images/generations", handler.imageGenerations) diff --git a/internal/chatgpt/files.go b/internal/chatgpt/files.go new file mode 100644 index 000000000..dfe5245bb --- /dev/null +++ b/internal/chatgpt/files.go @@ -0,0 +1,189 @@ +package chatgpt + +import ( + "aurora/httpclient" + "aurora/internal/tokens" + "bytes" + "encoding/json" + "fmt" + "io" + "mime" + "net/http" + "path/filepath" + "strings" + "sync" +) + +type UploadedFile struct { + ID string `json:"id,omitempty"` + FileID string `json:"file_id,omitempty"` + Object string `json:"object,omitempty"` + Bytes int64 `json:"bytes,omitempty"` + CreatedAt int64 `json:"created_at,omitempty"` + Filename string `json:"filename,omitempty"` + Purpose string `json:"purpose,omitempty"` + MimeType string `json:"mime_type,omitempty"` + LibraryFileID string `json:"library_file_id,omitempty"` +} + +type uploadMetaResponse struct { + FileID string `json:"file_id"` + UploadURL string `json:"upload_url"` + LibraryFileID string `json:"library_file_id"` +} + +var uploadedFiles sync.Map + +func RegisterUploadedFile(file UploadedFile) { + if file.FileID == "" { + file.FileID = file.ID + } + if file.ID == "" { + file.ID = file.FileID + } + if file.FileID == "" { + return + } + uploadedFiles.Store(file.FileID, file) +} + +func LookupUploadedFile(fileID string) (UploadedFile, bool) { + value, ok := uploadedFiles.Load(fileID) + if !ok { + return UploadedFile{}, false + } + file, ok := value.(UploadedFile) + return file, ok +} + +func UploadFile(client httpclient.AuroraHttpClient, secret *tokens.Secret, proxy, filename, contentType, purpose string, data []byte) (UploadedFile, int, error) { + if proxy != "" { + client.SetProxy(proxy) + } + if secret == nil || secret.Token == "" || secret.IsFree { + return UploadedFile{}, http.StatusBadRequest, fmt.Errorf("file upload requires a logged-in ChatGPT access token") + } + filename = strings.TrimSpace(filename) + if filename == "" { + filename = "upload.bin" + } + if contentType == "" { + contentType = mime.TypeByExtension(strings.ToLower(filepath.Ext(filename))) + } + if contentType == "" { + contentType = http.DetectContentType(data) + } + if purpose == "" { + purpose = "assistants" + } + + meta, status, err := createUpload(client, secret, filename, len(data)) + if err != nil { + return UploadedFile{}, status, err + } + if status, err := putUpload(client, meta.UploadURL, contentType, data); err != nil { + return UploadedFile{}, status, err + } + if status, err := confirmUpload(client, secret, meta.FileID); err != nil { + return UploadedFile{}, status, err + } + + result := UploadedFile{ + ID: meta.FileID, + FileID: meta.FileID, + Object: "file", + Bytes: int64(len(data)), + Filename: filename, + Purpose: purpose, + MimeType: contentType, + LibraryFileID: meta.LibraryFileID, + } + RegisterUploadedFile(result) + return result, http.StatusOK, nil +} + +func createUpload(client httpclient.AuroraHttpClient, secret *tokens.Secret, filename string, size int) (uploadMetaResponse, int, error) { + payload := map[string]interface{}{ + "file_name": filename, + "file_size": size, + "use_case": "multimodal", + "timezone_offset_min": -480, + "reset_rate_limits": false, + "store_in_library": true, + "library_persistence_mode": "opportunistic", + } + body, err := json.Marshal(payload) + if err != nil { + return uploadMetaResponse{}, http.StatusInternalServerError, err + } + header := createBaseHeader() + header.Set("accept", "application/json") + header.Set("content-type", "application/json") + if secret.Token != "" { + header.Set("Authorization", "Bearer "+secret.Token) + } + if secret.PUID != "" { + header.Set("Cookie", "_puid="+secret.PUID+";") + } + response, err := client.Request(http.MethodPost, BaseURL+"/files", header, nil, bytes.NewReader(body)) + if err != nil { + return uploadMetaResponse{}, http.StatusInternalServerError, err + } + defer response.Body.Close() + responseBody, _ := io.ReadAll(response.Body) + if response.StatusCode < 200 || response.StatusCode >= 300 { + return uploadMetaResponse{}, response.StatusCode, fmt.Errorf("create file upload failed: %s", string(responseBody)) + } + var meta uploadMetaResponse + if err := json.Unmarshal(responseBody, &meta); err != nil { + return uploadMetaResponse{}, response.StatusCode, err + } + if meta.FileID == "" || meta.UploadURL == "" { + return uploadMetaResponse{}, response.StatusCode, fmt.Errorf("invalid file upload response: %s", string(responseBody)) + } + return meta, response.StatusCode, nil +} + +func putUpload(client httpclient.AuroraHttpClient, uploadURL, contentType string, data []byte) (int, error) { + header := make(httpclient.AuroraHeaders) + header.Set("Content-Type", contentType) + header.Set("x-ms-blob-type", "BlockBlob") + header.Set("x-ms-version", "2020-04-08") + header.Set("Origin", "https://chatgpt.com") + header.Set("Referer", "https://chatgpt.com/") + header.Set("User-Agent", userAgent) + header.Set("Accept", "application/json, text/plain, */*") + header.Set("Accept-Language", "en-US,en;q=0.8") + response, err := client.Request(http.MethodPut, uploadURL, header, nil, bytes.NewReader(data)) + if err != nil { + return http.StatusInternalServerError, err + } + defer response.Body.Close() + responseBody, _ := io.ReadAll(response.Body) + if response.StatusCode < 200 || response.StatusCode >= 300 { + return response.StatusCode, fmt.Errorf("upload file data failed: %s", string(responseBody)) + } + return response.StatusCode, nil +} + +func confirmUpload(client httpclient.AuroraHttpClient, secret *tokens.Secret, fileID string) (int, error) { + header := createBaseHeader() + header.Set("accept", "application/json") + header.Set("content-type", "application/json") + if secret.Token != "" { + header.Set("Authorization", "Bearer "+secret.Token) + } + if secret.PUID != "" { + header.Set("Cookie", "_puid="+secret.PUID+";") + } + response, err := client.Request(http.MethodPost, BaseURL+"/files/"+fileID+"/uploaded", header, nil, strings.NewReader("{}")) + if err != nil { + return http.StatusInternalServerError, err + } + defer response.Body.Close() + responseBody, _ := io.ReadAll(response.Body) + if response.StatusCode < 200 || response.StatusCode >= 300 { + return response.StatusCode, fmt.Errorf("confirm file upload failed: %s", string(responseBody)) + } + return response.StatusCode, nil +} diff --git a/tests/test_file_qa.py b/tests/test_file_qa.py new file mode 100644 index 000000000..5fe9d62c3 --- /dev/null +++ b/tests/test_file_qa.py @@ -0,0 +1,154 @@ +import json +import os +import tempfile +import unittest +import urllib.error +import urllib.request +import uuid + + +BASE_URL = os.getenv("AURORA_BASE_URL", "http://127.0.0.1:8080").rstrip("/") + + +def auth_header() -> str: + header = os.getenv("AURORA_AUTH_HEADER", "").strip() + if header: + return header + + api_key = os.getenv("AURORA_API_KEY", "").strip() + access_token = os.getenv("AURORA_ACCESS_TOKEN", "").strip() + if api_key and access_token: + return f"Bearer {api_key} {access_token}" + if access_token: + return f"Bearer {access_token}" + if api_key: + return f"Bearer {api_key}" + return "" + + +def request_json(method: str, path: str, payload: dict, authorization: str) -> dict: + body = json.dumps(payload).encode("utf-8") + request = urllib.request.Request( + f"{BASE_URL}{path}", + data=body, + method=method, + headers={ + "Authorization": authorization, + "Content-Type": "application/json", + "Accept": "application/json", + }, + ) + try: + with urllib.request.urlopen(request, timeout=180) as response: + return json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + detail = exc.read().decode("utf-8", errors="replace") + raise AssertionError(f"{method} {path} failed: HTTP {exc.code}: {detail}") from exc + + +def upload_file(path: str, authorization: str, purpose: str = "assistants") -> dict: + boundary = f"----aurora-test-{uuid.uuid4().hex}" + filename = os.path.basename(path) + with open(path, "rb") as handle: + file_bytes = handle.read() + + fields = [ + ( + f"--{boundary}\r\n" + 'Content-Disposition: form-data; name="purpose"\r\n\r\n' + f"{purpose}\r\n" + ).encode("utf-8"), + ( + f"--{boundary}\r\n" + f'Content-Disposition: form-data; name="file"; filename="{filename}"\r\n' + "Content-Type: text/plain\r\n\r\n" + ).encode("utf-8"), + file_bytes, + f"\r\n--{boundary}--\r\n".encode("utf-8"), + ] + body = b"".join(fields) + + request = urllib.request.Request( + f"{BASE_URL}/v1/files", + data=body, + method="POST", + headers={ + "Authorization": authorization, + "Content-Type": f"multipart/form-data; boundary={boundary}", + "Accept": "application/json", + }, + ) + try: + with urllib.request.urlopen(request, timeout=180) as response: + return json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + detail = exc.read().decode("utf-8", errors="replace") + raise AssertionError(f"POST /v1/files failed: HTTP {exc.code}: {detail}") from exc + + +class FileQATest(unittest.TestCase): + def test_upload_file_then_ask_question(self): + authorization = auth_header() + if not authorization: + self.skipTest( + "set AURORA_ACCESS_TOKEN, AURORA_API_KEY, or AURORA_AUTH_HEADER to run this integration test" + ) + + sentinel = f"aurora-file-qa-{uuid.uuid4().hex[:10]}" + text = ( + "This is an Aurora file QA integration test.\n" + f"The sentinel value is: {sentinel}\n" + "When asked, answer with only the sentinel value.\n" + ) + + configured_path = os.getenv("AURORA_FILE_PATH", "").strip() + if configured_path: + file_path = configured_path + cleanup = False + else: + temp = tempfile.NamedTemporaryFile("w", suffix=".txt", encoding="utf-8", delete=False) + try: + temp.write(text) + file_path = temp.name + finally: + temp.close() + cleanup = True + + try: + uploaded = upload_file(file_path, authorization) + file_id = uploaded.get("id") or uploaded.get("file_id") + self.assertTrue(file_id, f"upload response did not include file id: {uploaded}") + + response = request_json( + "POST", + "/v1/chat/completions", + { + "model": os.getenv("AURORA_MODEL", "auto"), + "stream": False, + "messages": [ + { + "role": "user", + "content": [ + {"type": "input_file", "file_id": file_id}, + { + "type": "text", + "text": "What is the sentinel value in the uploaded file? Answer only the sentinel value.", + }, + ], + } + ], + }, + authorization, + ) + answer = response["choices"][0]["message"]["content"] + self.assertIn(sentinel, answer) + finally: + if cleanup: + try: + os.unlink(file_path) + except OSError: + pass + + +if __name__ == "__main__": + unittest.main() diff --git a/typings/chatgpt/request.go b/typings/chatgpt/request.go index 5ea7a3039..d04cfd0b4 100644 --- a/typings/chatgpt/request.go +++ b/typings/chatgpt/request.go @@ -7,14 +7,15 @@ import ( ) type chatgpt_message struct { - ID uuid.UUID `json:"id"` - Author chatgpt_author `json:"author"` - Content chatgpt_content `json:"content"` + ID uuid.UUID `json:"id"` + Author chatgpt_author `json:"author"` + Content chatgpt_content `json:"content"` + Metadata map[string]interface{} `json:"metadata,omitempty"` } type chatgpt_content struct { - ContentType string `json:"content_type"` - Parts []string `json:"parts"` + ContentType string `json:"content_type"` + Parts []interface{} `json:"parts"` } type chatgpt_author struct { @@ -52,7 +53,20 @@ func (c *ChatGPTRequest) AddMessage(role string, content string) { c.Messages = append(c.Messages, chatgpt_message{ ID: uuid.New(), Author: chatgpt_author{Role: role}, - Content: chatgpt_content{ContentType: "text", Parts: []string{content}}, + Content: chatgpt_content{ContentType: "text", Parts: []interface{}{content}}, + }) +} + +func (c *ChatGPTRequest) AddMultimodalMessage(role string, parts []interface{}, metadata map[string]interface{}) { + contentType := "text" + if len(parts) > 1 || (len(parts) == 1 && !isStringPart(parts[0])) { + contentType = "multimodal_text" + } + c.Messages = append(c.Messages, chatgpt_message{ + ID: uuid.New(), + Author: chatgpt_author{Role: role}, + Content: chatgpt_content{ContentType: contentType, Parts: parts}, + Metadata: metadata, }) } @@ -60,7 +74,12 @@ func (c *ChatGPTRequest) AddAssistantMessage(input string) { var msg = chatgpt_message{ ID: uuid.New(), Author: chatgpt_author{Role: "assistant"}, - Content: chatgpt_content{ContentType: "text", Parts: []string{input}}, + Content: chatgpt_content{ContentType: "text", Parts: []interface{}{input}}, } c.Messages = append(c.Messages, msg) } + +func isStringPart(part interface{}) bool { + _, ok := part.(string) + return ok +} diff --git a/typings/official/request.go b/typings/official/request.go index aae5670d9..73458d914 100644 --- a/typings/official/request.go +++ b/typings/official/request.go @@ -7,15 +7,150 @@ import ( ) type APIRequest struct { - Messages []api_message `json:"messages"` - Stream bool `json:"stream"` - Model string `json:"model"` - PluginIDs []string `json:"plugin_ids"` + Messages []APIMessage `json:"messages"` + Stream bool `json:"stream"` + Model string `json:"model"` + PluginIDs []string `json:"plugin_ids"` } -type api_message struct { - Role string `json:"role"` - Content string `json:"content"` +type APIMessage struct { + Role string `json:"role"` + Content MessageContent `json:"content"` + Attachments []FileAttachment `json:"attachments,omitempty"` +} + +type MessageContent struct { + TextValue string + Parts []MessageContentPart +} + +type MessageContentPart struct { + Type string `json:"type"` + Text string `json:"text,omitempty"` + FileID string `json:"file_id,omitempty"` + FileName string `json:"filename,omitempty"` + Name string `json:"name,omitempty"` + MimeType string `json:"mime_type,omitempty"` + MIMEType string `json:"mimeType,omitempty"` + Size int64 `json:"size,omitempty"` + Width int `json:"width,omitempty"` + Height int `json:"height,omitempty"` + File *FileAttachment `json:"file,omitempty"` +} + +type FileAttachment struct { + ID string `json:"id,omitempty"` + FileID string `json:"file_id,omitempty"` + Name string `json:"name,omitempty"` + FileName string `json:"file_name,omitempty"` + Filename string `json:"filename,omitempty"` + MimeType string `json:"mime_type,omitempty"` + MIMEType string `json:"mimeType,omitempty"` + Size int64 `json:"size,omitempty"` + Width int `json:"width,omitempty"` + Height int `json:"height,omitempty"` + LibraryFileID string `json:"library_file_id,omitempty"` + Source string `json:"source,omitempty"` +} + +func NewTextMessage(role, content string) APIMessage { + return APIMessage{Role: role, Content: MessageContent{TextValue: content}} +} + +func (c *MessageContent) UnmarshalJSON(data []byte) error { + var text string + if err := json.Unmarshal(data, &text); err == nil { + c.TextValue = text + c.Parts = nil + return nil + } + + var parts []MessageContentPart + if err := json.Unmarshal(data, &parts); err == nil { + c.TextValue = "" + c.Parts = parts + return nil + } + + var part MessageContentPart + if err := json.Unmarshal(data, &part); err == nil { + c.TextValue = "" + c.Parts = []MessageContentPart{part} + return nil + } + return fmt.Errorf("invalid message content") +} + +func (c MessageContent) MarshalJSON() ([]byte, error) { + if len(c.Parts) > 0 { + return json.Marshal(c.Parts) + } + return json.Marshal(c.TextValue) +} + +func (c MessageContent) Text() string { + if len(c.Parts) == 0 { + return c.TextValue + } + var texts []string + for _, part := range c.Parts { + switch part.Type { + case "text", "input_text", "output_text", "": + if part.Text != "" { + texts = append(texts, part.Text) + } + } + } + return strings.Join(texts, "") +} + +func (c MessageContent) Files() []FileAttachment { + var files []FileAttachment + for _, part := range c.Parts { + partType := strings.TrimSpace(part.Type) + if partType != "file" && partType != "input_file" && partType != "image" && partType != "input_image" { + continue + } + if part.File != nil { + files = append(files, *part.File) + continue + } + fileID := strings.TrimSpace(part.FileID) + if fileID == "" { + continue + } + files = append(files, FileAttachment{ + ID: fileID, + FileID: fileID, + Name: firstNonEmpty(part.Name, part.FileName), + Filename: firstNonEmpty(part.FileName, part.Name), + MimeType: firstNonEmpty(part.MimeType, part.MIMEType), + MIMEType: firstNonEmpty(part.MIMEType, part.MimeType), + Size: part.Size, + Width: part.Width, + Height: part.Height, + }) + } + return files +} + +func (m APIMessage) Text() string { + return m.Content.Text() +} + +func (m APIMessage) Files() []FileAttachment { + files := m.Content.Files() + files = append(files, m.Attachments...) + return files +} + +func firstNonEmpty(values ...string) string { + for _, value := range values { + if strings.TrimSpace(value) != "" { + return value + } + } + return "" } type ResponsesAPIRequest struct { @@ -42,10 +177,7 @@ func (r ResponsesAPIRequest) ToAPIRequest() (APIRequest, error) { } if instruction := rawText(r.Instructions); instruction != "" { - apiRequest.Messages = append(apiRequest.Messages, api_message{ - Role: "system", - Content: instruction, - }) + apiRequest.Messages = append(apiRequest.Messages, NewTextMessage("system", instruction)) } inputMessages, err := responsesInputToMessages(r.Input) @@ -71,13 +203,18 @@ func rawText(raw json.RawMessage) string { return strings.TrimSpace(string(raw)) } -func responsesInputToMessages(raw json.RawMessage) ([]api_message, error) { +func responsesInputToMessages(raw json.RawMessage) ([]APIMessage, error) { if len(raw) == 0 || string(raw) == "null" { return nil, nil } var text string if err := json.Unmarshal(raw, &text); err == nil { - return []api_message{{Role: "user", Content: text}}, nil + return []APIMessage{NewTextMessage("user", text)}, nil + } + + var content MessageContent + if err := json.Unmarshal(raw, &content); err == nil && (content.Text() != "" || len(content.Files()) > 0) { + return []APIMessage{{Role: "user", Content: content}}, nil } var messages []responseInputMessage @@ -85,21 +222,43 @@ func responsesInputToMessages(raw json.RawMessage) ([]api_message, error) { return nil, fmt.Errorf("invalid input") } - result := make([]api_message, 0, len(messages)) + result := make([]APIMessage, 0, len(messages)) for _, message := range messages { role := message.Role if role == "" { role = "user" } - content := responsesContentToText(message.Content) - if content == "" { + content, err := responseContentToMessageContent(message.Content) + if err != nil { + content = MessageContent{TextValue: responsesContentToText(message.Content)} + } + if content.Text() == "" && len(content.Files()) == 0 { continue } - result = append(result, api_message{Role: role, Content: content}) + result = append(result, APIMessage{Role: role, Content: content}) } return result, nil } +func responseContentToMessageContent(raw json.RawMessage) (MessageContent, error) { + var content MessageContent + if err := json.Unmarshal(raw, &content); err == nil { + return content, nil + } + var parts []responseInputContent + if err := json.Unmarshal(raw, &parts); err != nil { + return content, err + } + messageParts := make([]MessageContentPart, 0, len(parts)) + for _, part := range parts { + messageParts = append(messageParts, MessageContentPart{ + Type: part.Type, + Text: part.Text, + }) + } + return MessageContent{Parts: messageParts}, nil +} + func responsesContentToText(raw json.RawMessage) string { var text string if err := json.Unmarshal(raw, &text); err == nil { @@ -149,13 +308,8 @@ func (r ImageGenerationRequest) ToAPIRequest() APIRequest { } prompt := "Generate an image for this request. Return only the generated image, not a text description.\n\n" + r.Prompt return APIRequest{ - Model: model, - Messages: []api_message{ - { - Role: "user", - Content: prompt, - }, - }, + Model: model, + Messages: []APIMessage{NewTextMessage("user", prompt)}, } } From bbf6d5a6e45c9fa82a96f5c90e314b762fedf2fd Mon Sep 17 00:00:00 2001 From: xiaozhou26 Date: Sat, 6 Jun 2026 18:52:02 +0800 Subject: [PATCH 2/4] 1 --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 02d41c876..985aa6e43 100644 --- a/.gitignore +++ b/.gitignore @@ -13,4 +13,5 @@ tools/authenticator/.proxies.txt.swp /logs/ /target/ /bin/ -.gocache \ No newline at end of file +.gocache +chatgpt2api \ No newline at end of file From 3dda24b3ed7ca4eeafdadfb410255a00bc859392 Mon Sep 17 00:00:00 2001 From: xiaozhou26 Date: Sat, 6 Jun 2026 19:04:17 +0800 Subject: [PATCH 3/4] test --- .gitignore | 2 +- README.md | 23 +++++++++++++++++ conversion/requests/chatgpt/convert.go | 35 ++++++++++++++------------ 3 files changed, 43 insertions(+), 17 deletions(-) diff --git a/.gitignore b/.gitignore index 985aa6e43..e8d6ac706 100644 --- a/.gitignore +++ b/.gitignore @@ -14,4 +14,4 @@ tools/authenticator/.proxies.txt.swp /target/ /bin/ .gocache -chatgpt2api \ No newline at end of file +chatgpt2api/ \ No newline at end of file diff --git a/README.md b/README.md index a6f8c6dad..efa2b81d5 100644 --- a/README.md +++ b/README.md @@ -50,8 +50,31 @@ curl --location 'http://你的服务器ip:8080/v1/chat/completions' \ "stream": true }' ``` + ### 支持codex的api +### 携带文件问答 + +```bash +curl -X POST http://localhost:8080/v1/files \ + -H "Authorization: Bearer <你的key或access token>" \ + -F "purpose=assistants" \ + -F "file=@./test.pdf" +然后带 file_id 问答： +``` + +```bash +{ + "model": "auto", + "messages": [{ + "role": "user", + "content": [ + {"type": "input_file", "file_id": "file-xxx"}, + {"type": "text", "text": "总结这个文件"} + ] + }] +} +``` ### TTS 语音合成 ```bash diff --git a/conversion/requests/chatgpt/convert.go b/conversion/requests/chatgpt/convert.go index d60d29f37..c73a6741b 100644 --- a/conversion/requests/chatgpt/convert.go +++ b/conversion/requests/chatgpt/convert.go @@ -57,20 +57,22 @@ func buildMessageParts(message official_types.APIMessage) ([]interface{}, map[st if fileID == "" { continue } - part := map[string]interface{}{ - "content_type": filePartType(file), - "asset_pointer": "file-service://" + fileID, - } - if file.Size > 0 { - part["size_bytes"] = file.Size - } - if file.Width > 0 { - part["width"] = file.Width - } - if file.Height > 0 { - part["height"] = file.Height + if isImageFile(file) { + part := map[string]interface{}{ + "content_type": "image_asset_pointer", + "asset_pointer": "file-service://" + fileID, + } + if file.Size > 0 { + part["size_bytes"] = file.Size + } + if file.Width > 0 { + part["width"] = file.Width + } + if file.Height > 0 { + part["height"] = file.Height + } + parts = append(parts, part) } - parts = append(parts, part) attachment := map[string]interface{}{ "id": fileID, @@ -168,9 +170,10 @@ func fileMime(file official_types.FileAttachment) string { return strings.TrimSpace(file.MIMEType) } -func filePartType(file official_types.FileAttachment) string { +func isImageFile(file official_types.FileAttachment) bool { if strings.HasPrefix(strings.ToLower(fileMime(file)), "image/") { - return "image_asset_pointer" + return true } - return "file_asset_pointer" + name := strings.ToLower(fileName(file)) + return strings.HasSuffix(name, ".png") || strings.HasSuffix(name, ".jpg") || strings.HasSuffix(name, ".jpeg") || strings.HasSuffix(name, ".webp") || strings.HasSuffix(name, ".gif") } From 0c13c1f98a0b69656c370688289eb9b3c8afc734 Mon Sep 17 00:00:00 2001 From: xiaozhou26 Date: Sat, 6 Jun 2026 19:05:59 +0800 Subject: [PATCH 4/4] up --- .gitignore | 2 +- chatgpt2api | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) delete mode 160000 chatgpt2api diff --git a/.gitignore b/.gitignore index e8d6ac706..985aa6e43 100644 --- a/.gitignore +++ b/.gitignore @@ -14,4 +14,4 @@ tools/authenticator/.proxies.txt.swp /target/ /bin/ .gocache -chatgpt2api/ \ No newline at end of file +chatgpt2api \ No newline at end of file diff --git a/chatgpt2api b/chatgpt2api deleted file mode 160000 index faea4eb23..000000000 --- a/chatgpt2api +++ /dev/null @@ -1 +0,0 @@ -Subproject commit faea4eb237118125f206afaae3e239fcb95ff606