feat: added S3 universal interface

Bootstrap/higress/values.yaml

@@ -41,4 +41,9 @@ higress-core:
         - name: ssh
           port: 2222
           protocol: TCP
-          targetPort: 2222
+          targetPort: 2222
+  downstream:
+    connectionBufferLimits: 10485760    # 10 MB
+    http2:
+      initialConnectionWindowSize: 16777216  # 16 MB
+      initialStreamWindowSize: 16777216      # 16 MB

Lens/modules/skills-repository/go.mod

@@ -68,11 +68,11 @@ require (
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
 	golang.org/x/arch v0.8.0 // indirect
-	golang.org/x/crypto v0.40.0 // indirect
-	golang.org/x/net v0.42.0 // indirect
-	golang.org/x/sync v0.16.0 // indirect
-	golang.org/x/sys v0.34.0 // indirect
-	golang.org/x/text v0.27.0 // indirect
+	golang.org/x/crypto v0.46.0 // indirect
+	golang.org/x/net v0.48.0 // indirect
+	golang.org/x/sync v0.19.0 // indirect
+	golang.org/x/sys v0.39.0 // indirect
+	golang.org/x/text v0.32.0 // indirect
 	google.golang.org/protobuf v1.36.10 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
 )

Lens/modules/skills-repository/go.sum

@@ -178,18 +178,18 @@ go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
 golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
 golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc=
 golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys=
-golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM=
-golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY=
-golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs=
-golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8=
-golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw=
-golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
+golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU=
+golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0=
+golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU=
+golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY=
+golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
+golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA=
-golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
-golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4=
-golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU=
+golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk=
+golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU=
+golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY=
 google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE=
 google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

Lens/modules/skills-repository/pkg/api/handler.go

@@ -5,12 +5,17 @@ package api
 
 import (
 	"errors"
+	"io"
 	"log"
+	"mime"
 	"net/http"
+	"path/filepath"
 	"strconv"
+	"time"
 
 	"github.com/AMD-AGI/Primus-SaFE/Lens/skills-repository/pkg/safe"
 	"github.com/AMD-AGI/Primus-SaFE/Lens/skills-repository/pkg/service"
+	"github.com/AMD-AGI/Primus-SaFE/Lens/skills-repository/pkg/storage"
 	"github.com/gin-gonic/gin"
 )
 
@@ -22,6 +27,7 @@ type Handler struct {
 	runService     *service.RunService
 	toolsetService *service.ToolsetService
 	safeClient     *safe.UserClient
+	storage        storage.Storage
 }
 
 // NewHandler creates a new Handler
@@ -32,6 +38,7 @@ func NewHandler(
 	runSvc *service.RunService,
 	toolsetSvc *service.ToolsetService,
 	safeClient *safe.UserClient,
+	store storage.Storage,
 ) *Handler {
 	return &Handler{
 		toolService:    toolSvc,
@@ -40,6 +47,7 @@ func NewHandler(
 		runService:     runSvc,
 		toolsetService: toolsetSvc,
 		safeClient:     safeClient,
+		storage:        store,
 	}
 }
 
@@ -90,6 +98,11 @@ func RegisterRoutes(router *gin.Engine, h *Handler) {
 		// Get tool content (SKILL.md for skills)
 		auth.GET("/tools/:id/content", h.GetToolContent)
 
+		// Generic file storage (?path=xxx)
+		auth.POST("/files", h.UploadFile)
+		auth.GET("/files", h.DownloadFile)
+		auth.GET("/files/presign", h.GetPresignedURL)
+
 		// Toolsets
 		auth.GET("/toolsets", h.ListToolsets)
 		auth.POST("/toolsets", h.CreateToolset)
@@ -587,6 +600,150 @@ func (h *Handler) UploadIcon(c *gin.Context) {
 	c.JSON(http.StatusOK, gin.H{"icon_url": iconURL})
 }
 
+// GetPresignedURL handles GET /files/presign?filename=xxx&path=yyy
+func (h *Handler) GetPresignedURL(c *gin.Context) {
+	if h.storage == nil {
+		respondWithError(c, http.StatusServiceUnavailable, "STORAGE_NOT_CONFIGURED", "Storage backend not configured")
+		return
+	}
+
+	filename := c.Query("filename")
+	if filename == "" {
+		respondInvalidParameter(c, "filename", "Query parameter 'filename' is required")
+		return
+	}
+
+	prefix := c.Query("path")
+	key := filename
+	if prefix != "" {
+		key = prefix + "/" + filename
+	}
+
+	contentType := c.Query("content_type")
+	if contentType == "" {
+		contentType = c.Query("contentType")
+	}
+	if contentType == "" {
+		contentType = mime.TypeByExtension(filepath.Ext(filename))
+	}
+
+	userInfo := GetUserInfo(c)
+
+	// Generate presigned URL valid for 1 hour
+	uploadURL, err := h.storage.GeneratePresignedUploadURL(c.Request.Context(), key, contentType, time.Hour)
+	if err != nil {
+		log.Printf("[GetPresignedURL] user=%s key=%s error=%v", userInfo.UserID, key, err)
+		respondWithError(c, http.StatusInternalServerError, "PRESIGN_FAILED", err.Error())
+		return
+	}
+
+	downloadURL, _ := h.storage.GetURL(c.Request.Context(), key)
+
+	log.Printf("[GetPresignedURL] user=%s key=%s success", userInfo.UserID, key)
+	c.JSON(http.StatusOK, gin.H{
+		"upload_url":   uploadURL,
+		"download_url": downloadURL,
+		"key":          key,
+	})
+}
+
+// UploadFile handles POST /files?path=xxx - upload one or more files.
+// path is the directory prefix; filenames are taken from the uploaded files.
+func (h *Handler) UploadFile(c *gin.Context) {
+	if h.storage == nil {
+		respondWithError(c, http.StatusServiceUnavailable, "STORAGE_NOT_CONFIGURED", "Storage backend not configured")
+		return
+	}
+
+	if err := c.Request.ParseMultipartForm(32 << 20); err != nil {
+		respondBadRequest(c, "Invalid multipart form", err.Error())
+		return
+	}
+
+	files := c.Request.MultipartForm.File["file"]
+	if len(files) == 0 {
+		respondInvalidParameter(c, "file", "At least one file is required")
+		return
+	}
+
+	userInfo := GetUserInfo(c)
+	prefix := c.Query("path")
+
+	type fileResult struct {
+		Key string `json:"key"`
+		URL string `json:"url"`
+	}
+	results := make([]fileResult, 0, len(files))
+
+	for _, header := range files {
+		key := header.Filename
+		if prefix != "" {
+			key = prefix + "/" + header.Filename
+		}
+
+		file, err := header.Open()
+		if err != nil {
+			log.Printf("[UploadFile] user=%s key=%s open error=%v", userInfo.UserID, key, err)
+			respondWithError(c, http.StatusInternalServerError, "UPLOAD_FAILED", err.Error())
+			return
+		}
+
+		err = h.storage.Upload(c.Request.Context(), key, file)
+		file.Close()
+		if err != nil {
+			log.Printf("[UploadFile] user=%s key=%s error=%v", userInfo.UserID, key, err)
+			respondWithError(c, http.StatusInternalServerError, "UPLOAD_FAILED", err.Error())
+			return
+		}
+
+		url, _ := h.storage.GetURL(c.Request.Context(), key)
+		results = append(results, fileResult{Key: key, URL: url})
+		log.Printf("[UploadFile] user=%s key=%s size=%d success", userInfo.UserID, key, header.Size)
+	}
+
+	// Single file: flat response; multiple files: array response
+	if len(results) == 1 {
+		c.JSON(http.StatusOK, gin.H{"key": results[0].Key, "url": results[0].URL})
+	} else {
+		c.JSON(http.StatusOK, gin.H{"files": results})
+	}
+}
+
+// DownloadFile handles GET /files?path=xxx - download a file from storage
+func (h *Handler) DownloadFile(c *gin.Context) {
+	key := c.Query("path")
+	if key == "" {
+		respondInvalidParameter(c, "path", "Query parameter 'path' is required, e.g. ?path=models/weights.tar.gz")
+		return
+	}
+
+	if h.storage == nil {
+		respondWithError(c, http.StatusServiceUnavailable, "STORAGE_NOT_CONFIGURED", "Storage backend not configured")
+		return
+	}
+
+	userInfo := GetUserInfo(c)
+	log.Printf("[DownloadFile] user=%s key=%s", userInfo.UserID, key)
+
+	reader, err := h.storage.Download(c.Request.Context(), key)
+	if err != nil {
+		log.Printf("[DownloadFile] user=%s key=%s error=%v", userInfo.UserID, key, err)
+		respondWithError(c, http.StatusNotFound, "FILE_NOT_FOUND", err.Error())
+		return
+	}
+	defer reader.Close()
+
+	contentType := mime.TypeByExtension(filepath.Ext(key))
+	if contentType == "" {
+		contentType = "application/octet-stream"
+	}
+
+	c.Header("Content-Disposition", "inline; filename="+filepath.Base(key))
+	c.Header("Content-Type", contentType)
+	c.Status(http.StatusOK)
+	io.Copy(c.Writer, reader)
+}
+
 // GetToolContent handles GET /tools/:id/content - returns raw SKILL.md content
 func (h *Handler) GetToolContent(c *gin.Context) {
 	id, err := strconv.ParseInt(c.Param("id"), 10, 64)

Lens/modules/skills-repository/pkg/bootstrap/bootstrap.go

@@ -114,7 +114,7 @@ func (s *Server) Start() error {
 	}
 
 	// Create handler and register routes
-	handler := api.NewHandler(toolSvc, searchSvc, importSvc, runSvc, toolsetSvc, safeClient)
+	handler := api.NewHandler(toolSvc, searchSvc, importSvc, runSvc, toolsetSvc, safeClient, s.storage)
 	api.RegisterRoutes(router, handler)
 
 	s.httpServer = &http.Server{

Lens/modules/skills-repository/pkg/storage/local_storage.go

@@ -10,6 +10,7 @@ import (
 	"io"
 	"os"
 	"path/filepath"
+	"time"
 )
 
 // LocalStorage implements Storage interface for local filesystem
@@ -104,6 +105,11 @@ func (s *LocalStorage) GetURL(ctx context.Context, key string) (string, error) {
 	return fmt.Sprintf("%s/%s", s.baseURL, key), nil
 }
 
+// GeneratePresignedUploadURL generates a presigned URL for uploading a file directly
+func (s *LocalStorage) GeneratePresignedUploadURL(ctx context.Context, key string, contentType string, expire time.Duration) (string, error) {
+	return "", fmt.Errorf("presigned upload URL is not supported by local storage")
+}
+
 // ListObjects lists all objects with the given prefix
 func (s *LocalStorage) ListObjects(ctx context.Context, prefix string) ([]ObjectInfo, error) {
 	var objects []ObjectInfo