diff --git a/Bootstrap/higress/values.yaml b/Bootstrap/higress/values.yaml index 7a9c6d65f..9c2787304 100644 --- a/Bootstrap/higress/values.yaml +++ b/Bootstrap/higress/values.yaml @@ -41,4 +41,9 @@ higress-core: - name: ssh port: 2222 protocol: TCP - targetPort: 2222 \ No newline at end of file + targetPort: 2222 + downstream: + connectionBufferLimits: 10485760 # 10 MB + http2: + initialConnectionWindowSize: 16777216 # 16 MB + initialStreamWindowSize: 16777216 # 16 MB \ No newline at end of file diff --git a/Lens/modules/skills-repository/go.mod b/Lens/modules/skills-repository/go.mod index 0fbcf7a81..f25c2f64e 100644 --- a/Lens/modules/skills-repository/go.mod +++ b/Lens/modules/skills-repository/go.mod @@ -68,11 +68,11 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect golang.org/x/arch v0.8.0 // indirect - golang.org/x/crypto v0.40.0 // indirect - golang.org/x/net v0.42.0 // indirect - golang.org/x/sync v0.16.0 // indirect - golang.org/x/sys v0.34.0 // indirect - golang.org/x/text v0.27.0 // indirect + golang.org/x/crypto v0.46.0 // indirect + golang.org/x/net v0.48.0 // indirect + golang.org/x/sync v0.19.0 // indirect + golang.org/x/sys v0.39.0 // indirect + golang.org/x/text v0.32.0 // indirect google.golang.org/protobuf v1.36.10 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect ) diff --git a/Lens/modules/skills-repository/go.sum b/Lens/modules/skills-repository/go.sum index 24b11fec0..ad159977f 100644 --- a/Lens/modules/skills-repository/go.sum +++ b/Lens/modules/skills-repository/go.sum @@ -178,18 +178,18 @@ go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc= golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= -golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM= -golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY= -golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs= -golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8= -golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= -golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= +golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= +golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= +golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= -golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= -golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= -golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= +golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= +golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= +golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/Lens/modules/skills-repository/pkg/api/handler.go b/Lens/modules/skills-repository/pkg/api/handler.go index a9f4d9f0e..bc60002e0 100644 --- a/Lens/modules/skills-repository/pkg/api/handler.go +++ b/Lens/modules/skills-repository/pkg/api/handler.go @@ -5,12 +5,17 @@ package api import ( "errors" + "io" "log" + "mime" "net/http" + "path/filepath" "strconv" + "time" "github.com/AMD-AGI/Primus-SaFE/Lens/skills-repository/pkg/safe" "github.com/AMD-AGI/Primus-SaFE/Lens/skills-repository/pkg/service" + "github.com/AMD-AGI/Primus-SaFE/Lens/skills-repository/pkg/storage" "github.com/gin-gonic/gin" ) @@ -22,6 +27,7 @@ type Handler struct { runService *service.RunService toolsetService *service.ToolsetService safeClient *safe.UserClient + storage storage.Storage } // NewHandler creates a new Handler @@ -32,6 +38,7 @@ func NewHandler( runSvc *service.RunService, toolsetSvc *service.ToolsetService, safeClient *safe.UserClient, + store storage.Storage, ) *Handler { return &Handler{ toolService: toolSvc, @@ -40,6 +47,7 @@ func NewHandler( runService: runSvc, toolsetService: toolsetSvc, safeClient: safeClient, + storage: store, } } @@ -90,6 +98,11 @@ func RegisterRoutes(router *gin.Engine, h *Handler) { // Get tool content (SKILL.md for skills) auth.GET("/tools/:id/content", h.GetToolContent) + // Generic file storage (?path=xxx) + auth.POST("/files", h.UploadFile) + auth.GET("/files", h.DownloadFile) + auth.GET("/files/presign", h.GetPresignedURL) + // Toolsets auth.GET("/toolsets", h.ListToolsets) auth.POST("/toolsets", h.CreateToolset) @@ -587,6 +600,150 @@ func (h *Handler) UploadIcon(c *gin.Context) { c.JSON(http.StatusOK, gin.H{"icon_url": iconURL}) } +// GetPresignedURL handles GET /files/presign?filename=xxx&path=yyy +func (h *Handler) GetPresignedURL(c *gin.Context) { + if h.storage == nil { + respondWithError(c, http.StatusServiceUnavailable, "STORAGE_NOT_CONFIGURED", "Storage backend not configured") + return + } + + filename := c.Query("filename") + if filename == "" { + respondInvalidParameter(c, "filename", "Query parameter 'filename' is required") + return + } + + prefix := c.Query("path") + key := filename + if prefix != "" { + key = prefix + "/" + filename + } + + contentType := c.Query("content_type") + if contentType == "" { + contentType = c.Query("contentType") + } + if contentType == "" { + contentType = mime.TypeByExtension(filepath.Ext(filename)) + } + + userInfo := GetUserInfo(c) + + // Generate presigned URL valid for 1 hour + uploadURL, err := h.storage.GeneratePresignedUploadURL(c.Request.Context(), key, contentType, time.Hour) + if err != nil { + log.Printf("[GetPresignedURL] user=%s key=%s error=%v", userInfo.UserID, key, err) + respondWithError(c, http.StatusInternalServerError, "PRESIGN_FAILED", err.Error()) + return + } + + downloadURL, _ := h.storage.GetURL(c.Request.Context(), key) + + log.Printf("[GetPresignedURL] user=%s key=%s success", userInfo.UserID, key) + c.JSON(http.StatusOK, gin.H{ + "upload_url": uploadURL, + "download_url": downloadURL, + "key": key, + }) +} + +// UploadFile handles POST /files?path=xxx - upload one or more files. +// path is the directory prefix; filenames are taken from the uploaded files. +func (h *Handler) UploadFile(c *gin.Context) { + if h.storage == nil { + respondWithError(c, http.StatusServiceUnavailable, "STORAGE_NOT_CONFIGURED", "Storage backend not configured") + return + } + + if err := c.Request.ParseMultipartForm(32 << 20); err != nil { + respondBadRequest(c, "Invalid multipart form", err.Error()) + return + } + + files := c.Request.MultipartForm.File["file"] + if len(files) == 0 { + respondInvalidParameter(c, "file", "At least one file is required") + return + } + + userInfo := GetUserInfo(c) + prefix := c.Query("path") + + type fileResult struct { + Key string `json:"key"` + URL string `json:"url"` + } + results := make([]fileResult, 0, len(files)) + + for _, header := range files { + key := header.Filename + if prefix != "" { + key = prefix + "/" + header.Filename + } + + file, err := header.Open() + if err != nil { + log.Printf("[UploadFile] user=%s key=%s open error=%v", userInfo.UserID, key, err) + respondWithError(c, http.StatusInternalServerError, "UPLOAD_FAILED", err.Error()) + return + } + + err = h.storage.Upload(c.Request.Context(), key, file) + file.Close() + if err != nil { + log.Printf("[UploadFile] user=%s key=%s error=%v", userInfo.UserID, key, err) + respondWithError(c, http.StatusInternalServerError, "UPLOAD_FAILED", err.Error()) + return + } + + url, _ := h.storage.GetURL(c.Request.Context(), key) + results = append(results, fileResult{Key: key, URL: url}) + log.Printf("[UploadFile] user=%s key=%s size=%d success", userInfo.UserID, key, header.Size) + } + + // Single file: flat response; multiple files: array response + if len(results) == 1 { + c.JSON(http.StatusOK, gin.H{"key": results[0].Key, "url": results[0].URL}) + } else { + c.JSON(http.StatusOK, gin.H{"files": results}) + } +} + +// DownloadFile handles GET /files?path=xxx - download a file from storage +func (h *Handler) DownloadFile(c *gin.Context) { + key := c.Query("path") + if key == "" { + respondInvalidParameter(c, "path", "Query parameter 'path' is required, e.g. ?path=models/weights.tar.gz") + return + } + + if h.storage == nil { + respondWithError(c, http.StatusServiceUnavailable, "STORAGE_NOT_CONFIGURED", "Storage backend not configured") + return + } + + userInfo := GetUserInfo(c) + log.Printf("[DownloadFile] user=%s key=%s", userInfo.UserID, key) + + reader, err := h.storage.Download(c.Request.Context(), key) + if err != nil { + log.Printf("[DownloadFile] user=%s key=%s error=%v", userInfo.UserID, key, err) + respondWithError(c, http.StatusNotFound, "FILE_NOT_FOUND", err.Error()) + return + } + defer reader.Close() + + contentType := mime.TypeByExtension(filepath.Ext(key)) + if contentType == "" { + contentType = "application/octet-stream" + } + + c.Header("Content-Disposition", "inline; filename="+filepath.Base(key)) + c.Header("Content-Type", contentType) + c.Status(http.StatusOK) + io.Copy(c.Writer, reader) +} + // GetToolContent handles GET /tools/:id/content - returns raw SKILL.md content func (h *Handler) GetToolContent(c *gin.Context) { id, err := strconv.ParseInt(c.Param("id"), 10, 64) diff --git a/Lens/modules/skills-repository/pkg/bootstrap/bootstrap.go b/Lens/modules/skills-repository/pkg/bootstrap/bootstrap.go index ad661eeef..8e6c61f9c 100644 --- a/Lens/modules/skills-repository/pkg/bootstrap/bootstrap.go +++ b/Lens/modules/skills-repository/pkg/bootstrap/bootstrap.go @@ -114,7 +114,7 @@ func (s *Server) Start() error { } // Create handler and register routes - handler := api.NewHandler(toolSvc, searchSvc, importSvc, runSvc, toolsetSvc, safeClient) + handler := api.NewHandler(toolSvc, searchSvc, importSvc, runSvc, toolsetSvc, safeClient, s.storage) api.RegisterRoutes(router, handler) s.httpServer = &http.Server{ diff --git a/Lens/modules/skills-repository/pkg/storage/local_storage.go b/Lens/modules/skills-repository/pkg/storage/local_storage.go index 9cb780501..5fc62573c 100644 --- a/Lens/modules/skills-repository/pkg/storage/local_storage.go +++ b/Lens/modules/skills-repository/pkg/storage/local_storage.go @@ -10,6 +10,7 @@ import ( "io" "os" "path/filepath" + "time" ) // LocalStorage implements Storage interface for local filesystem @@ -104,6 +105,11 @@ func (s *LocalStorage) GetURL(ctx context.Context, key string) (string, error) { return fmt.Sprintf("%s/%s", s.baseURL, key), nil } +// GeneratePresignedUploadURL generates a presigned URL for uploading a file directly +func (s *LocalStorage) GeneratePresignedUploadURL(ctx context.Context, key string, contentType string, expire time.Duration) (string, error) { + return "", fmt.Errorf("presigned upload URL is not supported by local storage") +} + // ListObjects lists all objects with the given prefix func (s *LocalStorage) ListObjects(ctx context.Context, prefix string) ([]ObjectInfo, error) { var objects []ObjectInfo diff --git a/Lens/modules/skills-repository/pkg/storage/s3_storage.go b/Lens/modules/skills-repository/pkg/storage/s3_storage.go index 7c59b5d89..7077bcf87 100644 --- a/Lens/modules/skills-repository/pkg/storage/s3_storage.go +++ b/Lens/modules/skills-repository/pkg/storage/s3_storage.go @@ -7,6 +7,8 @@ import ( "bytes" "context" "io" + "net/url" + "strings" "time" "github.com/aws/aws-sdk-go-v2/aws" @@ -17,12 +19,14 @@ import ( // S3Storage implements Storage interface for S3/MinIO type S3Storage struct { - client *s3.Client - bucket string - endpoint string - publicURL string // Public URL for file access (optional, may include bucket path) - presigner *s3.PresignClient - urlExpiry time.Duration + client *s3.Client + bucket string + endpoint string + publicURL string // Public URL for file access (optional, may include bucket path) + publicURLPrefix string // Path prefix extracted from publicURL (e.g., "/s3") + presigner *s3.PresignClient + publicPresigner *s3.PresignClient // Presigner configured with public URL for generating upload links + urlExpiry time.Duration } // S3Config contains S3 configuration @@ -75,13 +79,64 @@ func NewS3Storage(cfg S3Config) (*S3Storage, error) { urlExpiry = 1 * time.Hour } + // Create a separate presigner for public URLs if configured + var publicPresigner *s3.PresignClient + var publicURLPrefix string + + if cfg.PublicURL != "" { + // Parse the public URL to extract the base endpoint and the prefix + // e.g. https://oci-slc.primus-safe.amd.com/s3/tools + u, err := url.Parse(cfg.PublicURL) + if err == nil { + // Base endpoint for signing MUST NOT include the prefix (e.g., /s3) + // otherwise the signature will be calculated for /s3/tools/... instead of /tools/... + publicEndpoint := u.Scheme + "://" + u.Host + + // Extract the prefix (e.g., /s3) + path := u.Path + bucketSuffix := "/" + cfg.Bucket + if strings.HasSuffix(path, bucketSuffix) { + publicURLPrefix = strings.TrimSuffix(path, bucketSuffix) + } else if path == cfg.Bucket { + publicURLPrefix = "" + } else { + publicURLPrefix = path + } + + publicResolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) { + return aws.Endpoint{ + URL: publicEndpoint, + HostnameImmutable: true, + }, nil + }) + + publicAwsCfg, _ := config.LoadDefaultConfig(context.Background(), + config.WithRegion(cfg.Region), + config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider( + cfg.AccessKeyID, + cfg.SecretAccessKey, + "", + )), + config.WithEndpointResolverWithOptions(publicResolver), + ) + publicClient := s3.NewFromConfig(publicAwsCfg, func(o *s3.Options) { + o.UsePathStyle = cfg.UsePathStyle + }) + publicPresigner = s3.NewPresignClient(publicClient) + } + } else { + publicPresigner = s3.NewPresignClient(client) + } + return &S3Storage{ - client: client, - bucket: cfg.Bucket, - endpoint: cfg.Endpoint, - publicURL: cfg.PublicURL, - presigner: s3.NewPresignClient(client), - urlExpiry: urlExpiry, + client: client, + bucket: cfg.Bucket, + endpoint: cfg.Endpoint, + publicURL: cfg.PublicURL, + publicURLPrefix: publicURLPrefix, + presigner: s3.NewPresignClient(client), + publicPresigner: publicPresigner, + urlExpiry: urlExpiry, }, nil } @@ -208,6 +263,35 @@ func (s *S3Storage) GetURL(ctx context.Context, key string) (string, error) { return url, nil } +// GeneratePresignedUploadURL generates a presigned URL for uploading a file directly to S3 +func (s *S3Storage) GeneratePresignedUploadURL(ctx context.Context, key string, contentType string, expire time.Duration) (string, error) { + input := &s3.PutObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(key), + } + if contentType != "" { + input.ContentType = aws.String(contentType) + } + + req, err := s.publicPresigner.PresignPutObject(ctx, input, func(opts *s3.PresignOptions) { + opts.Expires = expire + }) + if err != nil { + return "", err + } + + finalURL := req.URL + if s.publicURLPrefix != "" { + u, err := url.Parse(finalURL) + if err == nil { + u.Path = s.publicURLPrefix + u.Path + finalURL = u.String() + } + } + + return finalURL, nil +} + // ListObjects lists all objects with the given prefix func (s *S3Storage) ListObjects(ctx context.Context, prefix string) ([]ObjectInfo, error) { var objects []ObjectInfo diff --git a/Lens/modules/skills-repository/pkg/storage/storage.go b/Lens/modules/skills-repository/pkg/storage/storage.go index 261c2c598..96736f92f 100644 --- a/Lens/modules/skills-repository/pkg/storage/storage.go +++ b/Lens/modules/skills-repository/pkg/storage/storage.go @@ -36,6 +36,9 @@ type Storage interface { // GetURL returns a presigned URL for the file (for S3-compatible storage) GetURL(ctx context.Context, key string) (string, error) + // GeneratePresignedUploadURL generates a presigned URL for uploading a file directly to S3 + GeneratePresignedUploadURL(ctx context.Context, key string, contentType string, expire time.Duration) (string, error) + // ListObjects lists all objects with the given prefix ListObjects(ctx context.Context, prefix string) ([]ObjectInfo, error) }