Skip to content
Open
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
95 changes: 86 additions & 9 deletions internal/commands/storage/import.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,73 @@ import (
"github.com/spf13/pflag"
)

// supportedContentTypes maps file extensions to IANA-registered content types
// based on UpCloud Storage Import API documentation
var supportedContentTypes = map[string]string{
".gz": "application/gzip",
".xz": "application/x-xz",
".iso": "application/octet-stream",
".img": "application/octet-stream",
".raw": "application/octet-stream",
".qcow2": "application/octet-stream",
".tar": "application/x-tar",
".bz2": "application/x-bzip2",
".7z": "application/x-7z-compressed",
".zip": "application/zip",
Comment on lines +32 to +41
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Based on our our docs we only support raw images and gz and xz compressions, so no need for all of these 🤔

The supported data type is raw (extension .raw, .img or .iso). However, the data can be compressed either with GZip or XZ (Content-Types application/gzip and application/x-xz, typical extensions .gz and .xz respectively) in which case it will be transparently decompressed.

}

// getSupportedExtensionsText returns a formatted string of supported file extensions
func getSupportedExtensionsText() string {
var extensions []string
for ext := range supportedContentTypes {
extensions = append(extensions, ext)
}
// Sort for consistent output
for i := 0; i < len(extensions); i++ {
for j := i + 1; j < len(extensions); j++ {
if extensions[i] > extensions[j] {
extensions[i], extensions[j] = extensions[j], extensions[i]
}
}
}
var result string
for i, ext := range extensions {
if i > 0 {
result += ", "
}
result += ext
}
return result
}

// getSupportedContentTypesText returns a formatted string of supported content types
func getSupportedContentTypesText() string {
seen := make(map[string]bool)
var types []string
for _, ct := range supportedContentTypes {
if !seen[ct] {
types = append(types, ct)
seen[ct] = true
}
}
// Sort for consistent output
for i := 0; i < len(types); i++ {
for j := i + 1; j < len(types); j++ {
if types[i] > types[j] {
types[i], types[j] = types[j], types[i]
}
}
}
var result string
for i, ct := range types {
if i > 0 {
result += ", "
}
result += ct
}
return result
}

// ImportCommand creates the "storage import" command
func ImportCommand() commands.Command {
return &importCommand{
Expand Down Expand Up @@ -60,6 +127,7 @@ type importCommand struct {
existingStorageUUIDOrName string
noWait config.OptionalBoolean
wait config.OptionalBoolean
contentType string

createParams createParams

Expand All @@ -69,8 +137,9 @@ type importCommand struct {
// InitCommand implements Command.InitCommand
func (s *importCommand) InitCommand() {
flagSet := &pflag.FlagSet{}
flagSet.StringVar(&s.sourceLocation, "source-location", "", "Location of the source of the import. Can be a file or a URL.")
flagSet.StringVar(&s.sourceLocation, "source-location", "", fmt.Sprintf("Location of the source of the import. Can be a file or a URL. Supported file extensions: %s", getSupportedExtensionsText()))
flagSet.StringVar(&s.existingStorageUUIDOrName, "storage", "", "Import to an existing storage. Storage must be large enough and must be undetached or the server where the storage is attached must be in shutdown state.")
flagSet.StringVar(&s.contentType, "content-type", "", fmt.Sprintf("Content type of the file being imported. If not specified, it will be automatically detected based on file extension. Supported types: %s", getSupportedContentTypesText()))
config.AddToggleFlag(flagSet, &s.noWait, "no-wait", false, "When importing from remote url, do not wait until the import finishes or storage is in online state. If set, command will exit after import process has been initialized.")
config.AddToggleFlag(flagSet, &s.wait, "wait", false, "Wait for storage to be in online state before returning.")
applyCreateFlags(flagSet, &s.createParams, defaultCreateParams)
Expand Down Expand Up @@ -206,7 +275,7 @@ func (s *importCommand) ExecuteWithoutArguments(exec commands.Executor) (output.
if err != nil {
return commands.HandleError(exec, msg, fmt.Errorf("cannot open local file: %w", err))
}
go importLocalFile(exec, storageToImportTo.UUID, sourceFile, statusChan)
go importLocalFile(exec, storageToImportTo.UUID, sourceFile, s.contentType, statusChan)
}

// import has been triggered, read updates from the process
Expand Down Expand Up @@ -351,19 +420,27 @@ func pollStorageImportStatus(exec commands.Executor, uuid string, statusChan cha
}
}

func importLocalFile(exec commands.Executor, uuid string, file *os.File, statusChan chan<- storageImportStatus) {
func getContentType(filename string) string {
ext := filepath.Ext(filename)
if contentType, exists := supportedContentTypes[ext]; exists {
return contentType
}

// Default to octet-stream for unknown types
return "application/octet-stream"
}

func importLocalFile(exec commands.Executor, uuid string, file *os.File, userContentType string, statusChan chan<- storageImportStatus) {
// make sure we close the channel when exiting import
defer close(statusChan)
chDone := make(chan storageImportStatus)
reader := &readerCounter{source: file}

// figure out content type
contentType := "application/octet-stream"
switch filepath.Ext(file.Name()) {
case ".gz":
contentType = "application/gzip"
case ".xz":
contentType = "application/x-xz"
// use user-provided content type if specified, otherwise auto-detect
contentType := userContentType
if contentType == "" {
contentType = getContentType(file.Name())
}

go func() {
Expand Down
27 changes: 27 additions & 0 deletions internal/commands/storage/import_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,33 @@ func TestImportCommand(t *testing.T) {
}
}

func TestGetContentType(t *testing.T) {
tests := []struct {
filename string
expected string
}{
{"image.iso", "application/octet-stream"},
{"image.img", "application/octet-stream"},
{"image.raw", "application/octet-stream"},
{"image.qcow2", "application/octet-stream"},
{"archive.gz", "application/gzip"},
{"archive.xz", "application/x-xz"},
{"archive.tar", "application/x-tar"},
{"archive.bz2", "application/x-bzip2"},
{"archive.7z", "application/x-7z-compressed"},
{"archive.zip", "application/zip"},
{"unknown.bin", "application/octet-stream"},
{"noextension", "application/octet-stream"},
}

for _, test := range tests {
t.Run(test.filename, func(t *testing.T) {
result := getContentType(test.filename)
assert.Equal(t, test.expected, result)
})
}
}

func TestParseSource(t *testing.T) {
for _, test := range []struct {
name string
Expand Down