diff --git a/pkg/yqlib/candidate_node.go b/pkg/yqlib/candidate_node.go index 02168515ac..7c973494c6 100644 --- a/pkg/yqlib/candidate_node.go +++ b/pkg/yqlib/candidate_node.go @@ -100,6 +100,9 @@ type CandidateNode struct { // For formats like HCL and TOML: indicates that child entries should be emitted as separate blocks/tables // rather than consolidated into nested mappings (default behaviour) EncodeSeparate bool + // For formats like HCL: indicates that a blank line preceded this node in the original source, + // so the encoder should emit a blank line before it to preserve formatting. + BlankLineBefore bool } func (n *CandidateNode) CreateChild() *CandidateNode { @@ -411,7 +414,8 @@ func (n *CandidateNode) doCopy(cloneContent bool) *CandidateNode { EvaluateTogether: n.EvaluateTogether, IsMapKey: n.IsMapKey, - EncodeSeparate: n.EncodeSeparate, + EncodeSeparate: n.EncodeSeparate, + BlankLineBefore: n.BlankLineBefore, } if cloneContent { diff --git a/pkg/yqlib/decoder_hcl.go b/pkg/yqlib/decoder_hcl.go index 731f9ff3c9..72cc1a7e0b 100644 --- a/pkg/yqlib/decoder_hcl.go +++ b/pkg/yqlib/decoder_hcl.go @@ -43,6 +43,36 @@ type attributeWithName struct { Attr *hclsyntax.Attribute } +// bodyItem represents either an attribute or a block at a given byte position in the source, +// allowing attributes and blocks to be processed together in source order. +type bodyItem struct { + startByte int + attr *attributeWithName // non-nil for attributes + block *hclsyntax.Block // non-nil for blocks +} + +// sortedBodyItems returns attributes and blocks interleaved in source declaration order. +func sortedBodyItems(attrs hclsyntax.Attributes, blocks hclsyntax.Blocks) []bodyItem { + var items []bodyItem + for name, attr := range attrs { + items = append(items, bodyItem{ + startByte: attr.Range().Start.Byte, + attr: &attributeWithName{Name: name, Attr: attr}, + }) + } + for _, block := range blocks { + b := block + items = append(items, bodyItem{ + startByte: b.TypeRange.Start.Byte, + block: b, + }) + } + sort.Slice(items, func(i, j int) bool { + return items[i].startByte < items[j].startByte + }) + return items +} + // extractLineComment extracts any inline comment after the given position func extractLineComment(src []byte, endPos int) string { // Look for # comment after the token @@ -64,6 +94,59 @@ func extractLineComment(src []byte, endPos int) string { return "" } +// hasPrecedingBlankLine reports whether there is a blank line immediately before startPos, +// skipping over any immediately preceding comment lines and whitespace. +func hasPrecedingBlankLine(src []byte, startPos int) bool { + i := startPos - 1 + + // Skip trailing spaces/tabs on the current token's preceding content + for i >= 0 && (src[i] == ' ' || src[i] == '\t') { + i-- + } + + // We expect to be sitting just before a newline that ends the previous line. + // Walk backwards skipping comment lines until we find a blank line or a non-comment line. + for i >= 0 { + // We should be pointing at '\n' (end of previous line) or start of file. + if src[i] != '\n' { + return false + } + i-- // step past the '\n' + + // Skip '\r' for Windows line endings + if i >= 0 && src[i] == '\r' { + i-- + } + + // If immediately another '\n', this is a blank line. + if i < 0 || src[i] == '\n' { + return true + } + + // Read the previous line to see if it's a comment or blank. + lineEnd := i + for i >= 0 && src[i] != '\n' { + i-- + } + lineStart := i + 1 + line := strings.TrimSpace(string(src[lineStart : lineEnd+1])) + + if line == "" { + return true + } + + if strings.HasPrefix(line, "#") { + // This line is a comment belonging to the current element; keep scanning upward. + continue + } + + // A non-blank, non-comment line: no blank line precedes this element. + return false + } + + return false +} + // extractHeadComment extracts comments before a given start position func extractHeadComment(src []byte, startPos int) string { var comments []string @@ -136,39 +219,47 @@ func (dec *hclDecoder) Decode() (*CandidateNode, error) { root := &CandidateNode{Kind: MappingNode} - // process attributes in declaration order body := dec.file.Body.(*hclsyntax.Body) - firstAttr := true - for _, attrWithName := range sortedAttributes(body.Attributes) { - keyNode := createStringScalarNode(attrWithName.Name) - valNode := convertHclExprToNode(attrWithName.Attr.Expr, dec.fileBytes) - - // Attach comments if any - attrRange := attrWithName.Attr.Range() - headComment := extractHeadComment(dec.fileBytes, attrRange.Start.Byte) - if firstAttr && headComment != "" { - // For the first attribute, apply its head comment to the root - root.HeadComment = headComment - firstAttr = false - } else if headComment != "" { - keyNode.HeadComment = headComment - } - if lineComment := extractLineComment(dec.fileBytes, attrRange.End.Byte); lineComment != "" { - valNode.LineComment = lineComment - } - - root.AddKeyValueChild(keyNode, valNode) - } - // process blocks - // Count blocks by type at THIS level to detect multiple separate blocks + // Count blocks by type at THIS level to detect multiple separate blocks of the same type. blocksByType := make(map[string]int) for _, block := range body.Blocks { blocksByType[block.Type]++ } - for _, block := range body.Blocks { - addBlockToMapping(root, block, dec.fileBytes, blocksByType[block.Type] > 1) + // Process attributes and blocks together in source declaration order. + isFirst := true + for _, item := range sortedBodyItems(body.Attributes, body.Blocks) { + if item.attr != nil { + aw := item.attr + keyNode := createStringScalarNode(aw.Name) + valNode := convertHclExprToNode(aw.Attr.Expr, dec.fileBytes) + + attrRange := aw.Attr.Range() + headComment := extractHeadComment(dec.fileBytes, attrRange.Start.Byte) + if isFirst && headComment != "" { + // For the first element, apply its head comment to the root node + root.HeadComment = headComment + } else if headComment != "" { + keyNode.HeadComment = headComment + } + if lineComment := extractLineComment(dec.fileBytes, attrRange.End.Byte); lineComment != "" { + valNode.LineComment = lineComment + } + if !isFirst && hasPrecedingBlankLine(dec.fileBytes, attrRange.Start.Byte) { + keyNode.BlankLineBefore = true + } + + root.AddKeyValueChild(keyNode, valNode) + } else { + block := item.block + headComment := extractHeadComment(dec.fileBytes, block.TypeRange.Start.Byte) + if isFirst && headComment != "" { + root.HeadComment = headComment + } + addBlockToMappingOrdered(root, block, dec.fileBytes, blocksByType[block.Type] > 1, isFirst, headComment) + } + isFirst = false } dec.documentIndex++ @@ -178,71 +269,105 @@ func (dec *hclDecoder) Decode() (*CandidateNode, error) { func hclBodyToNode(body *hclsyntax.Body, src []byte) *CandidateNode { node := &CandidateNode{Kind: MappingNode} - for _, attrWithName := range sortedAttributes(body.Attributes) { - key := createStringScalarNode(attrWithName.Name) - val := convertHclExprToNode(attrWithName.Attr.Expr, src) - // Attach comments if any - attrRange := attrWithName.Attr.Range() - if headComment := extractHeadComment(src, attrRange.Start.Byte); headComment != "" { - key.HeadComment = headComment - } - if lineComment := extractLineComment(src, attrRange.End.Byte); lineComment != "" { - val.LineComment = lineComment - } - - node.AddKeyValueChild(key, val) - } - - // Process nested blocks, counting blocks by type at THIS level - // to detect which block types appear multiple times blocksByType := make(map[string]int) for _, block := range body.Blocks { blocksByType[block.Type]++ } - for _, block := range body.Blocks { - addBlockToMapping(node, block, src, blocksByType[block.Type] > 1) + isFirst := true + for _, item := range sortedBodyItems(body.Attributes, body.Blocks) { + if item.attr != nil { + aw := item.attr + key := createStringScalarNode(aw.Name) + val := convertHclExprToNode(aw.Attr.Expr, src) + + attrRange := aw.Attr.Range() + if headComment := extractHeadComment(src, attrRange.Start.Byte); headComment != "" { + key.HeadComment = headComment + } + if lineComment := extractLineComment(src, attrRange.End.Byte); lineComment != "" { + val.LineComment = lineComment + } + if !isFirst && hasPrecedingBlankLine(src, attrRange.Start.Byte) { + key.BlankLineBefore = true + } + + node.AddKeyValueChild(key, val) + } else { + block := item.block + headComment := extractHeadComment(src, block.TypeRange.Start.Byte) + addBlockToMappingOrdered(node, block, src, blocksByType[block.Type] > 1, isFirst, headComment) + } + isFirst = false } return node } -// addBlockToMapping nests block type and labels into the parent mapping, merging children. -// isMultipleBlocksOfType indicates if there are multiple blocks of this type at THIS level -func addBlockToMapping(parent *CandidateNode, block *hclsyntax.Block, src []byte, isMultipleBlocksOfType bool) { +// addBlockToMappingOrdered nests a block's type and labels into the parent mapping, merging children. +// isMultipleBlocksOfType: there are multiple blocks of this type at this level. +// isFirstInParent: this block is the first element in the parent (no preceding sibling). +// headComment: any comment extracted before this block's type keyword. +func addBlockToMappingOrdered(parent *CandidateNode, block *hclsyntax.Block, src []byte, isMultipleBlocksOfType bool, isFirstInParent bool, headComment string) { bodyNode := hclBodyToNode(block.Body, src) current := parent // ensure block type mapping exists var typeNode *CandidateNode + var typeKeyNode *CandidateNode for i := 0; i < len(current.Content); i += 2 { if current.Content[i].Value == block.Type { + typeKeyNode = current.Content[i] typeNode = current.Content[i+1] break } } if typeNode == nil { - _, typeNode = current.AddKeyValueChild(createStringScalarNode(block.Type), &CandidateNode{Kind: MappingNode}) - // Mark the type node if there are multiple blocks of this type at this level - // This tells the encoder to emit them as separate blocks rather than consolidating them + var newTypeKey *CandidateNode + newTypeKey, typeNode = current.AddKeyValueChild(createStringScalarNode(block.Type), &CandidateNode{Kind: MappingNode}) + typeKeyNode = newTypeKey + // Mark the type node if there are multiple blocks of this type at this level. + // This tells the encoder to emit them as separate blocks rather than consolidating them. if isMultipleBlocksOfType { typeNode.EncodeSeparate = true } + // Store the head comment on the type key (non-first elements only; first element's + // comment is handled by the caller and applied to the root node). + if !isFirstInParent && headComment != "" { + typeKeyNode.HeadComment = headComment + } + // Detect blank line before this block in the source. + // Only set it when this is not the first element (i.e. something already precedes it). + if !isFirstInParent && hasPrecedingBlankLine(src, block.TypeRange.Start.Byte) { + typeKeyNode.BlankLineBefore = true + } } current = typeNode // walk labels, creating/merging mappings - for _, label := range block.Labels { + for labelIdx, label := range block.Labels { var next *CandidateNode + var labelKey *CandidateNode for i := 0; i < len(current.Content); i += 2 { if current.Content[i].Value == label { + labelKey = current.Content[i] next = current.Content[i+1] break } } if next == nil { - _, next = current.AddKeyValueChild(createStringScalarNode(label), &CandidateNode{Kind: MappingNode}) + var newLabelKey *CandidateNode + newLabelKey, next = current.AddKeyValueChild(createStringScalarNode(label), &CandidateNode{Kind: MappingNode}) + labelKey = newLabelKey + // For same-type blocks: mark the first label key with BlankLineBefore when + // there is a blank line before this block in the source. + if labelIdx == 0 && len(current.Content) > 2 { + if hasPrecedingBlankLine(src, block.TypeRange.Start.Byte) { + labelKey.BlankLineBefore = true + } + } } + _ = labelKey current = next } diff --git a/pkg/yqlib/doc/usage/hcl.md b/pkg/yqlib/doc/usage/hcl.md index 7704ca4049..e4c5eed7c4 100644 --- a/pkg/yqlib/doc/usage/hcl.md +++ b/pkg/yqlib/doc/usage/hcl.md @@ -169,8 +169,10 @@ will output ```hcl # Arithmetic with literals and application-provided variables sum = 1 + addend + # String interpolation and templates message = "Hello, ${name}!" + # Application-provided functions shouty_message = upper(message) ``` @@ -199,3 +201,61 @@ resource "aws_instance" "db" { } ``` +## Roundtrip: blank lines between attributes are preserved +Given a sample.hcl file of: +```hcl +name = "app" + +version = 1 + +enabled = true + +``` +then +```bash +yq sample.hcl +``` +will output +```hcl +name = "app" + +version = 1 + +enabled = true +``` + +## Roundtrip: blank lines between blocks are preserved +Given a sample.hcl file of: +```hcl +terraform { + source = "git::https://example.com/module.git" +} + +include { + path = "../root.hcl" +} + +dependency "base" { + config_path = "../base" +} + +``` +then +```bash +yq sample.hcl +``` +will output +```hcl +terraform { + source = "git::https://example.com/module.git" +} + +include { + path = "../root.hcl" +} + +dependency "base" { + config_path = "../base" +} +``` + diff --git a/pkg/yqlib/encoder_hcl.go b/pkg/yqlib/encoder_hcl.go index 8dce06ea10..358692cbdd 100644 --- a/pkg/yqlib/encoder_hcl.go +++ b/pkg/yqlib/encoder_hcl.go @@ -50,9 +50,11 @@ func (he *hclEncoder) Encode(writer io.Writer, node *CandidateNode) error { f := hclwrite.NewEmptyFile() body := f.Body() - // Collect comments as we encode + // Collect comments and blank-line markers as we encode commentMap := make(map[string]string) + blankLineSet := make(map[string]bool) he.collectComments(node, "", commentMap) + he.collectBlankLines(node, blankLineSet) if err := he.encodeNode(body, node); err != nil { return fmt.Errorf("failed to encode HCL: %w", err) @@ -62,8 +64,12 @@ func (he *hclEncoder) Encode(writer io.Writer, node *CandidateNode) error { output := f.Bytes() compactOutput := he.compactSpacing(output) - // Inject comments back into the output - finalOutput := he.injectComments(compactOutput, commentMap) + // Inject comments first so that blank lines are inserted before comment+attribute groups. + withComments := he.injectComments(compactOutput, commentMap) + + // Inject blank lines before appropriate elements (after comments, so blanks appear before + // the comment that precedes an attribute, not between the comment and the attribute). + finalOutput := he.injectBlankLines(withComments, blankLineSet) if he.prefs.ColorsEnabled { colourized := he.colorizeHcl(finalOutput) @@ -123,6 +129,79 @@ func (he *hclEncoder) collectComments(node *CandidateNode, prefix string, commen } } +// collectBlankLines recursively collects keys that have BlankLineBefore set, so +// the encoder can re-insert blank lines into the output. +func (he *hclEncoder) collectBlankLines(node *CandidateNode, blankLineSet map[string]bool) { + if node == nil || node.Kind != MappingNode { + return + } + + for i := 0; i < len(node.Content); i += 2 { + keyNode := node.Content[i] + valueNode := node.Content[i+1] + key := keyNode.Value + + if keyNode.BlankLineBefore { + blankLineSet[key] = true + } + + if valueNode.Kind == MappingNode { + he.collectBlankLines(valueNode, blankLineSet) + } + } +} + +// injectBlankLines inserts a blank line before each HCL element (or its preceding comment block) +// whose key name is in blankLineSet. It scans lines in reverse: when it finds a key that needs a +// blank line, it walks backward over any immediately preceding comment lines and inserts the blank +// line before that comment block (so the blank line precedes the comment+key group, not between them). +func (he *hclEncoder) injectBlankLines(output []byte, blankLineSet map[string]bool) []byte { + if len(blankLineSet) == 0 { + return output + } + + // identRe captures the first identifier on a line (possibly indented). + identRe := regexp.MustCompile(`^\s*([A-Za-z_][A-Za-z0-9_-]*)(\s*(=|\{|"))`) + commentRe := regexp.MustCompile(`^\s*#`) + + lines := strings.Split(string(output), "\n") + + // Mark which line indices need a blank line inserted before them. + insertBefore := make(map[int]bool) + + for i, line := range lines { + m := identRe.FindStringSubmatch(line) + if m == nil || !blankLineSet[m[1]] { + continue + } + + // Walk backward over comment lines to find the insertion point. + insertAt := i + for insertAt > 0 && commentRe.MatchString(lines[insertAt-1]) { + insertAt-- + } + + // Only insert if the line before the insertion point is not already blank. + if insertAt > 0 && strings.TrimSpace(lines[insertAt-1]) != "" { + insertBefore[insertAt] = true + } + } + + if len(insertBefore) == 0 { + return output + } + + out := make([]string, 0, len(lines)+len(insertBefore)) + for i, line := range lines { + if insertBefore[i] { + out = append(out, "") + } + out = append(out, line) + } + + return []byte(strings.Join(out, "\n")) +} + // joinCommentPath concatenates path segments using commentPathSep, safely handling empty prefixes. func joinCommentPath(prefix, segment string) string { if prefix == "" { @@ -164,9 +243,16 @@ func (he *hclEncoder) injectComments(output []byte, commentMap map[string]string continue } - re := regexp.MustCompile(`(?m)^(\s*)` + regexp.QuoteMeta(key) + `\s*=`) - if re.MatchString(result) { - result = re.ReplaceAllString(result, "$1"+trimmed+"\n$0") + // Match both attribute assignments (key =) and block openers (key { or key "label"). + // Use ( *) instead of (\s*) to only capture indentation spaces, not newlines. + // Replace only the first occurrence to avoid duplicating comments before same-named keys. + re := regexp.MustCompile(`(?m)^( *)` + regexp.QuoteMeta(key) + `( *(=|\{|"))`) + submatches := re.FindStringSubmatchIndex(result) + if submatches != nil { + // submatches[2] and submatches[3] are the bounds of the ( *) indentation group. + indent := result[submatches[2]:submatches[3]] + insertPos := submatches[0] + result = result[:insertPos] + indent + trimmed + "\n" + result[insertPos:] } } @@ -345,6 +431,27 @@ func tokensForRawHCLExpr(expr string) (hclwrite.Tokens, error) { tokens = append(tokens, &hclwrite.Token{Type: hclsyntax.TokenStar, Bytes: []byte{'*'}}) case ch == '/': tokens = append(tokens, &hclwrite.Token{Type: hclsyntax.TokenSlash, Bytes: []byte{'/'}}) + case ch == '"': + // Quoted string literal: consume until closing quote, respecting backslash escapes. + start := i + i++ // skip opening quote + for i < len(expr) && expr[i] != '"' { + if expr[i] == '\\' { + i++ // skip escape character + } + i++ + } + if i < len(expr) { + i++ // skip closing quote + } + // Emit as a sequence of OQuote + QuotedLit + CQuote tokens so hclwrite renders it correctly. + raw := expr[start:i] + tokens = append(tokens, + &hclwrite.Token{Type: hclsyntax.TokenOQuote, Bytes: []byte{'"'}}, + &hclwrite.Token{Type: hclsyntax.TokenQuotedLit, Bytes: []byte(raw[1 : len(raw)-1])}, + &hclwrite.Token{Type: hclsyntax.TokenCQuote, Bytes: []byte{'"'}}, + ) + continue default: return nil, fmt.Errorf("unsupported character %q in raw HCL expression", ch) } @@ -353,6 +460,125 @@ func tokensForRawHCLExpr(expr string) (hclwrite.Tokens, error) { return tokens, nil } +// tokensForValue builds an hclwrite token stream for any node value, preserving raw HCL +// expressions (Style==0 !!str scalars) and recursing into mappings and sequences. +func tokensForValue(node *CandidateNode) (hclwrite.Tokens, error) { + switch node.Kind { + case ScalarNode: + if node.Tag == "!!str" { + if node.Style == 0 || node.Style&LiteralStyle != 0 { + // Raw HCL expression — emit without quotes. + return tokensForRawHCLExpr(node.Value) + } + if node.Style&DoubleQuotedStyle != 0 { + // Template or quoted string. + inner := node.Value + var toks hclwrite.Tokens + toks = append(toks, &hclwrite.Token{Type: hclsyntax.TokenOQuote, Bytes: []byte{'"'}}) + for i := 0; i < len(inner); { + if i < len(inner)-1 && inner[i] == '$' && inner[i+1] == '{' { + toks = append(toks, &hclwrite.Token{Type: hclsyntax.TokenTemplateInterp, Bytes: []byte("${")}) + i += 2 + start := i + depth := 1 + for i < len(inner) && depth > 0 { + switch inner[i] { + case '{': + depth++ + case '}': + depth-- + } + i++ + } + expr := inner[start : i-1] + toks = append(toks, &hclwrite.Token{Type: hclsyntax.TokenIdent, Bytes: []byte(expr)}) + toks = append(toks, &hclwrite.Token{Type: hclsyntax.TokenTemplateSeqEnd, Bytes: []byte("}")}) + } else { + toks = append(toks, &hclwrite.Token{Type: hclsyntax.TokenQuotedLit, Bytes: []byte{inner[i]}}) + i++ + } + } + toks = append(toks, &hclwrite.Token{Type: hclsyntax.TokenCQuote, Bytes: []byte{'"'}}) + return toks, nil + } + // Any other string style: emit as quoted string. + var toks hclwrite.Tokens + toks = append(toks, + &hclwrite.Token{Type: hclsyntax.TokenOQuote, Bytes: []byte{'"'}}, + &hclwrite.Token{Type: hclsyntax.TokenQuotedLit, Bytes: []byte(node.Value)}, + &hclwrite.Token{Type: hclsyntax.TokenCQuote, Bytes: []byte{'"'}}, + ) + return toks, nil + } + // Non-string scalar: use cty conversion and let hclwrite render it. + ctyVal, err := nodeToCtyValue(node) + if err != nil { + return nil, err + } + return hclwrite.TokensForValue(ctyVal), nil + case MappingNode: + // Build {\n key = value\n ...\n} as properly typed tokens so + // hclwrite's formatter counts brackets correctly and indents properly. + // Empty mappings are rendered as {} on a single line. + if len(node.Content) == 0 { + return hclwrite.Tokens{ + {Type: hclsyntax.TokenOBrace, Bytes: []byte("{")}, + {Type: hclsyntax.TokenCBrace, Bytes: []byte("}")}, + }, nil + } + var toks hclwrite.Tokens + toks = append(toks, + &hclwrite.Token{Type: hclsyntax.TokenOBrace, Bytes: []byte("{")}, + &hclwrite.Token{Type: hclsyntax.TokenNewline, Bytes: []byte("\n")}, + ) + for i := 0; i < len(node.Content); i += 2 { + keyNode := node.Content[i] + valNode := node.Content[i+1] + // Key token + if isValidHCLIdentifier(keyNode.Value) { + toks = append(toks, &hclwrite.Token{Type: hclsyntax.TokenIdent, Bytes: []byte(keyNode.Value)}) + } else { + toks = append(toks, + &hclwrite.Token{Type: hclsyntax.TokenOQuote, Bytes: []byte{'"'}}, + &hclwrite.Token{Type: hclsyntax.TokenQuotedLit, Bytes: []byte(keyNode.Value)}, + &hclwrite.Token{Type: hclsyntax.TokenCQuote, Bytes: []byte{'"'}}, + ) + } + // Equals token + toks = append(toks, &hclwrite.Token{Type: hclsyntax.TokenEqual, Bytes: []byte("=")}) + // Value tokens + valToks, err := tokensForValue(valNode) + if err != nil { + return nil, err + } + toks = append(toks, valToks...) + // Newline after each entry + toks = append(toks, &hclwrite.Token{Type: hclsyntax.TokenNewline, Bytes: []byte("\n")}) + } + toks = append(toks, + &hclwrite.Token{Type: hclsyntax.TokenCBrace, Bytes: []byte("}")}, + ) + return toks, nil + case SequenceNode: + var toks hclwrite.Tokens + toks = append(toks, &hclwrite.Token{Type: hclsyntax.TokenOBrack, Bytes: []byte("[")}) + for i, child := range node.Content { + if i > 0 { + toks = append(toks, &hclwrite.Token{Type: hclsyntax.TokenComma, Bytes: []byte(",")}) + } + childToks, err := tokensForValue(child) + if err != nil { + return nil, err + } + toks = append(toks, childToks...) + } + toks = append(toks, &hclwrite.Token{Type: hclsyntax.TokenCBrack, Bytes: []byte("]")}) + return toks, nil + default: + return nil, fmt.Errorf("unsupported node kind for HCL value: %v", kindToString(node.Kind)) + } +} + // encodeAttribute encodes a value as an HCL attribute func (he *hclEncoder) encodeAttribute(body *hclwrite.Body, key string, valueNode *CandidateNode) error { if valueNode.Kind == ScalarNode && valueNode.Tag == "!!str" { @@ -386,6 +612,15 @@ func (he *hclEncoder) encodeAttribute(body *hclwrite.Body, key string, valueNode return nil } } + // Flow-style mapping or sequence: use tokensForValue to preserve raw expressions inside. + if valueNode.Kind == MappingNode || valueNode.Kind == SequenceNode { + tokens, err := tokensForValue(valueNode) + if err != nil { + return err + } + body.SetAttributeRaw(key, tokens) + return nil + } // Default: use cty.Value for quoted strings and all other types ctyValue, err := nodeToCtyValue(valueNode) if err != nil { @@ -465,14 +700,12 @@ func (he *hclEncoder) encodeBlockIfMapping(body *hclwrite.Body, key string, valu if handled, err := he.encodeMappingChildrenAsBlocks(block.Body(), nestedType, bodyNode); err == nil && handled { return true } - if err := he.encodeNodeAttributes(block.Body(), bodyNode); err == nil { - return true - } - } - block := body.AppendNewBlock(key, labels) - if err := he.encodeNodeAttributes(block.Body(), bodyNode); err == nil { + _ = he.encodeNodeAttributes(block.Body(), bodyNode) return true } + block := body.AppendNewBlock(key, labels) + _ = he.encodeNodeAttributes(block.Body(), bodyNode) + return true } // If all child values are mappings, treat each child key as a labelled instance of this block type @@ -480,13 +713,12 @@ func (he *hclEncoder) encodeBlockIfMapping(body *hclwrite.Body, key string, valu return true } - // No labels detected, render as unlabelled block + // No labels detected, render as unlabelled block. + // Note: AppendNewBlock writes to the underlying buffer immediately, so we must return true + // regardless of whether encodeNodeAttributes succeeds to avoid double-emitting the key. block := body.AppendNewBlock(key, nil) - if err := he.encodeNodeAttributes(block.Body(), valueNode); err == nil { - return true - } - - return false + _ = he.encodeNodeAttributes(block.Body(), valueNode) + return true } // encodeNode encodes a CandidateNode directly to HCL, preserving style information diff --git a/pkg/yqlib/hcl_test.go b/pkg/yqlib/hcl_test.go index a8f2206554..bbb3a5afb0 100644 --- a/pkg/yqlib/hcl_test.go +++ b/pkg/yqlib/hcl_test.go @@ -71,8 +71,10 @@ shouty_message = upper(message)` var simpleSampleExpected = `# Arithmetic with literals and application-provided variables sum = 1 + addend + # String interpolation and templates message = "Hello, ${name}!" + # Application-provided functions shouty_message = upper(message) ` @@ -85,6 +87,51 @@ message: "Hello, ${name}!" shouty_message: upper(message) ` +var blankLinesBetweenAttributes = "name = \"app\"\n\nversion = 1\n\nenabled = true\n" + +var blankLinesBetweenBlocks = `terraform { + source = "git::https://example.com/module.git" +} + +include { + path = "../root.hcl" +} + +dependency "base" { + config_path = "../base" +} +` + +var blankLinesMixedAttributesAndBlocks = `# Root comment +name = "app" + +version = 1 + +terraform { + source = "git::https://example.com/module.git" +} + +dependency "base" { + config_path = "../base" +} +` + +var blocksWithCommentsAndBlankLines = `# First block comment +terraform { + source = "example" +} + +# Second block comment +include { + path = "../root.hcl" +} + +# Third block comment +dependencies { + paths = ["../base"] +} +` + var hclFormatScenarios = []formatScenario{ { description: "Parse HCL", @@ -472,6 +519,46 @@ var hclFormatScenarios = []formatScenario{ expected: "service {\n optional_field = null\n}\n", scenarioType: "roundtrip", }, + { + description: "block with function call containing quoted string argument", + skipDoc: true, + input: "include {\n path = find_in_parent_folders(\"root.hcl\")\n}\n", + expected: "include {\n path = find_in_parent_folders(\"root.hcl\")\n}\n", + scenarioType: "roundtrip", + }, + { + description: "Roundtrip: object attribute with traversal expression values", + skipDoc: true, + input: "inputs = {\n sub_id = dependency.base.outputs.subscription_id\n rg_name = dependency.base.outputs.resource_group_name\n}\n", + expected: "inputs = {\n sub_id = dependency.base.outputs.subscription_id\n rg_name = dependency.base.outputs.resource_group_name\n}\n", + scenarioType: "roundtrip", + }, + { + description: "Roundtrip: blank lines between attributes are preserved", + input: blankLinesBetweenAttributes, + expected: blankLinesBetweenAttributes, + scenarioType: "roundtrip", + }, + { + description: "Roundtrip: blank lines between blocks are preserved", + input: blankLinesBetweenBlocks, + expected: blankLinesBetweenBlocks, + scenarioType: "roundtrip", + }, + { + description: "Roundtrip: blank lines between mixed attributes and blocks are preserved", + skipDoc: true, + input: blankLinesMixedAttributesAndBlocks, + expected: blankLinesMixedAttributesAndBlocks, + scenarioType: "roundtrip", + }, + { + description: "Roundtrip: blocks with comments and blank lines are preserved", + skipDoc: true, + input: blocksWithCommentsAndBlankLines, + expected: blocksWithCommentsAndBlankLines, + scenarioType: "roundtrip", + }, } func testHclScenario(t *testing.T, s formatScenario) {