merge main into amd-staging

.github/CODEOWNERS

@@ -16,6 +16,7 @@
 /amd/device-libs/ @b-sumner @lamb-j
 /amd/hipcc/ @david-salinas @lamb-j
 
+/libc/ @llbm/reviewers-libc
 /libcxx/ @llvm/reviewers-libcxx
 /libcxxabi/ @llvm/reviewers-libcxxabi
 /libunwind/ @llvm/reviewers-libunwind

bolt/include/bolt/Profile/DataAggregator.h

@@ -234,21 +234,9 @@ class DataAggregator : public DataReader {
   /// Return a vector of offsets corresponding to a trace in a function
   /// if the trace is valid, std::nullopt otherwise.
   std::optional<SmallVector<std::pair<uint64_t, uint64_t>, 16>>
-  getFallthroughsInTrace(BinaryFunction &BF, const Trace &Trace, uint64_t Count,
+  getFallthroughsInTrace(BinaryFunction &BF, const Trace &Trace,
                          bool IsReturn) const;
 
-  /// Record external entry into the function \p BF.
-  ///
-  /// Return true if the entry is valid, false otherwise.
-  bool recordEntry(BinaryFunction &BF, uint64_t To, bool Mispred,
-                   uint64_t Count = 1) const;
-
-  /// Record exit from the function \p BF via a call or return.
-  ///
-  /// Return true if the exit point is valid, false otherwise.
-  bool recordExit(BinaryFunction &BF, uint64_t From, bool Mispred,
-                  uint64_t Count = 1) const;
-
   /// Branch stacks aggregation statistics
   uint64_t NumTraces{0};
   uint64_t NumInvalidTraces{0};

bolt/lib/Core/DIEBuilder.cpp

@@ -705,17 +705,17 @@ bool DIEBuilder::cloneExpression(const DataExtractor &Data,
          Description.Op[0] == Encoding::BaseTypeRef) ||
         (Description.Op.size() == 2 &&
          Description.Op[1] == Encoding::BaseTypeRef &&
-         Description.Op[0] != Encoding::Size1))
+         Description.Op[0] != Encoding::Size1 &&
+         Description.Op[0] != Encoding::SizeLEB))
       BC.outs() << "BOLT-WARNING: [internal-dwarf-error]: unsupported DW_OP "
                    "encoding.\n";
 
     if ((Description.Op.size() == 1 &&
          Description.Op[0] == Encoding::BaseTypeRef) ||
         (Description.Op.size() == 2 &&
          Description.Op[1] == Encoding::BaseTypeRef &&
-         Description.Op[0] == Encoding::Size1)) {
-      // This code assumes that the other non-typeref operand fits into 1
-      // byte.
+         (Description.Op[0] == Encoding::Size1 ||
+          Description.Op[0] == Encoding::SizeLEB))) {
       assert(OpOffset < Op.getEndOffset());
       const uint32_t ULEBsize = Op.getEndOffset() - OpOffset - 1;
       (void)ULEBsize;
@@ -727,7 +727,9 @@ bool DIEBuilder::cloneExpression(const DataExtractor &Data,
       if (Description.Op.size() == 1) {
         RefOffset = Op.getRawOperand(0);
       } else {
-        OutputBuffer.push_back(Op.getRawOperand(0));
+        const StringRef FirstOpBytes =
+            Data.getData().slice(OpOffset + 1, Op.getOperandEndOffset(0));
+        OutputBuffer.append(FirstOpBytes.begin(), FirstOpBytes.end());
         RefOffset = Op.getRawOperand(1);
       }
       uint32_t Offset = 0;
@@ -903,6 +905,7 @@ void DIEBuilder::cloneAttribute(
   case dwarf::DW_FORM_ref2:
   case dwarf::DW_FORM_ref4:
   case dwarf::DW_FORM_ref8:
+  case dwarf::DW_FORM_ref_udata:
     cloneDieOffsetReferenceAttribute(Die, U, InputDIE, AttrSpec,
                                      Val.getUnit()->getOffset() +
                                          *Val.getAsRelativeReference());

bolt/lib/Passes/LongJmp.cpp

@@ -367,6 +367,11 @@ LongJmpPass::tentativeLayoutRelocMode(const BinaryContext &BC,
   CurrentIndex = 0;
   bool ColdLayoutDone = false;
   auto runColdLayout = [&]() {
+    // Mirror the extra hugify alignment inserted by final section allocation
+    // after the last non-cold section. Account for it before assigning cold
+    // fragment addresses so range checks see the hot-to-cold gap.
+    if (opts::Hugify && !BC.HasFixedLoadAddress && !opts::HotFunctionsAtEnd)
+      DotAddress = alignTo(DotAddress, opts::AlignText);
     DotAddress = tentativeLayoutRelocColdPart(BC, SortedFunctions, DotAddress);
     ColdLayoutDone = true;
     if (opts::HotFunctionsAtEnd)

bolt/lib/Profile/DataAggregator.cpp

@@ -717,10 +717,8 @@ Error DataAggregator::preprocessProfile(BinaryContext &BC) {
 Error DataAggregator::readProfile(BinaryContext &BC) {
   processProfile(BC);
 
-  for (auto &BFI : BC.getBinaryFunctions()) {
-    BinaryFunction &Function = BFI.second;
-    convertBranchData(Function);
-  }
+  if (Error E = DataReader::readProfile(BC))
+    return E;
 
   if (opts::AggregateOnly) {
     if (opts::ProfileFormat == opts::ProfileFormatKind::PF_Fdata)
@@ -747,6 +745,12 @@ bool DataAggregator::mayHaveProfileData(const BinaryFunction &Function) {
 }
 
 void DataAggregator::processProfile(BinaryContext &BC) {
+  // Set for DataReader::readProfile
+  NoLBRMode = opts::BasicAggregation;
+
+  // Set for DataReader::recordBranch and evaluateProfileData
+  BATMode = usesBAT();
+
   if (opts::BasicAggregation)
     processBasicEvents();
   else
@@ -772,6 +776,9 @@ void DataAggregator::processProfile(BinaryContext &BC) {
     llvm::stable_sort(FuncBranches.second.EntryData);
   }
 
+  for (auto &FuncBasicSamples : NamesToBasicSamples)
+    llvm::stable_sort(FuncBasicSamples.second.Data);
+
   for (auto &MemEvents : NamesToMemEvents)
     llvm::stable_sort(MemEvents.second.Data);
 
@@ -880,8 +887,6 @@ bool DataAggregator::doInterBranch(BinaryFunction *FromFunc,
       FromAggrData->Name = SrcFunc;
       setBranchData(*FromFunc, FromAggrData);
     }
-
-    recordExit(*FromFunc, From, Mispreds, Count);
   }
   if (ToFunc) {
     DstFunc = getLocationName(*ToFunc, BAT);
@@ -891,8 +896,6 @@ bool DataAggregator::doInterBranch(BinaryFunction *FromFunc,
       ToAggrData->Name = DstFunc;
       setBranchData(*ToFunc, ToAggrData);
     }
-
-    recordEntry(*ToFunc, To, Mispreds, Count);
   }
 
   if (FromAggrData)
@@ -941,10 +944,8 @@ bool DataAggregator::doBranch(uint64_t From, uint64_t To, uint64_t Count,
     return false;
 
   // Treat recursive control transfers as inter-branches.
-  if (FromFunc == ToFunc && To != 0) {
-    recordBranch(*FromFunc, From, To, Count, Mispreds);
+  if (FromFunc == ToFunc && To != 0)
     return doIntraBranch(*FromFunc, From, To, Count, Mispreds);
-  }
 
   return doInterBranch(FromFunc, ToFunc, From, To, Count, Mispreds);
 }
@@ -976,7 +977,7 @@ bool DataAggregator::doTrace(const Trace &Trace, uint64_t Count,
   std::optional<BoltAddressTranslation::FallthroughListTy> FTs =
       BAT && BAT->isBATFunction(FuncAddress)
           ? BAT->getFallthroughsInTrace(FuncAddress, From - IsReturn, To)
-          : getFallthroughsInTrace(*FromFunc, Trace, Count, IsReturn);
+          : getFallthroughsInTrace(*FromFunc, Trace, IsReturn);
   if (!FTs) {
     LLVM_DEBUG(dbgs() << "Invalid trace " << Trace << '\n');
     NumInvalidTraces += Count;
@@ -993,7 +994,7 @@ bool DataAggregator::doTrace(const Trace &Trace, uint64_t Count,
 
 std::optional<SmallVector<std::pair<uint64_t, uint64_t>, 16>>
 DataAggregator::getFallthroughsInTrace(BinaryFunction &BF, const Trace &Trace,
-                                       uint64_t Count, bool IsReturn) const {
+                                       bool IsReturn) const {
   SmallVector<std::pair<uint64_t, uint64_t>, 16> Branches;
 
   BinaryContext &BC = BF.getBinaryContext();
@@ -1073,53 +1074,9 @@ DataAggregator::getFallthroughsInTrace(BinaryFunction &BF, const Trace &Trace,
     BB = NextBB;
   }
 
-  // Record fall-through jumps
-  for (const auto &[FromOffset, ToOffset] : Branches) {
-    BinaryBasicBlock *FromBB = BF.getBasicBlockContainingOffset(FromOffset);
-    BinaryBasicBlock *ToBB = BF.getBasicBlockAtOffset(ToOffset);
-    assert(FromBB && ToBB);
-    BinaryBasicBlock::BinaryBranchInfo &BI = FromBB->getBranchInfo(*ToBB);
-    BI.Count += Count;
-  }
-
   return Branches;
 }
 
-bool DataAggregator::recordEntry(BinaryFunction &BF, uint64_t To, bool Mispred,
-                                 uint64_t Count) const {
-  if (To > BF.getSize())
-    return false;
-
-  if (!BF.hasProfile())
-    BF.ExecutionCount = 0;
-
-  BinaryBasicBlock *EntryBB = nullptr;
-  if (To == 0) {
-    BF.ExecutionCount += Count;
-    if (!BF.empty())
-      EntryBB = &BF.front();
-  } else if (BinaryBasicBlock *BB = BF.getBasicBlockAtOffset(To)) {
-    if (BB->isEntryPoint())
-      EntryBB = BB;
-  }
-
-  if (EntryBB)
-    EntryBB->setExecutionCount(EntryBB->getKnownExecutionCount() + Count);
-
-  return true;
-}
-
-bool DataAggregator::recordExit(BinaryFunction &BF, uint64_t From, bool Mispred,
-                                uint64_t Count) const {
-  if (!BF.isSimple() || From > BF.getSize())
-    return false;
-
-  if (!BF.hasProfile())
-    BF.ExecutionCount = 0;
-
-  return true;
-}
-
 ErrorOr<DataAggregator::LBREntry> DataAggregator::parseLBREntry() {
   LBREntry Res;
   ErrorOr<StringRef> FromStrRes = parseString('/');

bolt/test/AArch64/long-jmp-hugify-fixup-out-of-range.s

@@ -1,13 +1,17 @@
-# Check that branches considered in-range during longjump
-# may go out of range at JITLink if hugify moves hot code.
+# The longjump pass may consider branch targets in range during tentative
+# layout and decide not to insert stubs for them. Later, final section
+# allocation may insert alignment padding after the last non-cold text section
+# when hugify is enabled. This moves the following cold section farther away,
+# resulting in relocation fixups going out of range at JITLink. Check that the
+# longjump pass accounts for this padding and inserts stubs when needed.
 
-# REQUIRES: system-linux, asserts
+# REQUIRES: system-linux, asserts, bolt-runtime, target=aarch64{{.*}}
 
 # RUN: %clang %cflags -Wl,-q %s -o %t
 # RUN: link_fdata --no-lbr %s %t %t.fdata
 # RUN: llvm-strip --strip-unneeded %t
-# RUN: not llvm-bolt %t -o %t.bolt --data %t.fdata -split-functions --hugify 2>&1 \
-# RUN:   | FileCheck %s
+# RUN: llvm-bolt %t -o %t.bolt --data %t.fdata -split-functions --hugify
+# RUN: llvm-objdump -d %t.bolt | FileCheck %s
 
   .globl foo
   .type foo, %function
@@ -34,4 +38,15 @@ _start:
 ## Force relocation mode.
 .reloc 0, R_AARCH64_NONE
 
-# CHECK: BOLT-ERROR: JITLink failed: In graph in-memory object file, section .text: relocation target {{0x[0-9a-f]+}} {{.*}} is out of range of CondBranch19PCRel fixup at address {{0x[0-9a-f]+}} {{.*}}
+# CHECK: Disassembly of section .text:
+
+# CHECK: <foo>:
+# CHECK-NEXT:            {{.*}} cbnz x0, 0x[[ADDR0:[0-9a-f]+]] <{{.*}}>
+# CHECK-NEXT:            {{.*}} b 0x[[ADDR1:[0-9a-f]+]] <{{.*}}>
+# CHECK-NEXT: [[ADDR0]]: {{.*}} b 0x[[ADDR2:[0-9a-f]+]] <{{.*}}>
+
+# CHECK: Disassembly of section .text.cold:
+
+# CHECK: <foo.cold.0>:
+# CHECK-NEXT: [[ADDR2]]: {{.*}} mov x0, #0x1 // =1
+# CHECK-NEXT: [[ADDR1]]: {{.*}} ret

bolt/test/X86/dwarf5-form-ref-udata.s

@@ -0,0 +1,70 @@
+# REQUIRES: system-linux
+
+# RUN: llvm-mc -dwarf-version=5 -filetype=obj -triple x86_64-unknown-linux %s -o %t.o
+# RUN: %clang %cflags -dwarf-5 %t.o -o %t.exe -Wl,-q
+# RUN: llvm-bolt %t.exe -o %t.bolt --update-debug-sections 2>&1 | \
+# RUN:   FileCheck %s --check-prefix CHECK-BOLT
+# RUN: llvm-dwarfdump --show-form --verbose --debug-info %t.bolt | FileCheck %s
+
+## Verify BOLT preserves DW_FORM_ref_udata (CU-relative ULEB128 DIE reference),
+## a form GCC may emit instead of DW_FORM_ref4.
+
+# CHECK:      DW_TAG_subprogram
+# CHECK:      DW_AT_type [DW_FORM_ref_udata]
+# CHECK-SAME: "int"
+
+# CHECK-BOLT-NOT: BOLT-WARNING
+
+	.text
+	.file	0 "." "main.cpp"
+	.globl	main
+main:
+.Lfunc_begin0:
+	.loc	0 1 0
+	xorl	%eax, %eax
+	retq
+.Lfunc_end0:
+	.size	main, .Lfunc_end0-main
+
+## Force relocations against .text
+.reloc 0, R_X86_64_NONE
+
+	.section	.debug_abbrev,"",@progbits
+	.byte	1, 17, 1                # CU, has children
+	.byte	17, 1                   # DW_AT_low_pc, DW_FORM_addr
+	.byte	18, 6                   # DW_AT_high_pc, DW_FORM_data4
+	.byte	16, 23                  # DW_AT_stmt_list, DW_FORM_sec_offset
+	.byte	0, 0
+	.byte	2, 46, 0                # subprogram, no children
+	.byte	17, 1                   # DW_AT_low_pc, DW_FORM_addr
+	.byte	18, 6                   # DW_AT_high_pc, DW_FORM_data4
+	.byte	73, 21                  # DW_AT_type, DW_FORM_ref_udata
+	.byte	0, 0
+	.byte	3, 36, 0                # base_type, no children
+	.byte	3, 8                    # DW_AT_name, DW_FORM_string
+	.byte	0, 0
+	.byte	0
+
+	.section	.debug_info,"",@progbits
+.Lcu_begin0:
+	.long	.Ldebug_info_end0-.Ldebug_info_start0
+.Ldebug_info_start0:
+	.short	5                       # DWARF version
+	.byte	1                       # DW_UT_compile
+	.byte	8                       # Address size
+	.long	.debug_abbrev           # Abbrev offset
+	.byte	1                       # CU
+	.quad	.Lfunc_begin0           # DW_AT_low_pc
+	.long	.Lfunc_end0-.Lfunc_begin0  # DW_AT_high_pc
+	.long	.Lline_table_start0     # DW_AT_stmt_list
+	.byte	2                       # subprogram
+	.quad	.Lfunc_begin0           # DW_AT_low_pc
+	.long	.Lfunc_end0-.Lfunc_begin0  # DW_AT_high_pc
+	.uleb128 .Ltype_int-.Lcu_begin0    # DW_AT_type (DW_FORM_ref_udata)
+.Ltype_int:
+	.byte	3                       # base_type
+	.asciz	"int"                   # DW_AT_name
+	.byte	0                       # End children of CU
+.Ldebug_info_end0:
+	.section	.debug_line,"",@progbits
+.Lline_table_start0: