Merge branch 'main' into users/el-ev/fold-masked-merge

Author: el-ev

.ci/metrics/metrics.py

@@ -21,16 +21,16 @@
 # Lists the Github workflows we want to track. Maps the Github job name to
 # the metric name prefix in grafana.
 # This metric name is also used as a key in the job->name map.
-GITHUB_WORKFLOW_TO_TRACK = {"LLVM Premerge Checks": "github_llvm_premerge_checks"}
+GITHUB_WORKFLOW_TO_TRACK = {"CI Checks": "github_llvm_premerge_checks"}
 
 # Lists the Github jobs to track for a given workflow. The key is the stable
 # name (metric name) of the workflow (see GITHUB_WORKFLOW_TO_TRACK).
 # Each value is a map to link the github job name to the corresponding metric
 # name.
 GITHUB_JOB_TO_TRACK = {
     "github_llvm_premerge_checks": {
-        "Linux Premerge Checks (Test Only - Please Ignore Results)": "premerge_linux",
-        "Windows Premerge Checks (Test Only - Please Ignore Results)": "premerge_windows",
+        "Build and Test Linux (Test Only - Please Ignore Results)": "premerge_linux",
+        "Build and Test Windows (Test Only - Please Ignore Results)": "premerge_windows",
     }
 }
 

.github/workflows/libc-fullbuild-tests.yml

@@ -19,13 +19,23 @@ jobs:
         include:
           - os: ubuntu-24.04
             ccache-variant: sccache
-            c_compiler: clang-20
-            cpp_compiler: clang++-20
+            c_compiler: clang-21
+            cpp_compiler: clang++-21
+            target: x86_64-unknown-linux-llvm
+            include_scudo: ON
           # TODO: remove ccache logic when https://github.com/hendrikmuhs/ccache-action/issues/279 is resolved.
           - os: ubuntu-24.04-arm
             ccache-variant: ccache
-            c_compiler: clang-20
-            cpp_compiler: clang++-20
+            c_compiler: clang-21
+            cpp_compiler: clang++-21
+            target: aarch64-unknown-linux-llvm
+            include_scudo: ON
+          - os: ubuntu-24.04
+            ccache-variant: ccache
+            c_compiler: clang-21
+            cpp_compiler: clang++-21
+            target: x86_64-unknown-uefi-llvm
+            include_scudo: OFF
           # TODO: add back gcc build when it is fixed
           # - c_compiler: gcc
           #   cpp_compiler: g++
@@ -53,7 +63,7 @@ jobs:
       run: |
         wget https://apt.llvm.org/llvm.sh
         chmod +x llvm.sh
-        sudo ./llvm.sh 20
+        sudo ./llvm.sh 21
         sudo apt-get update
         sudo apt-get install -y libmpfr-dev libgmp-dev libmpc-dev ninja-build linux-libc-dev
         sudo ln -sf /usr/include/$(uname -p)-linux-gnu/asm /usr/include/asm
@@ -68,22 +78,31 @@ jobs:
     # Configure libc fullbuild with scudo.
     # Use MinSizeRel to reduce the size of the build.
     - name: Configure CMake
-      run: >
-        cmake -B ${{ steps.strings.outputs.build-output-dir }}
-        -DCMAKE_CXX_COMPILER=${{ matrix.cpp_compiler }}
-        -DCMAKE_C_COMPILER=${{ matrix.c_compiler }}
-        -DCMAKE_BUILD_TYPE=${{ matrix.build_type }}
-        -DCMAKE_C_COMPILER_LAUNCHER=${{ matrix.ccache-variant }}
-        -DCMAKE_CXX_COMPILER_LAUNCHER=${{ matrix.ccache-variant }}
-        -DCMAKE_INSTALL_PREFIX=${{ steps.strings.outputs.build-install-dir }}
-        -DLLVM_ENABLE_RUNTIMES="libc;compiler-rt"
-        -DLLVM_LIBC_FULL_BUILD=ON
-        -DLLVM_LIBC_INCLUDE_SCUDO=ON
-        -DCOMPILER_RT_BUILD_SCUDO_STANDALONE_WITH_LLVM_LIBC=ON
-        -DCOMPILER_RT_BUILD_GWP_ASAN=OFF
-        -DCOMPILER_RT_SCUDO_STANDALONE_BUILD_SHARED=OFF
-        -G Ninja
-        -S ${{ github.workspace }}/runtimes
+      run: |
+        export RUNTIMES="libc"
+
+        if [[ ${{ matrix.include_scudo}} == "ON" ]]; then
+          export RUNTIMES="$RUNTIMES;compiler-rt"
+          export CMAKE_FLAGS="
+            -DLLVM_LIBC_INCLUDE_SCUDO=ON
+            -DCOMPILER_RT_BUILD_SCUDO_STANDALONE_WITH_LLVM_LIBC=ON
+            -DCOMPILER_RT_BUILD_GWP_ASAN=OFF
+            -DCOMPILER_RT_SCUDO_STANDALONE_BUILD_SHARED=OFF"
+        fi
+
+        cmake -B ${{ steps.strings.outputs.build-output-dir }} \
+        -DCMAKE_CXX_COMPILER=${{ matrix.cpp_compiler }} \
+        -DCMAKE_C_COMPILER=${{ matrix.c_compiler }} \
+        -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \
+        -DCMAKE_C_COMPILER_LAUNCHER=${{ matrix.ccache-variant }} \
+        -DCMAKE_CXX_COMPILER_LAUNCHER=${{ matrix.ccache-variant }} \
+        -DCMAKE_INSTALL_PREFIX=${{ steps.strings.outputs.build-install-dir }} \
+        -DLLVM_RUNTIMES_TARGET=${{ matrix.target }} \
+        -DLLVM_ENABLE_RUNTIMES="$RUNTIMES" \
+        -DLLVM_LIBC_FULL_BUILD=ON \
+        -G Ninja \
+        -S ${{ github.workspace }}/runtimes \
+        $CMAKE_FLAGS
 
     - name: Build
       run: >
@@ -93,6 +112,8 @@ jobs:
         --target install
 
     - name: Test
+      # Skip UEFI tests until we have testing set up.
+      if: ${{ ! endsWith(matrix.target, '-uefi-llvm') }}
       run: >
         cmake 
         --build ${{ steps.strings.outputs.build-output-dir }} 

.github/workflows/libcxx-build-and-test.yaml

@@ -37,7 +37,7 @@ jobs:
   stage1:
     if: github.repository_owner == 'llvm'
     runs-on: libcxx-self-hosted-linux
-    container: ghcr.io/llvm/libcxx-linux-builder:b6bb9dc5abd7c6452c13a53fa8949cb259db459b
+    container: ghcr.io/llvm/libcxx-linux-builder:2b57ebb50b6d418e70382e655feaa619b558e254
     continue-on-error: false
     strategy:
       fail-fast: false
@@ -75,7 +75,7 @@ jobs:
   stage2:
     if: github.repository_owner == 'llvm'
     runs-on: libcxx-self-hosted-linux
-    container: ghcr.io/llvm/libcxx-linux-builder:b6bb9dc5abd7c6452c13a53fa8949cb259db459b
+    container: ghcr.io/llvm/libcxx-linux-builder:2b57ebb50b6d418e70382e655feaa619b558e254
     needs: [ stage1 ]
     continue-on-error: false
     strategy:
@@ -163,7 +163,7 @@ jobs:
         - config: 'generic-msan'
           machine: libcxx-self-hosted-linux
     runs-on: ${{ matrix.machine }}
-    container: ghcr.io/llvm/libcxx-linux-builder:b6bb9dc5abd7c6452c13a53fa8949cb259db459b
+    container: ghcr.io/llvm/libcxx-linux-builder:2b57ebb50b6d418e70382e655feaa619b558e254
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - name: ${{ matrix.config }}

.github/workflows/premerge.yaml

@@ -1,4 +1,4 @@
-name: LLVM Premerge Checks
+name: CI Checks
 
 permissions:
   contents: read
@@ -25,7 +25,7 @@ concurrency:
 
 jobs:
   premerge-checks-linux:
-    name: Linux Premerge Checks (Test Only - Please Ignore Results)
+    name: Build and Test Linux (Test Only - Please Ignore Results)
     if: >-
         github.repository_owner == 'llvm' &&
         (github.event_name != 'pull_request' || github.event.action != 'closed')
@@ -74,7 +74,7 @@ jobs:
           include-hidden-files: 'true'
 
   premerge-checks-windows:
-    name: Windows Premerge Checks (Test Only - Please Ignore Results)
+    name: Build and Test Windows (Test Only - Please Ignore Results)
     if: >-
         github.repository_owner == 'llvm' &&
         (github.event_name != 'pull_request' || github.event.action != 'closed')

bolt/docs/BinaryAnalysis.md

@@ -180,12 +180,6 @@ The following are current known cases of false negatives:
    [prototype branch](
    https://github.com/llvm/llvm-project/compare/main...kbeyls:llvm-project:bolt-gadget-scanner-prototype).
 
-BOLT cannot currently handle functions with `cfi_negate_ra_state` correctly,
-i.e. any binaries built with `-mbranch-protection=pac-ret`. The scanner is meant
-to be used on specifically such binaries, so this is a major limitation! Work is
-going on in PR [#120064](https://github.com/llvm/llvm-project/pull/120064) to
-fix this.
-
 ## How to add your own binary analysis
 
 _TODO: this section needs to be written. Ideally, we should have a simple

bolt/include/bolt/Profile/DataAggregator.h

@@ -197,10 +197,6 @@ class DataAggregator : public DataReader {
 
   BoltAddressTranslation *BAT{nullptr};
 
-  /// Whether pre-aggregated profile needs to convert branch profile into call
-  /// to continuation fallthrough profile.
-  bool NeedsConvertRetProfileToCallCont{false};
-
   /// Update function execution profile with a recorded trace.
   /// A trace is region of code executed between two LBR entries supplied in
   /// execution order.

bolt/lib/Core/BinaryFunction.cpp

@@ -65,6 +65,8 @@ extern cl::opt<bool> StrictMode;
 extern cl::opt<bool> UpdateDebugSections;
 extern cl::opt<unsigned> Verbosity;
 
+extern bool BinaryAnalysisMode;
+extern bool HeatmapMode;
 extern bool processAllFunctions();
 
 static cl::opt<bool> CheckEncoding(
@@ -2760,13 +2762,19 @@ struct CFISnapshot {
     }
     case MCCFIInstruction::OpAdjustCfaOffset:
     case MCCFIInstruction::OpWindowSave:
-    case MCCFIInstruction::OpNegateRAState:
     case MCCFIInstruction::OpNegateRAStateWithPC:
     case MCCFIInstruction::OpLLVMDefAspaceCfa:
     case MCCFIInstruction::OpLabel:
     case MCCFIInstruction::OpValOffset:
       llvm_unreachable("unsupported CFI opcode");
       break;
+    case MCCFIInstruction::OpNegateRAState:
+      if (!(opts::BinaryAnalysisMode || opts::HeatmapMode)) {
+        llvm_unreachable("BOLT-ERROR: binaries using pac-ret hardening (e.g. "
+                         "as produced by '-mbranch-protection=pac-ret') are "
+                         "currently not supported by BOLT.");
+      }
+      break;
     case MCCFIInstruction::OpRememberState:
     case MCCFIInstruction::OpRestoreState:
     case MCCFIInstruction::OpGnuArgsSize:
@@ -2900,13 +2908,19 @@ struct CFISnapshotDiff : public CFISnapshot {
       return CFAReg == Instr.getRegister() && CFAOffset == Instr.getOffset();
     case MCCFIInstruction::OpAdjustCfaOffset:
     case MCCFIInstruction::OpWindowSave:
-    case MCCFIInstruction::OpNegateRAState:
     case MCCFIInstruction::OpNegateRAStateWithPC:
     case MCCFIInstruction::OpLLVMDefAspaceCfa:
     case MCCFIInstruction::OpLabel:
     case MCCFIInstruction::OpValOffset:
       llvm_unreachable("unsupported CFI opcode");
       return false;
+    case MCCFIInstruction::OpNegateRAState:
+      if (!(opts::BinaryAnalysisMode || opts::HeatmapMode)) {
+        llvm_unreachable("BOLT-ERROR: binaries using pac-ret hardening (e.g. "
+                         "as produced by '-mbranch-protection=pac-ret') are "
+                         "currently not supported by BOLT.");
+      }
+      break;
     case MCCFIInstruction::OpRememberState:
     case MCCFIInstruction::OpRestoreState:
     case MCCFIInstruction::OpGnuArgsSize:
@@ -3051,13 +3065,19 @@ BinaryFunction::unwindCFIState(int32_t FromState, int32_t ToState,
       break;
     case MCCFIInstruction::OpAdjustCfaOffset:
     case MCCFIInstruction::OpWindowSave:
-    case MCCFIInstruction::OpNegateRAState:
     case MCCFIInstruction::OpNegateRAStateWithPC:
     case MCCFIInstruction::OpLLVMDefAspaceCfa:
     case MCCFIInstruction::OpLabel:
     case MCCFIInstruction::OpValOffset:
       llvm_unreachable("unsupported CFI opcode");
       break;
+    case MCCFIInstruction::OpNegateRAState:
+      if (!(opts::BinaryAnalysisMode || opts::HeatmapMode)) {
+        llvm_unreachable("BOLT-ERROR: binaries using pac-ret hardening (e.g. "
+                         "as produced by '-mbranch-protection=pac-ret') are "
+                         "currently not supported by BOLT.");
+      }
+      break;
     case MCCFIInstruction::OpGnuArgsSize:
       // do not affect CFI state
       break;

bolt/lib/Passes/PatchEntries.cpp

@@ -98,21 +98,10 @@ Error PatchEntries::runOnFunctions(BinaryContext &BC) {
     });
 
     if (!Success) {
-      // We can't change output layout for AArch64 due to LongJmp pass
-      if (BC.isAArch64()) {
-        if (opts::ForcePatch) {
-          BC.errs() << "BOLT-ERROR: unable to patch entries in " << Function
-                    << "\n";
-          return createFatalBOLTError("");
-        }
-
-        continue;
-      }
-
       // If the original function entries cannot be patched, then we cannot
       // safely emit new function body.
       BC.errs() << "BOLT-WARNING: failed to patch entries in " << Function
-                << ". The function will not be optimized.\n";
+                << ". The function will not be optimized\n";
       Function.setIgnored();
       continue;
     }

bolt/lib/Profile/DataAggregator.cpp

@@ -720,23 +720,6 @@ bool DataAggregator::doBranch(uint64_t From, uint64_t To, uint64_t Count,
                : isReturn(Func.disassembleInstructionAtOffset(Offset));
   };
 
-  // Returns whether \p Offset in \p Func may be a call continuation excluding
-  // entry points and landing pads.
-  auto checkCallCont = [&](const BinaryFunction &Func, const uint64_t Offset) {
-    // No call continuation at a function start.
-    if (!Offset)
-      return false;
-
-    // FIXME: support BAT case where the function might be in empty state
-    // (split fragments declared non-simple).
-    if (!Func.hasCFG())
-      return false;
-
-    // The offset should not be an entry point or a landing pad.
-    const BinaryBasicBlock *ContBB = Func.getBasicBlockAtOffset(Offset);
-    return ContBB && !ContBB->isEntryPoint() && !ContBB->isLandingPad();
-  };
-
   // Mutates \p Addr to an offset into the containing function, performing BAT
   // offset translation and parent lookup.
   //
@@ -749,8 +732,7 @@ bool DataAggregator::doBranch(uint64_t From, uint64_t To, uint64_t Count,
 
     Addr -= Func->getAddress();
 
-    bool IsRetOrCallCont =
-        IsFrom ? checkReturn(*Func, Addr) : checkCallCont(*Func, Addr);
+    bool IsRet = IsFrom && checkReturn(*Func, Addr);
 
     if (BAT)
       Addr = BAT->translate(Func->getAddress(), Addr, IsFrom);
@@ -761,24 +743,16 @@ bool DataAggregator::doBranch(uint64_t From, uint64_t To, uint64_t Count,
       NumColdSamples += Count;
 
     if (!ParentFunc)
-      return std::pair{Func, IsRetOrCallCont};
+      return std::pair{Func, IsRet};
 
-    return std::pair{ParentFunc, IsRetOrCallCont};
+    return std::pair{ParentFunc, IsRet};
   };
 
-  uint64_t ToOrig = To;
   auto [FromFunc, IsReturn] = handleAddress(From, /*IsFrom*/ true);
-  auto [ToFunc, IsCallCont] = handleAddress(To, /*IsFrom*/ false);
+  auto [ToFunc, _] = handleAddress(To, /*IsFrom*/ false);
   if (!FromFunc && !ToFunc)
     return false;
 
-  // Record call to continuation trace.
-  if (NeedsConvertRetProfileToCallCont && FromFunc != ToFunc &&
-      (IsReturn || IsCallCont)) {
-    LBREntry First{ToOrig - 1, ToOrig - 1, false};
-    LBREntry Second{ToOrig, ToOrig, false};
-    return doTrace(First, Second, Count);
-  }
   // Ignore returns.
   if (IsReturn)
     return true;
@@ -1235,21 +1209,14 @@ std::error_code DataAggregator::parseAggregatedLBREntry() {
   ErrorOr<StringRef> TypeOrErr = parseString(FieldSeparator);
   if (std::error_code EC = TypeOrErr.getError())
     return EC;
-  // Pre-aggregated profile with branches and fallthroughs needs to convert
-  // return profile into call to continuation fall-through.
-  auto Type = AggregatedLBREntry::BRANCH;
-  if (TypeOrErr.get() == "B") {
-    NeedsConvertRetProfileToCallCont = true;
+  auto Type = AggregatedLBREntry::TRACE;
+  if (LLVM_LIKELY(TypeOrErr.get() == "T")) {
+  } else if (TypeOrErr.get() == "B") {
     Type = AggregatedLBREntry::BRANCH;
   } else if (TypeOrErr.get() == "F") {
-    NeedsConvertRetProfileToCallCont = true;
     Type = AggregatedLBREntry::FT;
   } else if (TypeOrErr.get() == "f") {
-    NeedsConvertRetProfileToCallCont = true;
     Type = AggregatedLBREntry::FT_EXTERNAL_ORIGIN;
-  } else if (TypeOrErr.get() == "T") {
-    // Trace is expanded into B and [Ff]
-    Type = AggregatedLBREntry::TRACE;
   } else {
     reportError("expected T, B, F or f");
     return make_error_code(llvm::errc::io_error);

bolt/lib/Rewrite/BinaryPassManager.cpp

@@ -497,6 +497,10 @@ Error BinaryFunctionPassManager::runAllPasses(BinaryContext &BC) {
   // memory profiling data.
   Manager.registerPass(std::make_unique<ReorderData>());
 
+  // Patch original function entries
+  if (BC.HasRelocations)
+    Manager.registerPass(std::make_unique<PatchEntries>());
+
   if (BC.isAArch64()) {
     Manager.registerPass(
         std::make_unique<ADRRelaxationPass>(PrintAdrRelaxation));
@@ -524,10 +528,6 @@ Error BinaryFunctionPassManager::runAllPasses(BinaryContext &BC) {
   // Assign each function an output section.
   Manager.registerPass(std::make_unique<AssignSections>());
 
-  // Patch original function entries
-  if (BC.HasRelocations)
-    Manager.registerPass(std::make_unique<PatchEntries>());
-
   // This pass turns tail calls into jumps which makes them invisible to
   // function reordering. It's unsafe to use any CFG or instruction analysis
   // after this point.