[AIX] PGO codegen changes for function-sections.

compiler-rt/test/profile/AIX/needs-garbage-collection.c

@@ -0,0 +1,45 @@
+// RUN: split-file %s %t
+// RUN: cd %t
+// RUN: %clang_pgogen -ffunction-sections main.c -c -o main.o
+// RUN: %clang_pgogen -ffunction-sections needs_gc.c -c -o needs_gc.o
+// RUN: %clang_pgogen main.o needs_gc.o -o needs_gc.out
+// RUN: env LLVM_PROFILE_FILE=needs_gc.profraw %run ./needs_gc.out > /dev/null
+// RUN: llvm-profdata show --all-functions needs_gc.profraw | FileCheck %s
+
+// CHECK-DAG: main
+// CHECK-DAG: baz
+// CHECK-DAG: get_message
+
+//--- main.c
+const char* get_message(void) {
+        return "Hello World!";
+}
+
+const char* baz();
+
+int printf(const char*, ...);
+
+int main(void) {
+        printf("%s\n", baz());
+}
+
+//--- needs_gc.c
+extern int not_def_one(const char *);
+extern double not_def_two(void);
+
+extern const char* get_message(void);
+
+char buf[512];
+int foo(const char *ptr, unsigned long size) {
+        void *memcpy(void *, const void *, unsigned long);
+        memcpy(buf, ptr, size);
+        return not_def_one(buf);
+}
+
+double bar(void) {
+        return not_def_two();
+}
+
+const char* baz() {
+        return get_message();
+}

llvm/include/llvm/IR/FixedMetadataKinds.def

@@ -53,3 +53,4 @@ LLVM_FIXED_MD_KIND(MD_DIAssignID, "DIAssignID", 38)
 LLVM_FIXED_MD_KIND(MD_coro_outside_frame, "coro.outside.frame", 39)
 LLVM_FIXED_MD_KIND(MD_mmra, "mmra", 40)
 LLVM_FIXED_MD_KIND(MD_noalias_addrspace, "noalias.addrspace", 41)
+LLVM_FIXED_MD_KIND(MD_pgo_associated, "pgo.associated", 42)

llvm/lib/CodeGen/GlobalMerge.cpp

@@ -87,6 +87,7 @@
 #include "llvm/InitializePasses.h"
 #include "llvm/MC/SectionKind.h"
 #include "llvm/Pass.h"
+#include "llvm/ProfileData/InstrProf.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
@@ -155,6 +156,7 @@ class GlobalMergeImpl {
   const TargetMachine *TM = nullptr;
   GlobalMergeOptions Opt;
   bool IsMachO = false;
+  bool IsAIX = false;
 
 private:
   bool doMerge(SmallVectorImpl<GlobalVariable *> &Globals, Module &M,
@@ -674,7 +676,9 @@ bool GlobalMergeImpl::run(Module &M) {
   if (!EnableGlobalMerge)
     return false;
 
-  IsMachO = M.getTargetTriple().isOSBinFormatMachO();
+  Triple T(M.getTargetTriple());
+  IsMachO = T.isOSBinFormatMachO();
+  IsAIX = T.isOSBinFormatXCOFF();
 
   auto &DL = M.getDataLayout();
   MapVector<std::pair<unsigned, StringRef>, SmallVector<GlobalVariable *, 0>>
@@ -717,6 +721,14 @@ bool GlobalMergeImpl::run(Module &M) {
         GV.getName().starts_with(".llvm.") || Section == "llvm.metadata")
       continue;
 
+    // Do not merge profiling counters as it will prevent us from breaking
+    // the __llvm_prf_cnts section into subsections, which in turn creates
+    // extra symbol dependencies that can break otherwise valid link steps.
+    if (IsAIX && TM && TM->getFunctionSections() && GV.hasSection() &&
+        Section.starts_with(
+            getInstrProfSectionName(IPSK_cnts, Triple::XCOFF, false)))
+      continue;
+
     // Ignore all "required" globals:
     if (isMustKeepGlobalVariable(&GV))
       continue;

llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp

@@ -2435,7 +2435,7 @@ MCSection *TargetLoweringObjectFileXCOFF::getExplicitSectionGlobal(
   if (!GO->hasSection())
     report_fatal_error("#pragma clang section is not yet supported");
 
-  StringRef SectionName = GO->getSection();
+  std::string SectionName(GO->getSection());
 
   // Handle the XCOFF::TD case first, then deal with the rest.
   if (const GlobalVariable *GVar = dyn_cast<GlobalVariable>(GO))
@@ -2458,6 +2458,25 @@ MCSection *TargetLoweringObjectFileXCOFF::getExplicitSectionGlobal(
   else
     report_fatal_error("XCOFF other section types not yet implemented.");
 
+  // The profiling instrumentation symbols are special in that we want to
+  // emit a unique CSECT for each when function sections are enabled, which
+  // are then renamed back to the CSECT name specified by the explicit section.
+  // This is to work around the limitation of not having section groups or a
+  // similar feature in XCOFF.
+  if (TM.getFunctionSections()) {
+    std::string ProfilingDataSectionName =
+        getInstrProfSectionName(IPSK_data, Triple::XCOFF, false);
+    std::string ProfilingCounterSectionName =
+        getInstrProfSectionName(IPSK_cnts, Triple::XCOFF, false);
+    if ((SectionName == ProfilingDataSectionName &&
+         GO->getName().starts_with(getInstrProfDataVarPrefix())) ||
+        (SectionName == ProfilingCounterSectionName &&
+         GO->getName().starts_with(getInstrProfCountersVarPrefix()))) {
+      SectionName += ".";
+      SectionName += GO->getName();
+    }
+  }
+
   return getContext().getXCOFFSection(
       SectionName, Kind, XCOFF::CsectProperties(MappingClass, XCOFF::XTY_SD),
       /* MultiSymbolsAllowed*/ true);

llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp

@@ -61,6 +61,7 @@
 #include "llvm/MC/MCSymbolXCOFF.h"
 #include "llvm/MC/SectionKind.h"
 #include "llvm/MC/TargetRegistry.h"
+#include "llvm/ProfileData/InstrProf.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/CodeGen.h"
 #include "llvm/Support/Debug.h"
@@ -253,6 +254,22 @@ class PPCAIXAsmPrinter : public PPCAsmPrinter {
   DenseMap<const GlobalObject *, SmallVector<const GlobalAlias *, 1>>
       GOAliasMap;
 
+  // The __profd_* symbol for the profiling instrumentation data and the
+  // corresponding __profc_* counters it references.
+  struct ProfilingSubSection {
+    MCSectionXCOFF *ProfD;
+    MCSectionXCOFF *ProfC;
+  };
+
+  // Collect the 'sub-sections' of the profile-generate symbols
+  // so we can:
+  // 1) rename to the common CSECT name after emission.
+  // 2) emit the refs from the profc_ symbol to the related CSECTs.
+  SmallVector<ProfilingSubSection> ProfGenSubSections;
+
+  void emitSharedSectionPGORefs(Module &M);
+  void emitSplitSectionPGORefs();
+
   uint16_t getNumberOfVRSaved();
   void emitTracebackTable();
 
@@ -2810,6 +2827,28 @@ void PPCAIXAsmPrinter::emitGlobalVariableHelper(const GlobalVariable *GV) {
   MCSectionXCOFF *Csect = cast<MCSectionXCOFF>(
       getObjFileLowering().SectionForGlobal(GV, GVKind, TM));
 
+  // When compiling with function sections enabled, we need some special
+  // codegen to rename the CSECTs. For each profiling data symbol find its
+  // associated profiling counters.
+  if (TM.getFunctionSections() &&
+      Csect->getName().starts_with("__llvm_prf_data.")) {
+    MDNode *MD = GV->getMetadata(LLVMContext::MD_pgo_associated);
+
+    assert(MD &&
+           "profiling data symbol must have an associated counter symbol");
+
+    const ValueAsMetadata *VAM = cast<ValueAsMetadata>(MD->getOperand(0).get());
+    const GlobalVariable *ProfCGV = cast<GlobalVariable>(VAM->getValue());
+
+    // Map the global variable to its CSECT.
+    SectionKind ProfCKind = getObjFileLowering().getKindForGlobal(GV, TM);
+
+    MCSectionXCOFF *ProfCCsect = cast<MCSectionXCOFF>(
+        getObjFileLowering().SectionForGlobal(ProfCGV, ProfCKind, TM));
+
+    ProfGenSubSections.push_back({Csect, ProfCCsect});
+  }
+
   // Switch to the containing csect.
   OutStreamer->switchSection(Csect);
 
@@ -2911,7 +2950,7 @@ void PPCAIXAsmPrinter::emitFunctionEntryLabel() {
         getObjFileLowering().getFunctionEntryPointSymbol(Alias, TM));
 }
 
-void PPCAIXAsmPrinter::emitPGORefs(Module &M) {
+void PPCAIXAsmPrinter::emitSharedSectionPGORefs(Module &M) {
   if (!OutContext.hasXCOFFSection(
           "__llvm_prf_cnts",
           XCOFF::CsectProperties(XCOFF::XMC_RW, XCOFF::XTY_SD)))
@@ -2960,6 +2999,65 @@ void PPCAIXAsmPrinter::emitPGORefs(Module &M) {
   }
 }
 
+void PPCAIXAsmPrinter::emitSplitSectionPGORefs() {
+  MCSymbol *NamesSym = nullptr;
+  MCSymbol *VNDSSym = nullptr;
+
+  auto profSectionName = [](InstrProfSectKind IPSK) -> std::string {
+    return getInstrProfSectionName(IPSK, Triple::XCOFF,
+                                   /* AddSegmentInfo */ false);
+  };
+
+  if (OutContext.hasXCOFFSection(
+          profSectionName(IPSK_name),
+          XCOFF::CsectProperties(XCOFF::XMC_RO, XCOFF::XTY_SD))) {
+    std::string SymName = profSectionName(IPSK_name);
+    SymName += "[RO]";
+    NamesSym = OutContext.getOrCreateSymbol(SymName);
+  }
+
+  if (OutContext.hasXCOFFSection(
+          profSectionName(IPSK_vnodes),
+          XCOFF::CsectProperties(XCOFF::XMC_RW, XCOFF::XTY_SD))) {
+    std::string SymName = profSectionName(IPSK_vnodes);
+    SymName += "[RW]";
+    VNDSSym = OutContext.getOrCreateSymbol(SymName);
+  }
+
+  for (auto SubSections : ProfGenSubSections) {
+    MCSectionXCOFF *ProfDCsect = SubSections.ProfD;
+    MCSectionXCOFF *ProfCCsect = SubSections.ProfC;
+
+    OutStreamer->switchSection(ProfCCsect);
+
+    if (NamesSym)
+      OutStreamer->emitXCOFFRefDirective(NamesSym);
+
+    if (VNDSSym)
+      OutStreamer->emitXCOFFRefDirective(VNDSSym);
+
+    OutStreamer->emitXCOFFRefDirective(ProfDCsect->getQualNameSymbol());
+
+    // Rename the subsection for the counters
+    OutStreamer->emitXCOFFRenameDirective(ProfCCsect->getQualNameSymbol(),
+                                          profSectionName(IPSK_cnts));
+    OutStreamer->addBlankLine();
+
+    // Rename the subsection for the data.
+    OutStreamer->switchSection(ProfDCsect);
+    OutStreamer->emitXCOFFRenameDirective(ProfDCsect->getQualNameSymbol(),
+                                          profSectionName(IPSK_data));
+    OutStreamer->addBlankLine();
+  }
+}
+
+void PPCAIXAsmPrinter::emitPGORefs(Module &M) {
+  if (!TM.getFunctionSections())
+    emitSharedSectionPGORefs(M);
+  else
+    emitSplitSectionPGORefs();
+}
+
 void PPCAIXAsmPrinter::emitGCOVRefs() {
   if (!OutContext.hasXCOFFSection(
           "__llvm_gcov_ctr_section",

llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp

@@ -1874,6 +1874,12 @@ void InstrLowerer::createDataVariable(InstrProfCntrInstBase *Inc) {
   Data->setAlignment(Align(INSTR_PROF_DATA_ALIGNMENT));
   maybeSetComdat(Data, Fn, CntsVarName);
 
+  if (TT.isOSBinFormatXCOFF()) {
+    Data->setMetadata(
+        LLVMContext::MD_pgo_associated,
+        MDNode::get(M.getContext(), ValueAsMetadata::get(PD.RegionCounters)));
+  }
+
   PD.DataVar = Data;
 
   // Mark the data variable as used so that it isn't stripped out.

llvm/test/CodeGen/PowerPC/aix-pgo-function-sections.ll

@@ -0,0 +1,64 @@
+; RUN: llc --function-sections -mtriple powerpc-ibm-aix-xcoff < %s | \
+; RUN: FileCheck %s
+
+; RUN: llc --function-sections -mtriple powerpc64-ibm-aix-xcoff < %s | \
+; RUN: FileCheck %s
+
+@i = external local_unnamed_addr global i32, align 4
+@__llvm_profile_raw_version = weak hidden local_unnamed_addr constant i64 72057594037927944
+@__profc_func1 = private global [1 x i64] zeroinitializer, section "__llvm_prf_cnts", align 8
+@__profd_func1 = private global { i64, i64, i32, ptr, ptr, i32, [4 x i16] } { i64 -2545542355363006406, i64 742261418966908927, i32 sub (i32 ptrtoint (ptr @__profc_func1 to i32), i32 ptrtoint (ptr @__profd_func1 to i32)), ptr @func1.local, ptr null, i32 1, [4 x i16] zeroinitializer }, section "__llvm_prf_data", align 8, !pgo.associated !0
+@__profc_func2 = private global [1 x i64] zeroinitializer, section "__llvm_prf_cnts", align 8
+@__profd_func2 = private global { i64, i64, i32, ptr, ptr, i32, [4 x i16] } { i64 -4377547752858689819, i64 742261418966908927, i32 sub (i32 ptrtoint (ptr @__profc_func2 to i32), i32 ptrtoint (ptr @__profd_func2 to i32)), ptr @func2.local, ptr null, i32 1, [4 x i16] zeroinitializer }, section "__llvm_prf_data", align 8, !pgo.associated !1
+@__llvm_prf_nm = private constant [13 x i8] c"\0B\00func1\01func2", section "__llvm_prf_names", align 1
+@__llvm_profile_filename = weak hidden local_unnamed_addr constant [19 x i8] c"default_%m.profraw\00"
+@llvm.used = appending global [3 x ptr] [ptr @__llvm_prf_nm, ptr @__profd_func1, ptr @__profd_func2], section "llvm.metadata"
+
+@func1.local = private alias i32 (), ptr @func1
+@func2.local = private alias i32 (), ptr @func2
+
+define i32 @func1() {
+entry:
+  %pgocount = load i64, ptr @__profc_func1, align 8
+  %0 = add i64 %pgocount, 1
+  store i64 %0, ptr @__profc_func1, align 8
+  %1 = load i32, ptr @i, align 4
+  ret i32 %1
+}
+
+define i32 @func2() {
+entry:
+  %pgocount = load i64, ptr @__profc_func2, align 8
+  %0 = add i64 %pgocount, 1
+  store i64 %0, ptr @__profc_func2, align 8
+  %1 = load i32, ptr @i, align 4
+  %call = tail call i32 @external_func(i32 noundef %1)
+  ret i32 %call
+}
+
+declare i32 @external_func(i32 noundef)
+
+!0 = !{ptr @__profc_func1}
+!1 = !{ptr @__profc_func2}
+
+; CHECK-DAG:        .csect __llvm_prf_cnts.__profc_func1[RW]
+; CHECK-DAG:        .csect __llvm_prf_data.__profd_func1[RW]
+; CHECK-DAG:        .csect __llvm_prf_cnts.__profc_func2[RW]
+; CHECK-DAG:        .csect __llvm_prf_data.__profd_func2[RW]
+; CHECK-DAG:        .csect __llvm_prf_names[RO]
+
+; CHECK:             .csect __llvm_prf_cnts.__profc_func1[RW]
+; CHECK-NEXT:        .ref __llvm_prf_names[RO]
+; CHECK-NEXT:        .ref __llvm_prf_data.__profd_func1[RW]
+; CHECK-NEXT:        .rename __llvm_prf_cnts.__profc_func1[RW],"__llvm_prf_cnts"
+
+; CHECK:             .csect __llvm_prf_data.__profd_func1[RW]
+; CHECK-NEXT:        .rename __llvm_prf_data.__profd_func1[RW],"__llvm_prf_data"
+
+; CHECK:             .csect __llvm_prf_cnts.__profc_func2[RW]
+; CHECK-NEXT:        .ref __llvm_prf_names[RO]
+; CHECK-NEXT:        .ref __llvm_prf_data.__profd_func2[RW]
+; CHECK-NEXT:        .rename __llvm_prf_cnts.__profc_func2[RW],"__llvm_prf_cnts"
+
+; CHECK:             .csect __llvm_prf_data.__profd_func2[RW]
+; CHECK-NEXT:        .rename __llvm_prf_data.__profd_func2[RW],"__llvm_prf_data"