From bae4eff5f5a6fe86470fbb359016a8b8b0ed29c7 Mon Sep 17 00:00:00 2001 From: mortimer Date: Tue, 1 Aug 2017 02:06:01 +0000 Subject: Use int3 trap padding between functions instead of trapsleds with a leading jump. ok deraadt@ --- gnu/llvm/include/llvm/CodeGen/AsmPrinter.h | 90 ++- gnu/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp | 640 ++++++++++++--------- gnu/llvm/lib/Target/X86/X86AsmPrinter.h | 42 +- gnu/llvm/lib/Target/X86/X86MCInstLower.cpp | 749 +++++++++++++++++-------- 4 files changed, 993 insertions(+), 528 deletions(-) (limited to 'gnu/llvm') diff --git a/gnu/llvm/include/llvm/CodeGen/AsmPrinter.h b/gnu/llvm/include/llvm/CodeGen/AsmPrinter.h index cf29fc9ef88..0e094c26d21 100644 --- a/gnu/llvm/include/llvm/CodeGen/AsmPrinter.h +++ b/gnu/llvm/include/llvm/CodeGen/AsmPrinter.h @@ -34,6 +34,7 @@ class ConstantArray; class DIE; class DIEAbbrev; class GCMetadataPrinter; +class GlobalIndirectSymbol; class GlobalValue; class GlobalVariable; class MachineBasicBlock; @@ -88,10 +89,6 @@ public: /// This is a pointer to the current MachineModuleInfo. MachineModuleInfo *MMI; - /// Name-mangler for global names. - /// - Mangler *Mang; - /// The symbol for the current function. This is recalculated at the beginning /// of each call to runOnMachineFunction(). /// @@ -125,11 +122,16 @@ private: struct HandlerInfo { AsmPrinterHandler *Handler; - const char *TimerName, *TimerGroupName; + const char *TimerName; + const char *TimerDescription; + const char *TimerGroupName; + const char *TimerGroupDescription; HandlerInfo(AsmPrinterHandler *Handler, const char *TimerName, - const char *TimerGroupName) + const char *TimerDescription, const char *TimerGroupName, + const char *TimerGroupDescription) : Handler(Handler), TimerName(TimerName), - TimerGroupName(TimerGroupName) {} + TimerDescription(TimerDescription), TimerGroupName(TimerGroupName), + TimerGroupDescription(TimerGroupDescription) {} }; /// A vector of all debug/EH info emitters we should use. This vector /// maintains ownership of the emitters. @@ -138,6 +140,9 @@ private: /// If the target supports dwarf debug info, this pointer is non-null. DwarfDebug *DD; + /// If the current module uses dwarf CFI annotations strictly for debugging. + bool isCFIMoveForDebugging; + protected: explicit AsmPrinter(TargetMachine &TM, std::unique_ptr Streamer); @@ -147,6 +152,11 @@ public: DwarfDebug *getDwarfDebug() { return DD; } DwarfDebug *getDwarfDebug() const { return DD; } + uint16_t getDwarfVersion() const; + void setDwarfVersion(uint16_t Version); + + bool isPositionIndependent() const; + /// Return true if assembly output should contain comments. /// bool isVerbose() const { return VerboseAsm; } @@ -173,9 +183,6 @@ public: void EmitToStreamer(MCStreamer &S, const MCInst &Inst); - /// Return the target triple string. - StringRef getTargetTriple() const; - /// Return the current section we are emitting to. const MCSection *getCurrentSection() const; @@ -184,6 +191,39 @@ public: MCSymbol *getSymbol(const GlobalValue *GV) const; + //===------------------------------------------------------------------===// + // XRay instrumentation implementation. + //===------------------------------------------------------------------===// +public: + // This describes the kind of sled we're storing in the XRay table. + enum class SledKind : uint8_t { + FUNCTION_ENTER = 0, + FUNCTION_EXIT = 1, + TAIL_CALL = 2, + }; + + // The table will contain these structs that point to the sled, the function + // containing the sled, and what kind of sled (and whether they should always + // be instrumented). + struct XRayFunctionEntry { + const MCSymbol *Sled; + const MCSymbol *Function; + SledKind Kind; + bool AlwaysInstrument; + const class Function *Fn; + + void emit(int, MCStreamer *, const MCSymbol *) const; + }; + + // All the sleds to be emitted. + std::vector Sleds; + + // Helper function to record a given XRay sled. + void recordSled(MCSymbol *Sled, const MachineInstr &MI, SledKind Kind); + + /// Emit a table with all XRay instrumentation points. + void emitXRayTable(); + //===------------------------------------------------------------------===// // MachineFunctionPass Implementation. //===------------------------------------------------------------------===// @@ -225,6 +265,10 @@ public: enum CFIMoveType { CFI_M_None, CFI_M_EH, CFI_M_Debug }; CFIMoveType needsCFIMoves(); + /// Returns false if needsCFIMoves() == CFI_M_EH for any function + /// in the module. + bool needsOnlyDebugCFIMoves() const { return isCFIMoveForDebugging; } + bool needsSEHMoves(); /// Print to the current output stream assembly representations of the @@ -238,11 +282,6 @@ public: /// virtual void EmitJumpTableInfo(); - /// Emit the control variable for an emulated TLS variable. - virtual void EmitEmulatedTLSControlVariable(const GlobalVariable *GV, - MCSymbol *EmittedSym, - bool AllZeroInitValue); - /// Emit the specified global variable to the .s file. virtual void EmitGlobalVariable(const GlobalVariable *GV); @@ -258,6 +297,11 @@ public: /// void EmitAlignment(unsigned NumBits, const GlobalObject *GO = nullptr) const; + /// Emit an alignment directive to the specified power of two boundary, + /// like EmitAlignment, but call EmitTrapToAlignment to fill with + /// trap instructions instead of NOPs. + void EmitTrapAlignment(unsigned NumBits, const GlobalObject *GO = nullptr) const; + /// Lower the specified LLVM Constant to an MCExpr. virtual const MCExpr *lowerConstant(const Constant *CV); @@ -316,6 +360,11 @@ public: llvm_unreachable("EmitInstruction not implemented"); } + /// Emit an alignment directive to the specified power + /// of two boundary, but use Trap instructions for alignment + /// sections that should never be executed. + virtual void EmitTrapToAlignment(unsigned NumBits) const; + /// Return the symbol for the specified constant pool entry. virtual MCSymbol *GetCPISymbol(unsigned CPID) const; @@ -441,9 +490,11 @@ public: /// Get the value for DW_AT_APPLE_isa. Zero if no isa encoding specified. virtual unsigned getISAEncoding() { return 0; } - /// EmitDwarfRegOp - Emit a dwarf register operation. - virtual void EmitDwarfRegOp(ByteStreamer &BS, - const MachineLocation &MLoc) const; + /// Emit the directive and value for debug thread local expression + /// + /// \p Value - The value to emit. + /// \p Size - The size of the integer (in bytes) to emit. + virtual void EmitDebugValue(const MCExpr *Value, unsigned Size) const; //===------------------------------------------------------------------===// // Dwarf Lowering Routines @@ -551,6 +602,9 @@ private: void EmitXXStructorList(const DataLayout &DL, const Constant *List, bool isCtor); GCMetadataPrinter *GetOrCreateGCPrinter(GCStrategy &C); + /// Emit GlobalAlias or GlobalIFunc. + void emitGlobalIndirectSymbol(Module &M, + const GlobalIndirectSymbol& GIS); }; } diff --git a/gnu/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/gnu/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp index 5f67d3daa97..88f2b1126d9 100644 --- a/gnu/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp +++ b/gnu/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp @@ -12,11 +12,10 @@ //===----------------------------------------------------------------------===// #include "llvm/CodeGen/AsmPrinter.h" +#include "CodeViewDebug.h" #include "DwarfDebug.h" #include "DwarfException.h" #include "WinException.h" -#include "WinCodeViewLineTables.h" -#include "llvm/ADT/SmallString.h" #include "llvm/ADT/Statistic.h" #include "llvm/Analysis/ConstantFolding.h" #include "llvm/CodeGen/Analysis.h" @@ -38,6 +37,8 @@ #include "llvm/MC/MCExpr.h" #include "llvm/MC/MCInst.h" #include "llvm/MC/MCSection.h" +#include "llvm/MC/MCSectionELF.h" +#include "llvm/MC/MCSectionMachO.h" #include "llvm/MC/MCStreamer.h" #include "llvm/MC/MCSymbolELF.h" #include "llvm/MC/MCValue.h" @@ -56,10 +57,15 @@ using namespace llvm; #define DEBUG_TYPE "asm-printer" -static const char *const DWARFGroupName = "DWARF Emission"; -static const char *const DbgTimerName = "Debug Info Emission"; -static const char *const EHTimerName = "DWARF Exception Writer"; -static const char *const CodeViewLineTablesGroupName = "CodeView Line Tables"; +static const char *const DWARFGroupName = "dwarf"; +static const char *const DWARFGroupDescription = "DWARF Emission"; +static const char *const DbgTimerName = "emit"; +static const char *const DbgTimerDescription = "Debug Info Emission"; +static const char *const EHTimerName = "write_exception"; +static const char *const EHTimerDescription = "DWARF Exception Writer"; +static const char *const CodeViewLineTablesGroupName = "linetables"; +static const char *const CodeViewLineTablesGroupDescription = + "CodeView Line Tables"; STATISTIC(EmittedInsts, "Number of machine instrs printed"); @@ -102,7 +108,7 @@ static unsigned getGVAlignmentLog2(const GlobalValue *GV, const DataLayout &DL, AsmPrinter::AsmPrinter(TargetMachine &tm, std::unique_ptr Streamer) : MachineFunctionPass(ID), TM(tm), MAI(tm.getMCAsmInfo()), OutContext(Streamer->getContext()), OutStreamer(std::move(Streamer)), - LastMI(nullptr), LastFn(0), Counter(~0U) { + isCFIMoveForDebugging(false), LastMI(nullptr), LastFn(0), Counter(~0U) { DD = nullptr; MMI = nullptr; LI = nullptr; @@ -125,6 +131,10 @@ AsmPrinter::~AsmPrinter() { } } +bool AsmPrinter::isPositionIndependent() const { + return TM.isPositionIndependent(); +} + /// getFunctionNumber - Return a unique ID for the current function. /// unsigned AsmPrinter::getFunctionNumber() const { @@ -140,7 +150,7 @@ const DataLayout &AsmPrinter::getDataLayout() const { } // Do not use the cached DataLayout because some client use it without a Module -// (llmv-dsymutil, llvm-dwarfdump). +// (llvm-dsymutil, llvm-dwarfdump). unsigned AsmPrinter::getPointerSize() const { return TM.getPointerSize(); } const MCSubtargetInfo &AsmPrinter::getSubtargetInfo() const { @@ -152,17 +162,11 @@ void AsmPrinter::EmitToStreamer(MCStreamer &S, const MCInst &Inst) { S.EmitInstruction(Inst, getSubtargetInfo()); } -StringRef AsmPrinter::getTargetTriple() const { - return TM.getTargetTriple().str(); -} - /// getCurrentSection() - Return the current section we are emitting to. const MCSection *AsmPrinter::getCurrentSection() const { - return OutStreamer->getCurrentSection().first; + return OutStreamer->getCurrentSectionOnly(); } - - void AsmPrinter::getAnalysisUsage(AnalysisUsage &AU) const { AU.setPreservesAll(); MachineFunctionPass::getAnalysisUsage(AU); @@ -181,8 +185,6 @@ bool AsmPrinter::doInitialization(Module &M) { OutStreamer->InitSections(false); - Mang = new Mangler(); - // Emit the version-min deplyment target directive if needed. // // FIXME: If we end up with a collection of these sorts of Darwin-specific @@ -191,7 +193,7 @@ bool AsmPrinter::doInitialization(Module &M) { // alternative is duplicated code in each of the target asm printers that // use the directive, where it would need the same conditionalization // anyway. - Triple TT(getTargetTriple()); + const Triple &TT = TM.getTargetTriple(); // If there is a version specified, Major will be non-zero. if (TT.isOSDarwin() && TT.getOSMajorVersion() != 0) { unsigned Major, Minor, Update; @@ -247,17 +249,43 @@ bool AsmPrinter::doInitialization(Module &M) { if (MAI->doesSupportDebugInformation()) { bool EmitCodeView = MMI->getModule()->getCodeViewFlag(); - if (EmitCodeView && TM.getTargetTriple().isKnownWindowsMSVCEnvironment()) { - Handlers.push_back(HandlerInfo(new WinCodeViewLineTables(this), - DbgTimerName, - CodeViewLineTablesGroupName)); + if (EmitCodeView && (TM.getTargetTriple().isKnownWindowsMSVCEnvironment() || + TM.getTargetTriple().isWindowsItaniumEnvironment())) { + Handlers.push_back(HandlerInfo(new CodeViewDebug(this), + DbgTimerName, DbgTimerDescription, + CodeViewLineTablesGroupName, + CodeViewLineTablesGroupDescription)); } if (!EmitCodeView || MMI->getModule()->getDwarfVersion()) { DD = new DwarfDebug(this, &M); - Handlers.push_back(HandlerInfo(DD, DbgTimerName, DWARFGroupName)); + DD->beginModule(); + Handlers.push_back(HandlerInfo(DD, DbgTimerName, DbgTimerDescription, + DWARFGroupName, DWARFGroupDescription)); } } + switch (MAI->getExceptionHandlingType()) { + case ExceptionHandling::SjLj: + case ExceptionHandling::DwarfCFI: + case ExceptionHandling::ARM: + isCFIMoveForDebugging = true; + if (MAI->getExceptionHandlingType() != ExceptionHandling::DwarfCFI) + break; + for (auto &F: M.getFunctionList()) { + // If the module contains any function with unwind data, + // .eh_frame has to be emitted. + // Ignore functions that won't get emitted. + if (!F.isDeclarationForLinker() && F.needsUnwindTableEntry()) { + isCFIMoveForDebugging = false; + break; + } + } + break; + default: + isCFIMoveForDebugging = false; + break; + } + EHStreamer *ES = nullptr; switch (MAI->getExceptionHandlingType()) { case ExceptionHandling::None: @@ -282,7 +310,8 @@ bool AsmPrinter::doInitialization(Module &M) { break; } if (ES) - Handlers.push_back(HandlerInfo(ES, EHTimerName, DWARFGroupName)); + Handlers.push_back(HandlerInfo(ES, EHTimerName, EHTimerDescription, + DWARFGroupName, DWARFGroupDescription)); return false; } @@ -319,79 +348,41 @@ void AsmPrinter::EmitLinkage(const GlobalValue *GV, MCSymbol *GVSym) const { OutStreamer->EmitSymbolAttribute(GVSym, MCSA_Weak); } return; - case GlobalValue::AppendingLinkage: - // FIXME: appending linkage variables should go into a section of - // their name or something. For now, just emit them as external. case GlobalValue::ExternalLinkage: - // If external or appending, declare as a global symbol. - // .globl _foo + // If external, declare as a global symbol: .globl _foo OutStreamer->EmitSymbolAttribute(GVSym, MCSA_Global); return; case GlobalValue::PrivateLinkage: case GlobalValue::InternalLinkage: return; + case GlobalValue::AppendingLinkage: case GlobalValue::AvailableExternallyLinkage: - llvm_unreachable("Should never emit this"); case GlobalValue::ExternalWeakLinkage: - llvm_unreachable("Don't know how to emit these"); + llvm_unreachable("Should never emit this"); } llvm_unreachable("Unknown linkage type!"); } void AsmPrinter::getNameWithPrefix(SmallVectorImpl &Name, const GlobalValue *GV) const { - TM.getNameWithPrefix(Name, GV, *Mang); + TM.getNameWithPrefix(Name, GV, getObjFileLowering().getMangler()); } MCSymbol *AsmPrinter::getSymbol(const GlobalValue *GV) const { - return TM.getSymbol(GV, *Mang); -} - -static MCSymbol *getOrCreateEmuTLSControlSym(MCSymbol *GVSym, MCContext &C) { - return C.getOrCreateSymbol(Twine("__emutls_v.") + GVSym->getName()); -} - -static MCSymbol *getOrCreateEmuTLSInitSym(MCSymbol *GVSym, MCContext &C) { - return C.getOrCreateSymbol(Twine("__emutls_t.") + GVSym->getName()); -} - -/// EmitEmulatedTLSControlVariable - Emit the control variable for an emulated TLS variable. -void AsmPrinter::EmitEmulatedTLSControlVariable(const GlobalVariable *GV, - MCSymbol *EmittedSym, - bool AllZeroInitValue) { - MCSection *TLSVarSection = getObjFileLowering().getDataSection(); - OutStreamer->SwitchSection(TLSVarSection); - MCSymbol *GVSym = getSymbol(GV); - EmitLinkage(GV, EmittedSym); // same linkage as GV - const DataLayout &DL = GV->getParent()->getDataLayout(); - uint64_t Size = DL.getTypeAllocSize(GV->getType()->getElementType()); - unsigned AlignLog = getGVAlignmentLog2(GV, DL); - unsigned WordSize = DL.getPointerSize(); - unsigned Alignment = DL.getPointerABIAlignment(); - EmitAlignment(Log2_32(Alignment)); - OutStreamer->EmitLabel(EmittedSym); - OutStreamer->EmitIntValue(Size, WordSize); - OutStreamer->EmitIntValue((1 << AlignLog), WordSize); - OutStreamer->EmitIntValue(0, WordSize); - if (GV->hasInitializer() && !AllZeroInitValue) { - OutStreamer->EmitSymbolValue( - getOrCreateEmuTLSInitSym(GVSym, OutContext), WordSize); - } else - OutStreamer->EmitIntValue(0, WordSize); - if (MAI->hasDotTypeDotSizeDirective()) - OutStreamer->emitELFSize(cast(EmittedSym), - MCConstantExpr::create(4 * WordSize, OutContext)); - OutStreamer->AddBlankLine(); // End of the __emutls_v.* variable. + return TM.getSymbol(GV); } /// EmitGlobalVariable - Emit the specified global variable to the .s file. void AsmPrinter::EmitGlobalVariable(const GlobalVariable *GV) { - bool IsEmuTLSVar = - GV->getThreadLocalMode() != llvm::GlobalVariable::NotThreadLocal && - TM.Options.EmulatedTLS; + bool IsEmuTLSVar = TM.Options.EmulatedTLS && GV->isThreadLocal(); assert(!(IsEmuTLSVar && GV->hasCommonLinkage()) && "No emulated TLS variables in the common section"); + // Never emit TLS variable xyz in emulated TLS model. + // The initialization value is in __emutls_t.xyz instead of xyz. + if (IsEmuTLSVar) + return; + if (GV->hasInitializer()) { // Check to see if this is a special global used by LLVM, if so, emit it. if (EmitSpecialLLVMGlobal(GV)) @@ -402,7 +393,7 @@ void AsmPrinter::EmitGlobalVariable(const GlobalVariable *GV) { if (GlobalGOTEquivs.count(getSymbol(GV))) return; - if (isVerbose() && !IsEmuTLSVar) { + if (isVerbose()) { // When printing the control variable __emutls_v.*, // we don't need to print the original TLS variable name. GV->printAsOperand(OutStreamer->GetCommentOS(), @@ -412,11 +403,11 @@ void AsmPrinter::EmitGlobalVariable(const GlobalVariable *GV) { } MCSymbol *GVSym = getSymbol(GV); - MCSymbol *EmittedSym = IsEmuTLSVar ? - getOrCreateEmuTLSControlSym(GVSym, OutContext) : GVSym; - // getOrCreateEmuTLSControlSym only creates the symbol with name and default attributes. - // GV's or GVSym's attributes will be used for the EmittedSym. + MCSymbol *EmittedSym = GVSym; + // getOrCreateEmuTLSControlSym only creates the symbol with name and default + // attributes. + // GV's or GVSym's attributes will be used for the EmittedSym. EmitVisibility(EmittedSym, GV->getVisibility(), !GV->isDeclaration()); if (!GV->hasInitializer()) // External globals require no extra code. @@ -440,48 +431,48 @@ void AsmPrinter::EmitGlobalVariable(const GlobalVariable *GV) { // sections and expected to be contiguous (e.g. ObjC metadata). unsigned AlignLog = getGVAlignmentLog2(GV, DL); - bool AllZeroInitValue = false; - const Constant *InitValue = GV->getInitializer(); - if (isa(InitValue)) - AllZeroInitValue = true; - else { - const ConstantInt *InitIntValue = dyn_cast(InitValue); - if (InitIntValue && InitIntValue->isZero()) - AllZeroInitValue = true; - } - if (IsEmuTLSVar) - EmitEmulatedTLSControlVariable(GV, EmittedSym, AllZeroInitValue); - for (const HandlerInfo &HI : Handlers) { - NamedRegionTimer T(HI.TimerName, HI.TimerGroupName, TimePassesIsEnabled); + NamedRegionTimer T(HI.TimerName, HI.TimerDescription, + HI.TimerGroupName, HI.TimerGroupDescription, + TimePassesIsEnabled); HI.Handler->setSymbolSize(GVSym, Size); } - // Handle common and BSS local symbols (.lcomm). - if (GVKind.isCommon() || GVKind.isBSSLocal()) { - assert(!(IsEmuTLSVar && GVKind.isCommon()) && - "No emulated TLS variables in the common section"); + // Handle common symbols + if (GVKind.isCommon()) { if (Size == 0) Size = 1; // .comm Foo, 0 is undefined, avoid it. unsigned Align = 1 << AlignLog; + if (!getObjFileLowering().getCommDirectiveSupportsAlignment()) + Align = 0; + + // .comm _foo, 42, 4 + OutStreamer->EmitCommonSymbol(GVSym, Size, Align); + return; + } - // Handle common symbols. - if (GVKind.isCommon()) { - if (!getObjFileLowering().getCommDirectiveSupportsAlignment()) - Align = 0; + // Determine to which section this global should be emitted. + MCSection *TheSection = getObjFileLowering().SectionForGlobal(GV, GVKind, TM); - // .comm _foo, 42, 4 - OutStreamer->EmitCommonSymbol(GVSym, Size, Align); - return; - } + // If we have a bss global going to a section that supports the + // zerofill directive, do so here. + if (GVKind.isBSS() && MAI->hasMachoZeroFillDirective() && + TheSection->isVirtualSection()) { + if (Size == 0) + Size = 1; // zerofill of 0 bytes is undefined. + unsigned Align = 1 << AlignLog; + EmitLinkage(GV, GVSym); + // .zerofill __DATA, __bss, _foo, 400, 5 + OutStreamer->EmitZerofill(TheSection, GVSym, Size, Align); + return; + } - // Handle local BSS symbols. - if (MAI->hasMachoZeroFillDirective()) { - MCSection *TheSection = - getObjFileLowering().SectionForGlobal(GV, GVKind, *Mang, TM); - // .zerofill __DATA, __bss, _foo, 400, 5 - OutStreamer->EmitZerofill(TheSection, GVSym, Size, Align); - return; - } + // If this is a BSS local symbol and we are emitting in the BSS + // section use .lcomm/.comm directive. + if (GVKind.isBSSLocal() && + getObjFileLowering().getBSSSection() == TheSection) { + if (Size == 0) + Size = 1; // .comm Foo, 0 is undefined, avoid it. + unsigned Align = 1 << AlignLog; // Use .lcomm only if it supports user-specified alignment. // Otherwise, while it would still be correct to use .lcomm in some @@ -505,30 +496,6 @@ void AsmPrinter::EmitGlobalVariable(const GlobalVariable *GV) { return; } - if (IsEmuTLSVar && AllZeroInitValue) - return; // No need of initialization values. - - MCSymbol *EmittedInitSym = IsEmuTLSVar ? - getOrCreateEmuTLSInitSym(GVSym, OutContext) : GVSym; - // getOrCreateEmuTLSInitSym only creates the symbol with name and default attributes. - // GV's or GVSym's attributes will be used for the EmittedInitSym. - - MCSection *TheSection = IsEmuTLSVar ? - getObjFileLowering().getReadOnlySection() : - getObjFileLowering().SectionForGlobal(GV, GVKind, *Mang, TM); - - // Handle the zerofill directive on darwin, which is a special form of BSS - // emission. - if (GVKind.isBSSExtern() && MAI->hasMachoZeroFillDirective() && !IsEmuTLSVar) { - if (Size == 0) Size = 1; // zerofill of 0 bytes is undefined. - - // .globl _foo - OutStreamer->EmitSymbolAttribute(GVSym, MCSA_Global); - // .zerofill __DATA, __common, _foo, 400, 5 - OutStreamer->EmitZerofill(TheSection, GVSym, Size, 1 << AlignLog); - return; - } - // Handle thread local data for mach-o which requires us to output an // additional structure of data and mangle the original symbol so that we // can reference it later. @@ -539,10 +506,10 @@ void AsmPrinter::EmitGlobalVariable(const GlobalVariable *GV) { // TLOF class. This will also make it more obvious that stuff like // MCStreamer::EmitTBSSSymbol is macho specific and only called from macho // specific code. - if (GVKind.isThreadLocal() && MAI->hasMachoTBSSDirective() && !IsEmuTLSVar) { + if (GVKind.isThreadLocal() && MAI->hasMachoTBSSDirective()) { // Emit the .tbss symbol MCSymbol *MangSym = - OutContext.getOrCreateSymbol(GVSym->getName() + Twine("$tlv$init")); + OutContext.getOrCreateSymbol(GVSym->getName() + Twine("$tlv$init")); if (GVKind.isThreadBSS()) { TheSection = getObjFileLowering().getTLSBSSSection(); @@ -581,11 +548,11 @@ void AsmPrinter::EmitGlobalVariable(const GlobalVariable *GV) { return; } + MCSymbol *EmittedInitSym = GVSym; + OutStreamer->SwitchSection(TheSection); - // emutls_t.* symbols are only used in the current compilation unit. - if (!IsEmuTLSVar) - EmitLinkage(GV, EmittedInitSym); + EmitLinkage(GV, EmittedInitSym); EmitAlignment(AlignLog, GV); OutStreamer->EmitLabel(EmittedInitSym); @@ -594,12 +561,21 @@ void AsmPrinter::EmitGlobalVariable(const GlobalVariable *GV) { if (MAI->hasDotTypeDotSizeDirective()) // .size foo, 42 - OutStreamer->emitELFSize(cast(EmittedInitSym), + OutStreamer->emitELFSize(EmittedInitSym, MCConstantExpr::create(Size, OutContext)); OutStreamer->AddBlankLine(); } +/// Emit the directive and value for debug thread local expression +/// +/// \p Value - The value to emit. +/// \p Size - The size of the integer (in bytes) to emit. +void AsmPrinter::EmitDebugValue(const MCExpr *Value, + unsigned Size) const { + OutStreamer->EmitValue(Value, Size); +} + /// EmitFunctionHeader - This method emits the header for the current /// function. void AsmPrinter::EmitFunctionHeader() { @@ -609,13 +585,12 @@ void AsmPrinter::EmitFunctionHeader() { // Print the 'header' of function. const Function *F = MF->getFunction(); - OutStreamer->SwitchSection( - getObjFileLowering().SectionForGlobal(F, *Mang, TM)); + OutStreamer->SwitchSection(getObjFileLowering().SectionForGlobal(F, TM)); EmitVisibility(CurrentFnSym, F->getVisibility()); EmitLinkage(F, CurrentFnSym); if (MAI->hasFunctionAlignment()) - EmitAlignment(MF->getAlignment(), F); + EmitTrapAlignment(MF->getAlignment(), F); if (MAI->hasDotTypeDotSizeDirective()) OutStreamer->EmitSymbolAttribute(CurrentFnSym, MCSA_ELF_TypeFunction); @@ -657,7 +632,8 @@ void AsmPrinter::EmitFunctionHeader() { // Emit pre-function debug and/or EH information. for (const HandlerInfo &HI : Handlers) { - NamedRegionTimer T(HI.TimerName, HI.TimerGroupName, TimePassesIsEnabled); + NamedRegionTimer T(HI.TimerName, HI.TimerDescription, HI.TimerGroupName, + HI.TimerGroupDescription, TimePassesIsEnabled); HI.Handler->beginFunction(MF); } @@ -691,26 +667,26 @@ static void emitComments(const MachineInstr &MI, raw_ostream &CommentOS) { // Check for spills and reloads int FI; - const MachineFrameInfo *FrameInfo = MF->getFrameInfo(); + const MachineFrameInfo &MFI = MF->getFrameInfo(); // We assume a single instruction only has a spill or reload, not // both. const MachineMemOperand *MMO; - if (TII->isLoadFromStackSlotPostFE(&MI, FI)) { - if (FrameInfo->isSpillSlotObjectIndex(FI)) { + if (TII->isLoadFromStackSlotPostFE(MI, FI)) { + if (MFI.isSpillSlotObjectIndex(FI)) { MMO = *MI.memoperands_begin(); CommentOS << MMO->getSize() << "-byte Reload\n"; } - } else if (TII->hasLoadFromStackSlot(&MI, MMO, FI)) { - if (FrameInfo->isSpillSlotObjectIndex(FI)) + } else if (TII->hasLoadFromStackSlot(MI, MMO, FI)) { + if (MFI.isSpillSlotObjectIndex(FI)) CommentOS << MMO->getSize() << "-byte Folded Reload\n"; - } else if (TII->isStoreToStackSlotPostFE(&MI, FI)) { - if (FrameInfo->isSpillSlotObjectIndex(FI)) { + } else if (TII->isStoreToStackSlotPostFE(MI, FI)) { + if (MFI.isSpillSlotObjectIndex(FI)) { MMO = *MI.memoperands_begin(); CommentOS << MMO->getSize() << "-byte Spill\n"; } - } else if (TII->hasStoreToStackSlot(&MI, MMO, FI)) { - if (FrameInfo->isSpillSlotObjectIndex(FI)) + } else if (TII->hasStoreToStackSlot(MI, MMO, FI)) { + if (MFI.isSpillSlotObjectIndex(FI)) CommentOS << MMO->getSize() << "-byte Folded Spill\n"; } @@ -745,7 +721,7 @@ static void emitKill(const MachineInstr *MI, AsmPrinter &AP) { AP.MF->getSubtarget().getRegisterInfo()) << (Op.isDef() ? "" : ""); } - AP.OutStreamer->AddComment(Str); + AP.OutStreamer->AddComment(OS.str()); AP.OutStreamer->AddBlankLine(); } @@ -770,9 +746,10 @@ static bool emitDebugValueComment(const MachineInstr *MI, AsmPrinter &AP) { OS << V->getName(); const DIExpression *Expr = MI->getDebugExpression(); - if (Expr->isBitPiece()) - OS << " [bit_piece offset=" << Expr->getBitPieceOffset() - << " size=" << Expr->getBitPieceSize() << "]"; + auto Fragment = Expr->getFragmentInfo(); + if (Fragment) + OS << " [fragment offset=" << Fragment->OffsetInBits + << " size=" << Fragment->SizeInBits << "]"; OS << " <- "; // The second operand is only an offset if it's an immediate. @@ -780,21 +757,21 @@ static bool emitDebugValueComment(const MachineInstr *MI, AsmPrinter &AP) { int64_t Offset = Deref ? MI->getOperand(1).getImm() : 0; for (unsigned i = 0; i < Expr->getNumElements(); ++i) { - if (Deref) { + uint64_t Op = Expr->getElement(i); + if (Op == dwarf::DW_OP_LLVM_fragment) { + // There can't be any operands after this in a valid expression + break; + } else if (Deref) { // We currently don't support extra Offsets or derefs after the first // one. Bail out early instead of emitting an incorrect comment OS << " [complex expression]"; AP.OutStreamer->emitRawComment(OS.str()); return true; - } - uint64_t Op = Expr->getElement(i); - if (Op == dwarf::DW_OP_deref) { + } else if (Op == dwarf::DW_OP_deref) { Deref = true; continue; - } else if (Op == dwarf::DW_OP_bit_piece) { - // There can't be any operands after this in a valid expression - break; } + uint64_t ExtraOffset = Expr->getElement(i++); if (Op == dwarf::DW_OP_plus) Offset += ExtraOffset; @@ -815,7 +792,7 @@ static bool emitDebugValueComment(const MachineInstr *MI, AsmPrinter &AP) { // There is no good way to print long double. Convert a copy to // double. Ah well, it's only a comment. bool ignored; - APF.convert(APFloat::IEEEdouble, APFloat::rmNearestTiesToEven, + APF.convert(APFloat::IEEEdouble(), APFloat::rmNearestTiesToEven, &ignored); OS << "(long double) " << APF.convertToDouble(); } @@ -878,8 +855,7 @@ void AsmPrinter::emitCFIInstruction(const MachineInstr &MI) { if (needsCFIMoves() == CFI_M_None) return; - const MachineModuleInfo &MMI = MF->getMMI(); - const std::vector &Instrs = MMI.getFrameInstructions(); + const std::vector &Instrs = MF->getFrameInstructions(); unsigned CFIIndex = MI.getOperand(0).getCFIIndex(); const MCCFIInstruction &CFI = Instrs[CFIIndex]; emitCFIInstruction(CFI); @@ -921,7 +897,8 @@ void AsmPrinter::EmitFunctionBody() { if (ShouldPrintDebugScopes) { for (const HandlerInfo &HI : Handlers) { - NamedRegionTimer T(HI.TimerName, HI.TimerGroupName, + NamedRegionTimer T(HI.TimerName, HI.TimerDescription, + HI.TimerGroupName, HI.TimerGroupDescription, TimePassesIsEnabled); HI.Handler->beginInstruction(&MI); } @@ -965,7 +942,8 @@ void AsmPrinter::EmitFunctionBody() { if (ShouldPrintDebugScopes) { for (const HandlerInfo &HI : Handlers) { - NamedRegionTimer T(HI.TimerName, HI.TimerGroupName, + NamedRegionTimer T(HI.TimerName, HI.TimerDescription, + HI.TimerGroupName, HI.TimerGroupDescription, TimePassesIsEnabled); HI.Handler->endInstruction(); } @@ -1003,8 +981,8 @@ void AsmPrinter::EmitFunctionBody() { // Emit target-specific gunk after the function body. EmitFunctionBodyEnd(); - if (!MMI->getLandingPads().empty() || MMI->hasDebugInfo() || - MMI->hasEHFunclets() || MAI->hasDotTypeDotSizeDirective()) { + if (!MF->getLandingPads().empty() || MMI->hasDebugInfo() || + MF->hasEHFunclets() || MAI->hasDotTypeDotSizeDirective()) { // Create a symbol for the end of function. CurrentFnEnd = createTempSymbol("func_end"); OutStreamer->EmitLabel(CurrentFnEnd); @@ -1018,12 +996,12 @@ void AsmPrinter::EmitFunctionBody() { const MCExpr *SizeExp = MCBinaryExpr::createSub( MCSymbolRefExpr::create(CurrentFnEnd, OutContext), MCSymbolRefExpr::create(CurrentFnSymForSize, OutContext), OutContext); - if (auto Sym = dyn_cast(CurrentFnSym)) - OutStreamer->emitELFSize(Sym, SizeExp); + OutStreamer->emitELFSize(CurrentFnSym, SizeExp); } for (const HandlerInfo &HI : Handlers) { - NamedRegionTimer T(HI.TimerName, HI.TimerGroupName, TimePassesIsEnabled); + NamedRegionTimer T(HI.TimerName, HI.TimerDescription, HI.TimerGroupName, + HI.TimerGroupDescription, TimePassesIsEnabled); HI.Handler->markFunctionEnd(); } @@ -1032,10 +1010,10 @@ void AsmPrinter::EmitFunctionBody() { // Emit post-function debug and/or EH information. for (const HandlerInfo &HI : Handlers) { - NamedRegionTimer T(HI.TimerName, HI.TimerGroupName, TimePassesIsEnabled); + NamedRegionTimer T(HI.TimerName, HI.TimerDescription, HI.TimerGroupName, + HI.TimerGroupDescription, TimePassesIsEnabled); HI.Handler->endFunction(MF); } - MMI->EndFunction(); OutStreamer->AddBlankLine(); } @@ -1065,8 +1043,9 @@ static bool isGOTEquivalentCandidate(const GlobalVariable *GV, // Global GOT equivalents are unnamed private globals with a constant // pointer initializer to another global symbol. They must point to a // GlobalVariable or Function, i.e., as GlobalValue. - if (!GV->hasUnnamedAddr() || !GV->hasInitializer() || !GV->isConstant() || - !GV->isDiscardableIfUnused() || !dyn_cast(GV->getOperand(0))) + if (!GV->hasGlobalUnnamedAddr() || !GV->hasInitializer() || + !GV->isConstant() || !GV->isDiscardableIfUnused() || + !dyn_cast(GV->getOperand(0))) return false; // To be a got equivalent, at least one of its users need to be a constant @@ -1118,6 +1097,51 @@ void AsmPrinter::emitGlobalGOTEquivs() { EmitGlobalVariable(GV); } +void AsmPrinter::emitGlobalIndirectSymbol(Module &M, + const GlobalIndirectSymbol& GIS) { + MCSymbol *Name = getSymbol(&GIS); + + if (GIS.hasExternalLinkage() || !MAI->getWeakRefDirective()) + OutStreamer->EmitSymbolAttribute(Name, MCSA_Global); + else if (GIS.hasWeakLinkage() || GIS.hasLinkOnceLinkage()) + OutStreamer->EmitSymbolAttribute(Name, MCSA_WeakReference); + else + assert(GIS.hasLocalLinkage() && "Invalid alias or ifunc linkage"); + + // Set the symbol type to function if the alias has a function type. + // This affects codegen when the aliasee is not a function. + if (GIS.getType()->getPointerElementType()->isFunctionTy()) { + OutStreamer->EmitSymbolAttribute(Name, MCSA_ELF_TypeFunction); + if (isa(GIS)) + OutStreamer->EmitSymbolAttribute(Name, MCSA_ELF_TypeIndFunction); + } + + EmitVisibility(Name, GIS.getVisibility()); + + const MCExpr *Expr = lowerConstant(GIS.getIndirectSymbol()); + + if (isa(&GIS) && MAI->hasAltEntry() && isa(Expr)) + OutStreamer->EmitSymbolAttribute(Name, MCSA_AltEntry); + + // Emit the directives as assignments aka .set: + OutStreamer->EmitAssignment(Name, Expr); + + if (auto *GA = dyn_cast(&GIS)) { + // If the aliasee does not correspond to a symbol in the output, i.e. the + // alias is not of an object or the aliased object is private, then set the + // size of the alias symbol from the type of the alias. We don't do this in + // other situations as the alias and aliasee having differing types but same + // size may be intentional. + const GlobalObject *BaseObject = GA->getBaseObject(); + if (MAI->hasDotTypeDotSizeDirective() && GA->getValueType()->isSized() && + (!BaseObject || BaseObject->hasPrivateLinkage())) { + const DataLayout &DL = M.getDataLayout(); + uint64_t Size = DL.getTypeAllocSize(GA->getValueType()); + OutStreamer->emitELFSize(Name, MCConstantExpr::create(Size, OutContext)); + } + } +} + bool AsmPrinter::doFinalization(Module &M) { // Set the MachineFunction to nullptr so that we can catch attempted // accesses to MF specific features at the module level and so that @@ -1155,7 +1179,7 @@ bool AsmPrinter::doFinalization(Module &M) { SmallVector ModuleFlags; M.getModuleFlagsMetadata(ModuleFlags); if (!ModuleFlags.empty()) - TLOF.emitModuleFlags(*OutStreamer, ModuleFlags, *Mang, TM); + TLOF.emitModuleFlags(*OutStreamer, ModuleFlags, TM); if (TM.getTargetTriple().isOSBinFormatELF()) { MachineModuleInfoELF &MMIELF = MMI->getObjFileInfo(); @@ -1176,8 +1200,8 @@ bool AsmPrinter::doFinalization(Module &M) { // Finalize debug and EH information. for (const HandlerInfo &HI : Handlers) { - NamedRegionTimer T(HI.TimerName, HI.TimerGroupName, - TimePassesIsEnabled); + NamedRegionTimer T(HI.TimerName, HI.TimerDescription, HI.TimerGroupName, + HI.TimerGroupDescription, TimePassesIsEnabled); HI.Handler->endModule(); delete HI.Handler; } @@ -1191,55 +1215,35 @@ bool AsmPrinter::doFinalization(Module &M) { // to notice uses in operands (due to constant exprs etc). This should // happen with the MC stuff eventually. - // Print out module-level global variables here. - for (const auto &G : M.globals()) { - if (!G.hasExternalWeakLinkage()) + // Print out module-level global objects here. + for (const auto &GO : M.global_objects()) { + if (!GO.hasExternalWeakLinkage()) continue; - OutStreamer->EmitSymbolAttribute(getSymbol(&G), MCSA_WeakReference); - } - - for (const auto &F : M) { - if (!F.hasExternalWeakLinkage()) - continue; - OutStreamer->EmitSymbolAttribute(getSymbol(&F), MCSA_WeakReference); + OutStreamer->EmitSymbolAttribute(getSymbol(&GO), MCSA_WeakReference); } } OutStreamer->AddBlankLine(); - for (const auto &Alias : M.aliases()) { - MCSymbol *Name = getSymbol(&Alias); - - if (Alias.hasExternalLinkage() || !MAI->getWeakRefDirective()) - OutStreamer->EmitSymbolAttribute(Name, MCSA_Global); - else if (Alias.hasWeakLinkage() || Alias.hasLinkOnceLinkage()) - OutStreamer->EmitSymbolAttribute(Name, MCSA_WeakReference); - else - assert(Alias.hasLocalLinkage() && "Invalid alias linkage"); - - // Set the symbol type to function if the alias has a function type. - // This affects codegen when the aliasee is not a function. - if (Alias.getType()->getPointerElementType()->isFunctionTy()) - OutStreamer->EmitSymbolAttribute(Name, MCSA_ELF_TypeFunction); - - EmitVisibility(Name, Alias.getVisibility()); - - // Emit the directives as assignments aka .set: - OutStreamer->EmitAssignment(Name, lowerConstant(Alias.getAliasee())); - // If the aliasee does not correspond to a symbol in the output, i.e. the - // alias is not of an object or the aliased object is private, then set the - // size of the alias symbol from the type of the alias. We don't do this in - // other situations as the alias and aliasee having differing types but same - // size may be intentional. - const GlobalObject *BaseObject = Alias.getBaseObject(); - if (MAI->hasDotTypeDotSizeDirective() && Alias.getValueType()->isSized() && - (!BaseObject || BaseObject->hasPrivateLinkage())) { - const DataLayout &DL = M.getDataLayout(); - uint64_t Size = DL.getTypeAllocSize(Alias.getValueType()); - OutStreamer->emitELFSize(cast(Name), - MCConstantExpr::create(Size, OutContext)); + // Print aliases in topological order, that is, for each alias a = b, + // b must be printed before a. + // This is because on some targets (e.g. PowerPC) linker expects aliases in + // such an order to generate correct TOC information. + SmallVector AliasStack; + SmallPtrSet AliasVisited; + for (const auto &Alias : M.aliases()) { + for (const GlobalAlias *Cur = &Alias; Cur; + Cur = dyn_cast(Cur->getAliasee())) { + if (!AliasVisited.insert(Cur).second) + break; + AliasStack.push_back(Cur); } + for (const GlobalAlias *AncestorAlias : reverse(AliasStack)) + emitGlobalIndirectSymbol(M, *AncestorAlias); + AliasStack.clear(); } + for (const auto &IFunc : M.ifuncs()) + emitGlobalIndirectSymbol(M, IFunc); GCModuleInfo *MI = getAnalysisIfAvailable(); assert(MI && "AsmPrinter didn't require GCModuleInfo?"); @@ -1252,9 +1256,10 @@ bool AsmPrinter::doFinalization(Module &M) { // Emit __morestack address if needed for indirect calls. if (MMI->usesMorestackAddr()) { + unsigned Align = 1; MCSection *ReadOnlySection = getObjFileLowering().getSectionForConstant( getDataLayout(), SectionKind::getReadOnly(), - /*C=*/nullptr); + /*C=*/nullptr, Align); OutStreamer->SwitchSection(ReadOnlySection); MCSymbol *AddrSymbol = @@ -1277,7 +1282,6 @@ bool AsmPrinter::doFinalization(Module &M) { // after everything else has gone out. EmitEndOfAsmFile(M); - delete Mang; Mang = nullptr; MMI = nullptr; OutStreamer->Finish(); @@ -1300,8 +1304,8 @@ void AsmPrinter::SetupMachineFunction(MachineFunction &MF) { CurrentFnBegin = nullptr; CurExceptionSym = nullptr; bool NeedsLocalForSize = MAI->needsLocalForSize(); - if (!MMI->getLandingPads().empty() || MMI->hasDebugInfo() || - MMI->hasEHFunclets() || NeedsLocalForSize) { + if (!MF.getLandingPads().empty() || MMI->hasDebugInfo() || + MF.hasEHFunclets() || NeedsLocalForSize) { CurrentFnBegin = createTempSymbol("func_begin"); if (NeedsLocalForSize) CurrentFnSymForSize = CurrentFnBegin; @@ -1344,8 +1348,8 @@ void AsmPrinter::EmitConstantPool() { if (!CPE.isMachineConstantPoolEntry()) C = CPE.Val.ConstVal; - MCSection *S = - getObjFileLowering().getSectionForConstant(getDataLayout(), Kind, C); + MCSection *S = getObjFileLowering().getSectionForConstant(getDataLayout(), + Kind, C, Align); // The number of sections are small, just do a linear search from the // last section to the first. @@ -1423,7 +1427,7 @@ void AsmPrinter::EmitJumpTableInfo() { *F); if (JTInDiffSection) { // Drop it in the readonly section. - MCSection *ReadOnlySection = TLOF.getSectionForJumpTable(*F, *Mang, TM); + MCSection *ReadOnlySection = TLOF.getSectionForJumpTable(*F, TM); OutStreamer->SwitchSection(ReadOnlySection); } @@ -1443,7 +1447,7 @@ void AsmPrinter::EmitJumpTableInfo() { // For the EK_LabelDifference32 entry, if using .set avoids a relocation, /// emit a .set directive for each unique entry. if (MJTI->getEntryKind() == MachineJumpTableInfo::EK_LabelDifference32 && - MAI->doesSetDirectiveSuppressesReloc()) { + MAI->doesSetDirectiveSuppressReloc()) { SmallPtrSet EmittedSets; const TargetLowering *TLI = MF->getSubtarget().getTargetLowering(); const MCExpr *Base = TLI->getPICJumpTableRelocBaseExpr(MF,JTI,OutContext); @@ -1524,7 +1528,7 @@ void AsmPrinter::EmitJumpTableEntry(const MachineJumpTableInfo *MJTI, // If the .set directive avoids relocations, this is emitted as: // .set L4_5_set_123, LBB123 - LJTI1_2 // .word L4_5_set_123 - if (MAI->doesSetDirectiveSuppressesReloc()) { + if (MAI->doesSetDirectiveSuppressReloc()) { Value = MCSymbolRefExpr::create(GetJTSetSymbol(UID, MBB->getNumber()), OutContext); break; @@ -1555,7 +1559,7 @@ bool AsmPrinter::EmitSpecialLLVMGlobal(const GlobalVariable *GV) { } // Ignore debug and non-emitted data. This handles llvm.compiler.used. - if (StringRef(GV->getSection()) == "llvm.metadata" || + if (GV->getSection() == "llvm.metadata" || GV->hasAvailableExternallyLinkage()) return true; @@ -1567,12 +1571,6 @@ bool AsmPrinter::EmitSpecialLLVMGlobal(const GlobalVariable *GV) { EmitXXStructorList(GV->getParent()->getDataLayout(), GV->getInitializer(), /* isCtor */ true); - if (TM.getRelocationModel() == Reloc::Static && - MAI->hasStaticCtorDtorReferenceInStaticMode()) { - StringRef Sym(".constructors_used"); - OutStreamer->EmitSymbolAttribute(OutContext.getOrCreateSymbol(Sym), - MCSA_Reference); - } return true; } @@ -1580,16 +1578,10 @@ bool AsmPrinter::EmitSpecialLLVMGlobal(const GlobalVariable *GV) { EmitXXStructorList(GV->getParent()->getDataLayout(), GV->getInitializer(), /* isCtor */ false); - if (TM.getRelocationModel() == Reloc::Static && - MAI->hasStaticCtorDtorReferenceInStaticMode()) { - StringRef Sym(".destructors_used"); - OutStreamer->EmitSymbolAttribute(OutContext.getOrCreateSymbol(Sym), - MCSA_Reference); - } return true; } - return false; + report_fatal_error("unknown special variable"); } /// EmitLLVMUsedList - For targets that define a MAI::UsedDirective, mark each @@ -1648,7 +1640,8 @@ void AsmPrinter::EmitXXStructorList(const DataLayout &DL, const Constant *List, S.Priority = Priority->getLimitedValue(65535); S.Func = CS->getOperand(1); if (ETy->getNumElements() == 3 && !CS->getOperand(2)->isNullValue()) - S.ComdatKey = dyn_cast(CS->getOperand(2)->stripPointerCasts()); + S.ComdatKey = + dyn_cast(CS->getOperand(2)->stripPointerCasts()); } // Emit the function pointers in the target-specific order @@ -1729,7 +1722,9 @@ void AsmPrinter::EmitLabelPlusOffset(const MCSymbol *Label, uint64_t Offset, unsigned Size, bool IsSectionRelative) const { if (MAI->needsDwarfSectionOffsetDirective() && IsSectionRelative) { - OutStreamer->EmitCOFFSecRel32(Label); + OutStreamer->EmitCOFFSecRel32(Label, Offset); + if (Size > 4) + OutStreamer->EmitZeros(Size - 4); return; } @@ -1765,6 +1760,33 @@ void AsmPrinter::EmitAlignment(unsigned NumBits, const GlobalObject *GV) const { OutStreamer->EmitValueToAlignment(1u << NumBits); } +//===----------------------------------------------------------------------===// +/// EmitTrapAlignment - Emit an alignment directive to the specified power of +/// two boundary, but call EmitTrapToAlignment to fill with Trap instructions +/// if the Target implements EmitTrapToAlignment. +void AsmPrinter::EmitTrapAlignment(unsigned NumBits, const GlobalObject *GV) const { + if (GV) + NumBits = getGVAlignmentLog2(GV, GV->getParent()->getDataLayout(), NumBits); + + if (NumBits == 0) return; // 1-byte aligned: no need to emit alignment. + + assert(NumBits < + static_cast(std::numeric_limits::digits) && + "undefined behavior"); + EmitTrapToAlignment(NumBits); +} + +//===----------------------------------------------------------------------===// +/// EmitTrapToAlignment - Emit an alignment directive to the specified power +/// of two boundary. This default implementation calls EmitCodeAlignment on +/// the OutStreamer, but can be overridden by Target implementations. +void AsmPrinter::EmitTrapToAlignment(unsigned NumBits) const { + if (NumBits == 0) return; + OutStreamer->EmitCodeAlignment(1u << NumBits); +} + + + //===----------------------------------------------------------------------===// // Constant emission. //===----------------------------------------------------------------------===// @@ -1789,16 +1811,12 @@ const MCExpr *AsmPrinter::lowerConstant(const Constant *CV) { llvm_unreachable("Unknown constant value to lower!"); } - if (const MCExpr *RelocExpr - = getObjFileLowering().getExecutableRelativeSymbol(CE, *Mang, TM)) - return RelocExpr; - switch (CE->getOpcode()) { default: // If the code isn't optimized, there may be outstanding folding // opportunities. Attempt to fold the expression using DataLayout as a // last resort before giving up. - if (Constant *C = ConstantFoldConstantExpression(CE, getDataLayout())) + if (Constant *C = ConstantFoldConstant(CE, getDataLayout())) if (C != CE) return lowerConstant(C); @@ -1830,7 +1848,7 @@ const MCExpr *AsmPrinter::lowerConstant(const Constant *CV) { // expression properly. This is important for differences between // blockaddress labels. Since the two labels are in the same function, it // is reasonable to treat their delta as a 32-bit value. - // FALL THROUGH. + LLVM_FALLTHROUGH; case Instruction::BitCast: return lowerConstant(CE->getOperand(0)); @@ -1868,10 +1886,34 @@ const MCExpr *AsmPrinter::lowerConstant(const Constant *CV) { return MCBinaryExpr::createAnd(OpExpr, MaskExpr, Ctx); } + case Instruction::Sub: { + GlobalValue *LHSGV; + APInt LHSOffset; + if (IsConstantOffsetFromGlobal(CE->getOperand(0), LHSGV, LHSOffset, + getDataLayout())) { + GlobalValue *RHSGV; + APInt RHSOffset; + if (IsConstantOffsetFromGlobal(CE->getOperand(1), RHSGV, RHSOffset, + getDataLayout())) { + const MCExpr *RelocExpr = + getObjFileLowering().lowerRelativeReference(LHSGV, RHSGV, TM); + if (!RelocExpr) + RelocExpr = MCBinaryExpr::createSub( + MCSymbolRefExpr::create(getSymbol(LHSGV), Ctx), + MCSymbolRefExpr::create(getSymbol(RHSGV), Ctx), Ctx); + int64_t Addend = (LHSOffset - RHSOffset).getSExtValue(); + if (Addend != 0) + RelocExpr = MCBinaryExpr::createAdd( + RelocExpr, MCConstantExpr::create(Addend, Ctx), Ctx); + return RelocExpr; + } + } + } + // else fallthrough + // The MC library also has a right-shift operator, but it isn't consistently // signed or unsigned between different targets. case Instruction::Add: - case Instruction::Sub: case Instruction::Mul: case Instruction::SDiv: case Instruction::SRem: @@ -1964,7 +2006,7 @@ static void emitGlobalConstantDataSequential(const DataLayout &DL, uint64_t Bytes = DL.getTypeAllocSize(CDS->getType()); // Don't emit a 1-byte object as a .fill. if (Bytes > 1) - return AP.OutStreamer->EmitFill(Bytes, Value); + return AP.OutStreamer->emitFill(Bytes, Value); } // If this can be emitted with .ascii/.asciz, emit it as such. @@ -2003,7 +2045,7 @@ static void emitGlobalConstantArray(const DataLayout &DL, if (Value != -1) { uint64_t Bytes = DL.getTypeAllocSize(CA->getType()); - AP.OutStreamer->EmitFill(Bytes, Value); + AP.OutStreamer->emitFill(Bytes, Value); } else { for (unsigned i = 0, e = CA->getNumOperands(); i != e; ++i) { @@ -2309,7 +2351,7 @@ static void emitGlobalConstantImpl(const DataLayout &DL, const Constant *CV, // If the constant expression's size is greater than 64-bits, then we have // to emit the value in chunks. Try to constant fold the value and emit it // that way. - Constant *New = ConstantFoldConstantExpression(CE, DL); + Constant *New = ConstantFoldConstant(CE, DL); if (New && New != CE) return emitGlobalConstantImpl(DL, New, AP); } @@ -2395,8 +2437,7 @@ MCSymbol *AsmPrinter::GetJTSetSymbol(unsigned UID, unsigned MBBID) const { MCSymbol *AsmPrinter::getSymbolWithGlobalValueBase(const GlobalValue *GV, StringRef Suffix) const { - return getObjFileLowering().getSymbolWithGlobalValueBase(GV, Suffix, *Mang, - TM); + return getObjFileLowering().getSymbolWithGlobalValueBase(GV, Suffix, TM); } /// Return the MCSymbol for the specified ExternalSymbol. @@ -2582,7 +2623,7 @@ isBlockOnlyReachableByFallthrough(const MachineBasicBlock *MBB) const { // If we are the operands of one of the branches, this is not a fall // through. Note that targets with delay slots will usually bundle // terminators with the delay slot instruction. - for (ConstMIBundleOperands OP(&MI); OP.isValid(); ++OP) { + for (ConstMIBundleOperands OP(MI); OP.isValid(); ++OP) { if (OP->isJTI()) return false; if (OP->isMBB() && OP->getMBB() == MBB) @@ -2609,12 +2650,12 @@ GCMetadataPrinter *AsmPrinter::GetOrCreateGCPrinter(GCStrategy &S) { if (GCPI != GCMap.end()) return GCPI->second.get(); - const char *Name = S.getName().c_str(); + auto Name = S.getName(); for (GCMetadataPrinterRegistry::iterator I = GCMetadataPrinterRegistry::begin(), E = GCMetadataPrinterRegistry::end(); I != E; ++I) - if (strcmp(Name, I->getName()) == 0) { + if (Name == I->getName()) { std::unique_ptr GMP = I->instantiate(); GMP->S = &S; auto IterBool = GCMap.insert(std::make_pair(&S, std::move(GMP))); @@ -2628,3 +2669,76 @@ GCMetadataPrinter *AsmPrinter::GetOrCreateGCPrinter(GCStrategy &S) { AsmPrinterHandler::~AsmPrinterHandler() {} void AsmPrinterHandler::markFunctionEnd() {} + +// In the binary's "xray_instr_map" section, an array of these function entries +// describes each instrumentation point. When XRay patches your code, the index +// into this table will be given to your handler as a patch point identifier. +void AsmPrinter::XRayFunctionEntry::emit(int Bytes, MCStreamer *Out, + const MCSymbol *CurrentFnSym) const { + Out->EmitSymbolValue(Sled, Bytes); + Out->EmitSymbolValue(CurrentFnSym, Bytes); + auto Kind8 = static_cast(Kind); + Out->EmitBytes(StringRef(reinterpret_cast(&Kind8), 1)); + Out->EmitBytes( + StringRef(reinterpret_cast(&AlwaysInstrument), 1)); + Out->EmitZeros(2 * Bytes - 2); // Pad the previous two entries +} + +void AsmPrinter::emitXRayTable() { + if (Sleds.empty()) + return; + + auto PrevSection = OutStreamer->getCurrentSectionOnly(); + auto Fn = MF->getFunction(); + MCSection *Section = nullptr; + if (MF->getSubtarget().getTargetTriple().isOSBinFormatELF()) { + if (Fn->hasComdat()) { + Section = OutContext.getELFSection("xray_instr_map", ELF::SHT_PROGBITS, + ELF::SHF_ALLOC | ELF::SHF_GROUP, 0, + Fn->getComdat()->getName()); + } else { + Section = OutContext.getELFSection("xray_instr_map", ELF::SHT_PROGBITS, + ELF::SHF_ALLOC); + } + } else if (MF->getSubtarget().getTargetTriple().isOSBinFormatMachO()) { + Section = OutContext.getMachOSection("__DATA", "xray_instr_map", 0, + SectionKind::getReadOnlyWithRel()); + } else { + llvm_unreachable("Unsupported target"); + } + + // Before we switch over, we force a reference to a label inside the + // xray_instr_map section. Since this function is always called just + // before the function's end, we assume that this is happening after + // the last return instruction. + + auto WordSizeBytes = TM.getPointerSize(); + MCSymbol *Tmp = OutContext.createTempSymbol("xray_synthetic_", true); + OutStreamer->EmitCodeAlignment(16); + OutStreamer->EmitSymbolValue(Tmp, WordSizeBytes, false); + OutStreamer->SwitchSection(Section); + OutStreamer->EmitLabel(Tmp); + for (const auto &Sled : Sleds) + Sled.emit(WordSizeBytes, OutStreamer.get(), CurrentFnSym); + + OutStreamer->SwitchSection(PrevSection); + Sleds.clear(); +} + +void AsmPrinter::recordSled(MCSymbol *Sled, const MachineInstr &MI, + SledKind Kind) { + auto Fn = MI.getParent()->getParent()->getFunction(); + auto Attr = Fn->getFnAttribute("function-instrument"); + bool AlwaysInstrument = + Attr.isStringAttribute() && Attr.getValueAsString() == "xray-always"; + Sleds.emplace_back( + XRayFunctionEntry{ Sled, CurrentFnSym, Kind, AlwaysInstrument, Fn }); +} + +uint16_t AsmPrinter::getDwarfVersion() const { + return OutStreamer->getContext().getDwarfVersion(); +} + +void AsmPrinter::setDwarfVersion(uint16_t Version) { + OutStreamer->getContext().setDwarfVersion(Version); +} diff --git a/gnu/llvm/lib/Target/X86/X86AsmPrinter.h b/gnu/llvm/lib/Target/X86/X86AsmPrinter.h index 9c8bd98dbad..af5b8098eb2 100644 --- a/gnu/llvm/lib/Target/X86/X86AsmPrinter.h +++ b/gnu/llvm/lib/Target/X86/X86AsmPrinter.h @@ -29,6 +29,7 @@ class LLVM_LIBRARY_VISIBILITY X86AsmPrinter : public AsmPrinter { const X86Subtarget *Subtarget; StackMaps SM; FaultMaps FM; + std::unique_ptr CodeEmitter; // This utility class tracks the length of a stackmap instruction's 'shadow'. // It is used by the X86AsmPrinter to ensure that the stackmap shadow @@ -40,10 +41,11 @@ class LLVM_LIBRARY_VISIBILITY X86AsmPrinter : public AsmPrinter { // few instruction bytes to cover the shadow are NOPs used for padding. class StackMapShadowTracker { public: - StackMapShadowTracker(TargetMachine &TM); - ~StackMapShadowTracker(); - void startFunction(MachineFunction &MF); - void count(MCInst &Inst, const MCSubtargetInfo &STI); + void startFunction(MachineFunction &MF) { + this->MF = &MF; + } + void count(MCInst &Inst, const MCSubtargetInfo &STI, + MCCodeEmitter *CodeEmitter); // Called to signal the start of a shadow of RequiredSize bytes. void reset(unsigned RequiredSize) { @@ -56,17 +58,15 @@ class LLVM_LIBRARY_VISIBILITY X86AsmPrinter : public AsmPrinter { // to emit any necessary padding-NOPs. void emitShadowPadding(MCStreamer &OutStreamer, const MCSubtargetInfo &STI); private: - TargetMachine &TM; const MachineFunction *MF; - std::unique_ptr CodeEmitter; - bool InShadow; + bool InShadow = false; // RequiredShadowSize holds the length of the shadow specified in the most // recently encountered STACKMAP instruction. // CurrentShadowSize counts the number of bytes encoded since the most // recently encountered STACKMAP, stopping when that number is greater than // or equal to RequiredShadowSize. - unsigned RequiredShadowSize, CurrentShadowSize; + unsigned RequiredShadowSize = 0, CurrentShadowSize = 0; }; StackMapShadowTracker SMShadowTracker; @@ -82,17 +82,27 @@ class LLVM_LIBRARY_VISIBILITY X86AsmPrinter : public AsmPrinter { void LowerPATCHPOINT(const MachineInstr &MI, X86MCInstLower &MCIL); void LowerSTATEPOINT(const MachineInstr &MI, X86MCInstLower &MCIL); void LowerFAULTING_LOAD_OP(const MachineInstr &MI, X86MCInstLower &MCIL); + void LowerPATCHABLE_OP(const MachineInstr &MI, X86MCInstLower &MCIL); void LowerTlsAddr(X86MCInstLower &MCInstLowering, const MachineInstr &MI); - public: - explicit X86AsmPrinter(TargetMachine &TM, - std::unique_ptr Streamer) - : AsmPrinter(TM, std::move(Streamer)), SM(*this), FM(*this), - SMShadowTracker(TM) {} + // XRay-specific lowering for X86. + void LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI, + X86MCInstLower &MCIL); + void LowerPATCHABLE_RET(const MachineInstr &MI, X86MCInstLower &MCIL); + void LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI, X86MCInstLower &MCIL); + + // Helper function that emits the XRay sleds we've collected for a particular + // function. + void EmitXRayTable(); - const char *getPassName() const override { - return "X86 Assembly / Object Emitter"; +public: + explicit X86AsmPrinter(TargetMachine &TM, + std::unique_ptr Streamer) + : AsmPrinter(TM, std::move(Streamer)), SM(*this), FM(*this) {} + + StringRef getPassName() const override { + return "X86 Assembly Printer"; } const X86Subtarget &getSubtarget() const { return *Subtarget; } @@ -103,6 +113,8 @@ class LLVM_LIBRARY_VISIBILITY X86AsmPrinter : public AsmPrinter { void EmitInstruction(const MachineInstr *MI) override; + void EmitTrapToAlignment(unsigned NumBits) const override; + void EmitBasicBlockEnd(const MachineBasicBlock &MBB) override { SMShadowTracker.emitShadowPadding(*OutStreamer, getSubtargetInfo()); } diff --git a/gnu/llvm/lib/Target/X86/X86MCInstLower.cpp b/gnu/llvm/lib/Target/X86/X86MCInstLower.cpp index e1ca558f0f2..b1db061482b 100644 --- a/gnu/llvm/lib/Target/X86/X86MCInstLower.cpp +++ b/gnu/llvm/lib/Target/X86/X86MCInstLower.cpp @@ -16,10 +16,12 @@ #include "X86RegisterInfo.h" #include "X86ShuffleDecodeConstantPool.h" #include "InstPrinter/X86ATTInstPrinter.h" +#include "InstPrinter/X86InstComments.h" #include "MCTargetDesc/X86BaseInfo.h" #include "Utils/X86ShuffleDecode.h" #include "llvm/ADT/Optional.h" #include "llvm/ADT/SmallString.h" +#include "llvm/ADT/iterator_range.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineConstantPool.h" #include "llvm/CodeGen/MachineOperand.h" @@ -35,9 +37,16 @@ #include "llvm/MC/MCFixup.h" #include "llvm/MC/MCInst.h" #include "llvm/MC/MCInstBuilder.h" +#include "llvm/MC/MCSection.h" #include "llvm/MC/MCStreamer.h" #include "llvm/MC/MCSymbol.h" +#include "llvm/MC/MCSymbolELF.h" +#include "llvm/MC/MCSectionELF.h" +#include "llvm/MC/MCSectionMachO.h" #include "llvm/Support/TargetRegistry.h" +#include "llvm/Support/ELF.h" +#include "llvm/Target/TargetLoweringObjectFile.h" + using namespace llvm; namespace { @@ -61,9 +70,6 @@ public: private: MachineModuleInfoMachO &getMachOMMI() const; - Mangler *getMang() const { - return AsmPrinter.Mang; - } }; } // end anonymous namespace @@ -72,47 +78,33 @@ private: static void EmitNops(MCStreamer &OS, unsigned NumBytes, bool Is64Bit, const MCSubtargetInfo &STI); -namespace llvm { - X86AsmPrinter::StackMapShadowTracker::StackMapShadowTracker(TargetMachine &TM) - : TM(TM), InShadow(false), RequiredShadowSize(0), CurrentShadowSize(0) {} - - X86AsmPrinter::StackMapShadowTracker::~StackMapShadowTracker() {} - - void - X86AsmPrinter::StackMapShadowTracker::startFunction(MachineFunction &F) { - MF = &F; - CodeEmitter.reset(TM.getTarget().createMCCodeEmitter( - *MF->getSubtarget().getInstrInfo(), - *MF->getSubtarget().getRegisterInfo(), MF->getContext())); - } - - void X86AsmPrinter::StackMapShadowTracker::count(MCInst &Inst, - const MCSubtargetInfo &STI) { - if (InShadow) { - SmallString<256> Code; - SmallVector Fixups; - raw_svector_ostream VecOS(Code); - CodeEmitter->encodeInstruction(Inst, VecOS, Fixups, STI); - CurrentShadowSize += Code.size(); - if (CurrentShadowSize >= RequiredShadowSize) - InShadow = false; // The shadow is big enough. Stop counting. - } +void X86AsmPrinter::StackMapShadowTracker::count(MCInst &Inst, + const MCSubtargetInfo &STI, + MCCodeEmitter *CodeEmitter) { + if (InShadow) { + SmallString<256> Code; + SmallVector Fixups; + raw_svector_ostream VecOS(Code); + CodeEmitter->encodeInstruction(Inst, VecOS, Fixups, STI); + CurrentShadowSize += Code.size(); + if (CurrentShadowSize >= RequiredShadowSize) + InShadow = false; // The shadow is big enough. Stop counting. } +} - void X86AsmPrinter::StackMapShadowTracker::emitShadowPadding( +void X86AsmPrinter::StackMapShadowTracker::emitShadowPadding( MCStreamer &OutStreamer, const MCSubtargetInfo &STI) { - if (InShadow && CurrentShadowSize < RequiredShadowSize) { - InShadow = false; - EmitNops(OutStreamer, RequiredShadowSize - CurrentShadowSize, - MF->getSubtarget().is64Bit(), STI); - } + if (InShadow && CurrentShadowSize < RequiredShadowSize) { + InShadow = false; + EmitNops(OutStreamer, RequiredShadowSize - CurrentShadowSize, + MF->getSubtarget().is64Bit(), STI); } +} - void X86AsmPrinter::EmitAndCountInstruction(MCInst &Inst) { - OutStreamer->EmitInstruction(Inst, getSubtargetInfo()); - SMShadowTracker.count(Inst, getSubtargetInfo()); - } -} // end llvm namespace +void X86AsmPrinter::EmitAndCountInstruction(MCInst &Inst) { + OutStreamer->EmitInstruction(Inst, getSubtargetInfo()); + SMShadowTracker.count(Inst, getSubtargetInfo(), CodeEmitter.get()); +} X86MCInstLower::X86MCInstLower(const MachineFunction &mf, X86AsmPrinter &asmprinter) @@ -140,12 +132,8 @@ GetSymbolFromOperand(const MachineOperand &MO) const { // Handle dllimport linkage. Name += "__imp_"; break; - case X86II::MO_DARWIN_STUB: - Suffix = "$stub"; - break; case X86II::MO_DARWIN_NONLAZY: case X86II::MO_DARWIN_NONLAZY_PIC_BASE: - case X86II::MO_DARWIN_HIDDEN_NONLAZY_PIC_BASE: Suffix = "$non_lazy_ptr"; break; } @@ -153,8 +141,6 @@ GetSymbolFromOperand(const MachineOperand &MO) const { if (!Suffix.empty()) Name += DL.getPrivateGlobalPrefix(); - unsigned PrefixLen = Name.size(); - if (MO.isGlobal()) { const GlobalValue *GV = MO.getGlobal(); AsmPrinter.getNameWithPrefix(Name, GV); @@ -164,14 +150,11 @@ GetSymbolFromOperand(const MachineOperand &MO) const { assert(Suffix.empty()); Sym = MO.getMBB()->getSymbol(); } - unsigned OrigLen = Name.size() - PrefixLen; Name += Suffix; if (!Sym) Sym = Ctx.getOrCreateSymbol(Name); - StringRef OrigName = StringRef(Name).substr(PrefixLen, OrigLen); - // If the target flags on the operand changes the name of the symbol, do that // before we return the symbol. switch (MO.getTargetFlags()) { @@ -189,36 +172,6 @@ GetSymbolFromOperand(const MachineOperand &MO) const { } break; } - case X86II::MO_DARWIN_HIDDEN_NONLAZY_PIC_BASE: { - MachineModuleInfoImpl::StubValueTy &StubSym = - getMachOMMI().getHiddenGVStubEntry(Sym); - if (!StubSym.getPointer()) { - assert(MO.isGlobal() && "Extern symbol not handled yet"); - StubSym = - MachineModuleInfoImpl:: - StubValueTy(AsmPrinter.getSymbol(MO.getGlobal()), - !MO.getGlobal()->hasInternalLinkage()); - } - break; - } - case X86II::MO_DARWIN_STUB: { - MachineModuleInfoImpl::StubValueTy &StubSym = - getMachOMMI().getFnStubEntry(Sym); - if (StubSym.getPointer()) - return Sym; - - if (MO.isGlobal()) { - StubSym = - MachineModuleInfoImpl:: - StubValueTy(AsmPrinter.getSymbol(MO.getGlobal()), - !MO.getGlobal()->hasInternalLinkage()); - } else { - StubSym = - MachineModuleInfoImpl:: - StubValueTy(Ctx.getOrCreateSymbol(OrigName), false); - } - break; - } } return Sym; @@ -237,7 +190,6 @@ MCOperand X86MCInstLower::LowerSymbolOperand(const MachineOperand &MO, // These affect the name of the symbol, not any suffix. case X86II::MO_DARWIN_NONLAZY: case X86II::MO_DLLIMPORT: - case X86II::MO_DARWIN_STUB: break; case X86II::MO_TLVP: RefKind = MCSymbolRefExpr::VK_TLVP; break; @@ -265,14 +217,13 @@ MCOperand X86MCInstLower::LowerSymbolOperand(const MachineOperand &MO, case X86II::MO_PLT: RefKind = MCSymbolRefExpr::VK_PLT; break; case X86II::MO_PIC_BASE_OFFSET: case X86II::MO_DARWIN_NONLAZY_PIC_BASE: - case X86II::MO_DARWIN_HIDDEN_NONLAZY_PIC_BASE: Expr = MCSymbolRefExpr::create(Sym, Ctx); // Subtract the pic base. Expr = MCBinaryExpr::createSub(Expr, MCSymbolRefExpr::create(MF.getPICBaseSymbol(), Ctx), Ctx); if (MO.isJTI()) { - assert(MAI.doesSetDirectiveSuppressesReloc()); + assert(MAI.doesSetDirectiveSuppressReloc()); // If .set directive is supported, use it to reduce the number of // relocations the assembler will generate for differences between // local labels. This is only safe when the symbols are in the same @@ -547,18 +498,13 @@ ReSimplify: break; } - // TAILJMPd, TAILJMPd64 - Lower to the correct jump instructions. - case X86::TAILJMPr: + // TAILJMPd, TAILJMPd64 - Lower to the correct jump instruction. + { unsigned Opcode; + case X86::TAILJMPr: Opcode = X86::JMP32r; goto SetTailJmpOpcode; case X86::TAILJMPd: - case X86::TAILJMPd64: { - unsigned Opcode; - switch (OutMI.getOpcode()) { - default: llvm_unreachable("Invalid opcode"); - case X86::TAILJMPr: Opcode = X86::JMP32r; break; - case X86::TAILJMPd: - case X86::TAILJMPd64: Opcode = X86::JMP_1; break; - } + case X86::TAILJMPd64: Opcode = X86::JMP_1; goto SetTailJmpOpcode; + SetTailJmpOpcode: MCOperand Saved = OutMI.getOperand(0); OutMI = MCInst(); OutMI.setOpcode(Opcode); @@ -653,50 +599,81 @@ ReSimplify: // MOV64ao8, MOV64o8a // XCHG16ar, XCHG32ar, XCHG64ar case X86::MOV8mr_NOREX: - case X86::MOV8mr: SimplifyShortMoveForm(AsmPrinter, OutMI, X86::MOV8o32a); break; + case X86::MOV8mr: case X86::MOV8rm_NOREX: - case X86::MOV8rm: SimplifyShortMoveForm(AsmPrinter, OutMI, X86::MOV8ao32); break; - case X86::MOV16mr: SimplifyShortMoveForm(AsmPrinter, OutMI, X86::MOV16o32a); break; - case X86::MOV16rm: SimplifyShortMoveForm(AsmPrinter, OutMI, X86::MOV16ao32); break; - case X86::MOV32mr: SimplifyShortMoveForm(AsmPrinter, OutMI, X86::MOV32o32a); break; - case X86::MOV32rm: SimplifyShortMoveForm(AsmPrinter, OutMI, X86::MOV32ao32); break; - - case X86::ADC8ri: SimplifyShortImmForm(OutMI, X86::ADC8i8); break; - case X86::ADC16ri: SimplifyShortImmForm(OutMI, X86::ADC16i16); break; - case X86::ADC32ri: SimplifyShortImmForm(OutMI, X86::ADC32i32); break; - case X86::ADC64ri32: SimplifyShortImmForm(OutMI, X86::ADC64i32); break; - case X86::ADD8ri: SimplifyShortImmForm(OutMI, X86::ADD8i8); break; - case X86::ADD16ri: SimplifyShortImmForm(OutMI, X86::ADD16i16); break; - case X86::ADD32ri: SimplifyShortImmForm(OutMI, X86::ADD32i32); break; - case X86::ADD64ri32: SimplifyShortImmForm(OutMI, X86::ADD64i32); break; - case X86::AND8ri: SimplifyShortImmForm(OutMI, X86::AND8i8); break; - case X86::AND16ri: SimplifyShortImmForm(OutMI, X86::AND16i16); break; - case X86::AND32ri: SimplifyShortImmForm(OutMI, X86::AND32i32); break; - case X86::AND64ri32: SimplifyShortImmForm(OutMI, X86::AND64i32); break; - case X86::CMP8ri: SimplifyShortImmForm(OutMI, X86::CMP8i8); break; - case X86::CMP16ri: SimplifyShortImmForm(OutMI, X86::CMP16i16); break; - case X86::CMP32ri: SimplifyShortImmForm(OutMI, X86::CMP32i32); break; - case X86::CMP64ri32: SimplifyShortImmForm(OutMI, X86::CMP64i32); break; - case X86::OR8ri: SimplifyShortImmForm(OutMI, X86::OR8i8); break; - case X86::OR16ri: SimplifyShortImmForm(OutMI, X86::OR16i16); break; - case X86::OR32ri: SimplifyShortImmForm(OutMI, X86::OR32i32); break; - case X86::OR64ri32: SimplifyShortImmForm(OutMI, X86::OR64i32); break; - case X86::SBB8ri: SimplifyShortImmForm(OutMI, X86::SBB8i8); break; - case X86::SBB16ri: SimplifyShortImmForm(OutMI, X86::SBB16i16); break; - case X86::SBB32ri: SimplifyShortImmForm(OutMI, X86::SBB32i32); break; - case X86::SBB64ri32: SimplifyShortImmForm(OutMI, X86::SBB64i32); break; - case X86::SUB8ri: SimplifyShortImmForm(OutMI, X86::SUB8i8); break; - case X86::SUB16ri: SimplifyShortImmForm(OutMI, X86::SUB16i16); break; - case X86::SUB32ri: SimplifyShortImmForm(OutMI, X86::SUB32i32); break; - case X86::SUB64ri32: SimplifyShortImmForm(OutMI, X86::SUB64i32); break; - case X86::TEST8ri: SimplifyShortImmForm(OutMI, X86::TEST8i8); break; - case X86::TEST16ri: SimplifyShortImmForm(OutMI, X86::TEST16i16); break; - case X86::TEST32ri: SimplifyShortImmForm(OutMI, X86::TEST32i32); break; - case X86::TEST64ri32: SimplifyShortImmForm(OutMI, X86::TEST64i32); break; - case X86::XOR8ri: SimplifyShortImmForm(OutMI, X86::XOR8i8); break; - case X86::XOR16ri: SimplifyShortImmForm(OutMI, X86::XOR16i16); break; - case X86::XOR32ri: SimplifyShortImmForm(OutMI, X86::XOR32i32); break; - case X86::XOR64ri32: SimplifyShortImmForm(OutMI, X86::XOR64i32); break; + case X86::MOV8rm: + case X86::MOV16mr: + case X86::MOV16rm: + case X86::MOV32mr: + case X86::MOV32rm: { + unsigned NewOpc; + switch (OutMI.getOpcode()) { + default: llvm_unreachable("Invalid opcode"); + case X86::MOV8mr_NOREX: + case X86::MOV8mr: NewOpc = X86::MOV8o32a; break; + case X86::MOV8rm_NOREX: + case X86::MOV8rm: NewOpc = X86::MOV8ao32; break; + case X86::MOV16mr: NewOpc = X86::MOV16o32a; break; + case X86::MOV16rm: NewOpc = X86::MOV16ao32; break; + case X86::MOV32mr: NewOpc = X86::MOV32o32a; break; + case X86::MOV32rm: NewOpc = X86::MOV32ao32; break; + } + SimplifyShortMoveForm(AsmPrinter, OutMI, NewOpc); + break; + } + + case X86::ADC8ri: case X86::ADC16ri: case X86::ADC32ri: case X86::ADC64ri32: + case X86::ADD8ri: case X86::ADD16ri: case X86::ADD32ri: case X86::ADD64ri32: + case X86::AND8ri: case X86::AND16ri: case X86::AND32ri: case X86::AND64ri32: + case X86::CMP8ri: case X86::CMP16ri: case X86::CMP32ri: case X86::CMP64ri32: + case X86::OR8ri: case X86::OR16ri: case X86::OR32ri: case X86::OR64ri32: + case X86::SBB8ri: case X86::SBB16ri: case X86::SBB32ri: case X86::SBB64ri32: + case X86::SUB8ri: case X86::SUB16ri: case X86::SUB32ri: case X86::SUB64ri32: + case X86::TEST8ri:case X86::TEST16ri:case X86::TEST32ri:case X86::TEST64ri32: + case X86::XOR8ri: case X86::XOR16ri: case X86::XOR32ri: case X86::XOR64ri32: { + unsigned NewOpc; + switch (OutMI.getOpcode()) { + default: llvm_unreachable("Invalid opcode"); + case X86::ADC8ri: NewOpc = X86::ADC8i8; break; + case X86::ADC16ri: NewOpc = X86::ADC16i16; break; + case X86::ADC32ri: NewOpc = X86::ADC32i32; break; + case X86::ADC64ri32: NewOpc = X86::ADC64i32; break; + case X86::ADD8ri: NewOpc = X86::ADD8i8; break; + case X86::ADD16ri: NewOpc = X86::ADD16i16; break; + case X86::ADD32ri: NewOpc = X86::ADD32i32; break; + case X86::ADD64ri32: NewOpc = X86::ADD64i32; break; + case X86::AND8ri: NewOpc = X86::AND8i8; break; + case X86::AND16ri: NewOpc = X86::AND16i16; break; + case X86::AND32ri: NewOpc = X86::AND32i32; break; + case X86::AND64ri32: NewOpc = X86::AND64i32; break; + case X86::CMP8ri: NewOpc = X86::CMP8i8; break; + case X86::CMP16ri: NewOpc = X86::CMP16i16; break; + case X86::CMP32ri: NewOpc = X86::CMP32i32; break; + case X86::CMP64ri32: NewOpc = X86::CMP64i32; break; + case X86::OR8ri: NewOpc = X86::OR8i8; break; + case X86::OR16ri: NewOpc = X86::OR16i16; break; + case X86::OR32ri: NewOpc = X86::OR32i32; break; + case X86::OR64ri32: NewOpc = X86::OR64i32; break; + case X86::SBB8ri: NewOpc = X86::SBB8i8; break; + case X86::SBB16ri: NewOpc = X86::SBB16i16; break; + case X86::SBB32ri: NewOpc = X86::SBB32i32; break; + case X86::SBB64ri32: NewOpc = X86::SBB64i32; break; + case X86::SUB8ri: NewOpc = X86::SUB8i8; break; + case X86::SUB16ri: NewOpc = X86::SUB16i16; break; + case X86::SUB32ri: NewOpc = X86::SUB32i32; break; + case X86::SUB64ri32: NewOpc = X86::SUB64i32; break; + case X86::TEST8ri: NewOpc = X86::TEST8i8; break; + case X86::TEST16ri: NewOpc = X86::TEST16i16; break; + case X86::TEST32ri: NewOpc = X86::TEST32i32; break; + case X86::TEST64ri32: NewOpc = X86::TEST64i32; break; + case X86::XOR8ri: NewOpc = X86::XOR8i8; break; + case X86::XOR16ri: NewOpc = X86::XOR16i16; break; + case X86::XOR32ri: NewOpc = X86::XOR32i32; break; + case X86::XOR64ri32: NewOpc = X86::XOR64i32; break; + } + SimplifyShortImmForm(OutMI, NewOpc); + break; + } // Try to shrink some forms of movsx. case X86::MOVSX16rr8: @@ -785,55 +762,77 @@ void X86AsmPrinter::LowerTlsAddr(X86MCInstLower &MCInstLowering, .addExpr(tlsRef)); } -/// \brief Emit the optimal amount of multi-byte nops on X86. -static void EmitNops(MCStreamer &OS, unsigned NumBytes, bool Is64Bit, const MCSubtargetInfo &STI) { +/// \brief Emit the largest nop instruction smaller than or equal to \p NumBytes +/// bytes. Return the size of nop emitted. +static unsigned EmitNop(MCStreamer &OS, unsigned NumBytes, bool Is64Bit, + const MCSubtargetInfo &STI) { // This works only for 64bit. For 32bit we have to do additional checking if // the CPU supports multi-byte nops. assert(Is64Bit && "EmitNops only supports X86-64"); - while (NumBytes) { - unsigned Opc, BaseReg, ScaleVal, IndexReg, Displacement, SegmentReg; - Opc = IndexReg = Displacement = SegmentReg = 0; - BaseReg = X86::RAX; ScaleVal = 1; - switch (NumBytes) { - case 0: llvm_unreachable("Zero nops?"); break; - case 1: NumBytes -= 1; Opc = X86::NOOP; break; - case 2: NumBytes -= 2; Opc = X86::XCHG16ar; break; - case 3: NumBytes -= 3; Opc = X86::NOOPL; break; - case 4: NumBytes -= 4; Opc = X86::NOOPL; Displacement = 8; break; - case 5: NumBytes -= 5; Opc = X86::NOOPL; Displacement = 8; - IndexReg = X86::RAX; break; - case 6: NumBytes -= 6; Opc = X86::NOOPW; Displacement = 8; - IndexReg = X86::RAX; break; - case 7: NumBytes -= 7; Opc = X86::NOOPL; Displacement = 512; break; - case 8: NumBytes -= 8; Opc = X86::NOOPL; Displacement = 512; - IndexReg = X86::RAX; break; - case 9: NumBytes -= 9; Opc = X86::NOOPW; Displacement = 512; - IndexReg = X86::RAX; break; - default: NumBytes -= 10; Opc = X86::NOOPW; Displacement = 512; - IndexReg = X86::RAX; SegmentReg = X86::CS; break; - } - unsigned NumPrefixes = std::min(NumBytes, 5U); - NumBytes -= NumPrefixes; - for (unsigned i = 0; i != NumPrefixes; ++i) - OS.EmitBytes("\x66"); + unsigned NopSize; + unsigned Opc, BaseReg, ScaleVal, IndexReg, Displacement, SegmentReg; + Opc = IndexReg = Displacement = SegmentReg = 0; + BaseReg = X86::RAX; + ScaleVal = 1; + switch (NumBytes) { + case 0: llvm_unreachable("Zero nops?"); break; + case 1: NopSize = 1; Opc = X86::NOOP; break; + case 2: NopSize = 2; Opc = X86::XCHG16ar; break; + case 3: NopSize = 3; Opc = X86::NOOPL; break; + case 4: NopSize = 4; Opc = X86::NOOPL; Displacement = 8; break; + case 5: NopSize = 5; Opc = X86::NOOPL; Displacement = 8; + IndexReg = X86::RAX; break; + case 6: NopSize = 6; Opc = X86::NOOPW; Displacement = 8; + IndexReg = X86::RAX; break; + case 7: NopSize = 7; Opc = X86::NOOPL; Displacement = 512; break; + case 8: NopSize = 8; Opc = X86::NOOPL; Displacement = 512; + IndexReg = X86::RAX; break; + case 9: NopSize = 9; Opc = X86::NOOPW; Displacement = 512; + IndexReg = X86::RAX; break; + default: NopSize = 10; Opc = X86::NOOPW; Displacement = 512; + IndexReg = X86::RAX; SegmentReg = X86::CS; break; + } - switch (Opc) { - default: llvm_unreachable("Unexpected opcode"); break; - case X86::NOOP: - OS.EmitInstruction(MCInstBuilder(Opc), STI); - break; - case X86::XCHG16ar: - OS.EmitInstruction(MCInstBuilder(Opc).addReg(X86::AX), STI); - break; - case X86::NOOPL: - case X86::NOOPW: - OS.EmitInstruction(MCInstBuilder(Opc).addReg(BaseReg) - .addImm(ScaleVal).addReg(IndexReg) - .addImm(Displacement).addReg(SegmentReg), STI); - break; - } - } // while (NumBytes) + unsigned NumPrefixes = std::min(NumBytes - NopSize, 5U); + NopSize += NumPrefixes; + for (unsigned i = 0; i != NumPrefixes; ++i) + OS.EmitBytes("\x66"); + + switch (Opc) { + default: + llvm_unreachable("Unexpected opcode"); + break; + case X86::NOOP: + OS.EmitInstruction(MCInstBuilder(Opc), STI); + break; + case X86::XCHG16ar: + OS.EmitInstruction(MCInstBuilder(Opc).addReg(X86::AX), STI); + break; + case X86::NOOPL: + case X86::NOOPW: + OS.EmitInstruction(MCInstBuilder(Opc) + .addReg(BaseReg) + .addImm(ScaleVal) + .addReg(IndexReg) + .addImm(Displacement) + .addReg(SegmentReg), + STI); + break; + } + assert(NopSize <= NumBytes && "We overemitted?"); + return NopSize; +} + +/// \brief Emit the optimal amount of multi-byte nops on X86. +static void EmitNops(MCStreamer &OS, unsigned NumBytes, bool Is64Bit, + const MCSubtargetInfo &STI) { + unsigned NopsToEmit = NumBytes; + (void)NopsToEmit; + while (NumBytes) { + NumBytes -= EmitNop(OS, NumBytes, Is64Bit, STI); + assert(NopsToEmit >= NumBytes && "Emitted more than I asked for!"); + } } void X86AsmPrinter::LowerSTATEPOINT(const MachineInstr &MI, @@ -891,10 +890,10 @@ void X86AsmPrinter::LowerSTATEPOINT(const MachineInstr &MI, void X86AsmPrinter::LowerFAULTING_LOAD_OP(const MachineInstr &MI, X86MCInstLower &MCIL) { - // FAULTING_LOAD_OP , , , + // FAULTING_LOAD_OP , , , unsigned LoadDefRegister = MI.getOperand(0).getReg(); - MCSymbol *HandlerLabel = MI.getOperand(1).getMCSymbol(); + MCSymbol *HandlerLabel = MI.getOperand(1).getMBB()->getSymbol(); unsigned LoadOpcode = MI.getOperand(2).getImm(); unsigned LoadOperandsBeginIdx = 3; @@ -915,6 +914,43 @@ void X86AsmPrinter::LowerFAULTING_LOAD_OP(const MachineInstr &MI, OutStreamer->EmitInstruction(LoadMI, getSubtargetInfo()); } +void X86AsmPrinter::LowerPATCHABLE_OP(const MachineInstr &MI, + X86MCInstLower &MCIL) { + // PATCHABLE_OP minsize, opcode, operands + + unsigned MinSize = MI.getOperand(0).getImm(); + unsigned Opcode = MI.getOperand(1).getImm(); + + MCInst MCI; + MCI.setOpcode(Opcode); + for (auto &MO : make_range(MI.operands_begin() + 2, MI.operands_end())) + if (auto MaybeOperand = MCIL.LowerMachineOperand(&MI, MO)) + MCI.addOperand(MaybeOperand.getValue()); + + SmallString<256> Code; + SmallVector Fixups; + raw_svector_ostream VecOS(Code); + CodeEmitter->encodeInstruction(MCI, VecOS, Fixups, getSubtargetInfo()); + + if (Code.size() < MinSize) { + if (MinSize == 2 && Opcode == X86::PUSH64r) { + // This is an optimization that lets us get away without emitting a nop in + // many cases. + // + // NB! In some cases the encoding for PUSH64r (e.g. PUSH64r %R9) takes two + // bytes too, so the check on MinSize is important. + MCI.setOpcode(X86::PUSH64rmr); + } else { + unsigned NopSize = EmitNop(*OutStreamer, MinSize, Subtarget->is64Bit(), + getSubtargetInfo()); + assert(NopSize == MinSize && "Could not implement MinSize!"); + (void) NopSize; + } + } + + OutStreamer->EmitInstruction(MCI, getSubtargetInfo()); +} + // Lower a stackmap of the form: // , , ... void X86AsmPrinter::LowerSTACKMAP(const MachineInstr &MI) { @@ -937,8 +973,7 @@ void X86AsmPrinter::LowerPATCHPOINT(const MachineInstr &MI, PatchPointOpers opers(&MI); unsigned ScratchIdx = opers.getNextScratchIdx(); unsigned EncodedBytes = 0; - const MachineOperand &CalleeMO = - opers.getMetaOper(PatchPointOpers::TargetPos); + const MachineOperand &CalleeMO = opers.getCallTarget(); // Check for null target. If target is non-null (i.e. is non-zero or is // symbolic) then emit a call. @@ -974,7 +1009,7 @@ void X86AsmPrinter::LowerPATCHPOINT(const MachineInstr &MI, } // Emit padding. - unsigned NumBytes = opers.getMetaOper(PatchPointOpers::NBytesPos).getImm(); + unsigned NumBytes = opers.getNumPatchBytes(); assert(NumBytes >= EncodedBytes && "Patchpoint can't request size less than the length of a call."); @@ -982,14 +1017,107 @@ void X86AsmPrinter::LowerPATCHPOINT(const MachineInstr &MI, getSubtargetInfo()); } +void X86AsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI, + X86MCInstLower &MCIL) { + // We want to emit the following pattern: + // + // .p2align 1, ... + // .Lxray_sled_N: + // jmp .tmpN + // # 9 bytes worth of noops + // .tmpN + // + // We need the 9 bytes because at runtime, we'd be patching over the full 11 + // bytes with the following pattern: + // + // mov %r10, // 6 bytes + // call // 5 bytes + // + auto CurSled = OutContext.createTempSymbol("xray_sled_", true); + OutStreamer->EmitCodeAlignment(2); + OutStreamer->EmitLabel(CurSled); + auto Target = OutContext.createTempSymbol(); + + // Use a two-byte `jmp`. This version of JMP takes an 8-bit relative offset as + // an operand (computed as an offset from the jmp instruction). + // FIXME: Find another less hacky way do force the relative jump. + OutStreamer->EmitBytes("\xeb\x09"); + EmitNops(*OutStreamer, 9, Subtarget->is64Bit(), getSubtargetInfo()); + OutStreamer->EmitLabel(Target); + recordSled(CurSled, MI, SledKind::FUNCTION_ENTER); +} + +void X86AsmPrinter::LowerPATCHABLE_RET(const MachineInstr &MI, + X86MCInstLower &MCIL) { + // Since PATCHABLE_RET takes the opcode of the return statement as an + // argument, we use that to emit the correct form of the RET that we want. + // i.e. when we see this: + // + // PATCHABLE_RET X86::RET ... + // + // We should emit the RET followed by sleds. + // + // .p2align 1, ... + // .Lxray_sled_N: + // ret # or equivalent instruction + // # 10 bytes worth of noops + // + // This just makes sure that the alignment for the next instruction is 2. + auto CurSled = OutContext.createTempSymbol("xray_sled_", true); + OutStreamer->EmitCodeAlignment(2); + OutStreamer->EmitLabel(CurSled); + unsigned OpCode = MI.getOperand(0).getImm(); + MCInst Ret; + Ret.setOpcode(OpCode); + for (auto &MO : make_range(MI.operands_begin() + 1, MI.operands_end())) + if (auto MaybeOperand = MCIL.LowerMachineOperand(&MI, MO)) + Ret.addOperand(MaybeOperand.getValue()); + OutStreamer->EmitInstruction(Ret, getSubtargetInfo()); + EmitNops(*OutStreamer, 10, Subtarget->is64Bit(), getSubtargetInfo()); + recordSled(CurSled, MI, SledKind::FUNCTION_EXIT); +} + +void X86AsmPrinter::LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI, X86MCInstLower &MCIL) { + // Like PATCHABLE_RET, we have the actual instruction in the operands to this + // instruction so we lower that particular instruction and its operands. + // Unlike PATCHABLE_RET though, we put the sled before the JMP, much like how + // we do it for PATCHABLE_FUNCTION_ENTER. The sled should be very similar to + // the PATCHABLE_FUNCTION_ENTER case, followed by the lowering of the actual + // tail call much like how we have it in PATCHABLE_RET. + auto CurSled = OutContext.createTempSymbol("xray_sled_", true); + OutStreamer->EmitCodeAlignment(2); + OutStreamer->EmitLabel(CurSled); + auto Target = OutContext.createTempSymbol(); + + // Use a two-byte `jmp`. This version of JMP takes an 8-bit relative offset as + // an operand (computed as an offset from the jmp instruction). + // FIXME: Find another less hacky way do force the relative jump. + OutStreamer->EmitBytes("\xeb\x09"); + EmitNops(*OutStreamer, 9, Subtarget->is64Bit(), getSubtargetInfo()); + OutStreamer->EmitLabel(Target); + recordSled(CurSled, MI, SledKind::TAIL_CALL); + + unsigned OpCode = MI.getOperand(0).getImm(); + MCInst TC; + TC.setOpcode(OpCode); + + // Before emitting the instruction, add a comment to indicate that this is + // indeed a tail call. + OutStreamer->AddComment("TAILCALL"); + for (auto &MO : make_range(MI.operands_begin() + 1, MI.operands_end())) + if (auto MaybeOperand = MCIL.LowerMachineOperand(&MI, MO)) + TC.addOperand(MaybeOperand.getValue()); + OutStreamer->EmitInstruction(TC, getSubtargetInfo()); +} + // Returns instruction preceding MBBI in MachineFunction. // If MBBI is the first instruction of the first basic block, returns null. static MachineBasicBlock::const_iterator PrevCrossBBInst(MachineBasicBlock::const_iterator MBBI) { const MachineBasicBlock *MBB = MBBI->getParent(); while (MBBI == MBB->begin()) { - if (MBB == MBB->getParent()->begin()) - return nullptr; + if (MBB == &MBB->getParent()->front()) + return MachineBasicBlock::const_iterator(); MBB = MBB->getPrevNode(); MBBI = MBB->end(); } @@ -1017,8 +1145,9 @@ static const Constant *getConstantFromPool(const MachineInstr &MI, return C; } -static std::string getShuffleComment(const MachineOperand &DstOp, - const MachineOperand &SrcOp, +static std::string getShuffleComment(const MachineInstr *MI, + unsigned SrcOp1Idx, + unsigned SrcOp2Idx, ArrayRef Mask) { std::string Comment; @@ -1031,40 +1160,73 @@ static std::string getShuffleComment(const MachineOperand &DstOp, return X86ATTInstPrinter::getRegisterName(RegNum); }; + const MachineOperand &DstOp = MI->getOperand(0); + const MachineOperand &SrcOp1 = MI->getOperand(SrcOp1Idx); + const MachineOperand &SrcOp2 = MI->getOperand(SrcOp2Idx); + StringRef DstName = DstOp.isReg() ? GetRegisterName(DstOp.getReg()) : "mem"; - StringRef SrcName = SrcOp.isReg() ? GetRegisterName(SrcOp.getReg()) : "mem"; + StringRef Src1Name = + SrcOp1.isReg() ? GetRegisterName(SrcOp1.getReg()) : "mem"; + StringRef Src2Name = + SrcOp2.isReg() ? GetRegisterName(SrcOp2.getReg()) : "mem"; + + // One source operand, fix the mask to print all elements in one span. + SmallVector ShuffleMask(Mask.begin(), Mask.end()); + if (Src1Name == Src2Name) + for (int i = 0, e = ShuffleMask.size(); i != e; ++i) + if (ShuffleMask[i] >= e) + ShuffleMask[i] -= e; raw_string_ostream CS(Comment); - CS << DstName << " = "; - bool NeedComma = false; - bool InSrc = false; - for (int M : Mask) { - // Wrap up any prior entry... - if (M == SM_SentinelZero && InSrc) { - InSrc = false; - CS << "]"; + CS << DstName; + + // Handle AVX512 MASK/MASXZ write mask comments. + // MASK: zmmX {%kY} + // MASKZ: zmmX {%kY} {z} + if (SrcOp1Idx > 1) { + assert((SrcOp1Idx == 2 || SrcOp1Idx == 3) && "Unexpected writemask"); + + const MachineOperand &WriteMaskOp = MI->getOperand(SrcOp1Idx - 1); + if (WriteMaskOp.isReg()) { + CS << " {%" << GetRegisterName(WriteMaskOp.getReg()) << "}"; + + if (SrcOp1Idx == 2) { + CS << " {z}"; + } } - if (NeedComma) - CS << ","; - else - NeedComma = true; + } - // Print this shuffle... - if (M == SM_SentinelZero) { + CS << " = "; + + for (int i = 0, e = ShuffleMask.size(); i != e; ++i) { + if (i != 0) + CS << ","; + if (ShuffleMask[i] == SM_SentinelZero) { CS << "zero"; - } else { - if (!InSrc) { - InSrc = true; - CS << SrcName << "["; - } - if (M == SM_SentinelUndef) + continue; + } + + // Otherwise, it must come from src1 or src2. Print the span of elements + // that comes from this src. + bool isSrc1 = ShuffleMask[i] < (int)e; + CS << (isSrc1 ? Src1Name : Src2Name) << '['; + + bool IsFirst = true; + while (i != e && ShuffleMask[i] != SM_SentinelZero && + (ShuffleMask[i] < (int)e) == isSrc1) { + if (!IsFirst) + CS << ','; + else + IsFirst = false; + if (ShuffleMask[i] == SM_SentinelUndef) CS << "u"; else - CS << M; + CS << ShuffleMask[i] % (int)e; + ++i; } + CS << ']'; + --i; // For loop increments element #. } - if (InSrc) - CS << "]"; CS.flush(); return Comment; @@ -1074,6 +1236,13 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) { X86MCInstLower MCInstLowering(*MF, *this); const X86RegisterInfo *RI = MF->getSubtarget().getRegisterInfo(); + // Add a comment about EVEX-2-VEX compression for AVX-512 instrs that + // are compressed from EVEX encoding to VEX encoding. + if (TM.Options.MCOptions.ShowMCEncoding) { + if (MI->getAsmPrinterFlags() & AC_EVEX_2_VEX) + OutStreamer->AddComment("EVEX TO VEX Compression ", false); + } + switch (MI->getOpcode()) { case TargetOpcode::DBG_VALUE: llvm_unreachable("Should be handled target independently"); @@ -1112,7 +1281,6 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) { case X86::TAILJMPd64: case X86::TAILJMPr64_REX: case X86::TAILJMPm64_REX: - case X86::TAILJMPd64_REX: // Lower these as normal, but add some comments. OutStreamer->AddComment("TAILCALL"); break; @@ -1202,12 +1370,24 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) { case TargetOpcode::FAULTING_LOAD_OP: return LowerFAULTING_LOAD_OP(*MI, MCInstLowering); + case TargetOpcode::PATCHABLE_OP: + return LowerPATCHABLE_OP(*MI, MCInstLowering); + case TargetOpcode::STACKMAP: return LowerSTACKMAP(*MI); case TargetOpcode::PATCHPOINT: return LowerPATCHPOINT(*MI, MCInstLowering); + case TargetOpcode::PATCHABLE_FUNCTION_ENTER: + return LowerPATCHABLE_FUNCTION_ENTER(*MI, MCInstLowering); + + case TargetOpcode::PATCHABLE_RET: + return LowerPATCHABLE_RET(*MI, MCInstLowering); + + case TargetOpcode::PATCHABLE_TAIL_CALL: + return LowerPATCHABLE_TAIL_CALL(*MI, MCInstLowering); + case X86::MORESTACK_RET: EmitAndCountInstruction(MCInstBuilder(getRetOpcode(*Subtarget))); return; @@ -1221,40 +1401,50 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) { return; case X86::SEH_PushReg: + assert(MF->hasWinCFI() && "SEH_ instruction in function without WinCFI?"); OutStreamer->EmitWinCFIPushReg(RI->getSEHRegNum(MI->getOperand(0).getImm())); return; case X86::SEH_SaveReg: + assert(MF->hasWinCFI() && "SEH_ instruction in function without WinCFI?"); OutStreamer->EmitWinCFISaveReg(RI->getSEHRegNum(MI->getOperand(0).getImm()), MI->getOperand(1).getImm()); return; case X86::SEH_SaveXMM: + assert(MF->hasWinCFI() && "SEH_ instruction in function without WinCFI?"); OutStreamer->EmitWinCFISaveXMM(RI->getSEHRegNum(MI->getOperand(0).getImm()), MI->getOperand(1).getImm()); return; case X86::SEH_StackAlloc: + assert(MF->hasWinCFI() && "SEH_ instruction in function without WinCFI?"); OutStreamer->EmitWinCFIAllocStack(MI->getOperand(0).getImm()); return; case X86::SEH_SetFrame: + assert(MF->hasWinCFI() && "SEH_ instruction in function without WinCFI?"); OutStreamer->EmitWinCFISetFrame(RI->getSEHRegNum(MI->getOperand(0).getImm()), MI->getOperand(1).getImm()); return; case X86::SEH_PushFrame: + assert(MF->hasWinCFI() && "SEH_ instruction in function without WinCFI?"); OutStreamer->EmitWinCFIPushFrame(MI->getOperand(0).getImm()); return; case X86::SEH_EndPrologue: + assert(MF->hasWinCFI() && "SEH_ instruction in function without WinCFI?"); OutStreamer->EmitWinCFIEndProlog(); return; case X86::SEH_Epilogue: { + assert(MF->hasWinCFI() && "SEH_ instruction in function without WinCFI?"); MachineBasicBlock::const_iterator MBBI(MI); // Check if preceded by a call and emit nop if so. - for (MBBI = PrevCrossBBInst(MBBI); MBBI; MBBI = PrevCrossBBInst(MBBI)) { + for (MBBI = PrevCrossBBInst(MBBI); + MBBI != MachineBasicBlock::const_iterator(); + MBBI = PrevCrossBBInst(MBBI)) { // Conservatively assume that pseudo instructions don't emit code and keep // looking for a call. We may emit an unnecessary nop in some cases. if (!MBBI->isPseudo()) { @@ -1305,42 +1495,130 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) { assert(MI->getNumOperands() >= 6 && "We should always have at least 6 operands!"); - const MachineOperand &DstOp = MI->getOperand(0); - const MachineOperand &SrcOp = MI->getOperand(SrcIdx); - const MachineOperand &MaskOp = MI->getOperand(MaskIdx); + const MachineOperand &MaskOp = MI->getOperand(MaskIdx); if (auto *C = getConstantFromPool(*MI, MaskOp)) { - SmallVector Mask; + SmallVector Mask; DecodePSHUFBMask(C, Mask); if (!Mask.empty()) - OutStreamer->AddComment(getShuffleComment(DstOp, SrcOp, Mask)); + OutStreamer->AddComment(getShuffleComment(MI, SrcIdx, SrcIdx, Mask)); } break; } + case X86::VPERMILPSrm: - case X86::VPERMILPDrm: case X86::VPERMILPSYrm: - case X86::VPERMILPDYrm: { + case X86::VPERMILPSZ128rm: + case X86::VPERMILPSZ128rmk: + case X86::VPERMILPSZ128rmkz: + case X86::VPERMILPSZ256rm: + case X86::VPERMILPSZ256rmk: + case X86::VPERMILPSZ256rmkz: + case X86::VPERMILPSZrm: + case X86::VPERMILPSZrmk: + case X86::VPERMILPSZrmkz: + case X86::VPERMILPDrm: + case X86::VPERMILPDYrm: + case X86::VPERMILPDZ128rm: + case X86::VPERMILPDZ128rmk: + case X86::VPERMILPDZ128rmkz: + case X86::VPERMILPDZ256rm: + case X86::VPERMILPDZ256rmk: + case X86::VPERMILPDZ256rmkz: + case X86::VPERMILPDZrm: + case X86::VPERMILPDZrmk: + case X86::VPERMILPDZrmkz: { if (!OutStreamer->isVerboseAsm()) break; - assert(MI->getNumOperands() > 5 && - "We should always have at least 5 operands!"); - const MachineOperand &DstOp = MI->getOperand(0); - const MachineOperand &SrcOp = MI->getOperand(1); - const MachineOperand &MaskOp = MI->getOperand(5); - + unsigned SrcIdx, MaskIdx; unsigned ElSize; switch (MI->getOpcode()) { default: llvm_unreachable("Invalid opcode"); - case X86::VPERMILPSrm: case X86::VPERMILPSYrm: ElSize = 32; break; - case X86::VPERMILPDrm: case X86::VPERMILPDYrm: ElSize = 64; break; + case X86::VPERMILPSrm: + case X86::VPERMILPSYrm: + case X86::VPERMILPSZ128rm: + case X86::VPERMILPSZ256rm: + case X86::VPERMILPSZrm: + SrcIdx = 1; MaskIdx = 5; ElSize = 32; break; + case X86::VPERMILPSZ128rmkz: + case X86::VPERMILPSZ256rmkz: + case X86::VPERMILPSZrmkz: + SrcIdx = 2; MaskIdx = 6; ElSize = 32; break; + case X86::VPERMILPSZ128rmk: + case X86::VPERMILPSZ256rmk: + case X86::VPERMILPSZrmk: + SrcIdx = 3; MaskIdx = 7; ElSize = 32; break; + case X86::VPERMILPDrm: + case X86::VPERMILPDYrm: + case X86::VPERMILPDZ128rm: + case X86::VPERMILPDZ256rm: + case X86::VPERMILPDZrm: + SrcIdx = 1; MaskIdx = 5; ElSize = 64; break; + case X86::VPERMILPDZ128rmkz: + case X86::VPERMILPDZ256rmkz: + case X86::VPERMILPDZrmkz: + SrcIdx = 2; MaskIdx = 6; ElSize = 64; break; + case X86::VPERMILPDZ128rmk: + case X86::VPERMILPDZ256rmk: + case X86::VPERMILPDZrmk: + SrcIdx = 3; MaskIdx = 7; ElSize = 64; break; } + assert(MI->getNumOperands() >= 6 && + "We should always have at least 6 operands!"); + + const MachineOperand &MaskOp = MI->getOperand(MaskIdx); if (auto *C = getConstantFromPool(*MI, MaskOp)) { SmallVector Mask; DecodeVPERMILPMask(C, ElSize, Mask); if (!Mask.empty()) - OutStreamer->AddComment(getShuffleComment(DstOp, SrcOp, Mask)); + OutStreamer->AddComment(getShuffleComment(MI, SrcIdx, SrcIdx, Mask)); + } + break; + } + + case X86::VPERMIL2PDrm: + case X86::VPERMIL2PSrm: + case X86::VPERMIL2PDrmY: + case X86::VPERMIL2PSrmY: { + if (!OutStreamer->isVerboseAsm()) + break; + assert(MI->getNumOperands() >= 8 && + "We should always have at least 8 operands!"); + + const MachineOperand &CtrlOp = MI->getOperand(MI->getNumOperands() - 1); + if (!CtrlOp.isImm()) + break; + + unsigned ElSize; + switch (MI->getOpcode()) { + default: llvm_unreachable("Invalid opcode"); + case X86::VPERMIL2PSrm: case X86::VPERMIL2PSrmY: ElSize = 32; break; + case X86::VPERMIL2PDrm: case X86::VPERMIL2PDrmY: ElSize = 64; break; + } + + const MachineOperand &MaskOp = MI->getOperand(6); + if (auto *C = getConstantFromPool(*MI, MaskOp)) { + SmallVector Mask; + DecodeVPERMIL2PMask(C, (unsigned)CtrlOp.getImm(), ElSize, Mask); + if (!Mask.empty()) + OutStreamer->AddComment(getShuffleComment(MI, 1, 2, Mask)); + } + break; + } + + case X86::VPPERMrrm: { + if (!OutStreamer->isVerboseAsm()) + break; + assert(MI->getNumOperands() >= 7 && + "We should always have at least 7 operands!"); + + const MachineOperand &MaskOp = MI->getOperand(6); + if (auto *C = getConstantFromPool(*MI, MaskOp)) { + SmallVector Mask; + DecodeVPPERMMask(C, Mask); + if (!Mask.empty()) + OutStreamer->AddComment(getShuffleComment(MI, 1, 2, Mask)); } break; } @@ -1378,7 +1656,8 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) { CASE_ALL_MOV_RM() if (!OutStreamer->isVerboseAsm()) break; - if (MI->getNumOperands() > 4) + if (MI->getNumOperands() <= 4) + break; if (auto *C = getConstantFromPool(*MI, MI->getOperand(4))) { std::string Comment; raw_string_ostream CS(Comment); @@ -1413,7 +1692,7 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) { CS << CI->getZExtValue(); } else { // print multi-word constant as (w0,w1) - auto Val = CI->getValue(); + const auto &Val = CI->getValue(); CS << "("; for (int i = 0, N = Val.getNumWords(); i < N; ++i) { if (i > 0) @@ -1446,7 +1725,7 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) { // is at the end of the shadow. if (MI->isCall()) { // Count then size of the call towards the shadow - SMShadowTracker.count(TmpInst, getSubtargetInfo()); + SMShadowTracker.count(TmpInst, getSubtargetInfo(), CodeEmitter.get()); // Then flush the shadow so that we fill with nops before the call, not // after it. SMShadowTracker.emitShadowPadding(*OutStreamer, getSubtargetInfo()); @@ -1457,3 +1736,9 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) { EmitAndCountInstruction(TmpInst); } + +/// Emit Trap bytes to the specified power of two alignment +void X86AsmPrinter::EmitTrapToAlignment(unsigned NumBits) const { + if (NumBits == 0) return; + OutStreamer->EmitValueToAlignment(1u << NumBits, 0xCC, 1); +} -- cgit v1.2.3