Add support for emitting Mach-O R2R images

docs/design/coreclr/botr/readytorun-format.md

@@ -135,6 +135,7 @@ struct READYTORUN_CORE_HEADER
 | READYTORUN_FLAG_COMPONENT                  | 0x00000020 | This is a component assembly of a composite R2R image
 | READYTORUN_FLAG_MULTIMODULE_VERSION_BUBBLE | 0x00000040 | This R2R module has multiple modules within its version bubble (For versions before version 6.3, all modules are assumed to possibly have this characteristic)
 | READYTORUN_FLAG_UNRELATED_R2R_CODE         | 0x00000080 | This R2R module has code in it that would not be naturally encoded into this module
+| READYTORUN_FLAG_PLATFORM_NATIVE_IMAGE              | 0x00000100 | The owning composite executable is in the platform native format
 
 ## READYTORUN_SECTION
 

src/coreclr/inc/readytorun.h

@@ -20,7 +20,7 @@
 // If you update this, ensure you run `git grep MINIMUM_READYTORUN_MAJOR_VERSION`
 // and handle pending work.
 #define READYTORUN_MAJOR_VERSION 16
-#define READYTORUN_MINOR_VERSION 0x0000
+#define READYTORUN_MINOR_VERSION 0x0001
 
 #define MINIMUM_READYTORUN_MAJOR_VERSION 16
 
@@ -47,6 +47,8 @@
 // R2R Version 14 changed x86 code generation to use funclets
 // R2R Version 15 removes double to int/uint helper calls
 // R2R Version 16 replaces the compression format for debug boundaries with a new format that is smaller and more efficient to parse
+//     R2R 16 is not backward compatible with 15.x or earlier.
+// R2R Version 16.1 adds the READYTORUN_FLAG_PLATFORM_NATIVE_IMAGE flag to specify that the R2R image pointed to by OwnerCompositeExecutable is in the platform native format.
 
 struct READYTORUN_CORE_HEADER
 {
@@ -83,6 +85,7 @@ enum ReadyToRunFlag
     READYTORUN_FLAG_COMPONENT                   = 0x00000020,   // This is the header describing a component assembly of composite R2R
     READYTORUN_FLAG_MULTIMODULE_VERSION_BUBBLE  = 0x00000040,   // This R2R module has multiple modules within its version bubble (For versions before version 6.2, all modules are assumed to possibly have this characteristic)
     READYTORUN_FLAG_UNRELATED_R2R_CODE          = 0x00000080,   // This R2R module has code in it that would not be naturally encoded into this module
+    READYTORUN_FLAG_PLATFORM_NATIVE_IMAGE         = 0x00000100,   // The owning composite executable is in the platform native format
 };
 
 enum class ReadyToRunSectionType : uint32_t

src/coreclr/nativeaot/Runtime/inc/ModuleHeaders.h

@@ -12,7 +12,7 @@ struct ReadyToRunHeaderConstants
     static const uint32_t Signature = 0x00525452; // 'RTR'
 
     static const uint32_t CurrentMajorVersion = 16;
-    static const uint32_t CurrentMinorVersion = 0;
+    static const uint32_t CurrentMinorVersion = 1;
 };
 
 struct ReadyToRunHeader

src/coreclr/tools/Common/Compiler/ObjectWriter/MachObjectWriter.cs

@@ -65,9 +65,15 @@ internal sealed partial class MachObjectWriter : UnixObjectWriter
         private readonly Dictionary<string, uint> _symbolNameToIndex = new();
         private readonly List<MachSymbol> _symbolTable = new();
         private readonly MachDynamicLinkEditSymbolTable _dySymbolTable = new();
+        private readonly Dictionary<string, (string StartNode, string EndNode)> _rangeSymbols = new();
 
-        public MachObjectWriter(NodeFactory factory, ObjectWritingOptions options)
-            : base(factory, options)
+        /// <summary>
+        /// Base symbol to use for <see cref="RelocType.IMAGE_REL_BASED_ADDR32NB"/> relocations.
+        /// </summary>
+        private readonly string _baseSymbolName;
+
+        public MachObjectWriter(NodeFactory factory, ObjectWritingOptions options, OutputInfoBuilder outputInfoBuilder = null)
+            : base(factory, options, outputInfoBuilder)
         {
             switch (factory.Target.Architecture)
             {
@@ -92,6 +98,12 @@ public MachObjectWriter(NodeFactory factory, ObjectWritingOptions options)
             _targetOS = factory.Target.OperatingSystem;
         }
 
+        public MachObjectWriter(NodeFactory factory, ObjectWritingOptions options, OutputInfoBuilder outputInfoBuilder, string baseSymbolName)
+            : this(factory, options, outputInfoBuilder)
+        {
+            _baseSymbolName = baseSymbolName;
+        }
+
         private protected override bool UsesSubsectionsViaSymbols => true;
 
         private protected override void EmitSectionsAndLayout()
@@ -372,6 +384,16 @@ protected internal override void UpdateSectionAlignment(int sectionIndex, int al
             machSection.Log2Alignment = Math.Max(machSection.Log2Alignment, (uint)BitOperations.Log2((uint)alignment));
         }
 
+        private protected override void EmitSymbolRangeDefinition(string rangeNodeName, string startSymbolName, string endSymbolName)
+        {
+            // Mach has a few characteristics that make range symbols more difficult to emit:
+            // - Emitting two symbols in the same location is not well supported by the Apple linker.
+            // - Mach-O makes it more difficult to preserve symbol layout in the way we like.
+            // However, it also has one thing that makes it easier:
+            // - Mach-O natively supports a relocation type that can represents the distance between two symbols.
+            _rangeSymbols.Add(rangeNodeName, (startSymbolName, endSymbolName));
+        }
+
         protected internal override unsafe void EmitRelocation(
             int sectionIndex,
             long offset,
@@ -380,6 +402,15 @@ protected internal override unsafe void EmitRelocation(
             string symbolName,
             long addend)
         {
+            // We don't emit the range node name into the image as it overlaps with another symbol.
+            // For relocs to it, instead target the start symbol.
+            // For the "symbol size" reloc, we'll handle it later when we emit relocations in the Mach format.
+            if (relocType != RelocType.IMAGE_REL_SYMBOL_SIZE
+                && _rangeSymbols.TryGetValue(symbolName, out (string StartNode, string) range))
+            {
+                symbolName = range.StartNode;
+            }
+
             // Mach-O doesn't use relocations between DWARF sections, so embed the offsets directly
             if (relocType is IMAGE_REL_BASED_DIR64 or IMAGE_REL_BASED_HIGHLOW &&
                 _sections[sectionIndex].IsDwarfSection)
@@ -501,6 +532,13 @@ private protected override void EmitSymbolTable(
             _dySymbolTable.ExternalSymbolsCount = (uint)definedSymbols.Count;
 
             uint savedSymbolIndex = symbolIndex;
+
+            // Add the base symbol as an undefined symbol.
+            if (_baseSymbolName is not null)
+            {
+                undefinedSymbols.Add(_baseSymbolName);
+            }
+
             foreach (string externSymbol in undefinedSymbols)
             {
                 if (!_symbolNameToIndex.ContainsKey(externSymbol))
@@ -548,6 +586,36 @@ private void EmitRelocationsX64(int sectionIndex, List<SymbolicRelocation> reloc
             relocationList.Reverse();
             foreach (SymbolicRelocation symbolicRelocation in relocationList)
             {
+                if (symbolicRelocation.Type == RelocType.IMAGE_REL_SYMBOL_SIZE
+                    && _rangeSymbols.TryGetValue(symbolicRelocation.SymbolName, out (string, string) range))
+                {
+                    // Represent as X86_64_RELOC_SUBTRACTOR + X86_64_RELOC_UNSIGNED.
+                    (string StartNode, string EndNode) = range;
+                    uint startSymbolIndex = _symbolNameToIndex[StartNode];
+                    uint endSymbolIndex = _symbolNameToIndex[EndNode];
+                    sectionRelocations.Add(
+                        new MachRelocation
+                        {
+                            Address = (int)symbolicRelocation.Offset,
+                            SymbolOrSectionIndex = endSymbolIndex,
+                            Length = 4,
+                            RelocationType = X86_64_RELOC_SUBTRACTOR,
+                            IsExternal = true,
+                            IsPCRelative = false,
+                        });
+                    sectionRelocations.Add(
+                        new MachRelocation
+                        {
+                            Address = (int)symbolicRelocation.Offset,
+                            SymbolOrSectionIndex = startSymbolIndex,
+                            Length = 4,
+                            RelocationType = X86_64_RELOC_UNSIGNED,
+                            IsExternal = true,
+                            IsPCRelative = false,
+                        });
+                    continue;
+                }
+
                 uint symbolIndex = _symbolNameToIndex[symbolicRelocation.SymbolName];
 
                 if (symbolicRelocation.Type == IMAGE_REL_BASED_DIR64)
@@ -600,6 +668,37 @@ private void EmitRelocationsX64(int sectionIndex, List<SymbolicRelocation> reloc
                             IsPCRelative = true,
                         });
                 }
+                else if (symbolicRelocation.Type == IMAGE_REL_BASED_ADDR32NB)
+                {
+                    if (_baseSymbolName is null)
+                    {
+                        throw new NotSupportedException("A base symbol name must be provided for IMAGE_REL_BASED_ADDR32NB relocations.");
+                    }
+
+                    Debug.Assert(_symbolNameToIndex.ContainsKey(_baseSymbolName));
+
+                    // Represent as X86_64_RELOC_SUBTRACTOR + X86_64_RELOC_UNSIGNED against the base symbol.
+                    sectionRelocations.Add(
+                        new MachRelocation
+                        {
+                            Address = (int)symbolicRelocation.Offset,
+                            SymbolOrSectionIndex = (uint)symbolIndex,
+                            Length = 4,
+                            RelocationType = X86_64_RELOC_SUBTRACTOR,
+                            IsExternal = true,
+                            IsPCRelative = false,
+                        });
+                    sectionRelocations.Add(
+                        new MachRelocation
+                        {
+                            Address = (int)symbolicRelocation.Offset,
+                            SymbolOrSectionIndex = _symbolNameToIndex[_baseSymbolName],
+                            Length = 4,
+                            RelocationType = X86_64_RELOC_UNSIGNED,
+                            IsExternal = true,
+                            IsPCRelative = false,
+                        });
+                }
                 else
                 {
                     throw new NotSupportedException("Unknown relocation type: " + symbolicRelocation.Type);
@@ -614,6 +713,36 @@ private void EmitRelocationsArm64(int sectionIndex, List<SymbolicRelocation> rel
             relocationList.Reverse();
             foreach (SymbolicRelocation symbolicRelocation in relocationList)
             {
+                if (symbolicRelocation.Type == RelocType.IMAGE_REL_SYMBOL_SIZE
+                    && _rangeSymbols.TryGetValue(symbolicRelocation.SymbolName, out (string, string) range))
+                {
+                    // Represent as ARM64_RELOC_SUBTRACTOR + ARM64_RELOC_UNSIGNED.
+                    (string StartNode, string EndNode) = range;
+                    uint startSymbolIndex = _symbolNameToIndex[StartNode];
+                    uint endSymbolIndex = _symbolNameToIndex[EndNode];
+                    sectionRelocations.Add(
+                        new MachRelocation
+                        {
+                            Address = (int)symbolicRelocation.Offset,
+                            SymbolOrSectionIndex = endSymbolIndex,
+                            Length = 4,
+                            RelocationType = ARM64_RELOC_SUBTRACTOR,
+                            IsExternal = true,
+                            IsPCRelative = false,
+                        });
+                    sectionRelocations.Add(
+                        new MachRelocation
+                        {
+                            Address = (int)symbolicRelocation.Offset,
+                            SymbolOrSectionIndex = startSymbolIndex,
+                            Length = 4,
+                            RelocationType = ARM64_RELOC_UNSIGNED,
+                            IsExternal = true,
+                            IsPCRelative = false,
+                        });
+                    continue;
+                }
+
                 uint symbolIndex = _symbolNameToIndex[symbolicRelocation.SymbolName];
 
                 if (symbolicRelocation.Type == IMAGE_REL_BASED_ARM64_BRANCH26)
@@ -701,6 +830,37 @@ private void EmitRelocationsArm64(int sectionIndex, List<SymbolicRelocation> rel
                             IsPCRelative = false,
                         });
                 }
+                else if (symbolicRelocation.Type == IMAGE_REL_BASED_ADDR32NB)
+                {
+                    if (_baseSymbolName is null)
+                    {
+                        throw new NotSupportedException("A base symbol name must be provided for IMAGE_REL_BASED_ADDR32NB relocations.");
+                    }
+
+                    Debug.Assert(_symbolNameToIndex.ContainsKey(_baseSymbolName));
+
+                    // Represent as ARM64_RELOC_SUBTRACTOR + ARM64_RELOC_UNSIGNED against the base symbol.
+                    sectionRelocations.Add(
+                        new MachRelocation
+                        {
+                            Address = (int)symbolicRelocation.Offset,
+                            SymbolOrSectionIndex = (uint)symbolIndex,
+                            Length = 4,
+                            RelocationType = ARM64_RELOC_SUBTRACTOR,
+                            IsExternal = true,
+                            IsPCRelative = false,
+                        });
+                    sectionRelocations.Add(
+                        new MachRelocation
+                        {
+                            Address = (int)symbolicRelocation.Offset,
+                            SymbolOrSectionIndex = _symbolNameToIndex[_baseSymbolName],
+                            Length = 4,
+                            RelocationType = ARM64_RELOC_UNSIGNED,
+                            IsExternal = true,
+                            IsPCRelative = false,
+                        });
+                }
                 else
                 {
                     throw new NotSupportedException("Unknown relocation type: " + symbolicRelocation.Type);

src/coreclr/tools/Common/Compiler/ObjectWriter/ObjectWriter.cs

@@ -506,19 +506,7 @@ public void EmitObject(Stream outputFileStream, IReadOnlyCollection<DependencyNo
 
                 string rangeNodeName = GetMangledName(range);
 
-                if (!_definedSymbols.TryGetValue(startNodeName, out var startSymbol)
-                    || !_definedSymbols.TryGetValue(endNodeName, out var endSymbol))
-                {
-                    throw new InvalidOperationException("The symbols defined by a symbol range must be emitted into the same object.");
-                }
-
-                if (startSymbol.SectionIndex != endSymbol.SectionIndex)
-                {
-                    throw new InvalidOperationException("The symbols that define a symbol range must be in the same section.");
-                }
-
-                // Don't use SectionWriter here as it emits symbols relative to the current writing position.
-                EmitSymbolDefinition(startSymbol.SectionIndex, rangeNodeName, startSymbol.Value, checked((int)(endSymbol.Value - startSymbol.Value)));
+                EmitSymbolRangeDefinition(rangeNodeName, startNodeName, endNodeName);
             }
 
             foreach (BlockToRelocate blockToRelocate in blocksToRelocate)
@@ -591,6 +579,22 @@ public void EmitObject(Stream outputFileStream, IReadOnlyCollection<DependencyNo
             }
         }
 
+        private protected virtual void EmitSymbolRangeDefinition(string rangeNodeName, string startNodeName, string endNodeName)
+        {
+            if (!_definedSymbols.TryGetValue(startNodeName, out var startSymbol)
+                    || !_definedSymbols.TryGetValue(endNodeName, out var endSymbol))
+            {
+                throw new InvalidOperationException("The symbols defined by a symbol range must be emitted into the same object.");
+            }
+
+            if (startSymbol.SectionIndex != endSymbol.SectionIndex)
+            {
+                throw new InvalidOperationException("The symbols that define a symbol range must be in the same section.");
+            }
+            // Don't use SectionWriter here as it emits symbols relative to the current writing position.
+            EmitSymbolDefinition(startSymbol.SectionIndex, rangeNodeName, startSymbol.Value, checked((int)(endSymbol.Value - startSymbol.Value)));
+        }
+
         private static string GetNodeTypeName(Type nodeType)
         {
             string name = nodeType.ToString();

src/coreclr/tools/Common/Compiler/ObjectWriter/UnixObjectWriter.cs

@@ -25,8 +25,8 @@ private sealed record UnixSectionDefinition(string SymbolName, Stream SectionStr
         private static readonly ObjectNodeSection LsdaSection = new ObjectNodeSection(".dotnet_eh_table", SectionType.ReadOnly);
         private static readonly ObjectNodeSection EhFrameSection = new ObjectNodeSection(".eh_frame", SectionType.UnwindData);
 
-        protected UnixObjectWriter(NodeFactory factory, ObjectWritingOptions options)
-            : base(factory, options)
+        protected UnixObjectWriter(NodeFactory factory, ObjectWritingOptions options, OutputInfoBuilder outputInfoBuilder = null)
+            : base(factory, options, outputInfoBuilder)
         {
         }
 

src/coreclr/tools/Common/Internal/Runtime/ModuleHeaders.cs

@@ -16,7 +16,7 @@ internal struct ReadyToRunHeaderConstants
         public const uint Signature = 0x00525452; // 'RTR'
 
         public const ushort CurrentMajorVersion = 16;
-        public const ushort CurrentMinorVersion = 0;
+        public const ushort CurrentMinorVersion = 1;
     }
 #if READYTORUN
 #pragma warning disable 0169

src/coreclr/tools/Common/Internal/Runtime/ReadyToRunConstants.cs

@@ -17,8 +17,9 @@ public enum ReadyToRunFlags
         READYTORUN_FLAG_NonSharedPInvokeStubs = 0x00000008,     // PInvoke stubs compiled into image are non-shareable (no secret parameter)
         READYTORUN_FLAG_EmbeddedMSIL = 0x00000010,              // MSIL is embedded in the composite R2R executable
         READYTORUN_FLAG_Component = 0x00000020,                 // This is the header describing a component assembly of composite R2R
-        READYTORUN_FLAG_MultiModuleVersionBubble = 0x00000040,   // This R2R module has multiple modules within its version bubble
+        READYTORUN_FLAG_MultiModuleVersionBubble = 0x00000040,  // This R2R module has multiple modules within its version bubble
         READYTORUN_FLAG_UnrelatedR2RCode = 0x00000080,          // This R2R module has generic code in it that would not be naturally encoded into this module
+        READYTORUN_FLAG_PlatformNativeImage = 0x00000100,       // The owning composite executable is in the platform native format
     }
 
     public enum ReadyToRunImportSectionType : byte

src/coreclr/tools/aot/ILCompiler.ReadyToRun/CodeGen/ReadyToRunContainerFormat.cs

@@ -9,6 +9,7 @@ namespace ILCompiler.DependencyAnalysis
 {
     public enum ReadyToRunContainerFormat
     {
-        PE
+        PE,
+        MachO,
     }
 }