1//===------------- JITLink.cpp - Core Run-time JIT linker APIs ------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/ExecutionEngine/JITLink/JITLink.h"
10
11#include "llvm/ADT/StringExtras.h"
12#include "llvm/BinaryFormat/Magic.h"
13#include "llvm/ExecutionEngine/JITLink/COFF.h"
14#include "llvm/ExecutionEngine/JITLink/ELF.h"
15#include "llvm/ExecutionEngine/JITLink/MachO.h"
16#include "llvm/ExecutionEngine/JITLink/aarch64.h"
17#include "llvm/ExecutionEngine/JITLink/i386.h"
18#include "llvm/ExecutionEngine/JITLink/loongarch.h"
19#include "llvm/ExecutionEngine/JITLink/x86_64.h"
20#include "llvm/Support/Format.h"
21#include "llvm/Support/MemoryBuffer.h"
22#include "llvm/Support/raw_ostream.h"
23
24using namespace llvm;
25using namespace llvm::object;
26
27#define DEBUG_TYPE "jitlink"
28
29namespace {
30
31enum JITLinkErrorCode { GenericJITLinkError = 1 };
32
33// FIXME: This class is only here to support the transition to llvm::Error. It
34// will be removed once this transition is complete. Clients should prefer to
35// deal with the Error value directly, rather than converting to error_code.
36class JITLinkerErrorCategory : public std::error_category {
37public:
38  const char *name() const noexcept override { return "runtimedyld"; }
39
40  std::string message(int Condition) const override {
41    switch (static_cast<JITLinkErrorCode>(Condition)) {
42    case GenericJITLinkError:
43      return "Generic JITLink error";
44    }
45    llvm_unreachable("Unrecognized JITLinkErrorCode");
46  }
47};
48
49} // namespace
50
51namespace llvm {
52namespace jitlink {
53
54char JITLinkError::ID = 0;
55
56void JITLinkError::log(raw_ostream &OS) const { OS << ErrMsg; }
57
58std::error_code JITLinkError::convertToErrorCode() const {
59  static JITLinkerErrorCategory TheJITLinkerErrorCategory;
60  return std::error_code(GenericJITLinkError, TheJITLinkerErrorCategory);
61}
62
63const char *getGenericEdgeKindName(Edge::Kind K) {
64  switch (K) {
65  case Edge::Invalid:
66    return "INVALID RELOCATION";
67  case Edge::KeepAlive:
68    return "Keep-Alive";
69  default:
70    return "<Unrecognized edge kind>";
71  }
72}
73
74const char *getLinkageName(Linkage L) {
75  switch (L) {
76  case Linkage::Strong:
77    return "strong";
78  case Linkage::Weak:
79    return "weak";
80  }
81  llvm_unreachable("Unrecognized llvm.jitlink.Linkage enum");
82}
83
84const char *getScopeName(Scope S) {
85  switch (S) {
86  case Scope::Default:
87    return "default";
88  case Scope::Hidden:
89    return "hidden";
90  case Scope::Local:
91    return "local";
92  }
93  llvm_unreachable("Unrecognized llvm.jitlink.Scope enum");
94}
95
96bool isCStringBlock(Block &B) {
97  if (B.getSize() == 0) // Empty blocks are not valid C-strings.
98    return false;
99
100  // Zero-fill blocks of size one are valid empty strings.
101  if (B.isZeroFill())
102    return B.getSize() == 1;
103
104  for (size_t I = 0; I != B.getSize() - 1; ++I)
105    if (B.getContent()[I] == '\0')
106      return false;
107
108  return B.getContent()[B.getSize() - 1] == '\0';
109}
110
111raw_ostream &operator<<(raw_ostream &OS, const Block &B) {
112  return OS << B.getAddress() << " -- " << (B.getAddress() + B.getSize())
113            << ": "
114            << "size = " << formatv("{0:x8}", B.getSize()) << ", "
115            << (B.isZeroFill() ? "zero-fill" : "content")
116            << ", align = " << B.getAlignment()
117            << ", align-ofs = " << B.getAlignmentOffset()
118            << ", section = " << B.getSection().getName();
119}
120
121raw_ostream &operator<<(raw_ostream &OS, const Symbol &Sym) {
122  OS << Sym.getAddress() << " (" << (Sym.isDefined() ? "block" : "addressable")
123     << " + " << formatv("{0:x8}", Sym.getOffset())
124     << "): size: " << formatv("{0:x8}", Sym.getSize())
125     << ", linkage: " << formatv("{0:6}", getLinkageName(Sym.getLinkage()))
126     << ", scope: " << formatv("{0:8}", getScopeName(Sym.getScope())) << ", "
127     << (Sym.isLive() ? "live" : "dead") << "  -   "
128     << (Sym.hasName() ? Sym.getName() : "<anonymous symbol>");
129  return OS;
130}
131
132void printEdge(raw_ostream &OS, const Block &B, const Edge &E,
133               StringRef EdgeKindName) {
134  OS << "edge@" << B.getAddress() + E.getOffset() << ": " << B.getAddress()
135     << " + " << formatv("{0:x}", E.getOffset()) << " -- " << EdgeKindName
136     << " -> ";
137
138  auto &TargetSym = E.getTarget();
139  if (TargetSym.hasName())
140    OS << TargetSym.getName();
141  else {
142    auto &TargetBlock = TargetSym.getBlock();
143    auto &TargetSec = TargetBlock.getSection();
144    orc::ExecutorAddr SecAddress(~uint64_t(0));
145    for (auto *B : TargetSec.blocks())
146      if (B->getAddress() < SecAddress)
147        SecAddress = B->getAddress();
148
149    orc::ExecutorAddrDiff SecDelta = TargetSym.getAddress() - SecAddress;
150    OS << TargetSym.getAddress() << " (section " << TargetSec.getName();
151    if (SecDelta)
152      OS << " + " << formatv("{0:x}", SecDelta);
153    OS << " / block " << TargetBlock.getAddress();
154    if (TargetSym.getOffset())
155      OS << " + " << formatv("{0:x}", TargetSym.getOffset());
156    OS << ")";
157  }
158
159  if (E.getAddend() != 0)
160    OS << " + " << E.getAddend();
161}
162
163Section::~Section() {
164  for (auto *Sym : Symbols)
165    Sym->~Symbol();
166  for (auto *B : Blocks)
167    B->~Block();
168}
169
170Block &LinkGraph::splitBlock(Block &B, size_t SplitIndex,
171                             SplitBlockCache *Cache) {
172
173  assert(SplitIndex > 0 && "splitBlock can not be called with SplitIndex == 0");
174
175  // If the split point covers all of B then just return B.
176  if (SplitIndex == B.getSize())
177    return B;
178
179  assert(SplitIndex < B.getSize() && "SplitIndex out of range");
180
181  // Create the new block covering [ 0, SplitIndex ).
182  auto &NewBlock =
183      B.isZeroFill()
184          ? createZeroFillBlock(B.getSection(), SplitIndex, B.getAddress(),
185                                B.getAlignment(), B.getAlignmentOffset())
186          : createContentBlock(
187                B.getSection(), B.getContent().slice(0, SplitIndex),
188                B.getAddress(), B.getAlignment(), B.getAlignmentOffset());
189
190  // Modify B to cover [ SplitIndex, B.size() ).
191  B.setAddress(B.getAddress() + SplitIndex);
192  B.setContent(B.getContent().slice(SplitIndex));
193  B.setAlignmentOffset((B.getAlignmentOffset() + SplitIndex) %
194                       B.getAlignment());
195
196  // Handle edge transfer/update.
197  {
198    // Copy edges to NewBlock (recording their iterators so that we can remove
199    // them from B), and update of Edges remaining on B.
200    std::vector<Block::edge_iterator> EdgesToRemove;
201    for (auto I = B.edges().begin(); I != B.edges().end();) {
202      if (I->getOffset() < SplitIndex) {
203        NewBlock.addEdge(*I);
204        I = B.removeEdge(I);
205      } else {
206        I->setOffset(I->getOffset() - SplitIndex);
207        ++I;
208      }
209    }
210  }
211
212  // Handle symbol transfer/update.
213  {
214    // Initialize the symbols cache if necessary.
215    SplitBlockCache LocalBlockSymbolsCache;
216    if (!Cache)
217      Cache = &LocalBlockSymbolsCache;
218    if (*Cache == std::nullopt) {
219      *Cache = SplitBlockCache::value_type();
220      for (auto *Sym : B.getSection().symbols())
221        if (&Sym->getBlock() == &B)
222          (*Cache)->push_back(Sym);
223
224      llvm::sort(**Cache, [](const Symbol *LHS, const Symbol *RHS) {
225        return LHS->getOffset() > RHS->getOffset();
226      });
227    }
228    auto &BlockSymbols = **Cache;
229
230    // Transfer all symbols with offset less than SplitIndex to NewBlock.
231    while (!BlockSymbols.empty() &&
232           BlockSymbols.back()->getOffset() < SplitIndex) {
233      auto *Sym = BlockSymbols.back();
234      // If the symbol extends beyond the split, update the size to be within
235      // the new block.
236      if (Sym->getOffset() + Sym->getSize() > SplitIndex)
237        Sym->setSize(SplitIndex - Sym->getOffset());
238      Sym->setBlock(NewBlock);
239      BlockSymbols.pop_back();
240    }
241
242    // Update offsets for all remaining symbols in B.
243    for (auto *Sym : BlockSymbols)
244      Sym->setOffset(Sym->getOffset() - SplitIndex);
245  }
246
247  return NewBlock;
248}
249
250void LinkGraph::dump(raw_ostream &OS) {
251  DenseMap<Block *, std::vector<Symbol *>> BlockSymbols;
252
253  // Map from blocks to the symbols pointing at them.
254  for (auto *Sym : defined_symbols())
255    BlockSymbols[&Sym->getBlock()].push_back(Sym);
256
257  // For each block, sort its symbols by something approximating
258  // relevance.
259  for (auto &KV : BlockSymbols)
260    llvm::sort(KV.second, [](const Symbol *LHS, const Symbol *RHS) {
261      if (LHS->getOffset() != RHS->getOffset())
262        return LHS->getOffset() < RHS->getOffset();
263      if (LHS->getLinkage() != RHS->getLinkage())
264        return LHS->getLinkage() < RHS->getLinkage();
265      if (LHS->getScope() != RHS->getScope())
266        return LHS->getScope() < RHS->getScope();
267      if (LHS->hasName()) {
268        if (!RHS->hasName())
269          return true;
270        return LHS->getName() < RHS->getName();
271      }
272      return false;
273    });
274
275  for (auto &Sec : sections()) {
276    OS << "section " << Sec.getName() << ":\n\n";
277
278    std::vector<Block *> SortedBlocks;
279    llvm::copy(Sec.blocks(), std::back_inserter(SortedBlocks));
280    llvm::sort(SortedBlocks, [](const Block *LHS, const Block *RHS) {
281      return LHS->getAddress() < RHS->getAddress();
282    });
283
284    for (auto *B : SortedBlocks) {
285      OS << "  block " << B->getAddress()
286         << " size = " << formatv("{0:x8}", B->getSize())
287         << ", align = " << B->getAlignment()
288         << ", alignment-offset = " << B->getAlignmentOffset();
289      if (B->isZeroFill())
290        OS << ", zero-fill";
291      OS << "\n";
292
293      auto BlockSymsI = BlockSymbols.find(B);
294      if (BlockSymsI != BlockSymbols.end()) {
295        OS << "    symbols:\n";
296        auto &Syms = BlockSymsI->second;
297        for (auto *Sym : Syms)
298          OS << "      " << *Sym << "\n";
299      } else
300        OS << "    no symbols\n";
301
302      if (!B->edges_empty()) {
303        OS << "    edges:\n";
304        std::vector<Edge> SortedEdges;
305        llvm::copy(B->edges(), std::back_inserter(SortedEdges));
306        llvm::sort(SortedEdges, [](const Edge &LHS, const Edge &RHS) {
307          return LHS.getOffset() < RHS.getOffset();
308        });
309        for (auto &E : SortedEdges) {
310          OS << "      " << B->getFixupAddress(E) << " (block + "
311             << formatv("{0:x8}", E.getOffset()) << "), addend = ";
312          if (E.getAddend() >= 0)
313            OS << formatv("+{0:x8}", E.getAddend());
314          else
315            OS << formatv("-{0:x8}", -E.getAddend());
316          OS << ", kind = " << getEdgeKindName(E.getKind()) << ", target = ";
317          if (E.getTarget().hasName())
318            OS << E.getTarget().getName();
319          else
320            OS << "addressable@"
321               << formatv("{0:x16}", E.getTarget().getAddress()) << "+"
322               << formatv("{0:x8}", E.getTarget().getOffset());
323          OS << "\n";
324        }
325      } else
326        OS << "    no edges\n";
327      OS << "\n";
328    }
329  }
330
331  OS << "Absolute symbols:\n";
332  if (!absolute_symbols().empty()) {
333    for (auto *Sym : absolute_symbols())
334      OS << "  " << Sym->getAddress() << ": " << *Sym << "\n";
335  } else
336    OS << "  none\n";
337
338  OS << "\nExternal symbols:\n";
339  if (!external_symbols().empty()) {
340    for (auto *Sym : external_symbols())
341      OS << "  " << Sym->getAddress() << ": " << *Sym << "\n";
342  } else
343    OS << "  none\n";
344}
345
346raw_ostream &operator<<(raw_ostream &OS, const SymbolLookupFlags &LF) {
347  switch (LF) {
348  case SymbolLookupFlags::RequiredSymbol:
349    return OS << "RequiredSymbol";
350  case SymbolLookupFlags::WeaklyReferencedSymbol:
351    return OS << "WeaklyReferencedSymbol";
352  }
353  llvm_unreachable("Unrecognized lookup flags");
354}
355
356void JITLinkAsyncLookupContinuation::anchor() {}
357
358JITLinkContext::~JITLinkContext() = default;
359
360bool JITLinkContext::shouldAddDefaultTargetPasses(const Triple &TT) const {
361  return true;
362}
363
364LinkGraphPassFunction JITLinkContext::getMarkLivePass(const Triple &TT) const {
365  return LinkGraphPassFunction();
366}
367
368Error JITLinkContext::modifyPassConfig(LinkGraph &G,
369                                       PassConfiguration &Config) {
370  return Error::success();
371}
372
373Error markAllSymbolsLive(LinkGraph &G) {
374  for (auto *Sym : G.defined_symbols())
375    Sym->setLive(true);
376  return Error::success();
377}
378
379Error makeTargetOutOfRangeError(const LinkGraph &G, const Block &B,
380                                const Edge &E) {
381  std::string ErrMsg;
382  {
383    raw_string_ostream ErrStream(ErrMsg);
384    Section &Sec = B.getSection();
385    ErrStream << "In graph " << G.getName() << ", section " << Sec.getName()
386              << ": relocation target ";
387    if (E.getTarget().hasName()) {
388      ErrStream << "\"" << E.getTarget().getName() << "\"";
389    } else
390      ErrStream << E.getTarget().getBlock().getSection().getName() << " + "
391                << formatv("{0:x}", E.getOffset());
392    ErrStream << " at address " << formatv("{0:x}", E.getTarget().getAddress())
393              << " is out of range of " << G.getEdgeKindName(E.getKind())
394              << " fixup at " << formatv("{0:x}", B.getFixupAddress(E)) << " (";
395
396    Symbol *BestSymbolForBlock = nullptr;
397    for (auto *Sym : Sec.symbols())
398      if (&Sym->getBlock() == &B && Sym->hasName() && Sym->getOffset() == 0 &&
399          (!BestSymbolForBlock ||
400           Sym->getScope() < BestSymbolForBlock->getScope() ||
401           Sym->getLinkage() < BestSymbolForBlock->getLinkage()))
402        BestSymbolForBlock = Sym;
403
404    if (BestSymbolForBlock)
405      ErrStream << BestSymbolForBlock->getName() << ", ";
406    else
407      ErrStream << "<anonymous block> @ ";
408
409    ErrStream << formatv("{0:x}", B.getAddress()) << " + "
410              << formatv("{0:x}", E.getOffset()) << ")";
411  }
412  return make_error<JITLinkError>(std::move(ErrMsg));
413}
414
415Error makeAlignmentError(llvm::orc::ExecutorAddr Loc, uint64_t Value, int N,
416                         const Edge &E) {
417  return make_error<JITLinkError>("0x" + llvm::utohexstr(Loc.getValue()) +
418                                  " improper alignment for relocation " +
419                                  formatv("{0:d}", E.getKind()) + ": 0x" +
420                                  llvm::utohexstr(Value) +
421                                  " is not aligned to " + Twine(N) + " bytes");
422}
423
424AnonymousPointerCreator getAnonymousPointerCreator(const Triple &TT) {
425  switch (TT.getArch()) {
426  case Triple::aarch64:
427    return aarch64::createAnonymousPointer;
428  case Triple::x86_64:
429    return x86_64::createAnonymousPointer;
430  case Triple::x86:
431    return i386::createAnonymousPointer;
432  case Triple::loongarch32:
433  case Triple::loongarch64:
434    return loongarch::createAnonymousPointer;
435  default:
436    return nullptr;
437  }
438}
439
440PointerJumpStubCreator getPointerJumpStubCreator(const Triple &TT) {
441  switch (TT.getArch()) {
442  case Triple::aarch64:
443    return aarch64::createAnonymousPointerJumpStub;
444  case Triple::x86_64:
445    return x86_64::createAnonymousPointerJumpStub;
446  case Triple::x86:
447    return i386::createAnonymousPointerJumpStub;
448  case Triple::loongarch32:
449  case Triple::loongarch64:
450    return loongarch::createAnonymousPointerJumpStub;
451  default:
452    return nullptr;
453  }
454}
455
456Expected<std::unique_ptr<LinkGraph>>
457createLinkGraphFromObject(MemoryBufferRef ObjectBuffer) {
458  auto Magic = identify_magic(ObjectBuffer.getBuffer());
459  switch (Magic) {
460  case file_magic::macho_object:
461    return createLinkGraphFromMachOObject(ObjectBuffer);
462  case file_magic::elf_relocatable:
463    return createLinkGraphFromELFObject(ObjectBuffer);
464  case file_magic::coff_object:
465    return createLinkGraphFromCOFFObject(ObjectBuffer);
466  default:
467    return make_error<JITLinkError>("Unsupported file format");
468  };
469}
470
471std::unique_ptr<LinkGraph> absoluteSymbolsLinkGraph(const Triple &TT,
472                                                    orc::SymbolMap Symbols) {
473  unsigned PointerSize;
474  endianness Endianness =
475      TT.isLittleEndian() ? endianness::little : endianness::big;
476  switch (TT.getArch()) {
477  case Triple::aarch64:
478  case llvm::Triple::riscv64:
479  case Triple::x86_64:
480    PointerSize = 8;
481    break;
482  case llvm::Triple::arm:
483  case llvm::Triple::riscv32:
484  case llvm::Triple::x86:
485    PointerSize = 4;
486    break;
487  default:
488    llvm::report_fatal_error("unhandled target architecture");
489  }
490
491  static std::atomic<uint64_t> Counter = {0};
492  auto Index = Counter.fetch_add(1, std::memory_order_relaxed);
493  auto G = std::make_unique<LinkGraph>(
494      "<Absolute Symbols " + std::to_string(Index) + ">", TT, PointerSize,
495      Endianness, /*GetEdgeKindName=*/nullptr);
496  for (auto &[Name, Def] : Symbols) {
497    auto &Sym =
498        G->addAbsoluteSymbol(*Name, Def.getAddress(), /*Size=*/0,
499                             Linkage::Strong, Scope::Default, /*IsLive=*/true);
500    Sym.setCallable(Def.getFlags().isCallable());
501  }
502
503  return G;
504}
505
506void link(std::unique_ptr<LinkGraph> G, std::unique_ptr<JITLinkContext> Ctx) {
507  switch (G->getTargetTriple().getObjectFormat()) {
508  case Triple::MachO:
509    return link_MachO(std::move(G), std::move(Ctx));
510  case Triple::ELF:
511    return link_ELF(std::move(G), std::move(Ctx));
512  case Triple::COFF:
513    return link_COFF(std::move(G), std::move(Ctx));
514  default:
515    Ctx->notifyFailed(make_error<JITLinkError>("Unsupported object format"));
516  };
517}
518
519} // end namespace jitlink
520} // end namespace llvm
521