Fix evaluation of expressions containing forward references

This commit fixes evaluation of expressions containing forward
references. At a high-level, eld evaluates assignments in sequence
and consequently the expressions containing forward references needs to
be reevaluated when the forward reference value is computed for the correct
computation results. For example:

  u = v1;    // Assignment 1
  v1 = v2;   // Assignment 2
  v2 = 0xa;  // Assignment 3

eld evaluates the assignments in order, that is, the evaluation happens
as: [Assignment 1, Assignment 2, Assignment 3]. If we follow this
evaluation order, the symbols `u` and `v1` will have incorrect values
because the value of `v2` is unknown when the assignments of `u` and
`v1` are evaluated.

This commit fixes evaluation of expressions containing forward
references by making the below two changes:

1) Assignments which cannot be completely evaluated during the
   sequential evaluation of expressions are marked as pending
   assignments. After the layout is done, but before the relaxation,
   these pending assignments are recursively reevaluated until there
   is no improvement in pending assignments or a MaxIteration limit is
   reached.

2) During a layout iteration, assignments may get evaluated multiple times
   as layout needs to reset itself based on a few conditions
   (new segment needs to be created, and so on...). It is important to reset
   the assignments and the symbol values if a layout gets reset. Let's
   see why it is important with the help of an example:

   SECTIONS {
     FOO : {
       u = v; // Assignment 1
       v = 0xa; // Assignment 2
       *(.text.foo)
     }
     BAR : {
       v = 0xb; // Assignment 3
     }
     v = 0xc; // Assignment 4
  }

  The sequential assignment evaluation order is: [A1, A2, A3, A4]. When
  A1 is evaluated, `v` is not defined, hence we mark A1 as pending
  assignment. However, if the layout gets reset after evaluating A2,
  then A1 will be evaluated again, but this time, `v` is defined (from
  assignment 2 evaluation) and thus A1 can be completely evaluated.
  This is wrong because A1 should get the value from the assignment 4
  instead of assignment 2!

  We fix this issue by resetting the assignments and the symbol values
  whenever a layout is reset.

  The same issue happens when the layout needs to be recomputed after a
  relaxation pass. And the same solution of resetting the assignments and
  the symbol values works for this case as well.

This commit also adds a new trace category 'pending-assignments' for
tracing pending assignment evaluation.

Closes #169

Signed-off-by: Parth Arora <[email protected]>

Author: parth-07

include/eld/Diagnostics/DiagVerbose.inc

@@ -163,3 +163,9 @@ DIAG(verbose_linker_script_output_file, DiagnosticEngine::Verbose,
      "Using output file name '%0' specified in the linker script '%1'")
 DIAG(verbose_rule_matching_cache_func_hash, DiagnosticEngine::Verbose,
      "Rule-matching cache-functionailty hash: %0")
+DIAG(verbose_performing_layout_iteration, DiagnosticEngine::Verbose,
+     "Performing layout iteration %0")
+DIAG(verbose_eval_pending_assignments, DiagnosticEngine::Verbose,
+     "Evaluating pending assignments")
+DIAG(verbose_reset_tracked_assignment, DiagnosticEngine::Verbose,
+     "Resetting tracked assignments")

include/eld/Diagnostics/DiagnosticPrinter.h

@@ -47,6 +47,7 @@ class DiagnosticPrinter {
     TraceMergeStrings = 0x8000,
     TraceLinkerScript = 0x10000,
     TraceSymDef = 0x100000,
+    TracePendingAssignments = 0x200000
   };
 
   enum Verbose : uint16_t { None = 0x0, Default = 0x1 };
@@ -68,6 +69,8 @@ class DiagnosticPrinter {
 
   bool traceAssignments() { return Trace & TraceAssignments; }
 
+  bool tracePendingAssignments() { return Trace & TracePendingAssignments; }
+
   bool traceFiles() { return Trace & TraceFiles; }
 
   bool traceSymbols() { return Trace & TraceSymbols; }

include/eld/Driver/GnuLinkerOptions.td

@@ -658,7 +658,8 @@ defm trace
           "\t\t\t --trace=trampolines : trace trampolines\n"
           "\t\t\t --trace=wrap-symbols : trace symbol wrap options\n"
           "\t\t\t --trace=symdef : trace symbol resolution from symdef files\n"
-          "\t\t\t --trace=dynamic-linking : trace dynamic linking">,
+          "\t\t\t --trace=dynamic-linking : trace dynamic linking\n"
+          "\t\t\t --trace=pending-assignments : trace pending symbol assignments evaluation">,
       MetaVarName<"<trace-type>">,
       Group<grp_diagopts>;
 defm trace_symbol : smDashTwoWithOpt<"y", "trace-symbol", "trace_symbol",

include/eld/Script/Assignment.h

@@ -83,7 +83,8 @@ class Assignment : public ScriptCommand {
   eld::Expected<void> activate(Module &CurModule) override;
 
   /// assign - evaluate the rhs and assign the result to lhs.
-  bool assign(Module &CurModule, const ELFSection *Section);
+  bool assign(Module &CurModule, const ELFSection *Section,
+              bool EvaluatePendingOnly);
 
   LDSymbol *symbol() const { return ThisSymbol; }
 
@@ -128,6 +129,9 @@ class Assignment : public ScriptCommand {
 
   bool isUsed() const { return IsUsed; }
 
+  /// Reset the assignment value and the expression node.
+  void reset();
+
 private:
   bool checkLinkerScript(Module &CurModule);
 

include/eld/Script/Expression.h

@@ -826,6 +826,28 @@ class GNULDBackend {
 
   const ResolveInfo *findAbsolutePLT(ResolveInfo *I) const;
 
+  /// Returns true if the symbol is partially evaluated. That
+  /// is, the symbol assignment expression was partially evaluated.
+  bool isPartiallyEvaluated(const ResolveInfo *RI) const {
+    return PartiallyEvaluatedSymbols.count(RI);
+  }
+
+  void addPartiallyEvaluatedSymbol(const ResolveInfo *RI) {
+    PartiallyEvaluatedSymbols.insert(RI);
+  }
+
+  void removePartiallyEvaluatedSymbol(const ResolveInfo *RI) {
+    PartiallyEvaluatedSymbols.erase(RI);
+  }
+
+  void resetPartiallyEvalAssignsAndSymbols() {
+    PartiallyEvaluatedAssignments.clear();
+    PartiallyEvaluatedSymbols.clear();
+  }
+
+  /// Recursively evaluates the partially evaluated assignments.
+  void evaluatePendingAssignments();
+
 protected:
   virtual int numReservedSegments() const { return m_NumReservedSegments; }
 
@@ -1000,6 +1022,34 @@ class GNULDBackend {
   // Setup TLS alignment and check for any layout issues
   bool setupTLS();
 
+  /// Evaluates the assignment and tracks the partially evaluated assignments.
+  /// Partially evaluated assignments can be requested to be (recursively)
+  /// reevaluated using evaluatePendingAssignments.
+  void evaluateAssignmentAndTrackPartiallyEvalAssignments(Assignment &A,
+                                                          const ELFSection *S);
+
+  /// Reset the tracked assignments and the TrackedAssignments vector.
+  /// Resetting an assignment resets the assignment node and all the
+  /// expression nodes contained in the assignment.
+  void resetTrackedAssignments();
+
+  /// Stores the snapshot of linker script symbols.
+  void takeLSSymbolsSnapshot();
+
+  /// Restores the value of linker script symbols using the linker script symbol
+  /// snapshot stored using takeLSSymbolsSnapshot.
+  void restoreLSSymbolsUsingSnapshot();
+
+  void takePartiallyEvalAssignsAndSymbolsSnapshot() {
+    PartiallyEvaluatedAssignmentsSnapshot = PartiallyEvaluatedAssignments;
+    PartiallyEvaluatedSymbolsSnapshot = PartiallyEvaluatedSymbols;
+  }
+
+  void restorePartiallyEvalAssignsAndSymbolsUsingSnapshot() {
+    PartiallyEvaluatedAssignments = PartiallyEvaluatedAssignmentsSnapshot;
+    PartiallyEvaluatedSymbols = PartiallyEvaluatedSymbolsSnapshot;
+  }
+
 protected:
   Module &m_Module;
 
@@ -1130,6 +1180,30 @@ class GNULDBackend {
   bool m_NeedEhdr = false;
 
   bool m_NeedPhdr = false;
+
+  std::unordered_set<const ResolveInfo *> PartiallyEvaluatedSymbols;
+
+  /// Stores the assignments which needs to be reevaluated later.
+  std::vector<std::pair<Assignment *, const ELFSection *>>
+      PartiallyEvaluatedAssignments;
+
+  std::unordered_set<const ResolveInfo *> PartiallyEvaluatedSymbolsSnapshot;
+
+  /// Stores the assignments which needs to be reevaluated later.
+  std::vector<std::pair<Assignment *, const ELFSection *>>
+      PartiallyEvaluatedAssignmentsSnapshot;
+
+  /// TrackedAssignments stores the assignments which gets evaluated if
+  /// TrackAssignments is true. It is used to reset the evaluated assignments.
+  bool TrackAssignments = false;
+  std::vector<Assignment *> TrackedAssignments;
+
+  struct LSSymbolInfo {
+    ResolveInfo *RI = nullptr;
+    uint64_t Value = 0;
+    bool HasScriptValue = false;
+  };
+  std::vector<LSSymbolInfo> LSSymbolsSnapshot;
 };
 
 } // namespace eld

include/eld/Target/GNULDBackend.h

@@ -230,6 +230,8 @@ eld::Expected<void> GeneralOptions::setTrace(const char *PTraceType) {
                   DiagEngine->getPrinter()->TraceDynamicLinking)
             .Case("linker-script", DiagEngine->getPrinter()->TraceLinkerScript)
             .Case("symdef", DiagEngine->getPrinter()->TraceSymDef)
+            .Case("pending-assignments",
+                  DiagEngine->getPrinter()->TracePendingAssignments)
             .Default(std::nullopt);
   }
   // Warn if trace category is unknown.

lib/Config/GeneralOptions.cpp

@@ -217,7 +217,9 @@ eld::Expected<void> LinkerWrapper::finishAssignOutputSections() {
 eld::Expected<void> LinkerWrapper::reassignVirtualAddresses() {
   if (getState() != State::CreatingSegments)
     RETURN_INVALID_LINK_STATE_ERR("CreatingSegments");
-  m_Module.getBackend().createScriptProgramHdrs();
+  auto &Backend = m_Module.getBackend();
+  Backend.createScriptProgramHdrs();
+  Backend.evaluatePendingAssignments();
   return {};
 }
 

lib/LinkerWrapper/LinkerWrapper.cpp

@@ -892,7 +892,10 @@ bool ObjectLinker::createOutputSection(ObjectBuilder &Builder,
 
   // force input alignment from ldscript if any
   if (Output->prolog().hasSubAlign()) {
-    Output->prolog().subAlign().eval();
+    // FIXME: eval() is an implementation function, it should not be
+    // used outside the component. Instead, evaluateAndReturnError and
+    // evaluateAndRaiseError should be used.
+    Output->prolog().subAlign().eval(/*EvaluatePendingOnly=*/false);
     Output->prolog().subAlign().commit();
     InAlign = Output->prolog().subAlign().result();
     HasSubAlign = true;

lib/Object/ObjectLinker.cpp

@@ -19,6 +19,7 @@
 #include "eld/Readers/Section.h"
 #include "eld/Script/Expression.h"
 #include "eld/SymbolResolver/LDSymbol.h"
+#include "eld/Target/GNULDBackend.h"
 #include "llvm/Support/Casting.h"
 #include <cassert>
 
@@ -205,7 +206,8 @@ void Assignment::getSymbols(std::vector<ResolveInfo *> &Symbols) const {
   ExpressionToEvaluate->getSymbols(Symbols);
 }
 
-bool Assignment::assign(Module &CurModule, const ELFSection *Section) {
+bool Assignment::assign(Module &CurModule, const ELFSection *Section,
+                        bool EvaluatePendingOnly) {
 
   if (Section && !Section->isAlloc() && isDot()) {
     CurModule.getConfig().raise(Diag::error_dot_lhs_in_non_alloc)
@@ -215,7 +217,11 @@ bool Assignment::assign(Module &CurModule, const ELFSection *Section) {
   }
 
   // evaluate, commit, then get the result of the expression
-  auto Result = ExpressionToEvaluate->evaluateAndRaiseError();
+  std::optional<uint64_t> Result;
+  if (EvaluatePendingOnly)
+    Result = ExpressionToEvaluate->evaluatePendingAndRaiseError();
+  else
+    Result = ExpressionToEvaluate->evaluateAndRaiseError();
   if (!Result)
     return false;
   ExpressionValue = *Result;
@@ -224,13 +230,23 @@ bool Assignment::assign(Module &CurModule, const ELFSection *Section) {
     return false;
 
   LDSymbol *Sym = CurModule.getNamePool().findSymbol(Name);
+  // ASSERT(Sym, "Sym must exist!");
+  // FIXME: Why is it okay for the Sym to be nullptr?
   if (Sym != nullptr) {
     ThisSymbol = Sym;
     ThisSymbol->setValue(ExpressionValue);
     ThisSymbol->setScriptValueDefined();
+    GNULDBackend &Backend = CurModule.getBackend();
+    const ResolveInfo *RI = ThisSymbol->resolveInfo();
+    if (ExpressionToEvaluate->hasPendingEvaluation())
+      Backend.addPartiallyEvaluatedSymbol(RI);
+    else
+      Backend.removePartiallyEvaluatedSymbol(RI);
   }
 
-  if (CurModule.getPrinter()->traceAssignments())
+  auto DP = CurModule.getPrinter();
+  if (DP->traceAssignments() ||
+      (EvaluatePendingOnly && DP->tracePendingAssignments()))
     trace(llvm::outs());
   return true;
 }
@@ -276,3 +292,10 @@ std::unordered_set<std::string> Assignment::getSymbolNames() const {
   ExpressionToEvaluate->getSymbolNames(SymbolNames);
   return SymbolNames;
 }
+
+void Assignment::reset() {
+  ExpressionValue = 0;
+  ThisSymbol = nullptr;
+  if (ExpressionToEvaluate)
+    ExpressionToEvaluate->resetRecursively();
+}