Skip to content

Ruby: add overlay annotations to AST/CFG/SSA layers #19989

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Jul 10, 2025
Merged

Ruby: add overlay annotations to AST/CFG/SSA layers #19989

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b51940d
Ruby: add overlay[local] annotations to AST/CFG/SSA layers
nickrolfe Jun 18, 2025
e1f2433
Ruby: make resolveConstant overlay[global]
nickrolfe Jul 1, 2025
c415795
Ruby: add changenote for overlay[local] annotations
nickrolfe Jul 9, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions ql/ql/src/codeql/Locations.qll
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
/** Provides classes for working with locations. */
overlay[local]
module;

import files.FileSystem

Expand Down
2 changes: 2 additions & 0 deletions ql/ql/src/codeql/files/FileSystem.qll
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
/** Provides classes for working with files and folders. */
overlay[local?]
module;

private import codeql_ql.ast.internal.TreeSitter
private import codeql.Locations
Expand Down
5 changes: 5 additions & 0 deletions ql/ql/src/codeql_ql/StructuredLogs.qll
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
overlay[local]
module;

private import ql
private import codeql_ql.ast.internal.TreeSitter
private import experimental.RA
Expand All @@ -23,6 +26,7 @@ private float stringToTimestamp(string str) {
}

bindingset[s]
overlay[global]
private Predicate getPredicateFromPosition(string s) {
exists(string r, string filepath, int startline | r = "(.*):(\\d+),(\\d+)-(\\d+),(\\d+)" |
filepath = s.regexpCapture(r, 1) and
Expand Down Expand Up @@ -397,6 +401,7 @@ module KindPredicatesLog {

string getPosition() { result = this.getString("position") }

overlay[global]
Predicate getPredicate() { result = getPredicateFromPosition(this.getPosition()) }

/**
Expand Down
12 changes: 4 additions & 8 deletions ql/ql/src/codeql_ql/ast/internal/TreeSitter.qll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ private predicate discardLocation(@location_default loc) {
)
}

overlay[local]
module QL {
/** The base class for all AST nodes */
class AstNode extends @ql_ast_node {
Expand Down Expand Up @@ -67,15 +68,13 @@ module QL {
}

/** Gets the file containing the given `node`. */
overlay[local]
private @file getNodeFile(@ql_ast_node node) {
exists(@location_default loc | ql_ast_node_location(node, loc) |
locations_default(loc, result, _, _, _, _)
)
}

/** Holds if `node` is in the `file` and is part of the overlay base database. */
overlay[local]
private predicate discardableAstNode(@file file, @ql_ast_node node) {
not isOverlay() and file = getNodeFile(node)
}
Expand Down Expand Up @@ -1315,6 +1314,7 @@ module QL {
}
}

overlay[local]
module Dbscheme {
/** The base class for all AST nodes */
class AstNode extends @dbscheme_ast_node {
Expand Down Expand Up @@ -1359,15 +1359,13 @@ module Dbscheme {
}

/** Gets the file containing the given `node`. */
overlay[local]
private @file getNodeFile(@dbscheme_ast_node node) {
exists(@location_default loc | dbscheme_ast_node_location(node, loc) |
locations_default(loc, result, _, _, _, _)
)
}

/** Holds if `node` is in the `file` and is part of the overlay base database. */
overlay[local]
private predicate discardableAstNode(@file file, @dbscheme_ast_node node) {
not isOverlay() and file = getNodeFile(node)
}
Expand Down Expand Up @@ -1673,6 +1671,7 @@ module Dbscheme {
}
}

overlay[local]
module Blame {
/** The base class for all AST nodes */
class AstNode extends @blame_ast_node {
Expand Down Expand Up @@ -1717,15 +1716,13 @@ module Blame {
}

/** Gets the file containing the given `node`. */
overlay[local]
private @file getNodeFile(@blame_ast_node node) {
exists(@location_default loc | blame_ast_node_location(node, loc) |
locations_default(loc, result, _, _, _, _)
)
}

/** Holds if `node` is in the `file` and is part of the overlay base database. */
overlay[local]
private predicate discardableAstNode(@file file, @blame_ast_node node) {
not isOverlay() and file = getNodeFile(node)
}
Expand Down Expand Up @@ -1808,6 +1805,7 @@ module Blame {
}
}

overlay[local]
module JSON {
/** The base class for all AST nodes */
class AstNode extends @json_ast_node {
Expand Down Expand Up @@ -1852,15 +1850,13 @@ module JSON {
}

/** Gets the file containing the given `node`. */
overlay[local]
private @file getNodeFile(@json_ast_node node) {
exists(@location_default loc | json_ast_node_location(node, loc) |
locations_default(loc, result, _, _, _, _)
)
}

/** Holds if `node` is in the `file` and is part of the overlay base database. */
overlay[local]
private predicate discardableAstNode(@file file, @json_ast_node node) {
not isOverlay() and file = getNodeFile(node)
}
Expand Down
5 changes: 2 additions & 3 deletions ql/ql/src/experimental/RA.qll
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
/**
* Parses RA expressions.
*/
overlay[local]
module;

/**
* A predicate that contains RA.
Expand Down
1 change: 1 addition & 0 deletions ql/ql/src/queries/performance/LargeTupleSum.ql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import ql
import codeql_ql.StructuredLogs
import KindPredicatesLog

overlay[local]
module SumCounts implements Fold<int> {
int base(PipeLineRun run) { result = sum(int i | | run.getCount(i)) }

Expand Down
1 change: 1 addition & 0 deletions ql/ql/test/experimental/raparser.ql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import experimental.RA

overlay[local]
class TestPredicate extends string {
TestPredicate() { this = "p1" }

Expand Down
4 changes: 4 additions & 0 deletions ruby/ql/lib/change-notes/2025-07-09-overlay-local.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
---
category: breaking
---
* Most classes and predicates in the AST, SSA, and control-flow-graph libraries are now annotated with `overlay[local]`, in preparation for incremental analysis. This could result in compiler errors for custom queries if they extend these classes. To mitigate such errors, look for ways to restructure custom QL code so it doesn't depend on changing the behavior of standard-library classes.
2 changes: 2 additions & 0 deletions ruby/ql/lib/codeql/Locations.qll
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
/** Provides classes for working with locations. */
overlay[local]
module;

import files.FileSystem

Expand Down
2 changes: 2 additions & 0 deletions ruby/ql/lib/codeql/files/FileSystem.qll
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
/** Provides classes for working with files and folders. */
overlay[local]
module;

private import codeql.Locations
private import codeql.util.FileSystem
Expand Down
3 changes: 3 additions & 0 deletions ruby/ql/lib/codeql/ruby/AST.qll
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
overlay[local]
module;

import codeql.Locations
import ast.Call
import ast.Control
Expand Down
2 changes: 2 additions & 0 deletions ruby/ql/lib/codeql/ruby/Diagnostics.qll
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
/** Provides classes relating to extraction diagnostics. */
overlay[local]
module;

private import codeql.Locations

Expand Down
8 changes: 7 additions & 1 deletion ruby/ql/lib/codeql/ruby/ast/Call.qll
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
overlay[local]
module;

private import codeql.ruby.AST
private import internal.AST
private import internal.Call
private import internal.Literal
private import internal.TreeSitter
private import codeql.ruby.dataflow.internal.DataFlowDispatch
private import codeql.ruby.dataflow.internal.DataFlowImplCommon
Expand Down Expand Up @@ -41,7 +45,7 @@ class Call extends Expr instanceof CallImpl {
final Expr getKeywordArgument(string keyword) {
exists(Pair p |
p = this.getAnArgument() and
p.getKey().getConstantValue().isSymbol(keyword) and
keyword = p.getKey().(SymbolLiteral).(StringlikeLiteralImpl).getStringValue() and
result = p.getValue()
)
}
Expand All @@ -52,6 +56,7 @@ class Call extends Expr instanceof CallImpl {
final int getNumberOfArguments() { result = super.getNumberOfArgumentsImpl() }

/** Gets a potential target of this call, if any. */
overlay[global]
final Callable getATarget() {
exists(DataFlowCall c |
this = c.asCall().getExpr() and
Expand Down Expand Up @@ -153,6 +158,7 @@ class MethodCall extends Call instanceof MethodCallImpl {
* TODO: When API Graphs is able to resolve calls to methods like `Kernel.send`
* this class is no longer necessary and should be removed.
*/
overlay[global]
class UnknownMethodCall extends MethodCall {
UnknownMethodCall() { not exists(this.(Call).getATarget()) }
}
Expand Down
9 changes: 9 additions & 0 deletions ruby/ql/lib/codeql/ruby/ast/Constant.qll
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
overlay[local]
module;

private import codeql.ruby.AST
private import internal.AST
private import internal.Constant
Expand All @@ -6,6 +9,7 @@ private import internal.Variable
private import internal.TreeSitter

/** A constant value. */
overlay[global]
class ConstantValue extends TConstantValue {
/** Gets a textual representation of this constant value. */
final string toString() { this.hasValueWithType(result, _) }
Expand Down Expand Up @@ -134,6 +138,7 @@ class ConstantValue extends TConstantValue {
}

/** Provides different sub classes of `ConstantValue`. */
overlay[global]
module ConstantValue {
/** A constant integer value. */
class ConstantIntegerValue extends ConstantValue, TInt { }
Expand Down Expand Up @@ -268,15 +273,18 @@ class ConstantReadAccess extends ConstantAccess {
*
* the value being read at `M::CONST` is `"const"`.
*/
overlay[global]
Expr getValue() { result = getConstantReadAccessValue(this) }

/**
* Gets a fully qualified name for this constant read, based on the context in
* which it occurs.
*/
overlay[global]
string getAQualifiedName() { result = resolveConstant(this) }

/** Gets the module that this read access resolves to, if any. */
overlay[global]
Module getModule() { result = resolveConstantReadAccess(this) }

final override string getAPrimaryQlClass() { result = "ConstantReadAccess" }
Expand Down Expand Up @@ -342,6 +350,7 @@ class ConstantWriteAccess extends ConstantAccess {
* constants up the namespace chain, the fully qualified name of a nested
* constant can be ambiguous from just statically looking at the AST.
*/
overlay[global]
string getAQualifiedName() { result = resolveConstantWrite(this) }
}

Expand Down
3 changes: 3 additions & 0 deletions ruby/ql/lib/codeql/ruby/ast/Control.qll
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
overlay[local]
module;

private import codeql.ruby.AST
private import internal.AST
private import internal.Control
Expand Down
3 changes: 3 additions & 0 deletions ruby/ql/lib/codeql/ruby/ast/Erb.qll
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
overlay[local]
module;

private import codeql.ruby.AST
private import internal.Erb
private import internal.TreeSitter
Expand Down
5 changes: 5 additions & 0 deletions ruby/ql/lib/codeql/ruby/ast/Expr.qll
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
overlay[local]
module;

private import codeql.ruby.AST
private import codeql.ruby.CFG
private import internal.AST
Expand All @@ -12,6 +15,7 @@ private import internal.TreeSitter
*/
class Expr extends Stmt, TExpr {
/** Gets the constant value of this expression, if any. */
overlay[global]
ConstantValue getConstantValue() { result = getConstantValueExpr(this) }
}

Expand Down Expand Up @@ -425,6 +429,7 @@ class StringConcatenation extends Expr, TStringConcatenation {
* "foo" "bar#{ n }"
* ```
*/
overlay[global]
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this can be made local:

  final string getConcatenatedValueText() {
    forall(StringLiteral c | c = this.getString(_) |
      exists(c.(StringlikeLiteralImpl).getStringValue())
    ) and
    result =
      concat(string valueText, int i |
        valueText = this.getString(i).(StringlikeLiteralImpl).getStringValue()
      |
        valueText order by i
      )
  }

(requires private import internal.Literal).

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The pragma still needs to be removed.

Copy link
Contributor Author

@nickrolfe nickrolfe Jul 8, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Unfortunately, the test failure shows that this change isn't equivalent. It no longer gives a result for "foo" "bar#{ 1 * 1 }" 'baz', since StringlikeLiteralImpl.getStringValue() doesn't give a result for the string with the interpolation.

The qldoc for getConcatenatedValueText() is actually wrong then, for the current behaviour.

I think I would prefer backing this change out — along with the other, similar changes — and keeping it global. At least in this PR. What do you think?

final string getConcatenatedValueText() {
forall(StringLiteral c | c = this.getString(_) |
exists(c.getConstantValue().getStringlikeValue())
Expand Down
Loading
Loading