Skip to content

Commit 822e6bb

Browse files
grooverdangrooverdan
committed
MDBF-143: Add Infer builder
This preforms static analysis on the MariaDB codebase by maintaining a git source repository as a shared volume. Because static analysis takes time, a lot of time, there is a shared cache volume to store build results from main branches of the codebase so that as much incremental usage can occur. Infer runs in to phases, a capture and an analyze. Infer output are in a result-dir this contains: * report.json - what infer tools use * report.txt - the human readable version of this * capture.db - the sqlite3 version presentation of captured files and the relation to functions definitions. * results.db - the analyze phase outputs Of these, the report.json is desirable as the long term record of vulnerabilities.
1 parent ac09186 commit 822e6bb

File tree

5 files changed

+305
-0
lines changed

5 files changed

+305
-0
lines changed

‎configuration/builders/sequences/helpers.py‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -391,3 +391,26 @@ def mtr_junit_reporter(
391391
warn_on_fail=True,
392392
),
393393
)
394+
395+
396+
def save_infer_logs(
397+
logs_path: PurePath = PurePath("infer_results"),
398+
step_wrapping_fn=lambda step: step,
399+
):
400+
return step_wrapping_fn(
401+
ShellStep(
402+
command=SaveCompressedTar(
403+
name="Save Infer artifacts/logs",
404+
workdir=logs_path,
405+
archive_name="logs",
406+
destination="/packages/%(prop:tarbuildnum)s/logs/%(prop:buildername)s",
407+
),
408+
url=URL(
409+
url=f"{os.environ['ARTIFACTS_URL']}/%(prop:tarbuildnum)s/logs/%(prop:buildername)s",
410+
url_text="Infer artifacts/logs",
411+
),
412+
options=StepOptions(
413+
alwaysRun=True,
414+
),
415+
),
416+
)

‎configuration/builders/sequences/sast.py‎

Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
from pathlib import PurePath
2+
3+
from configuration.builders.infra.runtime import (
4+
BuildSequence,
5+
DockerConfig,
6+
InContainer,
7+
)
8+
from configuration.builders.sequences.helpers import (
9+
save_infer_logs,
10+
)
11+
from configuration.steps.base import StepOptions
12+
from configuration.steps.commands.compile import CompileCMakeCommand
13+
from configuration.steps.commands.configure import ConfigureMariaDBCMake
14+
from configuration.steps.commands.download import GitInitFromCommit, GitFetch
15+
from configuration.steps.commands.util import InferScript, PrintEnvironmentDetails
16+
from configuration.steps.generators.cmake.compilers import ClangCompiler
17+
from configuration.steps.generators.cmake.generator import CMakeGenerator
18+
from configuration.steps.generators.cmake.options import (
19+
CMAKE,
20+
WITH,
21+
BuildType,
22+
CMakeOption,
23+
)
24+
from configuration.steps.remote import ShellStep
25+
26+
27+
def infer(
28+
config: DockerConfig,
29+
jobs: int,
30+
):
31+
sequence = BuildSequence()
32+
33+
sequence.add_step(ShellStep(command=PrintEnvironmentDetails()))
34+
# infer --version
35+
36+
sequence.add_step(
37+
InContainer(
38+
docker_environment=config,
39+
step=ShellStep(
40+
command=BashCommand(
41+
command="git clean -df", workdir=PurePath("/mnt", "src")
42+
),
43+
options=StepOptions(
44+
haltOnFailure=False,
45+
descriptionDone="git cleaned",
46+
),
47+
),
48+
)
49+
)
50+
51+
sequence.add_step(
52+
InContainer(
53+
ShellStep(
54+
command=GitInitFromCommit(
55+
repo_url="%(prop:repository)s",
56+
commit="%(prop:revision)s",
57+
jobs=jobs,
58+
depth=0,
59+
workdir=PurePath("/mnt", "src"),
60+
)
61+
),
62+
docker_environment=config,
63+
),
64+
)
65+
66+
sequence.add_step(
67+
InContainer(
68+
docker_environment=config,
69+
step=ShellStep(
70+
command=BashCommand(
71+
command="git diff --name-only FETCH_HEAD..%(prop:master_branch)s | tee $OLD_PWD/index.txt", workdir=PurePath("/mnt", "src")
72+
),
73+
options=StepOptions(
74+
haltOnFailure=False,
75+
descriptionDone="names of changed files",
76+
),
77+
),
78+
)
79+
)
80+
81+
flags = [
82+
# UBSAN is the only prevention of UNINIT_VAR(X) x= x
83+
# that generated lots of uninit read/write errors.
84+
CMakeOption(WITH.UBSAN, True),
85+
CMakeOption(CMAKE.EXPORT_COMPILE_COMMANDS, True),
86+
]
87+
88+
sequence.add_step(
89+
InContainer(
90+
docker_environment=config,
91+
step=ShellStep(
92+
command=ConfigureMariaDBCMake(
93+
name="configure",
94+
cmake_generator=CMakeGenerator(
95+
use_ccache=True,
96+
flags=flags,
97+
source_path="/mnt/src",
98+
builddir="bld",
99+
compiler=ClangCompiler(),
100+
),
101+
),
102+
options=StepOptions(descriptionDone="Configure"),
103+
),
104+
)
105+
)
106+
107+
# Some server code is generated, so these need to be generated to test
108+
sequence.add_step(
109+
InContainer(
110+
docker_environment=config,
111+
step=ShellStep(
112+
command=CompileCMakeCommand(
113+
builddir="bld",
114+
jobs=jobs,
115+
verbose=True,
116+
targets=[
117+
"GenError",
118+
"GenServerSource",
119+
"GenUnicodeDataSource",
120+
"GenFixPrivs",
121+
],
122+
),
123+
options=StepOptions(descriptionDone="compile"),
124+
),
125+
)
126+
)
127+
128+
env_vars = [
129+
(
130+
"JOBS",
131+
str(jobs)
132+
)
133+
]
134+
sequence.add_step(
135+
InContainer(
136+
docker_environment=config,
137+
step=ShellStep(
138+
command=InferScript(),
139+
command=BashScript("infer.sh", args=["%(prop:ldnum)s"]),
140+
options=StepOptions(
141+
descriptionDone="infer analysis complete",
142+
),
143+
env_vars=env_vars,
144+
),
145+
)
146+
)
147+
148+
sequence.add_step(
149+
save_infer_logs(
150+
step_wrapping_fn=lambda step: InContainer(
151+
docker_environment=config, step=step
152+
),
153+
)
154+
)
155+
return sequence

‎configuration/steps/commands/scripts/infer.sh‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
#!/bin/bash
2+
3+
# Infer script for performing
4+
# static analysis on the MariaDB codebase
5+
6+
set -x -e
7+
8+
if [ $# -lt 2 ]; then
9+
echo insufficient args >&2
10+
exit 1
11+
fi
12+
13+
# Our version
14+
branch=$1
15+
16+
shift
17+
# The base branch
18+
master_branch=$1
19+
20+
: "${JOBS:=4}"
21+
22+
result_dir=$PWD/infer_results
23+
24+
## Cases
25+
26+
if [ "$branch" == "$master_branch" ]; then
27+
if [ -d "/mnt/infer/$branch" ]; then
28+
# Update to a main branch
29+
pushd /mnt/src
30+
git diff --name-only "${master_branch}..$(< /mnt/infer/"$branch"/commit)" | tee "$OLDPWD"/index.txt
31+
popd
32+
else
33+
# New main branch
34+
result_dir=/mnt/infer/$branch
35+
rm index.txt
36+
fi
37+
else
38+
# Branch off from a main branch
39+
if [ ! -s index.txt ]; then
40+
echo "Empty changes - nothing necessary"
41+
exit 0
42+
fi
43+
44+
echo "Diff against $master_branch"
45+
fi
46+
47+
capture()
48+
{
49+
infer capture --compilation-database compile_commands.json --project-root /mnt/src --results-dir "${result_dir}" "$@"
50+
}
51+
52+
analyze()
53+
{
54+
infer analyze --project-root /mnt/src --results-dir "${result_dir}" --max-jobs "${JOBS}" "$@"
55+
}
56+
# Capture and analyze the feature of the files changes in index
57+
#
58+
cd bld
59+
capture
60+
61+
if [ ! -f ../index.txt ]; then
62+
echo "full run, this could take a while"
63+
analyze
64+
exit
65+
fi
66+
67+
# some form of incremental
68+
analyze --changed-files-index ../index.txt
69+
70+
# Preserve result
71+
cp "${result_dir}"/report.json ../report.json
72+
73+
pushd /mnt/src
74+
git checkout "$master_branch"
75+
popd
76+
77+
# just in case these have changed
78+
cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=1 .
79+
80+
capture --reactive --mark-unchanged-procs
81+
analyze --incremental-analysis --changed-files-index ../index.txt
82+
83+
infer reportdiff --report-current ../report.json --report-previous "${result_dir}"/report.json --project-root /mnt/src --results-dir "${result_dir}"

‎configuration/steps/commands/util.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -173,3 +173,13 @@ def __init__(
173173
):
174174
args = [f"{binary}:{','.join(libs)}" for binary, libs in binary_checks.items()]
175175
super().__init__(script_name="ldd_check.sh", args=args)
176+
177+
178+
class InferScript(BashScriptCommand):
179+
"""
180+
A command to run the infer analysis on the MariaDB codebase.
181+
"""
182+
183+
def __init__(self, output_file: str):
184+
args = [output_file]
185+
super().__init__(script_name="infer.sh", args=args)

‎master-migration/master.cfg‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ from configuration.builders.sequences.compile_only import (
1616
from configuration.builders.sequences.debug import openssl_fips
1717
from configuration.builders.sequences.release import deb_autobake, rpm_autobake
1818
from configuration.builders.sequences.sanitizers import asan_ubsan
19+
from configuration.builders.sequences.sast import infer
1920
from configuration.reporters import github_summary
2021
from configuration.workers import worker
2122
from master_common import base_master_config, IS_CHECKCONFIG
@@ -277,6 +278,39 @@ builder = "amd64-ubasan-clang-20"
277278
c["builders"].append(ubasan_builder(name=builder, debug=builder.endswith("debug")))
278279

279280

281+
## ------------------------------------------------------------------- ##
282+
## STATIC ANALYZERS BUILDERS ##
283+
## ------------------------------------------------------------------- ##
284+
285+
c["builders"].append(
286+
GenericBuilder(
287+
name="amd64-infer-clang-20",
288+
sequences=[
289+
infer(
290+
jobs=12,
291+
config=DockerConfig(
292+
repository=os.environ["CONTAINER_REGISTRY_URL"],
293+
image_tag="debian13-infer-clang-20",
294+
workdir=PurePath("/home/buildbot"),
295+
bind_mounts=[
296+
("/srv/buildbot/src", "/mnt/src"),
297+
("/srv/buildbot/infer", "/mnt/infer"),
298+
(f'{os.environ["MASTER_PACKAGES_DIR"]}/', "/packages"),
299+
],
300+
env_vars=[
301+
("ARTIFACTS_URL", os.environ["ARTIFACTS_URL"]),
302+
],
303+
),
304+
)
305+
],
306+
).get_config(
307+
workers=WORKER_POOL.get_workers_for_arch(arch="amd64"),
308+
next_build=nextBuild,
309+
can_start_build=canStartBuild,
310+
tags=["clang", "infer", "sast"],
311+
jobs=12,
312+
)
313+
)
280314

281315
## ------------------------------------------------------------------- ##
282316
## REPORTERS ##

0 commit comments

Comments
 (0)