diff --git a/docs/llvm/Dockerfile b/docs/llvm/Dockerfile new file mode 100644 index 00000000..f28dbc97 --- /dev/null +++ b/docs/llvm/Dockerfile @@ -0,0 +1,51 @@ +FROM ubuntu:24.04 + +ARG USERID + +RUN apt-get update && \ + apt-get install -y \ + cmake \ + git \ + ninja-build \ + clang-19 \ + lld-19 \ + sudo \ + adduser \ + tmux \ + bzip2 \ + curl \ + vim \ + software-properties-common + +RUN add-apt-repository ppa:deadsnakes/ppa +RUN apt-get update && \ + apt-get install -y \ + python3.11 \ + python3.11-venv + +RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.11 1 +RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1 +RUN update-alternatives --install /usr/bin/cc cc /usr/bin/clang-19 100 +RUN update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++-19 100 +RUN update-alternatives --install /usr/bin/ld ld /usr/bin/lld-19 100 + +RUN python -m ensurepip + +RUN mkdir -p /work/tflite +WORKDIR /work/tflite +RUN curl https://raw.githubusercontent.com/google/ml-compiler-opt/main/buildbot/build_tflite.sh | bash -s + +WORKDIR /work + +RUN git clone https://github.com/llvm/llvm-project.git +RUN git clone https://github.com/google/ml-compiler-opt + +WORKDIR /work/llvm-project +WORKDIR /work/ml-compiler-opt + +# we want a venv to avoid clashing with py3.11 system packages. +RUN python -m venv /work/pyenv +ENV PATH="/work/pyenv/bin:$PATH" + +RUN python -m pip install pipenv +RUN ./versioned_pipenv sync --system --categories "packages dev-packages ci" diff --git a/docs/llvm/build_clang_for_corpus.sh b/docs/llvm/build_clang_for_corpus.sh new file mode 100755 index 00000000..f293933b --- /dev/null +++ b/docs/llvm/build_clang_for_corpus.sh @@ -0,0 +1,29 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/bin/bash + +mkdir -p /work/llvm-corpus +cmake -B /work/llvm-corpus \ + -GNinja \ + -DCMAKE_EXPORT_COMPILE_COMMANDS=On \ + -DCMAKE_BUILD_TYPE=Release \ + -DLLVM_ENABLE_PROJECTS=clang \ + -DCMAKE_C_COMPILER=/work/llvm-train/bin/clang \ + -DCMAKE_CXX_COMPILER=/work/llvm-train/bin/clang++ \ + -DCMAKE_C_FLAGS="-Xclang=-fembed-bitcode=all" \ + -DCMAKE_CXX_FLAGS="-Xclang=-fembed-bitcode=all" \ + /work/llvm-project/llvm + +ninja -C /work/llvm-corpus diff --git a/docs/llvm/build_clang_for_training.sh b/docs/llvm/build_clang_for_training.sh new file mode 100755 index 00000000..3ff85f9a --- /dev/null +++ b/docs/llvm/build_clang_for_training.sh @@ -0,0 +1,25 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/bin/bash + +mkdir -p /work/llvm-train +cmake -B /work/llvm-train \ + -GNinja \ + -C /work/tflite/tflite.cmake \ + -DCMAKE_BUILD_TYPE=Release \ + -DLLVM_ENABLE_PROJECTS=clang \ + /work/llvm-project/llvm + +ninja -C /work/llvm-train diff --git a/docs/llvm/extract_corpus.sh b/docs/llvm/extract_corpus.sh new file mode 100755 index 00000000..1c72d795 --- /dev/null +++ b/docs/llvm/extract_corpus.sh @@ -0,0 +1,23 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/bin/bash + +PYTHONPATH=/work/llvm-project/llvm/utils/mlgo-utils:$PYTHONPATH \ + python /work/llvm-project/llvm/utils/mlgo-utils/extract_ir.py \ + --input /work/llvm-corpus/compile_commands.json \ + --input_type json \ + --output_dir /work/corpus/modules \ + --llvm_objcopy_path /work/llvm-train/bin/llvm-objcopy \ + --obj_base_dir /work/llvm-corpus diff --git a/docs/llvm/generate_default_trace.sh b/docs/llvm/generate_default_trace.sh new file mode 100755 index 00000000..2432ad36 --- /dev/null +++ b/docs/llvm/generate_default_trace.sh @@ -0,0 +1,24 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/bin/bash + +TF_CPP_MIN_LOG_LEVEL=3 PYTHONPATH=/work/ml-compiler-opt:$PYTHONPATH \ + python /work/ml-compiler-opt/compiler_opt/tools/generate_default_trace.py \ + --data_path /work/corpus/modules \ + --gin_files /work/ml-compiler-opt/compiler_opt/rl/inlining/gin_configs/common.gin \ + --gin_bindings clang_path="'/work/llvm-train/bin/clang'" \ + --gin_bindings llvm_size_path="'/work/llvm-train/bin/llvm-size'" \ + --compilation_timeout=600 \ + --output_path /work/corpus/default_trace diff --git a/docs/llvm/generate_vocab.sh b/docs/llvm/generate_vocab.sh new file mode 100755 index 00000000..8468836f --- /dev/null +++ b/docs/llvm/generate_vocab.sh @@ -0,0 +1,21 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/bin/bash + +TF_CPP_MIN_LOG_LEVEL=3 PYTHONPATH=/work/ml-compiler-opt:$PYTHONPATH \ + python /work/ml-compiler-opt/compiler_opt/tools/generate_vocab.py \ + --gin_files /work/ml-compiler-opt/compiler_opt/rl/inlining/gin_configs/common.gin \ + --input /work/corpus/default_trace \ + --output_dir /work/corpus/vocab diff --git a/docs/llvm/init.sh b/docs/llvm/init.sh new file mode 100755 index 00000000..23ba8d2f --- /dev/null +++ b/docs/llvm/init.sh @@ -0,0 +1,18 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/bin/bash + +cd /work/llvm-project +git fetch origin && git checkout 1ea201d73be2fdf03347e9c6be09ebed5f8e0e00 diff --git a/docs/llvm/run_everything.sh b/docs/llvm/run_everything.sh new file mode 100755 index 00000000..f10e1741 --- /dev/null +++ b/docs/llvm/run_everything.sh @@ -0,0 +1,30 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/bin/bash + +SCRIPT_DIR="$(dirname -- "${BASH_SOURCE[0]}")" + +SCRIPT_DIR="$(cd -- "$SCRIPT_DIR" && pwd)" +if [[ -z "$SCRIPT_DIR" ]] ; then + exit 1 +fi + +"${SCRIPT_DIR}/init.sh" +"${SCRIPT_DIR}/build_clang_for_training.sh" +"${SCRIPT_DIR}/build_clang_for_corpus.sh" +"${SCRIPT_DIR}/extract_corpus.sh" +"${SCRIPT_DIR}/generate_default_trace.sh" +"${SCRIPT_DIR}/generate_vocab.sh" +"${SCRIPT_DIR}/train_with_es.sh" diff --git a/docs/llvm/train_with_es.sh b/docs/llvm/train_with_es.sh new file mode 100755 index 00000000..53c4bf6d --- /dev/null +++ b/docs/llvm/train_with_es.sh @@ -0,0 +1,25 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/bin/bash + +TF_CPP_MIN_LOG_LEVEL=3 PYTHONPATH=/work/ml-compiler-opt:$PYTHONPATH \ + python /work/ml-compiler-opt/compiler_opt/es/es_trainer.py \ + --gin_files /work/ml-compiler-opt/compiler_opt/es/inlining/gin_configs/inlining.gin \ + --gin_files /work/ml-compiler-opt/compiler_opt/es/inlining/gin_configs/blackbox_learner.gin \ + --train_corpora /work/corpus/modules \ + --gin_bindings clang_path="'/work/llvm-train/bin/clang'" \ + --gin_bindings llvm_size_path="'/work/llvm-train/bin/llvm-size'" \ + --gin_bindings inlining.config.get_observation_processing_layer_creator.quantile_file_dir="'/work/corpus/vocab'" \ + --output_path /work/corpus/trained_model_es