demo docker

docs/llvm/Dockerfile

@@ -0,0 +1,51 @@
+FROM ubuntu:24.04
+
+ARG USERID
+
+RUN apt-get update && \
+    apt-get install -y \
+    cmake \
+    git \
+    ninja-build \
+    clang-19 \
+    lld-19 \
+    sudo \
+    adduser \
+    tmux \
+    bzip2 \
+    curl \
+    vim \
+    software-properties-common
+
+RUN add-apt-repository ppa:deadsnakes/ppa
+RUN apt-get update && \
+    apt-get install -y \
+    python3.11 \
+    python3.11-venv
+
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.11 1
+RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1
+RUN update-alternatives --install /usr/bin/cc cc /usr/bin/clang-19 100
+RUN update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++-19 100
+RUN update-alternatives --install /usr/bin/ld ld /usr/bin/lld-19  100
+
+RUN python -m ensurepip
+
+RUN mkdir -p /work/tflite
+WORKDIR /work/tflite
+RUN curl https://raw.githubusercontent.com/google/ml-compiler-opt/main/buildbot/build_tflite.sh | bash -s
+
+WORKDIR /work
+
+RUN git clone https://github.com/llvm/llvm-project.git
+RUN git clone https://github.com/google/ml-compiler-opt
+
+WORKDIR /work/llvm-project
+WORKDIR /work/ml-compiler-opt
+
+# we want a venv to avoid clashing with py3.11 system packages.
+RUN python -m venv /work/pyenv
+ENV PATH="/work/pyenv/bin:$PATH"
+
+RUN python -m pip install pipenv
+RUN ./versioned_pipenv sync --system --categories "packages dev-packages ci"

docs/llvm/build_clang_for_corpus.sh

@@ -0,0 +1,29 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#!/bin/bash
+
+mkdir -p /work/llvm-corpus
+cmake -B /work/llvm-corpus \
+  -GNinja \
+  -DCMAKE_EXPORT_COMPILE_COMMANDS=On \
+  -DCMAKE_BUILD_TYPE=Release \
+  -DLLVM_ENABLE_PROJECTS=clang \
+  -DCMAKE_C_COMPILER=/work/llvm-train/bin/clang \
+  -DCMAKE_CXX_COMPILER=/work/llvm-train/bin/clang++ \
+  -DCMAKE_C_FLAGS="-Xclang=-fembed-bitcode=all" \
+  -DCMAKE_CXX_FLAGS="-Xclang=-fembed-bitcode=all" \
+  /work/llvm-project/llvm
+
+ninja -C /work/llvm-corpus

docs/llvm/build_clang_for_training.sh

@@ -0,0 +1,25 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#!/bin/bash
+
+mkdir -p /work/llvm-train
+cmake -B /work/llvm-train \
+  -GNinja \
+  -C /work/tflite/tflite.cmake \
+  -DCMAKE_BUILD_TYPE=Release \
+  -DLLVM_ENABLE_PROJECTS=clang \
+  /work/llvm-project/llvm
+
+ninja -C /work/llvm-train

docs/llvm/extract_corpus.sh

@@ -0,0 +1,23 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#!/bin/bash
+
+PYTHONPATH=/work/llvm-project/llvm/utils/mlgo-utils:$PYTHONPATH \
+  python /work/llvm-project/llvm/utils/mlgo-utils/extract_ir.py \
+  --input /work/llvm-corpus/compile_commands.json \
+  --input_type json \
+  --output_dir /work/corpus/modules \
+  --llvm_objcopy_path /work/llvm-train/bin/llvm-objcopy \
+  --obj_base_dir /work/llvm-corpus

docs/llvm/generate_default_trace.sh

@@ -0,0 +1,24 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#!/bin/bash
+
+TF_CPP_MIN_LOG_LEVEL=3 PYTHONPATH=/work/ml-compiler-opt:$PYTHONPATH \
+  python /work/ml-compiler-opt/compiler_opt/tools/generate_default_trace.py \
+  --data_path /work/corpus/modules \
+  --gin_files /work/ml-compiler-opt/compiler_opt/rl/inlining/gin_configs/common.gin \
+  --gin_bindings clang_path="'/work/llvm-train/bin/clang'" \
+  --gin_bindings llvm_size_path="'/work/llvm-train/bin/llvm-size'" \
+  --compilation_timeout=600 \
+  --output_path /work/corpus/default_trace

docs/llvm/generate_vocab.sh

@@ -0,0 +1,21 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#!/bin/bash
+
+TF_CPP_MIN_LOG_LEVEL=3 PYTHONPATH=/work/ml-compiler-opt:$PYTHONPATH \
+  python /work/ml-compiler-opt/compiler_opt/tools/generate_vocab.py \
+  --gin_files /work/ml-compiler-opt/compiler_opt/rl/inlining/gin_configs/common.gin \
+  --input /work/corpus/default_trace \
+  --output_dir /work/corpus/vocab

docs/llvm/init.sh

@@ -0,0 +1,18 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#!/bin/bash
+
+cd /work/llvm-project
+git fetch origin && git checkout 1ea201d73be2fdf03347e9c6be09ebed5f8e0e00

docs/llvm/run_everything.sh

@@ -0,0 +1,30 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#!/bin/bash
+
+SCRIPT_DIR="$(dirname -- "${BASH_SOURCE[0]}")"
+
+SCRIPT_DIR="$(cd -- "$SCRIPT_DIR" && pwd)"
+if [[ -z "$SCRIPT_DIR" ]] ; then
+    exit 1
+fi
+
+"${SCRIPT_DIR}/init.sh"
+"${SCRIPT_DIR}/build_clang_for_training.sh"
+"${SCRIPT_DIR}/build_clang_for_corpus.sh"
+"${SCRIPT_DIR}/extract_corpus.sh"
+"${SCRIPT_DIR}/generate_default_trace.sh"
+"${SCRIPT_DIR}/generate_vocab.sh"
+"${SCRIPT_DIR}/train_with_es.sh"

docs/llvm/train_with_es.sh

@@ -0,0 +1,25 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#!/bin/bash
+
+TF_CPP_MIN_LOG_LEVEL=3 PYTHONPATH=/work/ml-compiler-opt:$PYTHONPATH \
+  python /work/ml-compiler-opt/compiler_opt/es/es_trainer.py \
+  --gin_files /work/ml-compiler-opt/compiler_opt/es/inlining/gin_configs/inlining.gin \
+  --gin_files /work/ml-compiler-opt/compiler_opt/es/inlining/gin_configs/blackbox_learner.gin \
+  --train_corpora /work/corpus/modules \
+  --gin_bindings clang_path="'/work/llvm-train/bin/clang'" \
+  --gin_bindings llvm_size_path="'/work/llvm-train/bin/llvm-size'" \
+  --gin_bindings inlining.config.get_observation_processing_layer_creator.quantile_file_dir="'/work/corpus/vocab'" \
+  --output_path /work/corpus/trained_model_es