From 73d8bc13ee0b393d6caa88654646d224fd16341c Mon Sep 17 00:00:00 2001 From: Anton Bondarenko Date: Tue, 11 Feb 2025 10:43:51 +0100 Subject: [PATCH 1/2] Limit job parallelization to avoid heavy memory usage Number of parallel jobs suitable for current CI runner configuration not always could be retrieved by standard system utilities. The value need to be hardcoded to avoid unexpected memory usage and could be aligned with current CI resources and configuration. Signed-off-by: Anton Bondarenko --- .gitlab-ci.yml | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d15f4e81..5bffd2c3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -10,6 +10,11 @@ stages: - analyze - deploy +variables: + # Number of parallel jobs suitable for current CI runner configuration. The value need to be hardcoded + # becaues in some cases standard system tools might not provide correct number of CPUs reserved for jobs + PARALLEL_JOBS: 8 + default: image: registry.gitlab.arm.com/kleidi/kleidiai/image:latest tags: @@ -39,7 +44,7 @@ build-clang: stage: build script: - cmake -G Ninja -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_CXX_FLAGS="-Werror" -DCMAKE_C_FLAGS="-Werror" -DCMAKE_BUILD_TYPE=Release -DKLEIDIAI_BUILD_TESTS=ON -S . -B ${CI_JOB_NAME_SLUG} - - cmake --build ${CI_JOB_NAME_SLUG} -j$(nproc) --verbose + - cmake --build ${CI_JOB_NAME_SLUG} -j${PARALLEL_JOBS} --verbose artifacts: expire_in: 1 day paths: @@ -51,7 +56,7 @@ build-clang-cov: stage: build script: - cmake -G Ninja -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_CXX_FLAGS="-Werror" -DCMAKE_C_FLAGS="-Werror" -DCMAKE_BUILD_TYPE=Release -DKLEIDIAI_BUILD_TESTS=ON -DCMAKE_C_FLAGS="--coverage" -DCMAKE_CXX_FLAGS="--coverage" -S . -B build - - cmake --build ./build -j$(nproc) --verbose + - cmake --build ./build -j${PARALLEL_JOBS} --verbose # save coverage build job path for later use since coverage processing requires files in exactly same location - echo "COVERAGE_PROJECT_PATH=${PWD}" > build/coverage_path.env artifacts: @@ -67,7 +72,7 @@ build-gcc: stage: build script: - cmake -G Ninja -DCMAKE_C_COMPILER=gcc -DCMAKE_CXX_COMPILER=g++ -DCMAKE_CXX_FLAGS="-Werror" -DCMAKE_C_FLAGS="-Werror" -DCMAKE_BUILD_TYPE=Release -DKLEIDIAI_BUILD_TESTS=ON -S . -B ${CI_JOB_NAME_SLUG} - - cmake --build ${CI_JOB_NAME_SLUG} -j$(nproc) --verbose + - cmake --build ${CI_JOB_NAME_SLUG} -j${PARALLEL_JOBS} --verbose artifacts: expire_in: 1 day paths: @@ -83,7 +88,7 @@ build-gcc-bazel: - /cache/bazelisk script: - bazelisk clean --expunge - - bazelisk build -c opt --copt="-Werror" --cxxopt="-Werror" --jobs=HOST_CPUS -k --subcommands --verbose_failures --curses=no //... + - bazelisk build -c opt --copt="-Werror" --cxxopt="-Werror" --jobs=${PARALLEL_JOBS} -k --subcommands --verbose_failures --curses=no //... - mkdir -p ${CI_JOB_NAME_SLUG} && cp bazel-bin/test/kleidiai_test ${CI_JOB_NAME_SLUG}/ artifacts: expire_in: 1 day @@ -101,7 +106,7 @@ build-clang-bazel: script: - bazelisk clean --expunge # explicitly disable layering_check feature - - CC=clang bazelisk build -c opt --copt="-Werror" --cxxopt="-Werror" --jobs=HOST_CPUS -k --subcommands --verbose_failures --compiler=clang --features=no-layering_check --curses=no //... + - CC=clang bazelisk build -c opt --copt="-Werror" --cxxopt="-Werror" --jobs=${PARALLEL_JOBS} -k --subcommands --verbose_failures --compiler=clang --features=no-layering_check --curses=no //... - mkdir -p ${CI_JOB_NAME_SLUG} && cp bazel-bin/test/kleidiai_test ${CI_JOB_NAME_SLUG}/ artifacts: expire_in: 1 day @@ -122,7 +127,7 @@ build-examples: echo "-----------------------------------------------------------" mkdir -p build_${EXAMPLE} cmake -G Ninja -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_CXX_FLAGS="-Werror" -DCMAKE_C_FLAGS="-Werror" -DCMAKE_BUILD_TYPE=Release -S examples/$EXAMPLE -B build_${EXAMPLE} - cmake --build build_${EXAMPLE} -j$(nproc) --verbose + cmake --build build_${EXAMPLE} -j${PARALLEL_JOBS} --verbose cp build_${EXAMPLE}/${EXAMPLE} build/ else echo "No build file found for ${EXAMPLE}" @@ -374,7 +379,7 @@ test-linux-aarch64-cov: script: - *test-linux-fvp - mkdir -p build/coverage - - gcovr --json=build/coverage/linux-aarch64-fvp.json -j --config gcovr.cfg + - gcovr --json=build/coverage/linux-aarch64-fvp.json -j ${PARALLEL_JOBS} --config gcovr.cfg artifacts: expire_in: 1 day paths: @@ -418,7 +423,9 @@ coverage: # - test-linux-aarch64-cov-fvp # disabled job script: - mkdir -p build/html/coverage - - gcovr --json-add-tracefile "build/coverage/*.json" --print-summary --cobertura=build/coverage.xml --html-details=build/html/coverage/coverage_report.html --html-title="KleidiAI Coverage Report" -j + - gcovr --json-add-tracefile "build/coverage/*.json" --print-summary + --cobertura=build/coverage.xml --html-details=build/html/coverage/coverage_report.html + --html-title="KleidiAI Coverage Report" -j ${PARALLEL_JOBS} artifacts: name: ${CI_JOB_NAME}-${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHA} expire_in: 1 day -- GitLab From 0da9730cef45646546aeff6c8e22cd47782a2b34 Mon Sep 17 00:00:00 2001 From: Anton Bondarenko Date: Wed, 12 Feb 2025 14:31:53 +0100 Subject: [PATCH 2/2] Address review comment with missing PARALLEL_JOBS Signed-off-by: Anton Bondarenko --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5bffd2c3..331b6d11 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -355,7 +355,7 @@ test-linux-aarch64-cov: script: - ./build/kleidiai_test --gtest_output=xml:kleidiai_test_results.xml - mkdir -p build/coverage - - gcovr --json=build/coverage/linux-aarch64.json -j --config gcovr.cfg + - gcovr --json=build/coverage/linux-aarch64.json -j ${PARALLEL_JOBS} --config gcovr.cfg artifacts: expire_in: 1 day paths: -- GitLab