From b207a85e3c0810c2cc6e0c2b8c3f4c649e325098 Mon Sep 17 00:00:00 2001 From: Drew Reed Date: Tue, 11 Oct 2022 10:39:56 +0000 Subject: [PATCH 1/5] ci: Switch to template yocto/kas build files Use the yocto/kas build template files to run the actual build and check layers steps Signed-off-by: Drew Reed --- .gitlab-ci.yml | 5 +- .gitlab/ci/cassini-build.yml | 104 ++++++++++++ .gitlab/ci/collate-results.yml | 23 --- .gitlab/ci/n1sdp-image-builds.yml | 32 +--- .gitlab/ci/rules.yml | 7 +- .gitlab/ci/templates/image_build.yml.j2 | 2 +- .gitlab/ci/trigger-image-builds.yml | 56 +------ .gitlab/ci/yocto-build.yml | 194 ----------------------- .gitlab/ci/yocto-qa.yml | 36 +---- .gitlab/scripts/child-job-test-report.py | 94 ----------- 10 files changed, 129 insertions(+), 424 deletions(-) create mode 100644 .gitlab/ci/cassini-build.yml delete mode 100644 .gitlab/ci/collate-results.yml delete mode 100644 .gitlab/ci/yocto-build.yml delete mode 100644 .gitlab/scripts/child-job-test-report.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7484c2c..d38289a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -36,7 +36,7 @@ variables: # include jobs include: - project: $PIPELINE_TEMPLATE_PROJECT - ref: v0.0.10 + ref: v0.0.12 file: - gitlab-ci/common_gitlab-ci.yml - gitlab-ci/docker-image-base_gitlab-ci.yml @@ -44,11 +44,12 @@ include: - gitlab-ci/changelog_gitlab-ci.yml - gitlab-ci/auto-release_gitlab-ci.yml - gitlab-ci/static-analysis_gitlab-ci.yml + - gitlab-ci/child-pipeline_gitlab-ci.yml - local: '.gitlab/ci/rules.yml' - local: '.gitlab/ci/docker-image-builds.yml' + - local: '.gitlab/ci/n1sdp-image-builds.yml' - local: '.gitlab/ci/trigger-image-builds.yml' - local: '.gitlab/ci/docs-build.yml' - - local: '.gitlab/ci/yocto-qa.yml' # Job to regenerate the changelog when the project version number is changed changelog: diff --git a/.gitlab/ci/cassini-build.yml b/.gitlab/ci/cassini-build.yml new file mode 100644 index 0000000..50a2199 --- /dev/null +++ b/.gitlab/ci/cassini-build.yml @@ -0,0 +1,104 @@ +# Copyright (c) 2022 Arm Limited or its affiliates. All rights reserved. +# +# SPDX-License-Identifier: MIT +--- + +# Modify the yocto/kas base build with Cassini specific settings and switch to +# the Cassini kas container +.yocto_base: + variables: + CACHE_DIR: /sstate-cache + SSTATE_DIR: $CACHE_DIR/$PROJECT_VERSION/yocto-sstate + DL_DIR: $CACHE_DIR/$PROJECT_VERSION/yocto-downloads + TOOLCHAIN_DIR: $CACHE_DIR/$PROJECT_VERSION/toolchains + +.kas_build: + image: $CI_REGISTRY/$CI_PROJECT_PATH/$DOCKER_IMAGE_NAME:$DOCKER_IMAGE_TAG + variables: + DOCKER_IMAGE_NAME: kas-build-image + KAS_REPO_REF_DIR: $CACHE_DIR/$PROJECT_VERSION/repos + +# Extend the kas base build to enab;e to use of repos that are not yet public +.kas_build_cassini: + extends: + - .kas_build + variables: + IMAGE_DIR: $CACHE_DIR/$PROJECT_VERSION/images + before_script: + - mkdir -p ~/.ssh + - eval "$(ssh-agent -s)" + - if [ -z "${SSH_PRIVATE_GITLAB}+x" ]; then + echo "${SSH_PRIVATE_GITLAB}" | tr -d '\r' | ssh-add - > /dev/null; + fi + - ssh-keyscan -t rsa ${CI_SERVER_HOST} >> ~/.ssh/known_hosts + # Ensure git is usable without prompts + - git config --global url.ssh://git@${CI_SERVER_HOST}.insteadOf + https://${CI_SERVER_HOST} + - git config --global user.email "you@example.com" + - git config --global user.name "Your Name" + # export parent pipeline global variables + - echo PARENT_PIPELINE_ID = $PARENT_PIPELINE_ID + - !reference [".kas_build", before_script] + # Setup workspace + - echo "Persistent cache volume could be empty, create directories required" + - mkdir --verbose --parents + $IMAGE_DIR + +# Delete stored images +Delete-Images: + extends: .kas_build_cassini + stage: Setup + needs: [] + allow_failure: true + rules: + - if: '$CI_PIPELINE_SOURCE == "parent_pipeline" && $CI_COMMIT_TAG' + - if: '$CI_PIPELINE_SOURCE == "parent_pipeline"' + when: manual + script: + - rm -rf $IMAGE_DIR/* + +# Report on disk usage including the image directory +Disk-Usage: + extends: .kas_build_cassini + needs: [] + rules: + - if: '$CI_PIPELINE_SOURCE == "parent_pipeline"' + when: manual + script: + - du -h -s $DL_DIR $SSTATE_DIR $TOOLCHAIN_DIR $KAS_REPO_REF_DIR $IMAGE_DIR + +# Suppress the use of the yocto/kas helper jobs as they +# should only be run in the child pipeline +Prune-Cache: + rules: + - if: '$CI_PIPELINE_SOURCE == "parent_pipeline" && $FREQUENCY == "nightly"' + - if: '$CI_PIPELINE_SOURCE == "parent_pipeline" && $FREQUENCY == "adhoc"' + when: manual + needs: [] + +Update-Repos: + rules: + - if: '$CI_PIPELINE_SOURCE == "parent_pipeline"' + +Delete-DL-Dir: + rules: + - if: '$CI_PIPELINE_SOURCE == "parent_pipeline" && $CI_COMMIT_TAG' + - if: '$CI_PIPELINE_SOURCE == "parent_pipeline"' + when: manual + +Delete-SState: + rules: + - if: '$CI_PIPELINE_SOURCE == "parent_pipeline" && $CI_COMMIT_TAG' + - if: '$CI_PIPELINE_SOURCE == "parent_pipeline"' + when: manual + +Delete-Toolchains: + rules: + - if: '$CI_PIPELINE_SOURCE == "parent_pipeline" && $CI_COMMIT_TAG' + - if: '$CI_PIPELINE_SOURCE == "parent_pipeline"' + when: manual + +Delete-Repo-Dir: + rules: + - if: '$CI_PIPELINE_SOURCE == "parent_pipeline"' + when: manual diff --git a/.gitlab/ci/collate-results.yml b/.gitlab/ci/collate-results.yml deleted file mode 100644 index 6ded770..0000000 --- a/.gitlab/ci/collate-results.yml +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright (c) 2022 Arm Limited or its affiliates. All rights reserved. -# -# SPDX-License-Identifier: MIT ---- -Collate-Results: - extends: .resource-request - image: "${MIRROR_DOCKER}/python:slim" - stage: Test - dependencies: - - merge_jobs - - trigger_jobs - before_script: - - pip install requests envparse junit_xml - script: - - python .gitlab/scripts/child-job-test-report.py - artifacts: - paths: - - TEST-ChildPipeline-*.xml - reports: - junit: - - TEST-ChildPipeline-*.xml - expire_in: 1 month - when: always diff --git a/.gitlab/ci/n1sdp-image-builds.yml b/.gitlab/ci/n1sdp-image-builds.yml index 217aa34..912722a 100644 --- a/.gitlab/ci/n1sdp-image-builds.yml +++ b/.gitlab/ci/n1sdp-image-builds.yml @@ -2,33 +2,15 @@ # # SPDX-License-Identifier: MIT --- -.generator: - extends: .resource-request - image: $CI_REGISTRY/$CI_PROJECT_PATH/$DOCKER_IMAGE_NAME:$DOCKER_IMAGE_TAG - stage: Setup +.generator-n1sdp: + extends: .generator variables: - DOCKER_IMAGE_NAME: utility-image MACHINE: n1sdp RUN_TESTS: 'none' - script: - - export CI_JOB_NAME_SLUG=${CI_JOB_NAME//\//-} - - mkdir tmp - - j2 .gitlab/ci/templates/image_build.yml.j2 -o tmp/${CI_JOB_NAME_SLUG}.yml; - - if [ "$CI_DEBUG_TRACE" == "true" ]; then - cat tmp/${CI_JOB_NAME_SLUG}.yml; - fi - needs: - - job: Build-Utility-Docker-Image - optional: true - artifacts: false - artifacts: - paths: - - tmp/*.yml - expire_in: '1 days' cassini/tests/n1sdp: - extends: .generator + extends: .generator-n1sdp variables: RUN_TESTS: ptest rules: @@ -38,7 +20,7 @@ cassini/tests/n1sdp: - !reference [".build-image:rules:n1sdp", rules] cassini/security/tests/n1sdp: - extends: .generator + extends: .generator-n1sdp variables: RUN_TESTS: ptest rules: @@ -48,7 +30,7 @@ cassini/security/tests/n1sdp: - !reference [".build-image:rules:n1sdp", rules] cassini-dev/tests/n1sdp: - extends: .generator + extends: .generator-n1sdp variables: RUN_TESTS: ptest rules: @@ -57,7 +39,7 @@ cassini-dev/tests/n1sdp: ($FREQUENCY == "nightly")' cassini-dev/security/tests/n1sdp: - extends: .generator + extends: .generator-n1sdp variables: RUN_TESTS: ptest rules: @@ -66,7 +48,7 @@ cassini-dev/security/tests/n1sdp: ($FREQUENCY == "nightly")' cassini-sdk/n1sdp: - extends: .generator + extends: .generator-n1sdp rules: - if: '($BUILD_IMAGE =~ /all-images/ || $BUILD_IMAGE =~ /k3s/) && ($BUILD_PLATFORM =~ /all-platforms/ || $BUILD_PLATFORM =~ /n1sdp/)' diff --git a/.gitlab/ci/rules.yml b/.gitlab/ci/rules.yml index 01bfe6d..008d3b7 100644 --- a/.gitlab/ci/rules.yml +++ b/.gitlab/ci/rules.yml @@ -12,10 +12,13 @@ # Changes patterns .ci-patterns: &ci-patterns - ".gitlab-ci.yml" + - ".gitlab/ci/cassini-build.yml" + - ".gitlab/ci/collate-results.yml" + - ".gitlab/ci/docs-build.yml" + - ".gitlab/ci/lava-test.yml" - ".gitlab/ci/rules.yml" - ".gitlab/ci/trigger-image-builds.yml" - - ".gitlab/ci/yocto-build.yml" - - ".gitlab/ci/lava-test.yml" + - ".gitlab/ci/yocto-qa.yml" - ".gitlab/ci/templates/image_build.yml.j2" - ".gitlab/scripts/**/*" diff --git a/.gitlab/ci/templates/image_build.yml.j2 b/.gitlab/ci/templates/image_build.yml.j2 index 081119e..674e60c 100644 --- a/.gitlab/ci/templates/image_build.yml.j2 +++ b/.gitlab/ci/templates/image_build.yml.j2 @@ -7,7 +7,7 @@ Image-Build-{{ CI_JOB_NAME_SLUG }}: tags: - x86_64 {% endif %} - extends: .yocto_build + extends: .kas_build_cassini # Store the build artifacts from the job. The download urls of these artifacts are later used by lava jobs to download and test the image. variables: KAS_CONFIGS: "{{ CI_JOB_NAME }}" diff --git a/.gitlab/ci/trigger-image-builds.yml b/.gitlab/ci/trigger-image-builds.yml index 3fd9c23..26cc461 100644 --- a/.gitlab/ci/trigger-image-builds.yml +++ b/.gitlab/ci/trigger-image-builds.yml @@ -2,17 +2,8 @@ # # SPDX-License-Identifier: MIT --- -include: - - local: '.gitlab/ci/n1sdp-image-builds.yml' - - local: '.gitlab/ci/collate-results.yml' merge_jobs: - extends: .resource-request - image: $CI_REGISTRY/$CI_PROJECT_PATH/$DOCKER_IMAGE_NAME:$DOCKER_IMAGE_TAG - stage: Setup - variables: - DOCKER_IMAGE_NAME: utility-image - CI_JOB_FILE: yocto-builds.yml needs: - job: Build-Utility-Docker-Image optional: true @@ -27,57 +18,24 @@ merge_jobs: optional: true - job: cassini-dev/security/tests/n1sdp optional: true - script: - - set -e + before_script: - | cat << EOF > ${CI_JOB_FILE} include: - project: $PIPELINE_TEMPLATE_PROJECT - ref: v0.0.10 + ref: v$PIPELINE_TEMPLATE_VERSION file: - gitlab-ci/common_gitlab-ci.yml + - gitlab-ci/yocto-build-base_gitlab-ci.yml + - gitlab-ci/kas-build-base_gitlab-ci.yml - gitlab-ci/lava-test-base_gitlab-ci.yml - - local: .gitlab/ci/yocto-build.yml + - local: .gitlab/ci/rules.yml + - local: .gitlab/ci/cassini-build.yml + - local: .gitlab/ci/yocto-qa.yml - local: .gitlab/ci/lava-test.yml - EOF - - if [ -d tmp ]; then - for file in tmp/*.yml; do - cat $file >> ${CI_JOB_FILE}; - echo -n >> ${CI_JOB_FILE}; - done - fi - # Add global variables to env file to export in child pipelines - - echo "CREATE_JOB_ID=${CI_JOB_ID}" > global_variables.env - - cat global_variables.env - - if [ "${CI_DEBUG_TRACE}" == "true" ]; then - cat ${CI_JOB_FILE}; - fi - artifacts: - name: "job-$CI_COMMIT_REF_NAME" - paths: - - $CI_JOB_FILE - - global_variables.env - expire_in: '1 days' trigger_jobs: - stage: Build - trigger: - include: - - artifact: yocto-builds.yml - job: merge_jobs - strategy: depend - forward: - pipeline_variables: true - yaml_variables: true - variables: - PARENT_PIPELINE_ID: $CI_PIPELINE_ID - CASSINI_VERSION: "vNext" - rules: - - if: $CI_COMMIT_TAG =~ /^v\d+\.\d+\.\d+/ - variables: - CASSINI_VERSION: $CI_COMMIT_TAG - - when: on_success needs: - merge_jobs - job: Build-Kas-Docker-Image diff --git a/.gitlab/ci/yocto-build.yml b/.gitlab/ci/yocto-build.yml deleted file mode 100644 index cd82fd6..0000000 --- a/.gitlab/ci/yocto-build.yml +++ /dev/null @@ -1,194 +0,0 @@ -# Copyright (c) 2022 Arm Limited or its affiliates. All rights reserved. -# -# SPDX-License-Identifier: MIT ---- -.yocto_base: - extends: .resource-request - variables: - DOCKER_IMAGE_NAME: kas-build-image - CACHE_DIR: /sstate-cache - KAS_WORK_DIR: $CI_PROJECT_DIR/work - KAS_REPO_REF_DIR: $CACHE_DIR/repos - SSTATE_DIR: $CACHE_DIR/$CASSINI_VERSION/yocto-sstate - DL_DIR: $CACHE_DIR/$CASSINI_VERSION/yocto-downloads - IMAGE_DIR: $CACHE_DIR/$CASSINI_VERSION/images - TOOLCHAIN_DIR: $CACHE_DIR/toolchains - image: $CI_REGISTRY/$CI_PROJECT_PATH/$DOCKER_IMAGE_NAME:$DOCKER_IMAGE_TAG - before_script: - - mkdir -p ~/.ssh - - eval "$(ssh-agent -s)" - - if [ -z "${SSH_PRIVATE_GITLAB}+x" ]; then - echo "${SSH_PRIVATE_GITLAB}" | tr -d '\r' | ssh-add - > /dev/null; - fi - - ssh-keyscan -t rsa ${CI_SERVER_HOST} >> ~/.ssh/known_hosts - # Ensure git is usable without prompts - - git config --global url.ssh://git@${CI_SERVER_HOST}.insteadOf - https://${CI_SERVER_HOST} - - git config --global user.email "you@example.com" - - git config --global user.name "Your Name" - # export parent pipeline global variables - - echo PARENT_PIPELINE_ID = $PARENT_PIPELINE_ID - - cat global_variables.env - - source global_variables.env - # Setup KAS workspace - - echo KAS_WORK_DIR = $KAS_WORK_DIR - - echo SSTATE_DIR = $SSTATE_DIR - - echo DL_DIR = $DL_DIR - - echo IMAGE_DIR = $IMAGE_DIR - - echo CI_PROJECT_DIR = $CI_PROJECT_DIR - - rm -rf $KAS_WORK_DIR - - echo "Persistent cache volume could be empty, create directories required" - - mkdir --verbose --parents - $KAS_WORK_DIR - $KAS_REPO_REF_DIR - $SSTATE_DIR - $DL_DIR - $IMAGE_DIR - $TOOLCHAIN_DIR - - if [ "$CI_DEBUG_TRACE" == "true" ]; then - env | sort; - fi - needs: - - pipeline: $PARENT_PIPELINE_ID - job: merge_jobs - artifacts: true - -.yocto_build: - stage: Build - extends: .yocto_base - needs: - - Update-Repos - - pipeline: $PARENT_PIPELINE_ID - job: merge_jobs - artifacts: true - variables: - KUBERNETES_CPU_REQUEST: '15' - KUBERNETES_MEMORY_REQUEST: 10Gi - KUBERNETES_EPHEMERAL_STORAGE_REQUEST: 30Gi - rules: - - when: on_success - script: - # Run build - - echo "Building target $KAS_CONFIGS" - - KASFILES=$(./.gitlab/scripts/jobs-to-kas ${KAS_CONFIGS}) - - echo KASFILES = $KASFILES - - export BB_NUMBER_THREADS="${KUBERNETES_CPU_REQUEST}" - - export PARALLEL_MAKE="-j ${KUBERNETES_CPU_REQUEST}" - - kas shell --update --force-checkout $KASFILES -c 'cat conf/*.conf' - - kas build $KASFILES; - artifacts: - paths: - - global_variables.env - expire_in: '1 days' - -Prune-Cache: - extends: .yocto_base - stage: Cleanup - # Can fail if build job are running in other pipelines - allow_failure: true - rules: - - if: '$KAS_CONFIGS == null && $FREQUENCY == "nightly"' - - if: '$KAS_CONFIGS == null && $FREQUENCY == "adhoc"' - when: manual - script: - - du -sh $SSTATE_DIR - - find $SSTATE_DIR -type f -atime +5 -delete - - echo "After cache cleanup" - - du -sh $SSTATE_DIR - - du -sh $DL_DIR - - find $DL_DIR -maxdepth 1 -type f -atime +5 -delete - - echo "After download cleanup" - - du -sh $DL_DIR - - -# -# Prep stage, update repositories once -# -Update-Repos: - extends: .yocto_base - stage: Setup - rules: - - if: '$KAS_CONFIGS == null' - script: - - flock --verbose --timeout 60 $KAS_REPO_REF_DIR - ./.gitlab/scripts/update-repos - -# -# Utility tasks, not executed automatically -# -Setup-Cache: - # Ensure this is not on an image with cassini-ci user - extends: .resource-request - stage: Setup - allow_failure: true - rules: - - if: '$KAS_CONFIGS == null' - when: manual - script: - - chmod 777 /sstate-cache - -Delete-DL-Dir: - extends: .yocto_base - stage: Setup - allow_failure: true - rules: - - if: '$KAS_CONFIGS == null' - when: manual - - if: '$KAS_CONFIGS == null && $CI_COMMIT_TAG' - script: - - rm -rf $DL_DIR/* - -Delete-Repo-Dir: - extends: .yocto_base - stage: Setup - allow_failure: true - rules: - - if: '$KAS_CONFIGS == null' - when: manual - script: - - rm -rf $KAS_REPO_REF_DIR/* - -# Delete all sstate -Delete-SState: - extends: .yocto_base - stage: Setup - allow_failure: true - rules: - - if: '$KAS_CONFIGS == null' - when: manual - - if: '$KAS_CONFIGS == null && $CI_COMMIT_TAG' - script: - - rm -rf $SSTATE_DIR/* - -# Delete stored images -Delete-Images: - extends: .yocto_base - stage: Setup - allow_failure: true - rules: - - if: '$KAS_CONFIGS == null' - when: manual - - if: '$KAS_CONFIGS == null && $CI_COMMIT_TAG' - script: - - rm -rf $IMAGE_DIR/* - -Delete-Toolchains: - extends: .yocto_base - stage: Setup - allow_failure: true - rules: - - if: '$KAS_CONFIGS == null' - when: manual - script: - - rm -rf $TOOLCHAIN_DIR/* - -# Report on disk usage -Disk-Usage: - extends: .yocto_base - stage: Setup - allow_failure: true - rules: - - if: '$KAS_CONFIGS == null' - when: manual - script: - - du -h -s $DL_DIR $SSTATE_DIR $IMAGE_DIR $KAS_REPO_REF_DIR $TOOLCHAIN_DIR diff --git a/.gitlab/ci/yocto-qa.yml b/.gitlab/ci/yocto-qa.yml index 952c222..5c0c21e 100644 --- a/.gitlab/ci/yocto-qa.yml +++ b/.gitlab/ci/yocto-qa.yml @@ -2,42 +2,10 @@ # # SPDX-License-Identifier: MIT --- + # Validate layers are Yocto Project Compatible check-layers: - extends: .resource-request - image: $CI_REGISTRY/$CI_PROJECT_PATH/$DOCKER_IMAGE_NAME:$DOCKER_IMAGE_TAG - stage: Build - needs: - - job: Build-Kas-Docker-Image - optional: true - variables: - DOCKER_IMAGE_NAME: kas-build-image - CACHE_DIR: /sstate-cache - KAS_WORK_DIR: $CI_PROJECT_DIR/work - KAS_REPO_REF_DIR: $CACHE_DIR/repos - SSTATE_DIR: $CACHE_DIR/$CASSINI_RELEASE_ID/yocto-sstate - DL_DIR: $CACHE_DIR/$CASSINI_RELEASE_ID/yocto-downloads - TOOLCHAIN_DIR: $CACHE_DIR/toolchains - before_script: - - if [ "$CI_DEBUG_TRACE" == "true" ]; then - env | sort; - fi - - mkdir -p ~/.ssh - - eval "$(ssh-agent -s)" - - if [ -z "${SSH_PRIVATE_GITLAB}+x" ]; then - echo "${SSH_PRIVATE_GITLAB}" | tr -d '\r' | ssh-add - > /dev/null; - fi - - ssh-keyscan -t rsa ${CI_SERVER_HOST} >> ~/.ssh/known_hosts - # Ensure git is usable without prompts - - git config --global url.ssh://git@${CI_SERVER_HOST}.insteadOf - https://${CI_SERVER_HOST} - - git config --global user.email "you@example.com" - - git config --global user.name "Your Name" - # Setup KAS workspace - - rm -rf $KAS_WORK_DIR - - echo "Persistent cache volume could be empty, create directories required" - - mkdir --verbose --parents $KAS_WORK_DIR $KAS_REPO_REF_DIR $SSTATE_DIR - $DL_DIR $IMAGE_DIR $TOOLCHAIN_DIR + extends: .kas_build_cassini script: - kas shell --update --force-checkout meta-cassini-config/kas/n1sdp.yml:meta-cassini-config/kas/tests.yml:meta-cassini-config/kas/cassini.yml:meta-cassini-config/kas/security.yml diff --git a/.gitlab/scripts/child-job-test-report.py b/.gitlab/scripts/child-job-test-report.py deleted file mode 100644 index 01b390f..0000000 --- a/.gitlab/scripts/child-job-test-report.py +++ /dev/null @@ -1,94 +0,0 @@ -# Copyright (c) 2022 Arm Limited or its affiliates. All rights reserved. -# -# SPDX-License-Identifier: MIT - -# Generates JUnit report for all child jobs - -import json -import sys -import requests - -from envparse import Env -from junit_xml import TestSuite, TestCase - - -def get_response(url, headers): - """ - Call the GitLab API and return the json formated result - """ - response = requests.get(url, headers=headers) - if not response.ok: - try: - generic_message = response.json().get('message', generic_message) - except json.decoder.JSONDecodeError: - generic_message = 'Invalid response from GitLab' - sys.stderr.write(generic_message + '\n') - exit(1) - return response.json() - - -def get_test_results(headers, project_url, pipeline_id): - """ - fetch a pipelines test report - """ - report_url = f'{project_url}/pipelines/{pipeline_id}/test_report' - - return get_response(f'{report_url}', headers) - - -def generate_junit_file(test_results, filename): - """ - Generate a Junit file from pipeline test results - """ - tss = [] - for test_suite in test_results['test_suites']: - tcs = [] - for test_case in test_suite['test_cases']: - tc = TestCase(test_case['name'], - elapsed_sec=test_case['execution_time'], - classname=test_suite['name'] + - "." + test_case['classname'], - stdout=test_case['system_output']) - - if test_case['status'] == 'error': - tc.add_failure_info("error") - elif test_case['status'] == 'skipped': - tc.add_skipped_info("skipped") - - tcs.append(tc) - - tss.append(TestSuite(test_suite['name'], tcs)) - - with open(filename, 'w') as f: - TestSuite.to_file(f, tss) - - -def main(): - """ - download artifacts and place them in the jobs directory - """ - env = Env() - - v4_origin = env('CI_API_V4_URL') - project_id = env("CI_PROJECT_ID") - pipeline_id = env('CI_PIPELINE_ID') - headers = { - 'PRIVATE-TOKEN': env('CI_BOT_API_TOKEN') - } - - project_url = f'{v4_origin}/projects/{project_id}' - page_url = f'{project_url}/pipelines/{pipeline_id}/bridges/' - json_response = get_response(f'{page_url}', headers) - - for bridge in json_response: - if bridge['downstream_pipeline']: - test_results = get_test_results(headers, - project_url, - bridge['downstream_pipeline']['id']) - - generate_junit_file(test_results, "TEST-ChildPipeline-" + - str(bridge['downstream_pipeline']['iid']) + ".xml") - - -if __name__ == '__main__': - main() -- GitLab From a167d47869218d3eb8500e356cdf2bcd862c268b Mon Sep 17 00:00:00 2001 From: Drew Reed Date: Thu, 3 Nov 2022 14:09:48 +0000 Subject: [PATCH 2/5] ci: Static Analysis exclude path update Adds the layers directory to the exclude path of all the codeclimate based static analysis plugins to allow the tools to be run locally on a complete source tree. Also removes the non existent meta-eawol directory from the headercheck exclusions Signed-off-by: Drew Reed --- .codeclimate.yml | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/.codeclimate.yml b/.codeclimate.yml index c64e18a..a2afc4d 100644 --- a/.codeclimate.yml +++ b/.codeclimate.yml @@ -4,24 +4,34 @@ --- version: "2" plugins: + duplication: + exclude_patterns: + - "/layers/" structure: exclude_patterns: - - documentation/variables.py + - "/layers/" + - "documentation/variables.py" pep8: enabled: true exclude_patterns: - - documentation/variables.py + - "/layers/" + - "documentation/variables.py" sonar-python: enabled: true + exclude_patterns: + - "/layers/" shellcheck: enabled: true file_types: - "shell script" - "bats script" - "bash script" + exclude_patterns: + - "/layers/" fixme: enabled: true exclude_patterns: + - "/layers/" - "/Dangerfile" spellcheck: enabled: true @@ -40,10 +50,12 @@ plugins: - "/documentation/index.rst" - "/documentation/requirements.txt" - "__pycache__" + - "/layers/" dict_path: "meta-cassini-config/qa-checks/cassini-dictionary" yamllint: enabled: true exclude_patterns: + - "/layers/" - ".rubocop.yml" - ".eslintrc.yml" headercheck: @@ -55,7 +67,7 @@ plugins: - ROOT exclude_patterns: - GITIGNORE_CONTENTS - - "meta-ewaol/" + - "/layers/" - "global_variables.env" - "meta-cassini-config/qa-checks/cassini-dictionary" - "/LICENSE.rst" -- GitLab From c339c8163746b1efa0d04dcd2a45b720876948a2 Mon Sep 17 00:00:00 2001 From: Drew Reed Date: Mon, 14 Nov 2022 14:24:53 +0000 Subject: [PATCH 3/5] ci: Update headercheck configuration Update the codeclimate headercheck plugin configuration to the new format as this allows us to setup the allowed licences for the repo Signed-off-by: Drew Reed --- .codeclimate.yml | 49 ++++++++++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/.codeclimate.yml b/.codeclimate.yml index a2afc4d..0d886e1 100644 --- a/.codeclimate.yml +++ b/.codeclimate.yml @@ -60,28 +60,27 @@ plugins: - ".eslintrc.yml" headercheck: enabled: true - config_options: - modules: - header: - paths: - - ROOT - exclude_patterns: - - GITIGNORE_CONTENTS - - "/layers/" - - "global_variables.env" - - "meta-cassini-config/qa-checks/cassini-dictionary" - - "/LICENSE.rst" - - "/CHANGELOG.md" - - "*.git" - - "*.gitignore" - - "*.pyc" - - "*.png" - - "*.cfg" - - "*.css" - - "*.scc" - - "*.patch" - - ".rubocop.yml" - - ".eslintignore" - - ".eslintrc.yml" - - ".csslintrc" - - "coffeelint.json" + original_works_licenses: + - "MIT" + external_works_licenses: + - "MIT" + exclude_patterns: + - GITIGNORE_CONTENTS + - "/layers/" + - "global_variables.env" + - "meta-cassini-config/qa-checks/cassini-dictionary" + - "/LICENSE.rst" + - "/CHANGELOG.md" + - "*.git" + - "*.gitignore" + - "*.pyc" + - "*.png" + - "*.cfg" + - "*.css" + - "*.scc" + - "*.patch" + - ".rubocop.yml" + - ".eslintignore" + - ".eslintrc.yml" + - ".csslintrc" + - "coffeelint.json" -- GitLab From 595151f7866670349f7cf87ca639a15ce11c62cb Mon Sep 17 00:00:00 2001 From: Drew Reed Date: Thu, 3 Nov 2022 14:10:44 +0000 Subject: [PATCH 4/5] ci: Update spellcheck configuration Update the codeclimate spellcheck plugin configuration to the new format and add to dictionary and exclude list so that it no longer flags false positives Signed-off-by: Drew Reed --- .codeclimate.yml | 40 ++++--- .../qa-checks/cassini-dictionary | 111 ++++++++++++++---- 2 files changed, 113 insertions(+), 38 deletions(-) diff --git a/.codeclimate.yml b/.codeclimate.yml index 0d886e1..821f402 100644 --- a/.codeclimate.yml +++ b/.codeclimate.yml @@ -35,23 +35,29 @@ plugins: - "/Dangerfile" spellcheck: enabled: true - config_options: - modules: - spell: - paths: - - "documentation" - exclude_patterns: - - GITIGNORE_CONTENTS - - "*.git" - - "*.png" - - "/documentation/conf.py" - - "/documentation/variables.py" - - "/documentation/Makefile" - - "/documentation/index.rst" - - "/documentation/requirements.txt" - - "__pycache__" - - "/layers/" - dict_path: "meta-cassini-config/qa-checks/cassini-dictionary" + exclude_patterns: + - GITIGNORE_CONTENTS + - "*.git" + - ".gitlab/" + - "*.yml" + - "*.yaml" + - "*.json" + - "/meta-cassini-*" + - "/gitlab/" + - "*.png" + - "/documentation/conf.py" + - "/documentation/variables.py" + - "/documentation/Makefile" + - "/documentation/index.rst" + - "/documentation/requirements.txt" + - "__pycache__" + - "/layers/" + - "CHANGELOG.md" + - "Dangerfile" + - ".csslintrc" + - ".eslintignore" + - ".gitignore" + dict_path: "meta-cassini-config/qa-checks/cassini-dictionary" yamllint: enabled: true exclude_patterns: diff --git a/meta-cassini-config/qa-checks/cassini-dictionary b/meta-cassini-config/qa-checks/cassini-dictionary index bf2b35b..0f34b06 100644 --- a/meta-cassini-config/qa-checks/cassini-dictionary +++ b/meta-cassini-config/qa-checks/cassini-dictionary @@ -1,37 +1,50 @@ -1.1.0+git0+b9460f26b4 1.0.0+rc93+git0+249bca0a13 -3.2.1+git0+ab4d0cf908 -4.16+stable0+f265444922 +1.1.0+git0+b9460f26b4 20.10.12+ce+git906f57f +3.2.1+git0+ab4d0cf908 3rd +4.16+stable0+f265444922 +512kb +CASSINI +CHANGELOG.md +FF-A +Signed-off-by +TF-A +a-useful-template-for-commit-messages aarch64 aem +api +api's +append +appending appends arm-machines.yml armv8-a armv8a-crc +aws b baremetal -baremetal_architecture.png baremetal-sdk.yml baremetal.yml +baremetal_architecture.png bblayers.conf bitbake bootloader bsp bsps -build_system build-time +build_system c cassini cassini's -CASSINI +cassini-config cassini_admin_group.in cassini_architecture.png cassini_layers_deps.png cassini_rel_dev_branches.png cassini_rel_yocto_overview.png cassini_rootfs_extra_space +cbea.ms ce_test_clean_env ce_test_guest_vm_name ce_test_image @@ -40,10 +53,14 @@ cgit cgit.cgi changelog cmd +codeinthehole.com codeline codeline_management +comma's commit_patch_message_guidelines committer +conditionally +config configs configurable configures @@ -53,23 +70,33 @@ containerized containers_kernelcfg_check.bbclass contributing_documentation_build_validation cortex-a +customizable customization customizations +cve +dangerfile datetime dbg +dependencies +deprecated +deprecation +developer-facing +developer.example.org developer_manual devicetree -deprecated diego.sueiro +distributions distro_features do_fetch docker's docs.lavasoftware.org dom0 domu -efi e.g +efi +endpoint extensible +f filenames filesystem filesystems @@ -83,11 +110,15 @@ fvp_base_revc-2xaemva_11.14_21.tgz gb gbytes generic-arm64 +git-scm.com +git-tools-rewriting-history +gitaly github gitlab -gitlab.arm.com gitlab-ci.yml +gitlab.arm.com go-fsnotify +greengrass h.twlc0ztnng3b hardknott high_level_overview.png @@ -101,45 +132,54 @@ image_rootfs_extra_space implementations independently-configurable index.html +init initializes introduction_use_cases_overview j k3s k3s-integration-tests.bb -k3s_git.bb -k3s_kernelcfg_check.bbclass k3s-killall.sh k3s.service +k3s_git.bb +k3s_kernelcfg_check.bbclass k3s_test_clean_env k3s_test_guest_vm_name k3s_test_log_dir kb kernel_features kirkstone +kube-system kubectl kubernetes -kube-system langdale layer_dependency_overview_label libpcre libpcre2 -licensees license.rst +licensees linter +linux +linux-based localhost loopback lts m +maintainers maintainers_label -maxdepth manual_build_system_target_platforms manual_yocto_layers_layer_dependency_overview +maxdepth mb +merchantability +merge_requests meta-arm-bsp +meta-cassini-config meta-openembedded meta-virtualization minicom modifiable +modifier +mr multiconfig n1 n1sdp @@ -148,17 +188,27 @@ n1sdp.yml neoverse nginx nodeport +noninfringement oci openembedded optionally ostree +parallaxsecond +parsec-cli-tests.sh +parsec-simple-e2e-tests +parsec-simple-e2e-tests.bb pc pep8 png podman ppa ppas +prefixed +prefixes project_root +ps +ps_test_clean_env +ps_test_log_dir ptest pycodestyle pyspellchecker @@ -169,20 +219,25 @@ quickstart r readability readme.md +rebase +rebases refactor refactored +refactoring refspec +remediation repository's requirements.txt restructuredtext return_code revc +reword rootfs rootfs_extra_space rpm rst -runc-opencontainers run-time +runc-opencontainers runtime runtime-integration-tests sc2086 @@ -197,19 +252,26 @@ security_hardening shellcheck shellcheck-py shortlog +sign-off soafee soafee.io sourceforge spdx spdx-license-identifier +spdx-specification-21-web-version +spdx.dev +spdx.org sphinx-build +squash_and_merge.html ssh sstate-cache +standalone standards-based stderr stdout sub-sections sub_test_name +sublicense submitting-patches.html sudo sudoers @@ -222,14 +284,18 @@ titlesonly toctree top_level_test_name ua_test_clean_env -ua_test_log_dir ua_test_guest_vm_name +ua_test_log_dir ubuntu ubuntu-based umask +upstreamed +upto +urls use-cases -user_accounts user-accounts-integration-tests.bb +user-facing +user_accounts user_guide user_guide_reproduce_environment_setup user_manual @@ -243,14 +309,15 @@ virt_test_clean_env virt_test_guest_vm_name virt_test_log_dir virtualization -virtualization_architecture.png virtualization-integration-tests.bb virtualization-sdk.yml virtualization.yml +virtualization_architecture.png virtualized vm vm's vms +vs w webserver wget @@ -259,16 +326,18 @@ with-libcap[-ng workloads www.kernel.org www.openembedded.org +x x86_64-linux xen -xenbr0 xen-devicetree.bb +xen_kernelcfg_check.bbclass +xenbr0 xendomains xendomains.service -xen_kernelcfg_check.bbclass xl yaml yamllint yocto -yocto_layers yocto's +yocto_layers +your-branch-name -- GitLab From 1e54d372d5bf3a828f1424835057d4e29316f75d Mon Sep 17 00:00:00 2001 From: Drew Reed Date: Mon, 14 Nov 2022 15:05:09 +0000 Subject: [PATCH 5/5] ci: Reduce the number of pipeline jobs By using the parallel feature we can reduce the number of jobs that need to be specified in the pipeline definition. This helps to make it easier to view to pipeline graph in the GitLab UI Signed-off-by: Drew Reed --- .gitlab/ci/docker-image-builds.yml | 90 +++++++++++------------------- 1 file changed, 33 insertions(+), 57 deletions(-) diff --git a/.gitlab/ci/docker-image-builds.yml b/.gitlab/ci/docker-image-builds.yml index 279972c..de26eab 100644 --- a/.gitlab/ci/docker-image-builds.yml +++ b/.gitlab/ci/docker-image-builds.yml @@ -2,101 +2,77 @@ # # SPDX-License-Identifier: MIT --- -.Build-Kas-Docker-Image: +Build-Kas-Docker-Arch-Image: extends: .build-docker-image - variables: + parallel: &Build-Matrix + matrix: + - BUILD_ARCH: x86_64 + ARCH_SUFFIX: -amd64 + - BUILD_ARCH: arm64 + ARCH_SUFFIX: -arm64v8 + tags: &Build-Kas-Docker-Arch-Image_tags + - ${BUILD_ARCH} + variables: &Build-Kas-Docker-Arch-Image_variables DOCKER_IMAGE_NAME: kas-build-image KANIKO_BUILD_CONTEXT: $CI_PROJECT_DIR/.gitlab/Dockerfiles/$DOCKER_IMAGE_NAME - rules: + rules: &Build-Kas-Docker-Arch-Image_rules - changes: - .gitlab-ci.yml - .gitlab/ci/docker-image-builds.yml - .gitlab/Dockerfiles/kas-build-image/**/* -Build-Kas-Docker-Image-x86_64: - extends: .Build-Kas-Docker-Image - variables: - ARCH_SUFFIX: -amd64 - -Build-Kas-Docker-Image-arm64: - tags: - - arm64 - extends: .Build-Kas-Docker-Image - variables: - ARCH_SUFFIX: -arm64v8 - Build-Kas-Docker-Image: extends: - - .Build-Kas-Docker-Image + - .build-docker-image - .build-docker-multiarch-image + variables: *Build-Kas-Docker-Arch-Image_variables + rules: *Build-Kas-Docker-Arch-Image_rules needs: - - job: Build-Kas-Docker-Image-x86_64 - artifacts: false - - job: Build-Kas-Docker-Image-arm64 + - job: Build-Kas-Docker-Arch-Image artifacts: false -.Build-Utility-Docker-Image: +Build-Utility-Docker-Arch-Image: extends: .build-docker-image - variables: + parallel: *Build-Matrix + tags: *Build-Kas-Docker-Arch-Image_tags + variables: &Build-Utility-Docker-Arch-Image_variables DOCKER_IMAGE_NAME: utility-image KANIKO_BUILD_CONTEXT: $CI_PROJECT_DIR/.gitlab/Dockerfiles/$DOCKER_IMAGE_NAME - rules: + rules: &Build-Utility-Docker-Arch-Image_rules - changes: - .gitlab-ci.yml - .gitlab/ci/docker-image-builds.yml - .gitlab/Dockerfiles/utility-image/**/* -Build-Utility-Docker-Image-x86_64: - extends: .Build-Utility-Docker-Image - variables: - ARCH_SUFFIX: -amd64 - -Build-Utility-Docker-Image-arm64: - tags: - - arm64 - extends: .Build-Utility-Docker-Image - variables: - ARCH_SUFFIX: -arm64v8 - Build-Utility-Docker-Image: extends: - - .Build-Utility-Docker-Image + - .build-docker-image - .build-docker-multiarch-image + variables: *Build-Utility-Docker-Arch-Image_variables + rules: *Build-Utility-Docker-Arch-Image_rules needs: - - job: Build-Utility-Docker-Image-x86_64 - artifacts: false - - job: Build-Utility-Docker-Image-arm64 + - job: Build-Utility-Docker-Arch-Image artifacts: false -.Build-Lava-Docker-Image: +Build-Lava-Docker-Arch-Image: extends: .build-docker-image - variables: + parallel: *Build-Matrix + tags: *Build-Kas-Docker-Arch-Image_tags + variables: &Build-Lava-Docker-Arch-Image_variables DOCKER_IMAGE_NAME: lava-test-image KANIKO_BUILD_CONTEXT: $CI_PROJECT_DIR/.gitlab/Dockerfiles/$DOCKER_IMAGE_NAME - rules: + rules: &Build-Lava-Docker-Arch-Image_rules - changes: - .gitlab-ci.yml - .gitlab/ci/docker-image-builds.yml - .gitlab/Dockerfiles/lava-test-image/**/* -Build-Lava-Docker-Image-x86_64: - extends: .Build-Lava-Docker-Image - variables: - ARCH_SUFFIX: -amd64 - -Build-Lava-Docker-Image-arm64: - tags: - - arm64 - extends: .Build-Lava-Docker-Image - variables: - ARCH_SUFFIX: -arm64v8 - Build-Lava-Docker-Image: extends: - - .Build-Lava-Docker-Image + - .build-docker-image - .build-docker-multiarch-image + variables: *Build-Lava-Docker-Arch-Image_variables + rules: *Build-Lava-Docker-Arch-Image_rules needs: - - job: Build-Lava-Docker-Image-x86_64 - artifacts: false - - job: Build-Lava-Docker-Image-arm64 + - job: Build-Lava-Docker-Arch-Image artifacts: false -- GitLab