diff --git a/.codeclimate.yml b/.codeclimate.yml index c64e18a7aab61c4548309b8c9e41ace7ad91b780..821f402ffce2a618d8fa4231757def3b2415f52e 100644 --- a/.codeclimate.yml +++ b/.codeclimate.yml @@ -4,72 +4,89 @@ --- version: "2" plugins: + duplication: + exclude_patterns: + - "/layers/" structure: exclude_patterns: - - documentation/variables.py + - "/layers/" + - "documentation/variables.py" pep8: enabled: true exclude_patterns: - - documentation/variables.py + - "/layers/" + - "documentation/variables.py" sonar-python: enabled: true + exclude_patterns: + - "/layers/" shellcheck: enabled: true file_types: - "shell script" - "bats script" - "bash script" + exclude_patterns: + - "/layers/" fixme: enabled: true exclude_patterns: + - "/layers/" - "/Dangerfile" spellcheck: enabled: true - config_options: - modules: - spell: - paths: - - "documentation" - exclude_patterns: - - GITIGNORE_CONTENTS - - "*.git" - - "*.png" - - "/documentation/conf.py" - - "/documentation/variables.py" - - "/documentation/Makefile" - - "/documentation/index.rst" - - "/documentation/requirements.txt" - - "__pycache__" - dict_path: "meta-cassini-config/qa-checks/cassini-dictionary" + exclude_patterns: + - GITIGNORE_CONTENTS + - "*.git" + - ".gitlab/" + - "*.yml" + - "*.yaml" + - "*.json" + - "/meta-cassini-*" + - "/gitlab/" + - "*.png" + - "/documentation/conf.py" + - "/documentation/variables.py" + - "/documentation/Makefile" + - "/documentation/index.rst" + - "/documentation/requirements.txt" + - "__pycache__" + - "/layers/" + - "CHANGELOG.md" + - "Dangerfile" + - ".csslintrc" + - ".eslintignore" + - ".gitignore" + dict_path: "meta-cassini-config/qa-checks/cassini-dictionary" yamllint: enabled: true exclude_patterns: + - "/layers/" - ".rubocop.yml" - ".eslintrc.yml" headercheck: enabled: true - config_options: - modules: - header: - paths: - - ROOT - exclude_patterns: - - GITIGNORE_CONTENTS - - "meta-ewaol/" - - "global_variables.env" - - "meta-cassini-config/qa-checks/cassini-dictionary" - - "/LICENSE.rst" - - "/CHANGELOG.md" - - "*.git" - - "*.gitignore" - - "*.pyc" - - "*.png" - - "*.cfg" - - "*.css" - - "*.scc" - - "*.patch" - - ".rubocop.yml" - - ".eslintignore" - - ".eslintrc.yml" - - ".csslintrc" - - "coffeelint.json" + original_works_licenses: + - "MIT" + external_works_licenses: + - "MIT" + exclude_patterns: + - GITIGNORE_CONTENTS + - "/layers/" + - "global_variables.env" + - "meta-cassini-config/qa-checks/cassini-dictionary" + - "/LICENSE.rst" + - "/CHANGELOG.md" + - "*.git" + - "*.gitignore" + - "*.pyc" + - "*.png" + - "*.cfg" + - "*.css" + - "*.scc" + - "*.patch" + - ".rubocop.yml" + - ".eslintignore" + - ".eslintrc.yml" + - ".csslintrc" + - "coffeelint.json" diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7484c2c5855cee35c2a5c7bb97a12cbd86f0ab48..d38289adb999ed176973c686cc3de2e4c854ff70 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -36,7 +36,7 @@ variables: # include jobs include: - project: $PIPELINE_TEMPLATE_PROJECT - ref: v0.0.10 + ref: v0.0.12 file: - gitlab-ci/common_gitlab-ci.yml - gitlab-ci/docker-image-base_gitlab-ci.yml @@ -44,11 +44,12 @@ include: - gitlab-ci/changelog_gitlab-ci.yml - gitlab-ci/auto-release_gitlab-ci.yml - gitlab-ci/static-analysis_gitlab-ci.yml + - gitlab-ci/child-pipeline_gitlab-ci.yml - local: '.gitlab/ci/rules.yml' - local: '.gitlab/ci/docker-image-builds.yml' + - local: '.gitlab/ci/n1sdp-image-builds.yml' - local: '.gitlab/ci/trigger-image-builds.yml' - local: '.gitlab/ci/docs-build.yml' - - local: '.gitlab/ci/yocto-qa.yml' # Job to regenerate the changelog when the project version number is changed changelog: diff --git a/.gitlab/ci/cassini-build.yml b/.gitlab/ci/cassini-build.yml new file mode 100644 index 0000000000000000000000000000000000000000..50a21993ecd43d4d919aa74bbe8a7eb5a634f8b1 --- /dev/null +++ b/.gitlab/ci/cassini-build.yml @@ -0,0 +1,104 @@ +# Copyright (c) 2022 Arm Limited or its affiliates. All rights reserved. +# +# SPDX-License-Identifier: MIT +--- + +# Modify the yocto/kas base build with Cassini specific settings and switch to +# the Cassini kas container +.yocto_base: + variables: + CACHE_DIR: /sstate-cache + SSTATE_DIR: $CACHE_DIR/$PROJECT_VERSION/yocto-sstate + DL_DIR: $CACHE_DIR/$PROJECT_VERSION/yocto-downloads + TOOLCHAIN_DIR: $CACHE_DIR/$PROJECT_VERSION/toolchains + +.kas_build: + image: $CI_REGISTRY/$CI_PROJECT_PATH/$DOCKER_IMAGE_NAME:$DOCKER_IMAGE_TAG + variables: + DOCKER_IMAGE_NAME: kas-build-image + KAS_REPO_REF_DIR: $CACHE_DIR/$PROJECT_VERSION/repos + +# Extend the kas base build to enab;e to use of repos that are not yet public +.kas_build_cassini: + extends: + - .kas_build + variables: + IMAGE_DIR: $CACHE_DIR/$PROJECT_VERSION/images + before_script: + - mkdir -p ~/.ssh + - eval "$(ssh-agent -s)" + - if [ -z "${SSH_PRIVATE_GITLAB}+x" ]; then + echo "${SSH_PRIVATE_GITLAB}" | tr -d '\r' | ssh-add - > /dev/null; + fi + - ssh-keyscan -t rsa ${CI_SERVER_HOST} >> ~/.ssh/known_hosts + # Ensure git is usable without prompts + - git config --global url.ssh://git@${CI_SERVER_HOST}.insteadOf + https://${CI_SERVER_HOST} + - git config --global user.email "you@example.com" + - git config --global user.name "Your Name" + # export parent pipeline global variables + - echo PARENT_PIPELINE_ID = $PARENT_PIPELINE_ID + - !reference [".kas_build", before_script] + # Setup workspace + - echo "Persistent cache volume could be empty, create directories required" + - mkdir --verbose --parents + $IMAGE_DIR + +# Delete stored images +Delete-Images: + extends: .kas_build_cassini + stage: Setup + needs: [] + allow_failure: true + rules: + - if: '$CI_PIPELINE_SOURCE == "parent_pipeline" && $CI_COMMIT_TAG' + - if: '$CI_PIPELINE_SOURCE == "parent_pipeline"' + when: manual + script: + - rm -rf $IMAGE_DIR/* + +# Report on disk usage including the image directory +Disk-Usage: + extends: .kas_build_cassini + needs: [] + rules: + - if: '$CI_PIPELINE_SOURCE == "parent_pipeline"' + when: manual + script: + - du -h -s $DL_DIR $SSTATE_DIR $TOOLCHAIN_DIR $KAS_REPO_REF_DIR $IMAGE_DIR + +# Suppress the use of the yocto/kas helper jobs as they +# should only be run in the child pipeline +Prune-Cache: + rules: + - if: '$CI_PIPELINE_SOURCE == "parent_pipeline" && $FREQUENCY == "nightly"' + - if: '$CI_PIPELINE_SOURCE == "parent_pipeline" && $FREQUENCY == "adhoc"' + when: manual + needs: [] + +Update-Repos: + rules: + - if: '$CI_PIPELINE_SOURCE == "parent_pipeline"' + +Delete-DL-Dir: + rules: + - if: '$CI_PIPELINE_SOURCE == "parent_pipeline" && $CI_COMMIT_TAG' + - if: '$CI_PIPELINE_SOURCE == "parent_pipeline"' + when: manual + +Delete-SState: + rules: + - if: '$CI_PIPELINE_SOURCE == "parent_pipeline" && $CI_COMMIT_TAG' + - if: '$CI_PIPELINE_SOURCE == "parent_pipeline"' + when: manual + +Delete-Toolchains: + rules: + - if: '$CI_PIPELINE_SOURCE == "parent_pipeline" && $CI_COMMIT_TAG' + - if: '$CI_PIPELINE_SOURCE == "parent_pipeline"' + when: manual + +Delete-Repo-Dir: + rules: + - if: '$CI_PIPELINE_SOURCE == "parent_pipeline"' + when: manual diff --git a/.gitlab/ci/collate-results.yml b/.gitlab/ci/collate-results.yml deleted file mode 100644 index 6ded770e625bf9c90b1347e381989a6382d3874a..0000000000000000000000000000000000000000 --- a/.gitlab/ci/collate-results.yml +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright (c) 2022 Arm Limited or its affiliates. All rights reserved. -# -# SPDX-License-Identifier: MIT ---- -Collate-Results: - extends: .resource-request - image: "${MIRROR_DOCKER}/python:slim" - stage: Test - dependencies: - - merge_jobs - - trigger_jobs - before_script: - - pip install requests envparse junit_xml - script: - - python .gitlab/scripts/child-job-test-report.py - artifacts: - paths: - - TEST-ChildPipeline-*.xml - reports: - junit: - - TEST-ChildPipeline-*.xml - expire_in: 1 month - when: always diff --git a/.gitlab/ci/docker-image-builds.yml b/.gitlab/ci/docker-image-builds.yml index 279972ccfc8261dd81510720ac7e97c4270584fc..de26eab104f76456e4e95b539f3cef7c7f8d5736 100644 --- a/.gitlab/ci/docker-image-builds.yml +++ b/.gitlab/ci/docker-image-builds.yml @@ -2,101 +2,77 @@ # # SPDX-License-Identifier: MIT --- -.Build-Kas-Docker-Image: +Build-Kas-Docker-Arch-Image: extends: .build-docker-image - variables: + parallel: &Build-Matrix + matrix: + - BUILD_ARCH: x86_64 + ARCH_SUFFIX: -amd64 + - BUILD_ARCH: arm64 + ARCH_SUFFIX: -arm64v8 + tags: &Build-Kas-Docker-Arch-Image_tags + - ${BUILD_ARCH} + variables: &Build-Kas-Docker-Arch-Image_variables DOCKER_IMAGE_NAME: kas-build-image KANIKO_BUILD_CONTEXT: $CI_PROJECT_DIR/.gitlab/Dockerfiles/$DOCKER_IMAGE_NAME - rules: + rules: &Build-Kas-Docker-Arch-Image_rules - changes: - .gitlab-ci.yml - .gitlab/ci/docker-image-builds.yml - .gitlab/Dockerfiles/kas-build-image/**/* -Build-Kas-Docker-Image-x86_64: - extends: .Build-Kas-Docker-Image - variables: - ARCH_SUFFIX: -amd64 - -Build-Kas-Docker-Image-arm64: - tags: - - arm64 - extends: .Build-Kas-Docker-Image - variables: - ARCH_SUFFIX: -arm64v8 - Build-Kas-Docker-Image: extends: - - .Build-Kas-Docker-Image + - .build-docker-image - .build-docker-multiarch-image + variables: *Build-Kas-Docker-Arch-Image_variables + rules: *Build-Kas-Docker-Arch-Image_rules needs: - - job: Build-Kas-Docker-Image-x86_64 - artifacts: false - - job: Build-Kas-Docker-Image-arm64 + - job: Build-Kas-Docker-Arch-Image artifacts: false -.Build-Utility-Docker-Image: +Build-Utility-Docker-Arch-Image: extends: .build-docker-image - variables: + parallel: *Build-Matrix + tags: *Build-Kas-Docker-Arch-Image_tags + variables: &Build-Utility-Docker-Arch-Image_variables DOCKER_IMAGE_NAME: utility-image KANIKO_BUILD_CONTEXT: $CI_PROJECT_DIR/.gitlab/Dockerfiles/$DOCKER_IMAGE_NAME - rules: + rules: &Build-Utility-Docker-Arch-Image_rules - changes: - .gitlab-ci.yml - .gitlab/ci/docker-image-builds.yml - .gitlab/Dockerfiles/utility-image/**/* -Build-Utility-Docker-Image-x86_64: - extends: .Build-Utility-Docker-Image - variables: - ARCH_SUFFIX: -amd64 - -Build-Utility-Docker-Image-arm64: - tags: - - arm64 - extends: .Build-Utility-Docker-Image - variables: - ARCH_SUFFIX: -arm64v8 - Build-Utility-Docker-Image: extends: - - .Build-Utility-Docker-Image + - .build-docker-image - .build-docker-multiarch-image + variables: *Build-Utility-Docker-Arch-Image_variables + rules: *Build-Utility-Docker-Arch-Image_rules needs: - - job: Build-Utility-Docker-Image-x86_64 - artifacts: false - - job: Build-Utility-Docker-Image-arm64 + - job: Build-Utility-Docker-Arch-Image artifacts: false -.Build-Lava-Docker-Image: +Build-Lava-Docker-Arch-Image: extends: .build-docker-image - variables: + parallel: *Build-Matrix + tags: *Build-Kas-Docker-Arch-Image_tags + variables: &Build-Lava-Docker-Arch-Image_variables DOCKER_IMAGE_NAME: lava-test-image KANIKO_BUILD_CONTEXT: $CI_PROJECT_DIR/.gitlab/Dockerfiles/$DOCKER_IMAGE_NAME - rules: + rules: &Build-Lava-Docker-Arch-Image_rules - changes: - .gitlab-ci.yml - .gitlab/ci/docker-image-builds.yml - .gitlab/Dockerfiles/lava-test-image/**/* -Build-Lava-Docker-Image-x86_64: - extends: .Build-Lava-Docker-Image - variables: - ARCH_SUFFIX: -amd64 - -Build-Lava-Docker-Image-arm64: - tags: - - arm64 - extends: .Build-Lava-Docker-Image - variables: - ARCH_SUFFIX: -arm64v8 - Build-Lava-Docker-Image: extends: - - .Build-Lava-Docker-Image + - .build-docker-image - .build-docker-multiarch-image + variables: *Build-Lava-Docker-Arch-Image_variables + rules: *Build-Lava-Docker-Arch-Image_rules needs: - - job: Build-Lava-Docker-Image-x86_64 - artifacts: false - - job: Build-Lava-Docker-Image-arm64 + - job: Build-Lava-Docker-Arch-Image artifacts: false diff --git a/.gitlab/ci/n1sdp-image-builds.yml b/.gitlab/ci/n1sdp-image-builds.yml index 217aa3482fa5a6a9793a1e551ea2a41ea3e59fe9..912722a453addbf639f23be660a30dc665ca6a08 100644 --- a/.gitlab/ci/n1sdp-image-builds.yml +++ b/.gitlab/ci/n1sdp-image-builds.yml @@ -2,33 +2,15 @@ # # SPDX-License-Identifier: MIT --- -.generator: - extends: .resource-request - image: $CI_REGISTRY/$CI_PROJECT_PATH/$DOCKER_IMAGE_NAME:$DOCKER_IMAGE_TAG - stage: Setup +.generator-n1sdp: + extends: .generator variables: - DOCKER_IMAGE_NAME: utility-image MACHINE: n1sdp RUN_TESTS: 'none' - script: - - export CI_JOB_NAME_SLUG=${CI_JOB_NAME//\//-} - - mkdir tmp - - j2 .gitlab/ci/templates/image_build.yml.j2 -o tmp/${CI_JOB_NAME_SLUG}.yml; - - if [ "$CI_DEBUG_TRACE" == "true" ]; then - cat tmp/${CI_JOB_NAME_SLUG}.yml; - fi - needs: - - job: Build-Utility-Docker-Image - optional: true - artifacts: false - artifacts: - paths: - - tmp/*.yml - expire_in: '1 days' cassini/tests/n1sdp: - extends: .generator + extends: .generator-n1sdp variables: RUN_TESTS: ptest rules: @@ -38,7 +20,7 @@ cassini/tests/n1sdp: - !reference [".build-image:rules:n1sdp", rules] cassini/security/tests/n1sdp: - extends: .generator + extends: .generator-n1sdp variables: RUN_TESTS: ptest rules: @@ -48,7 +30,7 @@ cassini/security/tests/n1sdp: - !reference [".build-image:rules:n1sdp", rules] cassini-dev/tests/n1sdp: - extends: .generator + extends: .generator-n1sdp variables: RUN_TESTS: ptest rules: @@ -57,7 +39,7 @@ cassini-dev/tests/n1sdp: ($FREQUENCY == "nightly")' cassini-dev/security/tests/n1sdp: - extends: .generator + extends: .generator-n1sdp variables: RUN_TESTS: ptest rules: @@ -66,7 +48,7 @@ cassini-dev/security/tests/n1sdp: ($FREQUENCY == "nightly")' cassini-sdk/n1sdp: - extends: .generator + extends: .generator-n1sdp rules: - if: '($BUILD_IMAGE =~ /all-images/ || $BUILD_IMAGE =~ /k3s/) && ($BUILD_PLATFORM =~ /all-platforms/ || $BUILD_PLATFORM =~ /n1sdp/)' diff --git a/.gitlab/ci/rules.yml b/.gitlab/ci/rules.yml index 01bfe6d3f12949cc13994808ff9e4a4fca3b6db9..008d3b71ae114a347f96efcae3375160b710f223 100644 --- a/.gitlab/ci/rules.yml +++ b/.gitlab/ci/rules.yml @@ -12,10 +12,13 @@ # Changes patterns .ci-patterns: &ci-patterns - ".gitlab-ci.yml" + - ".gitlab/ci/cassini-build.yml" + - ".gitlab/ci/collate-results.yml" + - ".gitlab/ci/docs-build.yml" + - ".gitlab/ci/lava-test.yml" - ".gitlab/ci/rules.yml" - ".gitlab/ci/trigger-image-builds.yml" - - ".gitlab/ci/yocto-build.yml" - - ".gitlab/ci/lava-test.yml" + - ".gitlab/ci/yocto-qa.yml" - ".gitlab/ci/templates/image_build.yml.j2" - ".gitlab/scripts/**/*" diff --git a/.gitlab/ci/templates/image_build.yml.j2 b/.gitlab/ci/templates/image_build.yml.j2 index 081119e6912184d9fa6898388c7184a8efd0f03f..674e60c5600eac09c064e2d4e3c663668629051b 100644 --- a/.gitlab/ci/templates/image_build.yml.j2 +++ b/.gitlab/ci/templates/image_build.yml.j2 @@ -7,7 +7,7 @@ Image-Build-{{ CI_JOB_NAME_SLUG }}: tags: - x86_64 {% endif %} - extends: .yocto_build + extends: .kas_build_cassini # Store the build artifacts from the job. The download urls of these artifacts are later used by lava jobs to download and test the image. variables: KAS_CONFIGS: "{{ CI_JOB_NAME }}" diff --git a/.gitlab/ci/trigger-image-builds.yml b/.gitlab/ci/trigger-image-builds.yml index 3fd9c2323f73575e0ce21bbe58a01d15d3c62257..26cc461d736351d0c1d9f303257fa1aabc8d4ac0 100644 --- a/.gitlab/ci/trigger-image-builds.yml +++ b/.gitlab/ci/trigger-image-builds.yml @@ -2,17 +2,8 @@ # # SPDX-License-Identifier: MIT --- -include: - - local: '.gitlab/ci/n1sdp-image-builds.yml' - - local: '.gitlab/ci/collate-results.yml' merge_jobs: - extends: .resource-request - image: $CI_REGISTRY/$CI_PROJECT_PATH/$DOCKER_IMAGE_NAME:$DOCKER_IMAGE_TAG - stage: Setup - variables: - DOCKER_IMAGE_NAME: utility-image - CI_JOB_FILE: yocto-builds.yml needs: - job: Build-Utility-Docker-Image optional: true @@ -27,57 +18,24 @@ merge_jobs: optional: true - job: cassini-dev/security/tests/n1sdp optional: true - script: - - set -e + before_script: - | cat << EOF > ${CI_JOB_FILE} include: - project: $PIPELINE_TEMPLATE_PROJECT - ref: v0.0.10 + ref: v$PIPELINE_TEMPLATE_VERSION file: - gitlab-ci/common_gitlab-ci.yml + - gitlab-ci/yocto-build-base_gitlab-ci.yml + - gitlab-ci/kas-build-base_gitlab-ci.yml - gitlab-ci/lava-test-base_gitlab-ci.yml - - local: .gitlab/ci/yocto-build.yml + - local: .gitlab/ci/rules.yml + - local: .gitlab/ci/cassini-build.yml + - local: .gitlab/ci/yocto-qa.yml - local: .gitlab/ci/lava-test.yml - EOF - - if [ -d tmp ]; then - for file in tmp/*.yml; do - cat $file >> ${CI_JOB_FILE}; - echo -n >> ${CI_JOB_FILE}; - done - fi - # Add global variables to env file to export in child pipelines - - echo "CREATE_JOB_ID=${CI_JOB_ID}" > global_variables.env - - cat global_variables.env - - if [ "${CI_DEBUG_TRACE}" == "true" ]; then - cat ${CI_JOB_FILE}; - fi - artifacts: - name: "job-$CI_COMMIT_REF_NAME" - paths: - - $CI_JOB_FILE - - global_variables.env - expire_in: '1 days' trigger_jobs: - stage: Build - trigger: - include: - - artifact: yocto-builds.yml - job: merge_jobs - strategy: depend - forward: - pipeline_variables: true - yaml_variables: true - variables: - PARENT_PIPELINE_ID: $CI_PIPELINE_ID - CASSINI_VERSION: "vNext" - rules: - - if: $CI_COMMIT_TAG =~ /^v\d+\.\d+\.\d+/ - variables: - CASSINI_VERSION: $CI_COMMIT_TAG - - when: on_success needs: - merge_jobs - job: Build-Kas-Docker-Image diff --git a/.gitlab/ci/yocto-build.yml b/.gitlab/ci/yocto-build.yml deleted file mode 100644 index cd82fd66f1756cd57114b2c0d13a8d37b972d6bb..0000000000000000000000000000000000000000 --- a/.gitlab/ci/yocto-build.yml +++ /dev/null @@ -1,194 +0,0 @@ -# Copyright (c) 2022 Arm Limited or its affiliates. All rights reserved. -# -# SPDX-License-Identifier: MIT ---- -.yocto_base: - extends: .resource-request - variables: - DOCKER_IMAGE_NAME: kas-build-image - CACHE_DIR: /sstate-cache - KAS_WORK_DIR: $CI_PROJECT_DIR/work - KAS_REPO_REF_DIR: $CACHE_DIR/repos - SSTATE_DIR: $CACHE_DIR/$CASSINI_VERSION/yocto-sstate - DL_DIR: $CACHE_DIR/$CASSINI_VERSION/yocto-downloads - IMAGE_DIR: $CACHE_DIR/$CASSINI_VERSION/images - TOOLCHAIN_DIR: $CACHE_DIR/toolchains - image: $CI_REGISTRY/$CI_PROJECT_PATH/$DOCKER_IMAGE_NAME:$DOCKER_IMAGE_TAG - before_script: - - mkdir -p ~/.ssh - - eval "$(ssh-agent -s)" - - if [ -z "${SSH_PRIVATE_GITLAB}+x" ]; then - echo "${SSH_PRIVATE_GITLAB}" | tr -d '\r' | ssh-add - > /dev/null; - fi - - ssh-keyscan -t rsa ${CI_SERVER_HOST} >> ~/.ssh/known_hosts - # Ensure git is usable without prompts - - git config --global url.ssh://git@${CI_SERVER_HOST}.insteadOf - https://${CI_SERVER_HOST} - - git config --global user.email "you@example.com" - - git config --global user.name "Your Name" - # export parent pipeline global variables - - echo PARENT_PIPELINE_ID = $PARENT_PIPELINE_ID - - cat global_variables.env - - source global_variables.env - # Setup KAS workspace - - echo KAS_WORK_DIR = $KAS_WORK_DIR - - echo SSTATE_DIR = $SSTATE_DIR - - echo DL_DIR = $DL_DIR - - echo IMAGE_DIR = $IMAGE_DIR - - echo CI_PROJECT_DIR = $CI_PROJECT_DIR - - rm -rf $KAS_WORK_DIR - - echo "Persistent cache volume could be empty, create directories required" - - mkdir --verbose --parents - $KAS_WORK_DIR - $KAS_REPO_REF_DIR - $SSTATE_DIR - $DL_DIR - $IMAGE_DIR - $TOOLCHAIN_DIR - - if [ "$CI_DEBUG_TRACE" == "true" ]; then - env | sort; - fi - needs: - - pipeline: $PARENT_PIPELINE_ID - job: merge_jobs - artifacts: true - -.yocto_build: - stage: Build - extends: .yocto_base - needs: - - Update-Repos - - pipeline: $PARENT_PIPELINE_ID - job: merge_jobs - artifacts: true - variables: - KUBERNETES_CPU_REQUEST: '15' - KUBERNETES_MEMORY_REQUEST: 10Gi - KUBERNETES_EPHEMERAL_STORAGE_REQUEST: 30Gi - rules: - - when: on_success - script: - # Run build - - echo "Building target $KAS_CONFIGS" - - KASFILES=$(./.gitlab/scripts/jobs-to-kas ${KAS_CONFIGS}) - - echo KASFILES = $KASFILES - - export BB_NUMBER_THREADS="${KUBERNETES_CPU_REQUEST}" - - export PARALLEL_MAKE="-j ${KUBERNETES_CPU_REQUEST}" - - kas shell --update --force-checkout $KASFILES -c 'cat conf/*.conf' - - kas build $KASFILES; - artifacts: - paths: - - global_variables.env - expire_in: '1 days' - -Prune-Cache: - extends: .yocto_base - stage: Cleanup - # Can fail if build job are running in other pipelines - allow_failure: true - rules: - - if: '$KAS_CONFIGS == null && $FREQUENCY == "nightly"' - - if: '$KAS_CONFIGS == null && $FREQUENCY == "adhoc"' - when: manual - script: - - du -sh $SSTATE_DIR - - find $SSTATE_DIR -type f -atime +5 -delete - - echo "After cache cleanup" - - du -sh $SSTATE_DIR - - du -sh $DL_DIR - - find $DL_DIR -maxdepth 1 -type f -atime +5 -delete - - echo "After download cleanup" - - du -sh $DL_DIR - - -# -# Prep stage, update repositories once -# -Update-Repos: - extends: .yocto_base - stage: Setup - rules: - - if: '$KAS_CONFIGS == null' - script: - - flock --verbose --timeout 60 $KAS_REPO_REF_DIR - ./.gitlab/scripts/update-repos - -# -# Utility tasks, not executed automatically -# -Setup-Cache: - # Ensure this is not on an image with cassini-ci user - extends: .resource-request - stage: Setup - allow_failure: true - rules: - - if: '$KAS_CONFIGS == null' - when: manual - script: - - chmod 777 /sstate-cache - -Delete-DL-Dir: - extends: .yocto_base - stage: Setup - allow_failure: true - rules: - - if: '$KAS_CONFIGS == null' - when: manual - - if: '$KAS_CONFIGS == null && $CI_COMMIT_TAG' - script: - - rm -rf $DL_DIR/* - -Delete-Repo-Dir: - extends: .yocto_base - stage: Setup - allow_failure: true - rules: - - if: '$KAS_CONFIGS == null' - when: manual - script: - - rm -rf $KAS_REPO_REF_DIR/* - -# Delete all sstate -Delete-SState: - extends: .yocto_base - stage: Setup - allow_failure: true - rules: - - if: '$KAS_CONFIGS == null' - when: manual - - if: '$KAS_CONFIGS == null && $CI_COMMIT_TAG' - script: - - rm -rf $SSTATE_DIR/* - -# Delete stored images -Delete-Images: - extends: .yocto_base - stage: Setup - allow_failure: true - rules: - - if: '$KAS_CONFIGS == null' - when: manual - - if: '$KAS_CONFIGS == null && $CI_COMMIT_TAG' - script: - - rm -rf $IMAGE_DIR/* - -Delete-Toolchains: - extends: .yocto_base - stage: Setup - allow_failure: true - rules: - - if: '$KAS_CONFIGS == null' - when: manual - script: - - rm -rf $TOOLCHAIN_DIR/* - -# Report on disk usage -Disk-Usage: - extends: .yocto_base - stage: Setup - allow_failure: true - rules: - - if: '$KAS_CONFIGS == null' - when: manual - script: - - du -h -s $DL_DIR $SSTATE_DIR $IMAGE_DIR $KAS_REPO_REF_DIR $TOOLCHAIN_DIR diff --git a/.gitlab/ci/yocto-qa.yml b/.gitlab/ci/yocto-qa.yml index 952c22253be07c58605a83020fad92610318a54d..5c0c21e21a844d11451e38007b476818789a74ca 100644 --- a/.gitlab/ci/yocto-qa.yml +++ b/.gitlab/ci/yocto-qa.yml @@ -2,42 +2,10 @@ # # SPDX-License-Identifier: MIT --- + # Validate layers are Yocto Project Compatible check-layers: - extends: .resource-request - image: $CI_REGISTRY/$CI_PROJECT_PATH/$DOCKER_IMAGE_NAME:$DOCKER_IMAGE_TAG - stage: Build - needs: - - job: Build-Kas-Docker-Image - optional: true - variables: - DOCKER_IMAGE_NAME: kas-build-image - CACHE_DIR: /sstate-cache - KAS_WORK_DIR: $CI_PROJECT_DIR/work - KAS_REPO_REF_DIR: $CACHE_DIR/repos - SSTATE_DIR: $CACHE_DIR/$CASSINI_RELEASE_ID/yocto-sstate - DL_DIR: $CACHE_DIR/$CASSINI_RELEASE_ID/yocto-downloads - TOOLCHAIN_DIR: $CACHE_DIR/toolchains - before_script: - - if [ "$CI_DEBUG_TRACE" == "true" ]; then - env | sort; - fi - - mkdir -p ~/.ssh - - eval "$(ssh-agent -s)" - - if [ -z "${SSH_PRIVATE_GITLAB}+x" ]; then - echo "${SSH_PRIVATE_GITLAB}" | tr -d '\r' | ssh-add - > /dev/null; - fi - - ssh-keyscan -t rsa ${CI_SERVER_HOST} >> ~/.ssh/known_hosts - # Ensure git is usable without prompts - - git config --global url.ssh://git@${CI_SERVER_HOST}.insteadOf - https://${CI_SERVER_HOST} - - git config --global user.email "you@example.com" - - git config --global user.name "Your Name" - # Setup KAS workspace - - rm -rf $KAS_WORK_DIR - - echo "Persistent cache volume could be empty, create directories required" - - mkdir --verbose --parents $KAS_WORK_DIR $KAS_REPO_REF_DIR $SSTATE_DIR - $DL_DIR $IMAGE_DIR $TOOLCHAIN_DIR + extends: .kas_build_cassini script: - kas shell --update --force-checkout meta-cassini-config/kas/n1sdp.yml:meta-cassini-config/kas/tests.yml:meta-cassini-config/kas/cassini.yml:meta-cassini-config/kas/security.yml diff --git a/.gitlab/scripts/child-job-test-report.py b/.gitlab/scripts/child-job-test-report.py deleted file mode 100644 index 01b390ffa90919aa62e550160fbacb0712ca5fba..0000000000000000000000000000000000000000 --- a/.gitlab/scripts/child-job-test-report.py +++ /dev/null @@ -1,94 +0,0 @@ -# Copyright (c) 2022 Arm Limited or its affiliates. All rights reserved. -# -# SPDX-License-Identifier: MIT - -# Generates JUnit report for all child jobs - -import json -import sys -import requests - -from envparse import Env -from junit_xml import TestSuite, TestCase - - -def get_response(url, headers): - """ - Call the GitLab API and return the json formated result - """ - response = requests.get(url, headers=headers) - if not response.ok: - try: - generic_message = response.json().get('message', generic_message) - except json.decoder.JSONDecodeError: - generic_message = 'Invalid response from GitLab' - sys.stderr.write(generic_message + '\n') - exit(1) - return response.json() - - -def get_test_results(headers, project_url, pipeline_id): - """ - fetch a pipelines test report - """ - report_url = f'{project_url}/pipelines/{pipeline_id}/test_report' - - return get_response(f'{report_url}', headers) - - -def generate_junit_file(test_results, filename): - """ - Generate a Junit file from pipeline test results - """ - tss = [] - for test_suite in test_results['test_suites']: - tcs = [] - for test_case in test_suite['test_cases']: - tc = TestCase(test_case['name'], - elapsed_sec=test_case['execution_time'], - classname=test_suite['name'] + - "." + test_case['classname'], - stdout=test_case['system_output']) - - if test_case['status'] == 'error': - tc.add_failure_info("error") - elif test_case['status'] == 'skipped': - tc.add_skipped_info("skipped") - - tcs.append(tc) - - tss.append(TestSuite(test_suite['name'], tcs)) - - with open(filename, 'w') as f: - TestSuite.to_file(f, tss) - - -def main(): - """ - download artifacts and place them in the jobs directory - """ - env = Env() - - v4_origin = env('CI_API_V4_URL') - project_id = env("CI_PROJECT_ID") - pipeline_id = env('CI_PIPELINE_ID') - headers = { - 'PRIVATE-TOKEN': env('CI_BOT_API_TOKEN') - } - - project_url = f'{v4_origin}/projects/{project_id}' - page_url = f'{project_url}/pipelines/{pipeline_id}/bridges/' - json_response = get_response(f'{page_url}', headers) - - for bridge in json_response: - if bridge['downstream_pipeline']: - test_results = get_test_results(headers, - project_url, - bridge['downstream_pipeline']['id']) - - generate_junit_file(test_results, "TEST-ChildPipeline-" + - str(bridge['downstream_pipeline']['iid']) + ".xml") - - -if __name__ == '__main__': - main() diff --git a/meta-cassini-config/qa-checks/cassini-dictionary b/meta-cassini-config/qa-checks/cassini-dictionary index bf2b35b48aa403d0dfc7d766fc1af661094f1bad..0f34b06bc325517cb259a4e53fe27893c02848b9 100644 --- a/meta-cassini-config/qa-checks/cassini-dictionary +++ b/meta-cassini-config/qa-checks/cassini-dictionary @@ -1,37 +1,50 @@ -1.1.0+git0+b9460f26b4 1.0.0+rc93+git0+249bca0a13 -3.2.1+git0+ab4d0cf908 -4.16+stable0+f265444922 +1.1.0+git0+b9460f26b4 20.10.12+ce+git906f57f +3.2.1+git0+ab4d0cf908 3rd +4.16+stable0+f265444922 +512kb +CASSINI +CHANGELOG.md +FF-A +Signed-off-by +TF-A +a-useful-template-for-commit-messages aarch64 aem +api +api's +append +appending appends arm-machines.yml armv8-a armv8a-crc +aws b baremetal -baremetal_architecture.png baremetal-sdk.yml baremetal.yml +baremetal_architecture.png bblayers.conf bitbake bootloader bsp bsps -build_system build-time +build_system c cassini cassini's -CASSINI +cassini-config cassini_admin_group.in cassini_architecture.png cassini_layers_deps.png cassini_rel_dev_branches.png cassini_rel_yocto_overview.png cassini_rootfs_extra_space +cbea.ms ce_test_clean_env ce_test_guest_vm_name ce_test_image @@ -40,10 +53,14 @@ cgit cgit.cgi changelog cmd +codeinthehole.com codeline codeline_management +comma's commit_patch_message_guidelines committer +conditionally +config configs configurable configures @@ -53,23 +70,33 @@ containerized containers_kernelcfg_check.bbclass contributing_documentation_build_validation cortex-a +customizable customization customizations +cve +dangerfile datetime dbg +dependencies +deprecated +deprecation +developer-facing +developer.example.org developer_manual devicetree -deprecated diego.sueiro +distributions distro_features do_fetch docker's docs.lavasoftware.org dom0 domu -efi e.g +efi +endpoint extensible +f filenames filesystem filesystems @@ -83,11 +110,15 @@ fvp_base_revc-2xaemva_11.14_21.tgz gb gbytes generic-arm64 +git-scm.com +git-tools-rewriting-history +gitaly github gitlab -gitlab.arm.com gitlab-ci.yml +gitlab.arm.com go-fsnotify +greengrass h.twlc0ztnng3b hardknott high_level_overview.png @@ -101,45 +132,54 @@ image_rootfs_extra_space implementations independently-configurable index.html +init initializes introduction_use_cases_overview j k3s k3s-integration-tests.bb -k3s_git.bb -k3s_kernelcfg_check.bbclass k3s-killall.sh k3s.service +k3s_git.bb +k3s_kernelcfg_check.bbclass k3s_test_clean_env k3s_test_guest_vm_name k3s_test_log_dir kb kernel_features kirkstone +kube-system kubectl kubernetes -kube-system langdale layer_dependency_overview_label libpcre libpcre2 -licensees license.rst +licensees linter +linux +linux-based localhost loopback lts m +maintainers maintainers_label -maxdepth manual_build_system_target_platforms manual_yocto_layers_layer_dependency_overview +maxdepth mb +merchantability +merge_requests meta-arm-bsp +meta-cassini-config meta-openembedded meta-virtualization minicom modifiable +modifier +mr multiconfig n1 n1sdp @@ -148,17 +188,27 @@ n1sdp.yml neoverse nginx nodeport +noninfringement oci openembedded optionally ostree +parallaxsecond +parsec-cli-tests.sh +parsec-simple-e2e-tests +parsec-simple-e2e-tests.bb pc pep8 png podman ppa ppas +prefixed +prefixes project_root +ps +ps_test_clean_env +ps_test_log_dir ptest pycodestyle pyspellchecker @@ -169,20 +219,25 @@ quickstart r readability readme.md +rebase +rebases refactor refactored +refactoring refspec +remediation repository's requirements.txt restructuredtext return_code revc +reword rootfs rootfs_extra_space rpm rst -runc-opencontainers run-time +runc-opencontainers runtime runtime-integration-tests sc2086 @@ -197,19 +252,26 @@ security_hardening shellcheck shellcheck-py shortlog +sign-off soafee soafee.io sourceforge spdx spdx-license-identifier +spdx-specification-21-web-version +spdx.dev +spdx.org sphinx-build +squash_and_merge.html ssh sstate-cache +standalone standards-based stderr stdout sub-sections sub_test_name +sublicense submitting-patches.html sudo sudoers @@ -222,14 +284,18 @@ titlesonly toctree top_level_test_name ua_test_clean_env -ua_test_log_dir ua_test_guest_vm_name +ua_test_log_dir ubuntu ubuntu-based umask +upstreamed +upto +urls use-cases -user_accounts user-accounts-integration-tests.bb +user-facing +user_accounts user_guide user_guide_reproduce_environment_setup user_manual @@ -243,14 +309,15 @@ virt_test_clean_env virt_test_guest_vm_name virt_test_log_dir virtualization -virtualization_architecture.png virtualization-integration-tests.bb virtualization-sdk.yml virtualization.yml +virtualization_architecture.png virtualized vm vm's vms +vs w webserver wget @@ -259,16 +326,18 @@ with-libcap[-ng workloads www.kernel.org www.openembedded.org +x x86_64-linux xen -xenbr0 xen-devicetree.bb +xen_kernelcfg_check.bbclass +xenbr0 xendomains xendomains.service -xen_kernelcfg_check.bbclass xl yaml yamllint yocto -yocto_layers yocto's +yocto_layers +your-branch-name