From 71469e8580f26bfa72a816487e39266822e47de1 Mon Sep 17 00:00:00 2001 From: Vikas Katariya Date: Thu, 4 May 2023 11:10:34 +0100 Subject: [PATCH 1/4] cassini-test: Move to latest version of bats Remove the old v1.6.1 compatibility and use v1.9.0 recipe from meta-oe Signed-off-by: Vikas Katariya --- .../recipes-tests/bats/bats_%.bbappend | 7 ++++ .../recipes-tests/bats/bats_1.6.1.bb | 41 ------------------- 2 files changed, 7 insertions(+), 41 deletions(-) create mode 100644 meta-cassini-tests/recipes-tests/bats/bats_%.bbappend delete mode 100644 meta-cassini-tests/recipes-tests/bats/bats_1.6.1.bb diff --git a/meta-cassini-tests/recipes-tests/bats/bats_%.bbappend b/meta-cassini-tests/recipes-tests/bats/bats_%.bbappend new file mode 100644 index 0000000..1f21cb8 --- /dev/null +++ b/meta-cassini-tests/recipes-tests/bats/bats_%.bbappend @@ -0,0 +1,7 @@ +# Copyright (c) 2023 Arm Limited and/or its affiliates. +# +# +# SPDX-License-Identifier: MIT + +inherit features_check +REQUIRED_DISTRO_FEATURES = "cassini-test" diff --git a/meta-cassini-tests/recipes-tests/bats/bats_1.6.1.bb b/meta-cassini-tests/recipes-tests/bats/bats_1.6.1.bb deleted file mode 100644 index ae0a8bd..0000000 --- a/meta-cassini-tests/recipes-tests/bats/bats_1.6.1.bb +++ /dev/null @@ -1,41 +0,0 @@ -# Based on: https://cgit.openembedded.org/meta-openembedded/commit/meta-oe/recipes-test/bats/bats_1.6.0.bb?id=76d8e526eae9fc9ab14ff5804ae0091230a8b39e -# In open-source project: https://git.openembedded.org/meta-openembedded -# -# Original file: Copyright (c) 2022 wangmy -# Modifications: Copyright (c) 2022 Arm Limited and Contributors. All rights reserved. -# -# SPDX-License-Identifier: MIT - -SUMMARY = "Bash Automated Testing System" -DESCRIPTION = "Bats is a TAP-compliant testing framework for Bash. It \ -provides a simple way to verify that the UNIX programs you write behave as expected." -AUTHOR = "Sam Stephenson & bats-core organization" -HOMEPAGE = "https://github.com/bats-core/bats-core" -LICENSE = "MIT" -LIC_FILES_CHKSUM = "file://LICENSE.md;md5=2970203aedf9e829edb96a137a4fe81b" - -SRC_URI = "git://github.com/bats-core/bats-core.git;branch=version/1.6.x;protocol=https" -# v1.6.1 -SRCREV = "1977254c2a7faa2e0af17355856f91dc471d1daa" - -S = "${WORKDIR}/git" - -do_configure:prepend() { - sed -i 's:\$BATS_ROOT/lib:\$BATS_ROOT/${baselib}:g' ${S}/libexec/bats-core/bats - sed -i 's:\$BATS_ROOT/lib:\$BATS_ROOT/${baselib}:g' ${S}/libexec/bats-core/bats-exec-file - sed -i 's:\$BATS_ROOT/lib:\$BATS_ROOT/${baselib}:g' ${S}/libexec/bats-core/bats-exec-test -} - -do_install() { - # Just a bunch of bash scripts to install - ${S}/install.sh ${D}${prefix} ${baselib} -} - -RDEPENDS:${PN} = "bash" -FILES:${PN} += "${libdir}/bats-core/*" - -PACKAGECONFIG ??= "pretty" -PACKAGECONFIG[pretty] = ",,,ncurses" - -inherit features_check -REQUIRED_DISTRO_FEATURES = "cassini-test" -- GitLab From 8b85dc40b513bdc7aa02b782d9b538ece0f505a0 Mon Sep 17 00:00:00 2001 From: Vikas Katariya Date: Tue, 16 May 2023 17:00:51 +0100 Subject: [PATCH 2/4] cassini-[ci,test]: Fix user access issue for tests Run the ptest-runner as a `test` user rather than root, this fixes the issue with the user access where `bats` is being invoked as a `test` user by ptest framework running as `root` All ptest-runner suites should be invoked as `test` user The `test` user should be able to access necessary drivers and not require `root` access Signed-off-by: Vikas Katariya --- .gitlab/lava/corstone1000-mps3/ptest.yml.j2 | 2 +- .gitlab/lava/n1sdp/ptest.yml.j2 | 2 +- .../runtime-integration-tests/files/run-test-suite | 5 +++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.gitlab/lava/corstone1000-mps3/ptest.yml.j2 b/.gitlab/lava/corstone1000-mps3/ptest.yml.j2 index 255b63a..0b0320e 100644 --- a/.gitlab/lava/corstone1000-mps3/ptest.yml.j2 +++ b/.gitlab/lava/corstone1000-mps3/ptest.yml.j2 @@ -163,7 +163,7 @@ actions: - functional run: steps: - - lava-test-case cassini-integration-tests --shell "ptest-runner - t 3600" + - lava-test-case cassini-integration-tests --shell "sudo -HEnu test ptest-runner - t 3600" - jfrog-cli config add artifactory-aws --interactive=false --artifactory-url={{ ARTIFACTORY_AWS_URL }} --user={{ ARTIFACTORY_USER }} --password={{ ARTIFACTORY_PASS }}; - | cat << EOF > ./upload_spec.json diff --git a/.gitlab/lava/n1sdp/ptest.yml.j2 b/.gitlab/lava/n1sdp/ptest.yml.j2 index de66af7..e434e9d 100644 --- a/.gitlab/lava/n1sdp/ptest.yml.j2 +++ b/.gitlab/lava/n1sdp/ptest.yml.j2 @@ -197,7 +197,7 @@ actions: - functional run: steps: - - lava-test-case cassini-integration-tests --shell "ptest-runner -t 1800" + - lava-test-case cassini-integration-tests --shell "sudo -HEnu test ptest-runner -t 1800" - jfrog-cli config add artifactory-aws --interactive=false --artifactory-url={{ ARTIFACTORY_AWS_URL }} --user={{ ARTIFACTORY_USER }} --password={{ ARTIFACTORY_PASS }}; - | cat << EOF > ./upload_spec.json diff --git a/meta-cassini-tests/recipes-tests/runtime-integration-tests/files/run-test-suite b/meta-cassini-tests/recipes-tests/runtime-integration-tests/files/run-test-suite index 8579453..47194ed 100644 --- a/meta-cassini-tests/recipes-tests/runtime-integration-tests/files/run-test-suite +++ b/meta-cassini-tests/recipes-tests/runtime-integration-tests/files/run-test-suite @@ -1,6 +1,7 @@ #!/usr/bin/env bash # -# Copyright (c) 2022 Arm Limited or its affiliates. All rights reserved. +# Copyright (c) 2022-2023 Arm Limited and/or its affiliates. +# # # SPDX-License-Identifier: MIT @@ -13,7 +14,7 @@ if ! cd /home ; then exit 1 fi -if sudo -HEnu "${CASSINI_TEST_ACCOUNT}" bats "${CASSINI_TEST_BATS_OPTIONS}" "${TEST_DIR}/${TEST_SUITE_NAME}.bats"; then +if bats "${CASSINI_TEST_BATS_OPTIONS}" "${TEST_DIR}/${TEST_SUITE_NAME}.bats"; then echo "PASS:${TEST_SUITE_NAME}" exit 0 else -- GitLab From 4e8a24570e95b8310661c997d68a9a114222125f Mon Sep 17 00:00:00 2001 From: Vikas Katariya Date: Tue, 16 May 2023 17:02:00 +0100 Subject: [PATCH 3/4] ci: Split `ptest` for nightly and merge requests Split the ptest for nightly/weekly and Merge request runs: The intention is to only run short-duration tests for each merge request raised and longer tests for nightly/weekly * `Merge request` run includes: user-accounts-integration-tests, container-engine-integration-tests and k3s-integration-tests * nightly/weekly run includes: parsec-simple-e2e-tests, psa-arch-tests, optee-xtests and all tests from `Merge request` run Signed-off-by: Vikas Katariya --- .gitlab/lava/corstone1000-mps3/ptest.yml.j2 | 24 ++++++++++++++++++--- .gitlab/lava/n1sdp/ptest.yml.j2 | 24 ++++++++++++++++++--- 2 files changed, 42 insertions(+), 6 deletions(-) diff --git a/.gitlab/lava/corstone1000-mps3/ptest.yml.j2 b/.gitlab/lava/corstone1000-mps3/ptest.yml.j2 index 0b0320e..0799ca4 100644 --- a/.gitlab/lava/corstone1000-mps3/ptest.yml.j2 +++ b/.gitlab/lava/corstone1000-mps3/ptest.yml.j2 @@ -3,11 +3,18 @@ # # SPDX-License-Identifier: MIT --- + +{% set should_run = ('nightly' in FREQUENCY or 'weekly' in FREQUENCY or 'all' in FORCE_TESTS or 'ptest' in FORCE_TESTS.split(',')) %} + device_type: mps3 job_name: {{ CI_JOB_NAME }} timeouts: job: - minutes: 300 +{% if should_run %} + minutes: 600 +{% else %} + minutes: 270 +{% endif %} metadata: source: {{ CI_PROJECT_URL }} @@ -150,7 +157,11 @@ actions: - test: namespace: secondary_media timeout: - minutes: 75 +{% if should_run %} + minutes: 360 +{% else %} + minutes: 15 +{% endif %} definitions: - repository: metadata: @@ -163,7 +174,14 @@ actions: - functional run: steps: - - lava-test-case cassini-integration-tests --shell "sudo -HEnu test ptest-runner - t 3600" + - lava-test-case cassini-integration-tests --shell "sudo -HEnu test ptest-runner -t 180 user-accounts-integration-tests" + - lava-test-case cassini-integration-tests --shell "sudo -HEnu test ptest-runner -t 180 container-engine-integration-tests" + - lava-test-case cassini-integration-tests --shell "sudo -HEnu test ptest-runner -t 180 k3s-integration-tests" +{% if should_run %} + - lava-test-case cassini-integration-tests --shell "sudo -HEnu test ptest-runner -t 3600 psa-arch-tests" + - lava-test-case cassini-integration-tests --shell "sudo -HEnu test ptest-runner -t 10800 optee-xtests" + - lava-test-case cassini-integration-tests --shell "sudo -HEnu test ptest-runner -t 1800 parsec-simple-e2e-tests" +{% endif %} - jfrog-cli config add artifactory-aws --interactive=false --artifactory-url={{ ARTIFACTORY_AWS_URL }} --user={{ ARTIFACTORY_USER }} --password={{ ARTIFACTORY_PASS }}; - | cat << EOF > ./upload_spec.json diff --git a/.gitlab/lava/n1sdp/ptest.yml.j2 b/.gitlab/lava/n1sdp/ptest.yml.j2 index e434e9d..461d0ac 100644 --- a/.gitlab/lava/n1sdp/ptest.yml.j2 +++ b/.gitlab/lava/n1sdp/ptest.yml.j2 @@ -3,11 +3,18 @@ # # SPDX-License-Identifier: MIT --- + +{% set should_run = ('nightly' in FREQUENCY or 'weekly' in FREQUENCY or 'all' in FORCE_TESTS or 'ptest' in FORCE_TESTS.split(',')) %} + device_type: n1sdp job_name: {{ CI_JOB_NAME }} timeouts: job: - minutes: 90 +{% if should_run %} + minutes: 450 +{% else %} + minutes: 80 +{% endif %} priority: medium visibility: group: @@ -184,7 +191,11 @@ actions: - test: namespace: secondary_media timeout: - minutes: 30 +{% if should_run %} + minutes: 360 +{% else %} + minutes: 15 +{% endif %} definitions: - repository: metadata: @@ -197,7 +208,14 @@ actions: - functional run: steps: - - lava-test-case cassini-integration-tests --shell "sudo -HEnu test ptest-runner -t 1800" + - lava-test-case cassini-integration-tests --shell "sudo -HEnu test ptest-runner -t 180 user-accounts-integration-tests" + - lava-test-case cassini-integration-tests --shell "sudo -HEnu test ptest-runner -t 180 container-engine-integration-tests" + - lava-test-case cassini-integration-tests --shell "sudo -HEnu test ptest-runner -t 180 k3s-integration-tests" +{% if should_run %} + - lava-test-case cassini-integration-tests --shell "sudo -HEnu test ptest-runner -t 3600 psa-arch-tests" + - lava-test-case cassini-integration-tests --shell "sudo -HEnu test ptest-runner -t 10800 optee-xtests" + - lava-test-case cassini-integration-tests --shell "sudo -HEnu test ptest-runner -t 1800 parsec-simple-e2e-tests" +{% endif %} - jfrog-cli config add artifactory-aws --interactive=false --artifactory-url={{ ARTIFACTORY_AWS_URL }} --user={{ ARTIFACTORY_USER }} --password={{ ARTIFACTORY_PASS }}; - | cat << EOF > ./upload_spec.json -- GitLab From 1051d7783a998955afc82b597cb484c16b6e7792 Mon Sep 17 00:00:00 2001 From: Vikas Katariya Date: Fri, 19 May 2023 08:39:41 +0100 Subject: [PATCH 4/4] ci: Fix the n1sdp lava job prompts Lava uses pexpect to search for prompts, which includes all the output on the connection. Having a prompt with a wildcard like (*.) in beginning of the search string, the pexpect is asked to match against an extremely large and growing input string, which causes significant delays in processing job output. This fixes the sanity and ptest job which takes more than time on lava farm to reach the boot stage. Also update the timeouts of tasks in the job. Signed-off-by: Vikas Katariya --- .gitlab/lava/n1sdp/ptest.yml.j2 | 7 ++++--- .gitlab/lava/n1sdp/sanity_job.yml.j2 | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.gitlab/lava/n1sdp/ptest.yml.j2 b/.gitlab/lava/n1sdp/ptest.yml.j2 index 461d0ac..f5c016d 100644 --- a/.gitlab/lava/n1sdp/ptest.yml.j2 +++ b/.gitlab/lava/n1sdp/ptest.yml.j2 @@ -171,10 +171,10 @@ actions: - boot: namespace: secondary_media timeout: - minutes: 15 + minutes: 10 method: minimal auto_login: - login_prompt: '(.*)login:' + login_prompt: 'n1sdp login:.*' username: cassini login_commands: - cassini123 @@ -183,7 +183,8 @@ actions: prompts: - 'New password: ' - 'Re-enter new password: ' - - '(.*)n1sdp:' + - '@n1sdp:.+\$ ' + - '@n1sdp:.+# ' transfer_overlay: download_command: wget -S unpack_command: tar -C / -xzf diff --git a/.gitlab/lava/n1sdp/sanity_job.yml.j2 b/.gitlab/lava/n1sdp/sanity_job.yml.j2 index d409b69..ffee45b 100644 --- a/.gitlab/lava/n1sdp/sanity_job.yml.j2 +++ b/.gitlab/lava/n1sdp/sanity_job.yml.j2 @@ -162,10 +162,10 @@ actions: - boot: namespace: secondary_media timeout: - minutes: 15 + minutes: 10 method: minimal auto_login: - login_prompt: '(.*)login:' + login_prompt: 'n1sdp login:.*' username: cassini login_commands: - cassini123 @@ -173,7 +173,8 @@ actions: prompts: - 'New password: ' - 'Re-enter new password: ' - - '(.*)n1sdp:' + - '@n1sdp:.+\$ ' + - '@n1sdp:.+# ' transfer_overlay: download_command: wget -S unpack_command: tar -C / -xzf -- GitLab