From 19479fab7202e2d7044403af68b5eb97644dd3a4 Mon Sep 17 00:00:00 2001 From: Vikas Katariya Date: Thu, 4 May 2023 11:10:34 +0100 Subject: [PATCH 1/4] cassini-test: Move to latest version of bats Remove the old v1.6.1 compatibility and use v1.9.0 recipe from meta-oe Signed-off-by: Vikas Katariya --- .../recipes-tests/bats/bats_%.bbappend | 7 ++++ .../recipes-tests/bats/bats_1.6.1.bb | 41 ------------------- 2 files changed, 7 insertions(+), 41 deletions(-) create mode 100644 meta-cassini-tests/recipes-tests/bats/bats_%.bbappend delete mode 100644 meta-cassini-tests/recipes-tests/bats/bats_1.6.1.bb diff --git a/meta-cassini-tests/recipes-tests/bats/bats_%.bbappend b/meta-cassini-tests/recipes-tests/bats/bats_%.bbappend new file mode 100644 index 0000000..1f21cb8 --- /dev/null +++ b/meta-cassini-tests/recipes-tests/bats/bats_%.bbappend @@ -0,0 +1,7 @@ +# Copyright (c) 2023 Arm Limited and/or its affiliates. +# +# +# SPDX-License-Identifier: MIT + +inherit features_check +REQUIRED_DISTRO_FEATURES = "cassini-test" diff --git a/meta-cassini-tests/recipes-tests/bats/bats_1.6.1.bb b/meta-cassini-tests/recipes-tests/bats/bats_1.6.1.bb deleted file mode 100644 index ae0a8bd..0000000 --- a/meta-cassini-tests/recipes-tests/bats/bats_1.6.1.bb +++ /dev/null @@ -1,41 +0,0 @@ -# Based on: https://cgit.openembedded.org/meta-openembedded/commit/meta-oe/recipes-test/bats/bats_1.6.0.bb?id=76d8e526eae9fc9ab14ff5804ae0091230a8b39e -# In open-source project: https://git.openembedded.org/meta-openembedded -# -# Original file: Copyright (c) 2022 wangmy -# Modifications: Copyright (c) 2022 Arm Limited and Contributors. All rights reserved. -# -# SPDX-License-Identifier: MIT - -SUMMARY = "Bash Automated Testing System" -DESCRIPTION = "Bats is a TAP-compliant testing framework for Bash. It \ -provides a simple way to verify that the UNIX programs you write behave as expected." -AUTHOR = "Sam Stephenson & bats-core organization" -HOMEPAGE = "https://github.com/bats-core/bats-core" -LICENSE = "MIT" -LIC_FILES_CHKSUM = "file://LICENSE.md;md5=2970203aedf9e829edb96a137a4fe81b" - -SRC_URI = "git://github.com/bats-core/bats-core.git;branch=version/1.6.x;protocol=https" -# v1.6.1 -SRCREV = "1977254c2a7faa2e0af17355856f91dc471d1daa" - -S = "${WORKDIR}/git" - -do_configure:prepend() { - sed -i 's:\$BATS_ROOT/lib:\$BATS_ROOT/${baselib}:g' ${S}/libexec/bats-core/bats - sed -i 's:\$BATS_ROOT/lib:\$BATS_ROOT/${baselib}:g' ${S}/libexec/bats-core/bats-exec-file - sed -i 's:\$BATS_ROOT/lib:\$BATS_ROOT/${baselib}:g' ${S}/libexec/bats-core/bats-exec-test -} - -do_install() { - # Just a bunch of bash scripts to install - ${S}/install.sh ${D}${prefix} ${baselib} -} - -RDEPENDS:${PN} = "bash" -FILES:${PN} += "${libdir}/bats-core/*" - -PACKAGECONFIG ??= "pretty" -PACKAGECONFIG[pretty] = ",,,ncurses" - -inherit features_check -REQUIRED_DISTRO_FEATURES = "cassini-test" -- GitLab From e0b10d3aaf6d7be59e3593c44ee3bed3f9c77d50 Mon Sep 17 00:00:00 2001 From: Vikas Katariya Date: Tue, 16 May 2023 17:00:51 +0100 Subject: [PATCH 2/4] cassini-[ci,test]: Fix user access issue for tests Run the ptest-runner as a `test` user rather than root, this fixes the issue with the user access where `bats` is being invoked as a `test` user by ptest framework running as `root` All ptest-runner suites should be invoked as `test` user The `test` user should be able to access necessary drivers and not require `root` access Signed-off-by: Vikas Katariya --- .gitlab/lava/corstone1000-mps3/ptest.yml.j2 | 2 +- .gitlab/lava/n1sdp/ptest.yml.j2 | 2 +- .../runtime-integration-tests/files/run-test-suite | 5 +++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.gitlab/lava/corstone1000-mps3/ptest.yml.j2 b/.gitlab/lava/corstone1000-mps3/ptest.yml.j2 index 255b63a..0b0320e 100644 --- a/.gitlab/lava/corstone1000-mps3/ptest.yml.j2 +++ b/.gitlab/lava/corstone1000-mps3/ptest.yml.j2 @@ -163,7 +163,7 @@ actions: - functional run: steps: - - lava-test-case cassini-integration-tests --shell "ptest-runner - t 3600" + - lava-test-case cassini-integration-tests --shell "sudo -HEnu test ptest-runner - t 3600" - jfrog-cli config add artifactory-aws --interactive=false --artifactory-url={{ ARTIFACTORY_AWS_URL }} --user={{ ARTIFACTORY_USER }} --password={{ ARTIFACTORY_PASS }}; - | cat << EOF > ./upload_spec.json diff --git a/.gitlab/lava/n1sdp/ptest.yml.j2 b/.gitlab/lava/n1sdp/ptest.yml.j2 index 6f666c9..4a1274a 100644 --- a/.gitlab/lava/n1sdp/ptest.yml.j2 +++ b/.gitlab/lava/n1sdp/ptest.yml.j2 @@ -195,7 +195,7 @@ actions: - functional run: steps: - - lava-test-case cassini-integration-tests --shell "ptest-runner -t 1800" + - lava-test-case cassini-integration-tests --shell "sudo -HEnu test ptest-runner -t 1800" - jfrog-cli config add artifactory-aws --interactive=false --artifactory-url={{ ARTIFACTORY_AWS_URL }} --user={{ ARTIFACTORY_USER }} --password={{ ARTIFACTORY_PASS }}; - | cat << EOF > ./upload_spec.json diff --git a/meta-cassini-tests/recipes-tests/runtime-integration-tests/files/run-test-suite b/meta-cassini-tests/recipes-tests/runtime-integration-tests/files/run-test-suite index 8579453..47194ed 100644 --- a/meta-cassini-tests/recipes-tests/runtime-integration-tests/files/run-test-suite +++ b/meta-cassini-tests/recipes-tests/runtime-integration-tests/files/run-test-suite @@ -1,6 +1,7 @@ #!/usr/bin/env bash # -# Copyright (c) 2022 Arm Limited or its affiliates. All rights reserved. +# Copyright (c) 2022-2023 Arm Limited and/or its affiliates. +# # # SPDX-License-Identifier: MIT @@ -13,7 +14,7 @@ if ! cd /home ; then exit 1 fi -if sudo -HEnu "${CASSINI_TEST_ACCOUNT}" bats "${CASSINI_TEST_BATS_OPTIONS}" "${TEST_DIR}/${TEST_SUITE_NAME}.bats"; then +if bats "${CASSINI_TEST_BATS_OPTIONS}" "${TEST_DIR}/${TEST_SUITE_NAME}.bats"; then echo "PASS:${TEST_SUITE_NAME}" exit 0 else -- GitLab From 0a1ca391292061616c3e024960cb593bb67eefaf Mon Sep 17 00:00:00 2001 From: Vikas Katariya Date: Tue, 16 May 2023 17:02:00 +0100 Subject: [PATCH 3/4] ci: Split `ptest` for nightly and merge requests Split the ptest for nightly/weekly and Merge request runs: The intention is to only run short-duration tests for each merge request raised and longer tests for nightly/weekly * `Merge request` run includes: user-accounts-integration-tests, container-engine-integration-tests and k3s-integration-tests * nightly/weekly run include parsec-simple-e2e-tests and all tests from `Merge request` run Signed-off-by: Vikas Katariya --- .gitlab/lava/corstone1000-mps3/ptest.yml.j2 | 20 +++++++++++++++++-- .gitlab/lava/n1sdp/ptest.yml.j2 | 22 ++++++++++++++++++--- 2 files changed, 37 insertions(+), 5 deletions(-) diff --git a/.gitlab/lava/corstone1000-mps3/ptest.yml.j2 b/.gitlab/lava/corstone1000-mps3/ptest.yml.j2 index 0b0320e..b45251e 100644 --- a/.gitlab/lava/corstone1000-mps3/ptest.yml.j2 +++ b/.gitlab/lava/corstone1000-mps3/ptest.yml.j2 @@ -3,11 +3,18 @@ # # SPDX-License-Identifier: MIT --- + +{% set should_run = ('nightly' in FREQUENCY or 'weekly' in FREQUENCY or 'all' in FORCE_TESTS or 'ptest' in FORCE_TESTS.split(',')) %} + device_type: mps3 job_name: {{ CI_JOB_NAME }} timeouts: job: - minutes: 300 +{% if should_run %} + minutes: 360 +{% else %} + minutes: 270 +{% endif %} metadata: source: {{ CI_PROJECT_URL }} @@ -150,7 +157,11 @@ actions: - test: namespace: secondary_media timeout: +{% if should_run %} minutes: 75 +{% else %} + minutes: 15 +{% endif %} definitions: - repository: metadata: @@ -163,7 +174,12 @@ actions: - functional run: steps: - - lava-test-case cassini-integration-tests --shell "sudo -HEnu test ptest-runner - t 3600" + - lava-test-case cassini-integration-tests --shell "sudo -HEnu test ptest-runner -t 180 user-accounts-integration-tests" + - lava-test-case cassini-integration-tests --shell "sudo -HEnu test ptest-runner -t 180 container-engine-integration-tests" + - lava-test-case cassini-integration-tests --shell "sudo -HEnu test ptest-runner -t 180 k3s-integration-tests" +{% if should_run %} + - lava-test-case cassini-integration-tests --shell "sudo -HEnu test ptest-runner -t 1800 parsec-simple-e2e-tests" +{% endif %} - jfrog-cli config add artifactory-aws --interactive=false --artifactory-url={{ ARTIFACTORY_AWS_URL }} --user={{ ARTIFACTORY_USER }} --password={{ ARTIFACTORY_PASS }}; - | cat << EOF > ./upload_spec.json diff --git a/.gitlab/lava/n1sdp/ptest.yml.j2 b/.gitlab/lava/n1sdp/ptest.yml.j2 index 4a1274a..863384a 100644 --- a/.gitlab/lava/n1sdp/ptest.yml.j2 +++ b/.gitlab/lava/n1sdp/ptest.yml.j2 @@ -3,11 +3,18 @@ # # SPDX-License-Identifier: MIT --- + +{% set should_run = ('nightly' in FREQUENCY or 'weekly' in FREQUENCY or 'all' in FORCE_TESTS or 'ptest' in FORCE_TESTS.split(',')) %} + device_type: n1sdp job_name: {{ CI_JOB_NAME }} timeouts: job: - minutes: 90 +{% if should_run %} + minutes: 130 +{% else %} + minutes: 80 +{% endif %} priority: medium visibility: group: @@ -182,7 +189,11 @@ actions: - test: namespace: secondary_media timeout: - minutes: 30 +{% if should_run %} + minutes: 45 +{% else %} + minutes: 15 +{% endif %} definitions: - repository: metadata: @@ -195,7 +206,12 @@ actions: - functional run: steps: - - lava-test-case cassini-integration-tests --shell "sudo -HEnu test ptest-runner -t 1800" + - lava-test-case cassini-integration-tests --shell "sudo -HEnu test ptest-runner -t 180 user-accounts-integration-tests" + - lava-test-case cassini-integration-tests --shell "sudo -HEnu test ptest-runner -t 180 container-engine-integration-tests" + - lava-test-case cassini-integration-tests --shell "sudo -HEnu test ptest-runner -t 180 k3s-integration-tests" +{% if should_run %} + - lava-test-case cassini-integration-tests --shell "sudo -HEnu test ptest-runner -t 1800 parsec-simple-e2e-tests" +{% endif %} - jfrog-cli config add artifactory-aws --interactive=false --artifactory-url={{ ARTIFACTORY_AWS_URL }} --user={{ ARTIFACTORY_USER }} --password={{ ARTIFACTORY_PASS }}; - | cat << EOF > ./upload_spec.json -- GitLab From 65289aaf052ee61a60e27ed34dda5dd86a8e1b85 Mon Sep 17 00:00:00 2001 From: Vikas Katariya Date: Fri, 19 May 2023 08:39:41 +0100 Subject: [PATCH 4/4] ci: Fix the n1sdp lava job prompts Lava uses pexpect to search for prompts, which includes all the output on the connection. Having a prompt with a wildcard like (*.) in beginning of the search string, the pexpect is asked to match against an extremely large and growing input string, which causes significant delays in processing job output. This fixes the sanity and ptest job which takes more than time on lava farm to reach the boot stage. Also update the timeouts of tasks in the job. Signed-off-by: Vikas Katariya --- .gitlab/lava/n1sdp/ptest.yml.j2 | 7 ++++--- .gitlab/lava/n1sdp/sanity_job.yml.j2 | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.gitlab/lava/n1sdp/ptest.yml.j2 b/.gitlab/lava/n1sdp/ptest.yml.j2 index 863384a..ad9b42b 100644 --- a/.gitlab/lava/n1sdp/ptest.yml.j2 +++ b/.gitlab/lava/n1sdp/ptest.yml.j2 @@ -169,10 +169,10 @@ actions: - boot: namespace: secondary_media timeout: - minutes: 15 + minutes: 10 method: minimal auto_login: - login_prompt: '(.*)login:' + login_prompt: 'n1sdp login:.*' username: cassini login_commands: - cassini123 @@ -181,7 +181,8 @@ actions: prompts: - 'New password: ' - 'Re-enter new password: ' - - '(.*)n1sdp:' + - '@n1sdp:.+\$ ' + - '@n1sdp:.+# ' transfer_overlay: download_command: wget -S unpack_command: tar -C / -xzf diff --git a/.gitlab/lava/n1sdp/sanity_job.yml.j2 b/.gitlab/lava/n1sdp/sanity_job.yml.j2 index d409b69..ffee45b 100644 --- a/.gitlab/lava/n1sdp/sanity_job.yml.j2 +++ b/.gitlab/lava/n1sdp/sanity_job.yml.j2 @@ -162,10 +162,10 @@ actions: - boot: namespace: secondary_media timeout: - minutes: 15 + minutes: 10 method: minimal auto_login: - login_prompt: '(.*)login:' + login_prompt: 'n1sdp login:.*' username: cassini login_commands: - cassini123 @@ -173,7 +173,8 @@ actions: prompts: - 'New password: ' - 'Re-enter new password: ' - - '(.*)n1sdp:' + - '@n1sdp:.+\$ ' + - '@n1sdp:.+# ' transfer_overlay: download_command: wget -S unpack_command: tar -C / -xzf -- GitLab