diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000000000000000000000000000000000000..98dbb27ee053b03db62174d241a03769eb74928d --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,112 @@ +# +# SPDX-License-Identifier: Apache-2.0 +# +# Copyright (C) 2024, Arm Limited and contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +default: + tags: + - ubuntu_22_04_amd64 + # cancel a job if a new pipeline is triggered on the same branch + interruptible: true + +# Required for Manual runs through UI +variables: + ENABLE_TEST: + value: "False" + options: + - "True" + - "False" + description: True to run test job. Defaults to false. + ENABLE_AUTOBRANCH: + value: "False" + options: + - "True" + - "False" + description: True to run Autobranch job. Defaults to false. Autobranch is allowed only on main. + +stages: + - Test + - Deploy + +Test: + stage: Test + rules: + - if: $CI_PIPELINE_SOURCE == "web" && $ENABLE_TEST == "True" + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + parallel: + matrix: + - PYVERSION: [8, 11] + before_script: + - export DEBIAN_FRONTEND=noninteractive + - sudo apt update -y + - sudo apt install software-properties-common -y + - sudo add-apt-repository ppa:deadsnakes/ppa -y + - sudo apt-get update -y + - sudo apt-get -y install python3.${PYVERSION}-venv python3.${PYVERSION}-tk python3.${PYVERSION}-full + script: + - export LISA_PYTHON=python3.${PYVERSION} + - sudo sh -c 'echo "APT::Acquire::Retries=3;" >> /etc/apt/apt.conf.d/99retry-downloads' + - sudo ./install_base.sh --install-all + - echo "$(python3 --version)" + - sudo chmod o+rx /sys/kernel/debug + - bash ./tools/tests.sh + +Autobranch: + stage: Deploy + variables: + GIT_STRATEGY: none #Required to enable the autobranch run from a specific branch:$CI_DEFAULT_BRANCH + rules: + - if: $CI_PIPELINE_SOURCE == "web" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $ENABLE_AUTOBRANCH == "True" + - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && '$CI_MERGE_REQUEST_LABELS =~ /^autobranch$/' #regex to pickup 'autobranch' label alone + resource_group: autobranch + before_script: + - sudo apt update -y + - sudo apt install tzdata -y + - sudo apt install software-properties-common -y + - sudo apt-get update -y + - sudo apt-get install python3.10 git python3.10-venv -y + - sudo apt-get install gcc python3-dev -y + - | + git clone -b $CI_DEFAULT_BRANCH $CI_REPOSITORY_URL origin-repo + + cd origin-repo && + + git config --global user.name 'Gitlab CI' + git config --global user.email 'gitlab-ci@arm.com' + + git remote -v + git branch --list --remotes + script: + - | + set -e + + export LC_ALL=C + + source init_env && + pip install requests # Required for calling Gitlab API + source ci/autobranch-job-script.sh + + ret=0 + function keepgoing { + "$@" || ret=1 + } + + keepgoing update_branch for-master-autobranch master master-force + keepgoing update_branch for-preview-autobranch preview preview-force + + exit $ret diff --git a/ci/autobranch-job-script.sh b/ci/autobranch-job-script.sh new file mode 100755 index 0000000000000000000000000000000000000000..de2930628f261f4f1afef19d30644158e2cc428c --- /dev/null +++ b/ci/autobranch-job-script.sh @@ -0,0 +1,51 @@ +# +# SPDX-License-Identifier: Apache-2.0 +# +# Copyright (C) 2024, Arm Limited and contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +function update_branch() { + local label=$1 + local branch=$2 + local force_branch=$3 + + local worktree=../${branch}-repo + local patch=${branch}-update.patch + + lisa-combine-gitlab-mr --repo "$CI_PROJECT_PATH" --mr-label "$label" --branch "$force_branch" && + + git fetch origin "$branch" && + + # Work in a separate worktree so that there is no risk of folders + # added to PATH by init_env being manipulated + git worktree add "$worktree" --checkout "$branch" && + + git -C "$worktree" diff --binary "HEAD..$force_branch" > "$patch" && + + if [[ -s "$patch" ]]; then + # Apply the patch to the index as well, so that any file created + # is automatically added to the commit we are about to create. + git -C "$worktree" apply --index "../origin-repo/$patch" && + git -C "$worktree" commit --all -m "Autocommit to $branch branch on $(date) tracking $force_branch" + + git remote set-url origin https://gitlab-ci:${GITLAB_REPO_TOKEN}@${CI_SERVER_HOST}/${CI_PROJECT_PATH}.git + + git push --force origin "$force_branch" + git push origin "$branch" + else + echo "Empty patch, $branch and $force_branch branches are up to date." + fi +} diff --git a/tools/lisa-combine-gitlab-mr b/tools/lisa-combine-gitlab-mr new file mode 100755 index 0000000000000000000000000000000000000000..17d3b1fe59c3486fcfe43da0a8d433a3b5d138fc --- /dev/null +++ b/tools/lisa-combine-gitlab-mr @@ -0,0 +1,155 @@ +#! /usr/bin/env python3 +# +# SPDX-License-Identifier: Apache-2.0 +# +# Copyright (C) 2024, Arm Limited and contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import subprocess +from itertools import chain +from tempfile import NamedTemporaryFile +import json, os +from collections import ChainMap +from operator import itemgetter +import argparse +import logging +import requests + +# Use HTTP APIs of GitLab to retrieve associated mrs +# of this project. +def get_gitlab_mrs(state="opened", scope="all", labels=""): + def _call_gitlab_api(endpoint): + api_token = os.environ.get("GITLAB_REPO_TOKEN") + api_url = os.environ.get("CI_API_V4_URL") + headers = {"PRIVATE-TOKEN": api_token} + r = requests.get("/".join([api_url,endpoint]), headers=headers) + if r.status_code != 200: + raise Exception(r.text) + return r + + this_project_id = int(os.environ.get("CI_PROJECT_ID")) + page_number = 1 # obvioulsy at start 1 + + results = [] + while True: + mr_response = _call_gitlab_api( + f"merge_requests?state={state}&scope={scope}&labels={labels}&page={page_number}" + ) + for this_mr in mr_response.json(): + if this_mr.get("project_id") == this_project_id: + # populate commits count - use another api + mr_commit_response = _call_gitlab_api( + f"projects/{this_project_id}/merge_requests/{this_mr['iid']}/commits" + ) + this_mr["commits_count"] = len(mr_commit_response.json()) + + # this_mr could be from a fork - use another api + project_response = _call_gitlab_api( + f"projects/{this_mr['source_project_id']}" + ) + this_mr["http_url_to_repo"] = project_response.json()[ + "http_url_to_repo" + ] + + results.append(this_mr) + + # handle paging - only required for mr + if not mr_response.headers["X-Next-Page"]: + break + page_number = mr_response.headers["X-Next-Page"] + + return results + +def main(): + parser = argparse.ArgumentParser( + description=""" + Combine gitlab merge requests with the given tag into a branch, rebasing all + MRs on top of each other. + """, + ) + + parser.add_argument('--repo', required=True, help='Gitlab repository as owner/name') + parser.add_argument('--mr-label', action='append', required=True, help='Merge request labels to look for') + parser.add_argument('--branch', required=True, help='Name of the branch to be created. If the branch exists, it will be forcefully updated') + + args = parser.parse_args() + + project = args.repo + owner, repo = args.repo.split('/', 1) + labels = ','.join(args.mr_label) + branch = args.branch + + logging.basicConfig(level=logging.INFO) + + server_host_ssh = os.environ.get('CI_SERVER_SHELL_SSH_HOST') + + gl_mrs = get_gitlab_mrs(labels=labels) + + def make_topic(mr): + remote = f'remote_{mr["sha"]}' + return ( + { + remote: { + 'url': mr["http_url_to_repo"] + } + }, + { + 'name': mr["source_branch"], + 'remote': remote, + 'nr-commits': mr["commits_count"], + 'tip': mr["source_branch"], + } + ) + + topics = [] + + for mr in gl_mrs: + topics += [ make_topic(mr) ] + + remotes, topics = zip(*topics) if topics else ([], []) + remotes = dict(ChainMap(*chain( + [{ + 'gitlab': { + 'url': f'https://{server_host_ssh}/{owner}/{repo}.git' + } + }], + remotes + ))) + + conf = { + 'rebase-conf': { + 'rr-cache': './rr-cache', + 'remotes': remotes, + 'base': { + 'remote': 'gitlab', + 'ref': 'main', + }, + 'topics': sorted(topics, key=itemgetter('name')) + } + } + conf = json.dumps(conf, indent=4) + logging.info(conf) + + with NamedTemporaryFile(mode='w+', suffix='.manifest.json') as f: + f.write(conf) + f.flush() + + manifest = f.name + + cmd = ['batch-rebase', 'create', '.', '--manifest', manifest, '--create-branch', branch] + logging.info(f'Running {" ".join(map(str, cmd))}') + subprocess.check_call(cmd) + +main()