diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 98dbb27ee053b03db62174d241a03769eb74928d..78a83a561fa76a8d316f8dc7077e9add88c2790d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -24,82 +24,132 @@ default: # Required for Manual runs through UI variables: - ENABLE_TEST: - value: "False" + PYVERSION: + value: 11 + description: Default version of Python to use for non-test jobs + + WEB_ENABLE_TEST: + value: "True" options: - "True" - "False" - description: True to run test job. Defaults to false. - ENABLE_AUTOBRANCH: - value: "False" + description: True to run test job when triggered via Web UI. + WEB_ENABLE_AUTOBRANCH: + value: "True" options: - "True" - "False" - description: True to run Autobranch job. Defaults to false. Autobranch is allowed only on main. - -stages: - - Test - - Deploy + description: True to run test job when triggered via Web UI. Test: - stage: Test rules: - - if: $CI_PIPELINE_SOURCE == "web" && $ENABLE_TEST == "True" + - if: $CI_PIPELINE_SOURCE == "web" && $WEB_ENABLE_TEST == "True" - if: $CI_PIPELINE_SOURCE == "merge_request_event" - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH parallel: matrix: - PYVERSION: [8, 11] before_script: + - sudo sh -c 'echo "APT::Acquire::Retries=3;" >> /etc/apt/apt.conf.d/99retry-downloads' - export DEBIAN_FRONTEND=noninteractive - - sudo apt update -y - - sudo apt install software-properties-common -y + - sudo apt-get update -y + - sudo apt-get upgrade -y + - sudo apt-get install software-properties-common -y - sudo add-apt-repository ppa:deadsnakes/ppa -y - sudo apt-get update -y - - sudo apt-get -y install python3.${PYVERSION}-venv python3.${PYVERSION}-tk python3.${PYVERSION}-full + - sudo apt-get -y install python3.${PYVERSION}-full + - sudo chmod o+rx /sys/kernel/debug script: - export LISA_PYTHON=python3.${PYVERSION} - - sudo sh -c 'echo "APT::Acquire::Retries=3;" >> /etc/apt/apt.conf.d/99retry-downloads' - sudo ./install_base.sh --install-all - - echo "$(python3 --version)" - - sudo chmod o+rx /sys/kernel/debug - bash ./tools/tests.sh Autobranch: - stage: Deploy + needs: ['Test'] + environment: + # This grants the GITLAB_REPO_TOKEN variable allowing to push to the repo + name: repo_write_access + variables: - GIT_STRATEGY: none #Required to enable the autobranch run from a specific branch:$CI_DEFAULT_BRANCH + # Do not clone or fetch anything, as we will do it manually so that the job + # runs with the code from the default branch (main) rather than the merge + # request branch for safety reasons. + GIT_STRATEGY: none rules: - - if: $CI_PIPELINE_SOURCE == "web" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $ENABLE_AUTOBRANCH == "True" + - if: $CI_PIPELINE_SOURCE == "web" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $WEB_ENABLE_AUTOBRANCH == "True" - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - - if: $CI_PIPELINE_SOURCE == "merge_request_event" && '$CI_MERGE_REQUEST_LABELS =~ /^autobranch$/' #regex to pickup 'autobranch' label alone + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_LABELS =~ /(^|,)autobranch($|,)/ resource_group: autobranch before_script: - - sudo apt update -y - - sudo apt install tzdata -y - - sudo apt install software-properties-common -y + - sudo sh -c 'echo "APT::Acquire::Retries=3;" >> /etc/apt/apt.conf.d/99retry-downloads' + - export DEBIAN_FRONTEND=noninteractive + - sudo apt-get update -y + - sudo apt-get upgrade -y + - sudo apt-get install software-properties-common -y + - sudo add-apt-repository ppa:deadsnakes/ppa -y - sudo apt-get update -y - - sudo apt-get install python3.10 git python3.10-venv -y - - sudo apt-get install gcc python3-dev -y + - sudo apt-get -y install python3.${PYVERSION}-full + - export LISA_PYTHON=python3.${PYVERSION} + - | - git clone -b $CI_DEFAULT_BRANCH $CI_REPOSITORY_URL origin-repo + # Avoid using /tmp since it's probably in tmpfs and would consume RAM + export TMPDIR=$(readlink -f .)/tmp + mkdir -p "$TMPDIR" - cd origin-repo && + origin_repo=$(mktemp -d) + git clone -b "$CI_DEFAULT_BRANCH" "$CI_REPOSITORY_URL" "$origin_repo" - git config --global user.name 'Gitlab CI' - git config --global user.email 'gitlab-ci@arm.com' + # This is critical for safety: We want to be executing the code from the + # main branch to avoid any manipulation from the MR branch + cd "$origin_repo" || exit 1 + - | + git remote set-url origin "https://gitlab-ci:${GITLAB_REPO_TOKEN}@${CI_SERVER_HOST}/${CI_PROJECT_PATH}.git" git remote -v git branch --list --remotes + + git config --global user.name 'Gitlab CI' + git config --global user.email 'gitlab-ci@arm.com' + script: - | set -e - export LC_ALL=C + venv=$(mktemp -d) + python3 -m venv "$venv" && source "$venv/bin/activate" + $LISA_PYTHON -m pip install --upgrade pip + $LISA_PYTHON -m pip install ./tools/lisa-combine-gitlab-mr + + function update_branch() { + local label=$1 + local branch=$2 + local force_branch=$3 + + local worktree=$(mktemp -d --suffix "${branch}-repo") + local patch=$(mktemp --suffix "$branch.patch") - source init_env && - pip install requests # Required for calling Gitlab API - source ci/autobranch-job-script.sh + lisa-combine-gitlab-mr --server "$CI_SERVER_HOST" --repo "$CI_PROJECT_PATH" --project-id "$CI_PROJECT_ID" --api-url "$CI_API_V4_URL" --mr-label "$label" --branch "$force_branch" && + + git fetch origin "$branch" && + + # Work in a separate worktree so that there is no risk of folders + # added to PATH by init_env being manipulated + git worktree add "$worktree" --checkout "$branch" && + + git -C "$worktree" diff --binary "HEAD..$force_branch" > "$patch" && + + if [[ -s "$patch" ]]; then + # Apply the patch to the index as well, so that any file created + # is automatically added to the commit we are about to create. + git -C "$worktree" apply --index "$patch" && + git -C "$worktree" commit --all -m "Autocommit to $branch branch on $(date) tracking $force_branch" + + git push --force origin "$force_branch" + git push origin "$branch" + else + echo "Empty patch, $branch and $force_branch branches are up to date." + fi + } ret=0 function keepgoing { @@ -110,3 +160,35 @@ Autobranch: keepgoing update_branch for-preview-autobranch preview preview-force exit $ret + + +pages: + needs: ['Autobranch', 'Test'] + resource_group: pages + + artifacts: + paths: + - doc/public + publish: doc/public + + rules: + - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + + before_script: + - sudo sh -c 'echo "APT::Acquire::Retries=3;" >> /etc/apt/apt.conf.d/99retry-downloads' + - export DEBIAN_FRONTEND=noninteractive + - sudo apt-get update -y + - sudo apt-get upgrade -y + - sudo apt-get install software-properties-common -y + - sudo add-apt-repository ppa:deadsnakes/ppa -y + - sudo apt-get update -y + - sudo apt-get -y install python3.${PYVERSION}-full + - export LISA_PYTHON=python3.${PYVERSION} + + script: + - sudo ./install_base.sh --install-doc-extras + - source init_env + - | + set -e + cd doc/ + sphinx-build -b html . "public/$CI_COMMIT_REF_NAME" diff --git a/README.rst b/README.rst index 0638fa123da0542a12b1693cc2a8649be2585310..2b95fabca95515125ae310282e2d2a1db92c78b5 100644 --- a/README.rst +++ b/README.rst @@ -1,8 +1,11 @@ +⚠️ LISA has moved from GitHub to GitLab and is now available at: +https://gitlab.arm.com/tooling/lisa. Please update your clone URLs and note +that the GitHub repository will not be used for any pull requests or issue +management. ⚠️ ⚠️ ``lisa_tests`` package will be moved to another repository in January 2024. When more details are available, this pull request will be -updated: https://github.com/ARM-software/lisa/pull/2140 ⚠️ - +updated: https://gitlab.arm.com/tooling/lisa/-/merge_requests/2140/ ⚠️ Introduction |CI status| |Documentation Status| =================================================== @@ -76,7 +79,7 @@ Here are some noteworthy sections: How to reach us =============== -Bug reports should be raised against the `GitHub issue tracker `__. +Bug reports should be raised against the `GitLab issue tracker `__. External Links ============== @@ -113,7 +116,7 @@ where you have authored all of the code. If you do this on work time make sure your employer is cool with this. We also have a `Contributor Guide `__ -.. |CI status| image:: https://github.com/ARM-software/lisa/workflows/test/badge.svg?branch=main - :target: https://github.com/ARM-software/lisa/actions +.. |CI status| image:: https://gitlab.arm.com/tooling/lisa/badges/main/pipeline.svg + :target: https://gitlab.arm.com/tooling/lisa/-/commits/main .. |Documentation Status| image:: https://readthedocs.org/projects/lisa-linux-integrated-system-analysis/badge/?version=main :target: https://lisa-linux-integrated-system-analysis.readthedocs.io/en/main/ diff --git a/ci/autobranch-job-script.sh b/ci/autobranch-job-script.sh deleted file mode 100755 index de2930628f261f4f1afef19d30644158e2cc428c..0000000000000000000000000000000000000000 --- a/ci/autobranch-job-script.sh +++ /dev/null @@ -1,51 +0,0 @@ -# -# SPDX-License-Identifier: Apache-2.0 -# -# Copyright (C) 2024, Arm Limited and contributors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -function update_branch() { - local label=$1 - local branch=$2 - local force_branch=$3 - - local worktree=../${branch}-repo - local patch=${branch}-update.patch - - lisa-combine-gitlab-mr --repo "$CI_PROJECT_PATH" --mr-label "$label" --branch "$force_branch" && - - git fetch origin "$branch" && - - # Work in a separate worktree so that there is no risk of folders - # added to PATH by init_env being manipulated - git worktree add "$worktree" --checkout "$branch" && - - git -C "$worktree" diff --binary "HEAD..$force_branch" > "$patch" && - - if [[ -s "$patch" ]]; then - # Apply the patch to the index as well, so that any file created - # is automatically added to the commit we are about to create. - git -C "$worktree" apply --index "../origin-repo/$patch" && - git -C "$worktree" commit --all -m "Autocommit to $branch branch on $(date) tracking $force_branch" - - git remote set-url origin https://gitlab-ci:${GITLAB_REPO_TOKEN}@${CI_SERVER_HOST}/${CI_PROJECT_PATH}.git - - git push --force origin "$force_branch" - git push origin "$branch" - else - echo "Empty patch, $branch and $force_branch branches are up to date." - fi -} diff --git a/devmode_requirements.txt b/devmode_requirements.txt index fe5224395ea1bca4c21be9125be8cb57fade2374..c97069d6af2addd6a1dd4b1c9fce8bb7354e1214 100644 --- a/devmode_requirements.txt +++ b/devmode_requirements.txt @@ -12,3 +12,4 @@ -e ./[all] -e ./tools/bisector +-e ./tools/lisa-combine-gitlab-mr diff --git a/doc/changes.rst b/doc/changes.rst index 54ed9cc664be24b74b1d64736744cd696a077c57..8fc678ed69b304e19e8f0936f7f24b21f1a21644 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -24,7 +24,7 @@ Everything else is private. important API laws. This is unfortunately not future-proof, as new versions can add new methods that would also require being overridden and kept in sync. If for some reason subclassing is required, please get in touch in the - `github issue tracker `_ + `GitLab issue tracker `_ before relying on that for production. .. note:: Instance attributes are considered public following the same diff --git a/doc/conf.py b/doc/conf.py index b5a8622aacaf9bc0bc9b2559a0600572ca354312..132cae867751d4e58f730f87e0291a4ceffbf01d 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -61,6 +61,7 @@ if RTD: # history to be able to generate breaking change list and any other # git-based documentation subprocess.run(['git', 'fetch', '--unshallow'], check=False) + subprocess.run(['git', 'fetch', '--tags'], check=False) # Ensure we have the changelog notes that supplement commit messages, as # sometimes the markers such as FEATURE were forgotten and later added diff --git a/doc/contributors_guide.rst b/doc/contributors_guide.rst index cd2d251c60699be10d3c12ed60b67e40a5a6745c..2909077d9e77632a727ebc608da20d321abab76a 100644 --- a/doc/contributors_guide.rst +++ b/doc/contributors_guide.rst @@ -10,7 +10,7 @@ How to reach us =============== If you’re hitting an error/bug and need help, it’s best to raise an -issue on `GitHub `__. +issue on `GitLab `__. Coding style ============ diff --git a/doc/index.rst b/doc/index.rst index 2e2ef21ac175fb66d9ebf797d4e4331ae28ead48..32b9a9dd0c3ef18a789dd6831692c05f8cff54b5 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -11,14 +11,14 @@ LISA Documentation LISA - "Linux Integrated System Analysis" is a toolkit for interactive analysis and automated regression testing of Linux kernel behaviour. -- See the README on the project's `Github home page`__ for an overview. +- See the README on the project's `GitLab home page`__ for an overview. - Once you have LISA running, take a look at the tutorial and example notebooks included with the installation. -__ https://github.com/ARM-software/lisa +__ https://gitlab.arm.com/tooling/lisa Contributions to LISA and its documentation are very welcome, and handled -via Github pull requests. +via GitLab merge requests. .. _Readme: diff --git a/doc/setup.rst b/doc/setup.rst index 1ff986a4faaf03c25b9ef46e5d2be1617f6d6f51..45ded3dd5ab135cf4a3f81dab95f63c9d43639ea 100644 --- a/doc/setup.rst +++ b/doc/setup.rst @@ -26,10 +26,10 @@ LISA is available on `PyPI `_: them. Alternatively, the installation from the git repository allows setting up a full environment. -From GitHub +From GitLab ----------- -LISA is hosted at `github `_. +LISA is hosted on `GitLab `_. The following references are available: * ``main`` branch: Main development branch where pull requests are merged as they @@ -76,7 +76,7 @@ need. .. code:: shell - git clone https://github.com/ARM-software/lisa.git + git clone https://gitlab.arm.com/tooling/lisa # Jump into the cloned repo directory cd lisa # This will provide a more accurate changelog when building the doc @@ -142,7 +142,7 @@ Once these two components are available on your machine, issue these commands: .. code:: shell - git clone https://github.com/ARM-software/lisa.git + git clone https://gitlab.arm.com/tooling/lisa cd lisa vagrant up diff --git a/doc/test_example.py b/doc/test_example.py deleted file mode 120000 index ccd801e51d250edd8af347c10cb77f1e4144eaca..0000000000000000000000000000000000000000 --- a/doc/test_example.py +++ /dev/null @@ -1 +0,0 @@ -../lisa/test_example.py \ No newline at end of file diff --git a/doc/workflows/notebook.rst b/doc/workflows/notebook.rst index da4f76e89ddc5ab76e4da73323983aede98a3eac..71f306ce086b5d47242c8bfd2a4844dfb80b6de8 100644 --- a/doc/workflows/notebook.rst +++ b/doc/workflows/notebook.rst @@ -33,7 +33,7 @@ Once the server is started you can have a look at the provided tutorial notebooks are accessible by following this `link `__. This initial tutorial can be seen (but not executed) also on `github -`__. +`__. Notebooks as development environment ==================================== diff --git a/lisa/energy_meter.py b/lisa/energy_meter.py index 3807a6a2a0c9479ef38dd0bcd4a582025f58b6e9..1fbb0e62d3ead7dadc55fd732798a67571c9ac66 100644 --- a/lisa/energy_meter.py +++ b/lisa/energy_meter.py @@ -389,18 +389,6 @@ class Monsoon(_DevlibContinuousEnergyMeter): self._instrument.reset() -_acme_install_instructions = ''' - - If you need to measure energy using an ACME EnergyProbe, - please do follow installation instructions available here: - https://github.com/ARM-software/lisa/wiki/Energy-Meters-Requirements#iiocapture---baylibre-acme-cape - - Othwerwise, please select a different energy meter in your - configuration file. - -''' - - class ACMEConf(SimpleMultiSrcConf, HideExekallID): """ Configuration class for :class:`ACME`. @@ -461,7 +449,6 @@ class ACME(EnergyMeter): subprocess.call([self._iiocapturebin, '-h'], stdout=PIPE, stderr=STDOUT) except FileNotFoundError as e: logger.error(f'iio-capture binary {self._iiocapturebin} not available') - logger.warning(_acme_install_instructions) raise FileNotFoundError('Missing iio-capture binary') from e def sample(self): diff --git a/requirements.txt b/requirements.txt index 16687201bf47dbf4eb61f053d219b327068aedc2..c1361b79859fd933f7ff5aa9398c94230eae1cce 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ ./tools/exekall ./[all] ./tools/bisector +./tools/lisa-combine-gitlab-mr diff --git a/setup.py b/setup.py index a6584c179cf2e4c2adaa9900d704de6ff3472693..7d7aeea2aa8a9f110f74d952d9e8ada67cdf2545 100755 --- a/setup.py +++ b/setup.py @@ -81,7 +81,6 @@ extras_require={ "pytest", "build", "twine", - "github3.py", ], "wa": [ @@ -117,11 +116,11 @@ if __name__ == "__main__": version=lisa_version, maintainer='Arm Ltd.', packages=packages, - url='https://github.com/ARM-software/lisa', + url='https://gitlab.arm.com/tooling/lisa', project_urls={ - "Bug Tracker": "https://github.com/ARM-software/lisa/issues", "Documentation": "https://lisa-linux-integrated-system-analysis.readthedocs.io/", - "Source Code": "https://github.com/ARM-software/lisa", + "Bug Tracker": "https://gitlab.arm.com/tooling/lisa/-/issues", + "Source Code": "https://gitlab.arm.com/tooling/lisa", }, description='A stick to probe the kernel with', long_description=long_description, diff --git a/shell/lisa_shell b/shell/lisa_shell index 7a1003391d01d9aa1e8c35f477d15406747fe4b7..dc1074585385b3b10a520330d8ed92690a6bfc25 100755 --- a/shell/lisa_shell +++ b/shell/lisa_shell @@ -46,8 +46,18 @@ export LISA_HOST_ABI=${LISA_HOST_ABI:-$(uname -m | sed 's/aarch64/arm64/')} export _DOC_LISA_USE_SYSTEM_BIN="Use the system binaries if 1, will use the ones shipped with LISA if 0" export LISA_USE_SYSTEM_BIN=${LISA_USE_SYSTEM_BIN:-0} -_ANDROID_BUILD_TOOLS=$(find "$ANDROID_HOME/build-tools/"* -maxdepth 0 -type d -print -quit) -_LISA_PATH="$LISA_HOME/shell/:$LISA_HOME/tools/$LISA_HOST_ABI:$LISA_HOME/tools/:$ANDROID_HOME/tools/:$ANDROID_HOME/platform-tools/:$_ANDROID_BUILD_TOOLS" +_LISA_PATH="$LISA_HOME/shell/:$LISA_HOME/tools/$LISA_HOST_ABI:$LISA_HOME/tools/" + +if [[ -d "$ANDROID_HOME" ]]; then + _LISA_PATH="$_LISA_PATH:$ANDROID_HOME/tools/:$ANDROID_HOME/platform-tools" + + _ANDROID_BUILD_TOOLS="$ANDROID_HOME/build-tools/" + if [[ -d "$_ANDROID_BUILD_TOOLS" ]]; then + _ANDROID_BUILD_TOOLS=$(find "$_ANDROID_BUILD_TOOLS"* -maxdepth 0 -type d -print -quit || printf "%s" "$_ANDROID_BUILD_TOOLS") + _LISA_PATH="$_LISA_PATH:$_ANDROID_BUILD_TOOLS" + fi +fi + if [[ "$LISA_USE_SYSTEM_BIN" == 1 ]]; then export PATH=$PATH:$_LISA_PATH else diff --git a/tools/batch-rebase/LICENSE.txt b/tools/batch-rebase/LICENSE.txt new file mode 120000 index 0000000000000000000000000000000000000000..1ef648f64b34ffd4b08b00bc2d967f9b4387c8f4 --- /dev/null +++ b/tools/batch-rebase/LICENSE.txt @@ -0,0 +1 @@ +../../LICENSE.txt \ No newline at end of file diff --git a/tools/batch-rebase/pyproject.toml b/tools/batch-rebase/pyproject.toml new file mode 100644 index 0000000000000000000000000000000000000000..e9b06e07b9b1d98ab68a60a3be865be14bffff84 --- /dev/null +++ b/tools/batch-rebase/pyproject.toml @@ -0,0 +1,54 @@ +[project] +name = "batch-rebase" # Required +version = "1.0.0" # Required +description = "Merge several branches by rebasing them on top of each other" +requires-python = ">=3.8" +license = {file = "LICENSE.txt"} +keywords = ["git", "development"] + +# readme = "README.md" +authors = [ + {name = "Douglas RAILLARD"} +] +maintainers = [ + {name = "Arm Ltd." } +] + +# Classifiers help users find your project by categorizing it. +# +# For a list of valid classifiers, see https://pypi.org/classifiers/ +classifiers = [ # Optional + # How mature is this project? Common values are + # 3 - Alpha + # 4 - Beta + # 5 - Production/Stable + "Development Status :: 4 - Beta", + + # Indicate who your project is intended for + "Intended Audience :: Developers", + "Topic :: Software Development :: Build Tools", + + # This is not a standard classifier, as there is nothing defined for + # Apache 2.0 yet: + # https://pypi.org/classifiers/ +] + +dependencies = [ + "ruamel.yaml", +] + +[project.urls] +"Homepage" = "https://gitlab.arm.com/tooling/lisa" +"Source" = "https://gitlab.arm.com/tooling/lisa" +"Documentation" = "https://lisa-linux-integrated-system-analysis.readthedocs.io/" +"Bug Tracker" = "https://gitlab.arm.com/tooling/lisa/-/issues" + +# The following would provide a command line executable called `sample` +# which executes the function `main` from this package when invoked. +[project.scripts] # Optional +batch-rebase = "batch_rebase.main:main" + +[build-system] +requires = ["pdm-backend"] +build-backend = "pdm.backend" + diff --git a/tools/batch-rebase b/tools/batch-rebase/src/batch_rebase/main.py similarity index 85% rename from tools/batch-rebase rename to tools/batch-rebase/src/batch_rebase/main.py index d5cc3a3d7276d6b3ac9ec739f8318458888a67cb..a87109031b5f17472db298faacfc7ffbdd3ac9dc 100755 --- a/tools/batch-rebase +++ b/tools/batch-rebase/src/batch_rebase/main.py @@ -33,8 +33,7 @@ import textwrap import logging import contextlib import typing - -from lisa.conf import SimpleMultiSrcConf, KeyDesc, LevelKeyDesc, TopLevelKeyDesc +import json def info(msg): logging.info(msg) @@ -42,36 +41,55 @@ def info(msg): def warn(msg): logging.warning(msg) -class BatchRebaseManifest(SimpleMultiSrcConf): - """ - Configuration of batch-rebase +def get_nested_key(conf, key): + for k in key: + conf = conf[k] + return conf + +def load_conf(path): + + def postprocess(conf): + return conf['rebase-conf'] + + try: + from ruamel.yaml import YAML + except ImportError: + try_yaml = False + else: + try_yaml = True + + def load_yaml(path): + yaml = YAML(typ='safe') + with open(path) as f: + return yaml.load(f) + + def load_json(path): + with open(path) as f: + return json.load(f) + + loaders = [ + load_json, + *([load_yaml] if try_yaml else []) + ] + + last_excep = ValueError('No loader selected') + for loader in loaders: + try: + conf = loader(path) + except Exception as e: + last_excep = e + error(e) + else: + return postprocess(conf) + + raise last_excep + + +def dump_conf(conf, path): + conf = {'rebase-conf': conf} + with open(path, 'w') as f: + json.dump(conf, f) - {generated_help} - """ - STRUCTURE = TopLevelKeyDesc('rebase-conf', 'Batch rebase configuration', ( - KeyDesc('topics', 'List of topics. Each topic is described by a mapping with "name", "remote", "base" (or "nr-commits) and "tip" git references keys. Also, a tag can be added between topics with "action: tag" key and "name: tag-name".', [list]), - KeyDesc('remotes', 'Git remotes. Keys are remote name, values are a mapping with an "url" key', [Mapping]), - KeyDesc('rr-cache', 'Path to git rr-cache. Relative paths are relative to that manifest file', [str, None]), - LevelKeyDesc('base', 'Base branche spec', [ - KeyDesc('remote', 'remote where the base branch is located', [str]), - KeyDesc('ref', 'Name of the base branch', [str]), - ]), - LevelKeyDesc('tip', 'New branche spec', [ - KeyDesc('ref', 'Name of the base branch. --refs-suffix can be used to tweak the name', [str]), - KeyDesc('tags', 'Default of --tags', [bool]), - ]), - LevelKeyDesc('resume', 'Internal state used for resuming after a conflict', [ - KeyDesc('conflict-topic', 'Topic where the conflict happened', [str]), - KeyDesc('persistent-refs', 'List of references that needs to be pushed back to the main repo', [typing.Sequence[str]]), - LevelKeyDesc('tags', 'Topic branch tags', [ - KeyDesc('persistent', 'whether tags should be pushed back or not', [bool]), - KeyDesc('suffix', 'suffix to use for tags, or None', [str]), - ]), - KeyDesc('repo', 'Path to the main repo', [str]), - KeyDesc('branch', 'Branch that is being created', [str]), - KeyDesc('rr-cache', 'Source of the rr-cache', [str, None]), - ]) - )) RESUME_MANIFEST_NAME = '.batch-rebase-state' @@ -216,13 +234,13 @@ def do_cherry_pick(repo, temp_repo, conf, persistent_tags, tags_suffix, branch, has_conflict, persistent_refs = _do_cherry_pick(temp_repo, conf, persistent_tags, tags_suffix) if has_conflict: - conf['resume'].add_src('conflict', { + conf['resume'] = { 'repo': str(repo), 'rr-cache': str(rr_cache) if rr_cache else rr_cache, 'branch': branch, - }) + } # Save the augmented manifest for later resumption - conf.to_yaml_map(temp_repo/RESUME_MANIFEST_NAME) + dump_conf(conf, temp_repo/RESUME_MANIFEST_NAME) else: # Copy back the rr-cache if rr_cache: @@ -282,7 +300,7 @@ def _do_cherry_pick(repo, conf, persistent_tags, tags_suffix): ) # If we are resuming after a conflict - if 'conflict-topic' in conf['resume']: + if 'conflict-topic' in conf.get('resume', {}): resume_topic = conf['resume']['conflict-topic'] persistent_refs.update(conf['resume']['persistent-refs']) @@ -365,17 +383,14 @@ def _do_cherry_pick(repo, conf, persistent_tags, tags_suffix): if not cherry_pick_ref(repo, range_ref): # Save the current state for later resumption - conf['resume'].add_src( - 'conflict', - { - 'conflict-topic': name, - 'persistent-refs': sorted(persistent_refs), - 'tags': { - 'persistent': persistent_tags, - 'suffix': tags_suffix, - } - }, - ) + conf['resume'] = { + 'conflict-topic': name, + 'persistent-refs': sorted(persistent_refs), + 'tags': { + 'persistent': persistent_tags, + 'suffix': tags_suffix, + } + } info(textwrap.dedent(""" A conflict occured while cherry-picking topic "{topic}" ({range_ref}) In order to fix it, please: @@ -518,7 +533,37 @@ def main(): The manifest is a YAML file following that structure: - {manifest_doc} + ├ rebase-conf: Batch rebase configuration + ├ topics (list): + List of topics. Each topic is described by a mapping with + "name", "remote", "base" (or "nr-commits) and "tip" git + references keys. Also, a tag can be added between topics + with "action: tag" key and "name: tag-name".. + ├ remotes (Mapping): + Git remotes. Keys are remote name, values are a mapping with + an "url" key. + ├ rr-cache (str or None): + Path to git rr-cache. Relative paths are relative to that + manifest file. + ├ base: Base branche spec + ├ remote (str): remote where the base branch is located. + └ ref (str): Name of the base branch. + ├ tip: New branche spec + ├ ref (str): + Name of the base branch. --refs-suffix can be used to tweak + the name. + └ tags (bool): Default of --tags. + └ resume: Internal state used for resuming after a conflict + ├ conflict-topic (str): Topic where the conflict happened. + ├ persistent-refs (typing.Sequence[str]): + List of references that needs to be pushed back to the main + repo. + ├ tags: Topic branch tags + ├ persistent (bool): whether tags should be pushed back or not. + └ suffix (str): suffix to use for tags, or None. + ├ repo (str): Path to the main repo. + ├ branch (str): Branch that is being created. + └ rr-cache (str or None): Source of the rr-cache. EXAMPLE @@ -555,12 +600,12 @@ def main(): url: git://linux-arm.org/linux-power.git - ''').format(manifest_doc=BatchRebaseManifest.get_help()), + '''), formatter_class=argparse.RawDescriptionHelpFormatter, ) - subparsers = parser.add_subparsers(title='subcommands', dest='subcommand') + subparsers = parser.add_subparsers(title='subcommands', dest='subcommand', required=True) create_parser = subparsers.add_parser('create', help='Create a branch' ) @@ -635,10 +680,10 @@ def main(): temp_repo = None try: if args.subcommand == 'create': - conf = BatchRebaseManifest.from_yaml_map(args.manifest) + conf = load_conf(args.manifest) def default_from_conf(x, key): try: - default = conf.get_nested_key(key) + default = get_nested_key(conf, key) except KeyError: return x else: @@ -686,7 +731,7 @@ def main(): elif args.subcommand == 'resume': temp_repo = repo - conf = BatchRebaseManifest.from_yaml_map(temp_repo/RESUME_MANIFEST_NAME) + conf = load_conf(temp_repo/RESUME_MANIFEST_NAME) delete_temp_repo, ret = do_resume(temp_repo, conf) finally: diff --git a/tools/bisector/LICENSE.txt b/tools/bisector/LICENSE.txt deleted file mode 100644 index d645695673349e3947e8e5ae42332d0ac3164cd7..0000000000000000000000000000000000000000 --- a/tools/bisector/LICENSE.txt +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/tools/bisector/LICENSE.txt b/tools/bisector/LICENSE.txt new file mode 120000 index 0000000000000000000000000000000000000000..1ef648f64b34ffd4b08b00bc2d967f9b4387c8f4 --- /dev/null +++ b/tools/bisector/LICENSE.txt @@ -0,0 +1 @@ +../../LICENSE.txt \ No newline at end of file diff --git a/tools/bisector/doc/api.rst b/tools/bisector/doc/api.rst index 4b6a29dd9552ef4a4642167d77e62023ffaaca73..2c06c4baee1032d19c5e5382847298e810a71c94 100644 --- a/tools/bisector/doc/api.rst +++ b/tools/bisector/doc/api.rst @@ -9,6 +9,6 @@ In order to support report edition (``bisector edit``) and more stable serialized format, new steps constructors must be written in a similar way to exisiting ones. -.. automodule:: bisector.bisector +.. automodule:: bisector.main :members: diff --git a/tools/bisector/doc/index.rst b/tools/bisector/doc/index.rst index 7d12d5883dda293511fa6b90bfb895b0546a4ee9..9d5858f6edb866f8e6fc264ad42cae97da3fcf87 100644 --- a/tools/bisector/doc/index.rst +++ b/tools/bisector/doc/index.rst @@ -13,7 +13,7 @@ Overview ======== Bisector is a ``git bisect run`` compatible tool used in LISA. Check out the -project's `Github`__ for some guides to installation and setup. +project's `GitLab`__ for some guides to installation and setup. ``bisector`` allows setting up the steps of a test iteration, repeating them an infinite number of times (by default). These steps can involve flashing @@ -32,7 +32,7 @@ that happen on long running sessions. ``bisector`` will never leave you with an inconsistent report, or worse, no report at all. A new report is saved after each iteration and can be inspected as the execution goes on. -__ https://github.com/ARM-software/lisa +__ https://gitlab.arm.com/tooling/lisa .. [#] https://git-scm.com/docs/git-bisect Contents diff --git a/tools/bisector/pyproject.toml b/tools/bisector/pyproject.toml new file mode 100644 index 0000000000000000000000000000000000000000..757299468415bb3d700ad0721756a3db91015eab --- /dev/null +++ b/tools/bisector/pyproject.toml @@ -0,0 +1,61 @@ +[project] +name = "bisector" # Required +version = "1.0.0" # Required +description = "Command execution sequencer" +requires-python = ">=3.8" +license = {file = "LICENSE.txt"} +keywords = ["development"] + +readme = "README.rst" +authors = [ + {name = "Douglas RAILLARD"} +] +maintainers = [ + {name = "Arm Ltd." } +] + +# Classifiers help users find your project by categorizing it. +# +# For a list of valid classifiers, see https://pypi.org/classifiers/ +classifiers = [ # Optional + # How mature is this project? Common values are + # 3 - Alpha + # 4 - Beta + # 5 - Production/Stable + "Development Status :: 4 - Beta", + + "Programming Language :: Python :: 3 :: Only", + # This is not a standard classifier, as there is nothing defined for + # Apache 2.0 yet: + # https://pypi.org/classifiers/ + "License :: OSI Approved :: Apache 2.0", + # It has not been tested under any other OS + "Operating System :: POSIX :: Linux", + "Topic :: Software Development :: Testing", + "Intended Audience :: Developers", +] + +dependencies = [ + # Older versions will have troubles with serializing complex nested + # objects hierarchy implementing custom __getstate__ and __setstate__ + "ruamel.yaml >= 0.15.72", + "pandas", + "scipy", + "requests", +] + +[project.urls] +"Homepage" = "https://gitlab.arm.com/tooling/lisa" +"Source" = "https://gitlab.arm.com/tooling/lisa" +"Documentation" = "https://lisa-linux-integrated-system-analysis.readthedocs.io/" +"Bug Tracker" = "https://gitlab.arm.com/tooling/lisa/-/issues" + +# The following would provide a command line executable called `sample` +# which executes the function `main` from this package when invoked. +[project.scripts] # Optional +bisector = "bisector.main:main" + +[build-system] +requires = ["pdm-backend"] +build-backend = "pdm.backend" + diff --git a/tools/bisector/setup.py b/tools/bisector/setup.py deleted file mode 100755 index c9e7d07bd410ca044bed6fb9bb3a9624a4123dd4..0000000000000000000000000000000000000000 --- a/tools/bisector/setup.py +++ /dev/null @@ -1,73 +0,0 @@ -#! /usr/bin/env python3 -# SPDX-License-Identifier: Apache-2.0 -# -# Copyright (C) 2018, Arm Limited and contributors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from setuptools import setup - -with open('README.rst') as fh: - long_description = fh.read() - -setup( - name='bisector', - version='1.0', - author='Arm Ltd', - # TODO: figure out which email to put here - # author_email= - packages=['bisector'], - # url='http://pypi.python.org/pypi/TowelStuff/', - license='LICENSE.txt', - description='Command execution sequencer', - long_description=long_description, - entry_points={ - 'console_scripts': [ - 'bisector=bisector.bisector:main', - ], - }, - python_requires='>= 3.5', - install_requires=[ - # Older versions will have troubles with serializing complex nested - # objects hierarchy implementing custom __getstate__ and __setstate__ - "ruamel.yaml >= 0.15.72", - "pandas", - "scipy", - "requests", - ], - - extras_require={ - 'dbus': [ - 'pydbus', - 'pygobject', - # You will also need gobject-introspection package from your - # distribution - ] - }, - - classifiers=[ - "Programming Language :: Python :: 3 :: Only", - # This is not a standard classifier, as there is nothing defined for - # Apache 2.0 yet: - # https://pypi.org/classifiers/ - "License :: OSI Approved :: Apache 2.0", - # It has not been tested under any other OS - "Operating System :: POSIX :: Linux", - - "Topic :: Software Development :: Testing", - "Intended Audience :: Developers", - ], -) - -# vim :set tabstop=4 shiftwidth=4 textwidth=80 expandtab diff --git a/tools/bisector/bisector/bisector.py b/tools/bisector/src/bisector/main.py similarity index 100% rename from tools/bisector/bisector/bisector.py rename to tools/bisector/src/bisector/main.py diff --git a/tools/exekall/LICENSE.txt b/tools/exekall/LICENSE.txt deleted file mode 100644 index d645695673349e3947e8e5ae42332d0ac3164cd7..0000000000000000000000000000000000000000 --- a/tools/exekall/LICENSE.txt +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/tools/exekall/LICENSE.txt b/tools/exekall/LICENSE.txt new file mode 120000 index 0000000000000000000000000000000000000000..1ef648f64b34ffd4b08b00bc2d967f9b4387c8f4 --- /dev/null +++ b/tools/exekall/LICENSE.txt @@ -0,0 +1 @@ +../../LICENSE.txt \ No newline at end of file diff --git a/tools/exekall/README.rst b/tools/exekall/README.rst index 878aebd79f5b783bb40f1f76f63ca1bd7e327ac4..e7b593863613f59add7eb7b644335409047f5f99 100644 --- a/tools/exekall/README.rst +++ b/tools/exekall/README.rst @@ -1,7 +1,7 @@ -Exekall is the test runner of LISA. Check out the project's `Github`__ for some +Exekall is the test runner of LISA. Check out the project's `GitLab`__ for some guides to installation and setup. ``exekall`` runs a set of expressions that are discovered from Python sources, allowing to build test suites in a highly modular way. -__ https://github.com/ARM-software/lisa +__ https://gitlab.arm.com/tooling/lisa diff --git a/tools/exekall/doc/index.rst b/tools/exekall/doc/index.rst index eb40215bae5d1edf6f5444c882a3c27093d6fe6e..bb2ed80876fe000226e7530709249877a9df8664 100644 --- a/tools/exekall/doc/index.rst +++ b/tools/exekall/doc/index.rst @@ -12,13 +12,13 @@ Exekall Documentation Overview ======== -Exekall is the test runner of LISA. Check out the project's `Github`__ for some +Exekall is the test runner of LISA. Check out the project's `GitLab`__ for some guides to installation and setup. ``exekall`` runs a set of expressions that are discovered from Python sources, allowing to build test suites in a highly modular way. -__ https://github.com/ARM-software/lisa +__ https://gitlab.arm.com/tooling/lisa Contents ======== diff --git a/tools/exekall/pyproject.toml b/tools/exekall/pyproject.toml new file mode 100644 index 0000000000000000000000000000000000000000..da3d279d78c6ada43fa6dea6ac1ced33ac6afc71 --- /dev/null +++ b/tools/exekall/pyproject.toml @@ -0,0 +1,53 @@ +[project] +name = "exekall" # Required +version = "1.0.0" # Required +description = "Python expression execution engine" +requires-python = ">=3.8" +license = {file = "LICENSE.txt"} +keywords = ["development"] + +readme = "README.rst" +authors = [ + {name = "Douglas RAILLARD"} +] +maintainers = [ + {name = "Arm Ltd." } +] + +# Classifiers help users find your project by categorizing it. +# +# For a list of valid classifiers, see https://pypi.org/classifiers/ +classifiers = [ # Optional + # How mature is this project? Common values are + # 3 - Alpha + # 4 - Beta + # 5 - Production/Stable + "Development Status :: 4 - Beta", + + "Programming Language :: Python :: 3 :: Only", + # This is not a standard classifier, as there is nothing defined for + # Apache 2.0 yet: + # https://pypi.org/classifiers/ + # It has not been tested under any other OS + "Operating System :: POSIX :: Linux", + + "Intended Audience :: Developers", +] + +dependencies = [] + +[project.urls] +"Homepage" = "https://gitlab.arm.com/tooling/lisa" +"Source" = "https://gitlab.arm.com/tooling/lisa" +"Documentation" = "https://lisa-linux-integrated-system-analysis.readthedocs.io/" +"Bug Tracker" = "https://gitlab.arm.com/tooling/lisa/-/issues" + +# The following would provide a command line executable called `sample` +# which executes the function `main` from this package when invoked. +[project.scripts] # Optional +exekall = "exekall.main:main" + +[build-system] +requires = ["pdm-backend"] +build-backend = "pdm.backend" + diff --git a/tools/exekall/setup.py b/tools/exekall/setup.py deleted file mode 100755 index 374a4cae709c6301bcef3038a4cccc777b9a9b1c..0000000000000000000000000000000000000000 --- a/tools/exekall/setup.py +++ /dev/null @@ -1,54 +0,0 @@ -#! /usr/bin/env python3 -# SPDX-License-Identifier: Apache-2.0 -# -# Copyright (C) 2021, Arm Limited and contributors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from setuptools import setup - -with open('README.rst') as fh: - long_description = fh.read() - -setup( - name='exekall', - version='1.0', - maintainer='Arm Ltd.', - packages=['exekall'], - url='https://github.com/ARM-software/lisa', - project_urls={ - "Bug Tracker": "https://github.com/ARM-software/lisa/issues", - "Documentation": "https://lisa-linux-integrated-system-analysis.readthedocs.io/", - "Source Code": "https://github.com/ARM-software/lisa", - }, - license='Apache License 2.0', - description='Python expression execution engine', - long_description=long_description, - entry_points={ - 'console_scripts': ['exekall=exekall.main:main'], - }, - python_requires='>= 3.5', - classifiers=[ - "Programming Language :: Python :: 3 :: Only", - # This is not a standard classifier, as there is nothing defined for - # Apache 2.0 yet: - # https://pypi.org/classifiers/ - # It has not been tested under any other OS - "Operating System :: POSIX :: Linux", - - "Intended Audience :: Developers", - ], -) - -# vim :set tabstop=4 shiftwidth=4 textwidth=80 expandtab diff --git a/tools/exekall/exekall/__init__.py b/tools/exekall/src/exekall/__init__.py similarity index 100% rename from tools/exekall/exekall/__init__.py rename to tools/exekall/src/exekall/__init__.py diff --git a/tools/exekall/exekall/tests/exekall_customize.py b/tools/exekall/src/exekall/_tests/exekall_customize.py similarity index 96% rename from tools/exekall/exekall/tests/exekall_customize.py rename to tools/exekall/src/exekall/_tests/exekall_customize.py index 9234e0a2a0e627a44f21dfc22ec2c77c1084841d..78f6967e3dfa7e5dd7407f084a361a8a3fc26fdf 100644 --- a/tools/exekall/exekall/tests/exekall_customize.py +++ b/tools/exekall/src/exekall/_tests/exekall_customize.py @@ -18,7 +18,7 @@ from exekall.utils import flatten_seq from exekall.customization import AdaptorBase -from exekall.tests.suite import TestCaseABC, TestResult, TestResultStatus +from exekall._tests.suite import TestCaseABC, TestResult, TestResultStatus class SelfTestAdaptor(AdaptorBase): diff --git a/tools/exekall/exekall/tests/suite.py b/tools/exekall/src/exekall/_tests/suite.py similarity index 91% rename from tools/exekall/exekall/tests/suite.py rename to tools/exekall/src/exekall/_tests/suite.py index 044fc50edabaf68c1982f9e6389fe0eb099d801b..bbaa6f2255f27627bce03d3da5cb4d991f33188d 100644 --- a/tools/exekall/exekall/tests/suite.py +++ b/tools/exekall/src/exekall/_tests/suite.py @@ -25,7 +25,7 @@ import shutil import exekall.utils as utils import exekall.engine as engine -from exekall.tests.utils import indent +from exekall._tests.utils import indent class TestResultStatus(enum.Enum): @@ -478,11 +478,11 @@ class SingleExprTestCase(NoExcepTestCase): # key: get_id() kwargs dict as a tuple to be passed to dict() # val: ID EXPR_VAL_ID = { - (('qual', True),): 'exekall.tests.suite.init:exekall.tests.suite.middle[tag1=val1][tag2=val2]:exekall.tests.suite.final(b2=exekall.tests.suite.init:exekall.tests.suite.middle2,b3=exekall.tests.suite.init:exekall.tests.suite.middle2:exekall.tests.suite.middle3)', + (('qual', True),): 'exekall._tests.suite.init:exekall._tests.suite.middle[tag1=val1][tag2=val2]:exekall._tests.suite.final(b2=exekall._tests.suite.init:exekall._tests.suite.middle2,b3=exekall._tests.suite.init:exekall._tests.suite.middle2:exekall._tests.suite.middle3)', (('qual', False),): 'init:middle[tag1=val1][tag2=val2]:final(b2=init:middle2,b3=init:middle2:middle3)', } EXPR_ID = { - (('qual', True),): 'exekall.tests.suite.init:exekall.tests.suite.middle:exekall.tests.suite.final(b2=exekall.tests.suite.init:exekall.tests.suite.middle2,b3=exekall.tests.suite.init:exekall.tests.suite.middle2:exekall.tests.suite.middle3)', + (('qual', True),): 'exekall._tests.suite.init:exekall._tests.suite.middle:exekall._tests.suite.final(b2=exekall._tests.suite.init:exekall._tests.suite.middle2,b3=exekall._tests.suite.init:exekall._tests.suite.middle2:exekall._tests.suite.middle3)', (('qual', False),): 'init:middle:final(b2=init:middle2,b3=init:middle2:middle3)', } @@ -530,7 +530,7 @@ def final_derived(b: B) -> Final: class InheritanceTestCase(NoExcepTestCase): CALLABLES = {init, middle_derived, final_derived} EXPR_ID = { - (('qual', True),): 'exekall.tests.suite.init:exekall.tests.suite.middle_derived:exekall.tests.suite.final_derived', + (('qual', True),): 'exekall._tests.suite.init:exekall._tests.suite.middle_derived:exekall._tests.suite.final_derived', (('qual', False),): 'init:middle_derived:final_derived', } # no tags used @@ -552,7 +552,7 @@ class ConsumerTestCase(NoExcepTestCase): ] EXPR_ID = { - (('qual', True),): 'exekall.tests.suite.init_consumer:exekall.tests.suite.middle:exekall.tests.suite.final(b2=exekall.tests.suite.init_consumer:exekall.tests.suite.middle2,b3=exekall.tests.suite.init_consumer:exekall.tests.suite.middle2:exekall.tests.suite.middle3)', + (('qual', True),): 'exekall._tests.suite.init_consumer:exekall._tests.suite.middle:exekall._tests.suite.final(b2=exekall._tests.suite.init_consumer:exekall._tests.suite.middle2,b3=exekall._tests.suite.init_consumer:exekall._tests.suite.middle2:exekall._tests.suite.middle3)', (('qual', False),): 'init_consumer:middle:final(b2=init_consumer:middle2,b3=init_consumer:middle2:middle3)', } # no tags used @@ -571,7 +571,7 @@ class ExceptionTestCase(TestCaseBase): CALLABLES = {init, middle_excep, middle2, final_excep} EXPR_ID = { - (('qual', True),): 'exekall.tests.suite.init:exekall.tests.suite.middle_excep:exekall.tests.suite.final_excep(b2=exekall.tests.suite.init:exekall.tests.suite.middle2)', + (('qual', True),): 'exekall._tests.suite.init:exekall._tests.suite.middle_excep:exekall._tests.suite.final_excep(b2=exekall._tests.suite.init:exekall._tests.suite.middle2)', (('qual', False),): 'init:middle_excep:final_excep(b2=init:middle2)', } # no tags used @@ -610,7 +610,7 @@ class CloneTestCase(NoExcepTestCase): ] EXPR_ID = { - (('qual', True),): 'exekall.tests.suite.init:exekall.tests.suite.middle:exekall.tests.suite.final(b2=exekall.tests.suite.init:exekall.tests.suite.middle2,b3=exekall.tests.suite.init:exekall.tests.suite.middle2:exekall.tests.suite.middle3)', + (('qual', True),): 'exekall._tests.suite.init:exekall._tests.suite.middle:exekall._tests.suite.final(b2=exekall._tests.suite.init:exekall._tests.suite.middle2,b3=exekall._tests.suite.init:exekall._tests.suite.middle2:exekall._tests.suite.middle3)', (('qual', False),): 'init:middle:final(b2=init:middle2,b3=init:middle2:middle3)', } # no tags used @@ -653,7 +653,7 @@ class AssociatedTypesTestCase(NoExcepTestCase): AssociatedDerived1_CLS.final, } EXPR_ID = { - (('qual', True),): 'exekall.tests.suite.AssociatedDerived1.make:exekall.tests.suite.AssociatedBase.make_associated:exekall.tests.suite.AssociatedDerived1_CLS.final', + (('qual', True),): 'exekall._tests.suite.AssociatedDerived1.make:exekall._tests.suite.AssociatedBase.make_associated:exekall._tests.suite.AssociatedDerived1_CLS.final', (('qual', False),): 'AssociatedDerived1.make:AssociatedBase.make_associated:AssociatedDerived1_CLS.final', } # no tags used @@ -686,7 +686,7 @@ def consume_HavingFactoryMethodDerived(x: HavingFactoryMethodDerived) -> Final: class BasicFactoryTestCase(NoExcepTestCase): CALLABLES = {HavingFactoryMethodDerived.factory_basic, consume_HavingFactoryMethodDerived} EXPR_ID = { - (('qual', True),): 'exekall.tests.suite.HavingFactoryMethodDerived.factory_basic:exekall.tests.suite.consume_HavingFactoryMethodDerived', + (('qual', True),): 'exekall._tests.suite.HavingFactoryMethodDerived.factory_basic:exekall._tests.suite.consume_HavingFactoryMethodDerived', (('qual', False),): 'HavingFactoryMethodDerived.factory_basic:consume_HavingFactoryMethodDerived', } # no tags used @@ -697,7 +697,7 @@ if test_typing_self: class SelfFactoryTestCase(NoExcepTestCase): CALLABLES = {HavingFactoryMethodDerived.factory_self, consume_HavingFactoryMethodDerived} EXPR_ID = { - (('qual', True),): 'exekall.tests.suite.HavingFactoryMethodDerived.factory_self:exekall.tests.suite.consume_HavingFactoryMethodDerived', + (('qual', True),): 'exekall._tests.suite.HavingFactoryMethodDerived.factory_self:exekall._tests.suite.consume_HavingFactoryMethodDerived', (('qual', False),): 'HavingFactoryMethodDerived.factory_self:consume_HavingFactoryMethodDerived', } # no tags used diff --git a/tools/exekall/exekall/tests/utils.py b/tools/exekall/src/exekall/_tests/utils.py similarity index 100% rename from tools/exekall/exekall/tests/utils.py rename to tools/exekall/src/exekall/_tests/utils.py diff --git a/tools/exekall/exekall/_utils.py b/tools/exekall/src/exekall/_utils.py similarity index 100% rename from tools/exekall/exekall/_utils.py rename to tools/exekall/src/exekall/_utils.py diff --git a/tools/exekall/exekall/customization.py b/tools/exekall/src/exekall/customization.py similarity index 100% rename from tools/exekall/exekall/customization.py rename to tools/exekall/src/exekall/customization.py diff --git a/tools/exekall/exekall/engine.py b/tools/exekall/src/exekall/engine.py similarity index 100% rename from tools/exekall/exekall/engine.py rename to tools/exekall/src/exekall/engine.py diff --git a/tools/exekall/exekall/main.py b/tools/exekall/src/exekall/main.py similarity index 100% rename from tools/exekall/exekall/main.py rename to tools/exekall/src/exekall/main.py diff --git a/tools/exekall/exekall/utils.py b/tools/exekall/src/exekall/utils.py similarity index 100% rename from tools/exekall/exekall/utils.py rename to tools/exekall/src/exekall/utils.py diff --git a/tools/exekall/exekall/valuedb.py b/tools/exekall/src/exekall/valuedb.py similarity index 100% rename from tools/exekall/exekall/valuedb.py rename to tools/exekall/src/exekall/valuedb.py diff --git a/tools/kmodules/lisa-in-tree/fetch_lisa_module.py b/tools/kmodules/lisa-in-tree/fetch_lisa_module.py index 8c20dca55cfc3be83d71d7ff3d892a7a77bcab88..229090aca01f1d0b23588a2b10000d7c5680cc9a 100755 --- a/tools/kmodules/lisa-in-tree/fetch_lisa_module.py +++ b/tools/kmodules/lisa-in-tree/fetch_lisa_module.py @@ -30,7 +30,7 @@ def main(): parser.add_argument('--module-kernel-path', help='Path relative to the kernel tree root where the module will be stored') parser.add_argument('--git-remote', help='Git remote to pull the module from', - default='https://github.com/ARM-software/lisa.git') + default='https://gitlab.arm.com/tooling/lisa') args = parser.parse_args() module_kernel_path = Path(args.module_kernel_path).resolve() diff --git a/tools/lisa-combine-gitlab-mr/LICENSE.txt b/tools/lisa-combine-gitlab-mr/LICENSE.txt new file mode 120000 index 0000000000000000000000000000000000000000..1ef648f64b34ffd4b08b00bc2d967f9b4387c8f4 --- /dev/null +++ b/tools/lisa-combine-gitlab-mr/LICENSE.txt @@ -0,0 +1 @@ +../../LICENSE.txt \ No newline at end of file diff --git a/tools/lisa-combine-gitlab-mr/pyproject.toml b/tools/lisa-combine-gitlab-mr/pyproject.toml new file mode 100644 index 0000000000000000000000000000000000000000..8b86939ed09234cc2a0dd1193e57fe2a29da5445 --- /dev/null +++ b/tools/lisa-combine-gitlab-mr/pyproject.toml @@ -0,0 +1,55 @@ +[project] +name = "lisa-combine-gitlab-mr" # Required +version = "1.0.0" # Required +description = "Combine GitLab merge requests with the given labels for the specified project" +requires-python = ">=3.8" +license = {file = "LICENSE.txt"} +keywords = ["git", "development"] + +# readme = "README.md" +authors = [ + {name = "Douglas RAILLARD"} +] +maintainers = [ + {name = "Arm Ltd." } +] + +# Classifiers help users find your project by categorizing it. +# +# For a list of valid classifiers, see https://pypi.org/classifiers/ +classifiers = [ # Optional + # How mature is this project? Common values are + # 3 - Alpha + # 4 - Beta + # 5 - Production/Stable + "Development Status :: 4 - Beta", + + # Indicate who your project is intended for + "Intended Audience :: Developers", + "Topic :: Software Development :: Build Tools", + + # This is not a standard classifier, as there is nothing defined for + # Apache 2.0 yet: + # https://pypi.org/classifiers/ +] + +dependencies = [ + # To query GitLab API + "requests", + "batch-rebase @ file:///${PROJECT_ROOT}/../batch-rebase/", +] + +[project.urls] +"Homepage" = "https://gitlab.arm.com/tooling/lisa" +"Source" = "https://gitlab.arm.com/tooling/lisa" +"Documentation" = "https://lisa-linux-integrated-system-analysis.readthedocs.io/" +"Bug Tracker" = "https://gitlab.arm.com/tooling/lisa/-/issues" + +# The following would provide a command line executable called `sample` +# which executes the function `main` from this package when invoked. +[project.scripts] # Optional +lisa-combine-gitlab-mr = "lisa_combine_gitlab_mr.main:main" + +[build-system] +requires = ["pdm-backend"] +build-backend = "pdm.backend" diff --git a/tools/lisa-combine-gitlab-mr b/tools/lisa-combine-gitlab-mr/src/lisa_combine_gitlab_mr/main.py similarity index 57% rename from tools/lisa-combine-gitlab-mr rename to tools/lisa-combine-gitlab-mr/src/lisa_combine_gitlab_mr/main.py index 17d3b1fe59c3486fcfe43da0a8d433a3b5d138fc..98807c30b7f9d4530639479c9a5fc549988aea99 100755 --- a/tools/lisa-combine-gitlab-mr +++ b/tools/lisa-combine-gitlab-mr/src/lisa_combine_gitlab_mr/main.py @@ -20,50 +20,67 @@ import subprocess from itertools import chain from tempfile import NamedTemporaryFile -import json, os +import json from collections import ChainMap from operator import itemgetter import argparse import logging + import requests # Use HTTP APIs of GitLab to retrieve associated mrs # of this project. -def get_gitlab_mrs(state="opened", scope="all", labels=""): - def _call_gitlab_api(endpoint): - api_token = os.environ.get("GITLAB_REPO_TOKEN") - api_url = os.environ.get("CI_API_V4_URL") - headers = {"PRIVATE-TOKEN": api_token} - r = requests.get("/".join([api_url,endpoint]), headers=headers) - if r.status_code != 200: - raise Exception(r.text) +def get_gitlab_mrs(api_url, project_id, api_token=None, state="opened", scope="all", labels=None): + if labels: + labels = ','.join(labels) + labels = f'&labels={labels}' + else: + labels = '' + + def call_gitlab_api(endpoint): + headers = { + k: v + for k, v in [('PRIVATE-TOKEN', api_token)] + if v + } + r = requests.get(f'{api_url}/{endpoint}', headers=headers) + r.raise_for_status() return r - this_project_id = int(os.environ.get("CI_PROJECT_ID")) - page_number = 1 # obvioulsy at start 1 + def get_mr(mr): + # populate commits count - use another api + mr_commit_response = call_gitlab_api( + f"projects/{project_id}/merge_requests/{mr['iid']}/commits" + ) + mr_commit_response = mr_commit_response.json() + assert isinstance(mr_commit_response, list) + commits_count = len(mr_commit_response) + + # mr could be from a fork - use another api + project_response = call_gitlab_api( + f"projects/{mr['source_project_id']}" + ) + clone_url = project_response.json()['http_url_to_repo'] + + return dict( + sha=mr['sha'], + source_branch=mr['source_branch'], + commits_count=commits_count, + clone_url=clone_url, + ) results = [] + # obvioulsy at start 1 + page_number = 1 while True: - mr_response = _call_gitlab_api( - f"merge_requests?state={state}&scope={scope}&labels={labels}&page={page_number}" + mr_response = call_gitlab_api( + f"merge_requests?state={state}&scope={scope}{labels}&page={page_number}" + ) + results.extend( + get_mr(mr) + for mr in mr_response.json() + if mr.get("project_id") == project_id ) - for this_mr in mr_response.json(): - if this_mr.get("project_id") == this_project_id: - # populate commits count - use another api - mr_commit_response = _call_gitlab_api( - f"projects/{this_project_id}/merge_requests/{this_mr['iid']}/commits" - ) - this_mr["commits_count"] = len(mr_commit_response.json()) - - # this_mr could be from a fork - use another api - project_response = _call_gitlab_api( - f"projects/{this_mr['source_project_id']}" - ) - this_mr["http_url_to_repo"] = project_response.json()[ - "http_url_to_repo" - ] - - results.append(this_mr) # handle paging - only required for mr if not mr_response.headers["X-Next-Page"]: @@ -80,29 +97,32 @@ def main(): """, ) + parser.add_argument('--server', required=True, help='Gitlab server URL') + parser.add_argument('--api-url', required=True, help='Gitlab API URL') + parser.add_argument('--api-token', help='Gitlab API token. If omitted, anonymous requests will be used which may fail') + parser.add_argument('--project-id', required=True, help='Gitlab Project ID') parser.add_argument('--repo', required=True, help='Gitlab repository as owner/name') parser.add_argument('--mr-label', action='append', required=True, help='Merge request labels to look for') parser.add_argument('--branch', required=True, help='Name of the branch to be created. If the branch exists, it will be forcefully updated') args = parser.parse_args() - project = args.repo owner, repo = args.repo.split('/', 1) - labels = ','.join(args.mr_label) + labels = args.mr_label branch = args.branch + server = args.server + api_url = args.api_url + api_token = args.api_token + project_id = args.project_id logging.basicConfig(level=logging.INFO) - server_host_ssh = os.environ.get('CI_SERVER_SHELL_SSH_HOST') - - gl_mrs = get_gitlab_mrs(labels=labels) - def make_topic(mr): remote = f'remote_{mr["sha"]}' return ( { remote: { - 'url': mr["http_url_to_repo"] + 'url': mr["clone_url"] } }, { @@ -112,17 +132,22 @@ def main(): 'tip': mr["source_branch"], } ) - - topics = [] - - for mr in gl_mrs: - topics += [ make_topic(mr) ] + + topics = [ + make_topic(mr) + for mr in get_gitlab_mrs( + api_url=api_url, + api_token=api_token, + project_id=project_id, + labels=labels + ) + ] remotes, topics = zip(*topics) if topics else ([], []) remotes = dict(ChainMap(*chain( [{ 'gitlab': { - 'url': f'https://{server_host_ssh}/{owner}/{repo}.git' + 'url': f'https://{server}/{owner}/{repo}.git' } }], remotes diff --git a/tools/lisa-combine-pr b/tools/lisa-combine-pr deleted file mode 100755 index d1d1dca09038be8ef24ec82fc7d21ea286ba50e1..0000000000000000000000000000000000000000 --- a/tools/lisa-combine-pr +++ /dev/null @@ -1,123 +0,0 @@ -#! /usr/bin/env python3 -# -# SPDX-License-Identifier: Apache-2.0 -# -# Copyright (C) 2023, Arm Limited and contributors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import subprocess -from itertools import starmap, chain -from tempfile import NamedTemporaryFile -import json -from collections import ChainMap -from operator import itemgetter -import argparse -import logging - -from github3 import GitHub - -def main(): - parser = argparse.ArgumentParser( - description=""" - Combine github pull requests with the given tag into a branch, rebasing all - PRs on top of each other. - """, - ) - - parser.add_argument('--repo', required=True, help='Github repository as owner/name') - parser.add_argument('--pr-label', action='append', required=True, help='Pull request labels to look for') - parser.add_argument('--branch', required=True, help='Name of the branch to be created. If the branch exists, it will be forcefully updated') - - args = parser.parse_args() - - owner, repo = args.repo.split('/', 1) - labels = args.pr_label - branch = args.branch - - logging.basicConfig(level=logging.INFO) - - gh = GitHub() - issues = gh.issues_on( - username=owner, - repository=repo, - state='open', - labels=labels, - ) - - prs = [ - ( - issue, - gh.pull_request( - owner=owner, - repository=repo, - number=issue.number - ) - ) - for issue in issues - if issue.pull_request_urls - ] - - def make_topic(issue, pr): - remote = f'remote_{pr.head.sha}' - return ( - { - remote: { - 'url': pr.head.repository.clone_url - } - }, - { - 'name': pr.head.ref, - 'remote': remote, - 'nr-commits': pr.commits_count, - 'tip': pr.head.ref, - } - ) - - topics = list(starmap(make_topic, prs)) - remotes, topics = zip(*topics) if topics else ([], []) - remotes = dict(ChainMap(*chain( - [{ - 'github': { - 'url': f'https://github.com/{owner}/{repo}.git' - } - }], - remotes - ))) - - conf = { - 'rebase-conf': { - 'rr-cache': './rr-cache', - 'remotes': remotes, - 'base': { - 'remote': 'github', - 'ref': 'main', - }, - 'topics': sorted(topics, key=itemgetter('name')) - } - } - conf = json.dumps(conf, indent=4) - logging.info(conf) - - with NamedTemporaryFile(mode='w+', suffix='.manifest.json') as f: - f.write(conf) - f.flush() - - manifest = f.name - - cmd = ['batch-rebase', 'create', '.', '--manifest', manifest, '--create-branch', branch] - logging.info(f'Running {" ".join(map(str, cmd))}') - subprocess.check_call(cmd) - -main() diff --git a/tools/tests.sh b/tools/tests.sh index c125f08efb64edc2e73c05dffaafb68e5c21164b..3c86e393d3caed7a637c976bd762e95304373285 100755 --- a/tools/tests.sh +++ b/tools/tests.sh @@ -33,7 +33,7 @@ echo "Starting self tests ..." timeout -s INT 1h python3 -m pytest -vv echo "Starting exekall self tests" -exekall run "$LISA_HOME/tools/exekall/exekall/tests" +exekall run "$LISA_HOME/tools/exekall/src/exekall/_tests" echo "Available LISA tests:" lisa-test --list diff --git a/vagrant_requirements.txt b/vagrant_requirements.txt index 701196a52980287aa3aeee744e85e5fdbe56e3f2..f92ef0d8baaa04ac94f65845357820ea7f814423 100644 --- a/vagrant_requirements.txt +++ b/vagrant_requirements.txt @@ -3,3 +3,4 @@ ./external/workload-automation/ -e ./[all] -e ./tools/bisector +-e ./tools/lisa-combine-gitlab-mr