From 779c4c2b21d284b2018c03a07ff743a3a706332f Mon Sep 17 00:00:00 2001 From: Edwin James Date: Thu, 1 Feb 2024 18:30:47 +0000 Subject: [PATCH 01/23] CI rule for autobranch label Conditions within '' are not picked up by the gitlab CI --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 98dbb27ee..e68561a32 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -72,7 +72,7 @@ Autobranch: rules: - if: $CI_PIPELINE_SOURCE == "web" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $ENABLE_AUTOBRANCH == "True" - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - - if: $CI_PIPELINE_SOURCE == "merge_request_event" && '$CI_MERGE_REQUEST_LABELS =~ /^autobranch$/' #regex to pickup 'autobranch' label alone + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_LABELS =~ /^autobranch$/ #regex to pickup 'autobranch' label alone resource_group: autobranch before_script: - sudo apt update -y -- GitLab From cd67619860a70553a061bf78e2d72d57b1db2513 Mon Sep 17 00:00:00 2001 From: Douglas Raillard Date: Thu, 1 Feb 2024 21:11:27 +0000 Subject: [PATCH 02/23] shell/lisa_shell: Avoid error if ANDROID_HOME does not exist FIX --- shell/lisa_shell | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/shell/lisa_shell b/shell/lisa_shell index 7a1003391..dc1074585 100755 --- a/shell/lisa_shell +++ b/shell/lisa_shell @@ -46,8 +46,18 @@ export LISA_HOST_ABI=${LISA_HOST_ABI:-$(uname -m | sed 's/aarch64/arm64/')} export _DOC_LISA_USE_SYSTEM_BIN="Use the system binaries if 1, will use the ones shipped with LISA if 0" export LISA_USE_SYSTEM_BIN=${LISA_USE_SYSTEM_BIN:-0} -_ANDROID_BUILD_TOOLS=$(find "$ANDROID_HOME/build-tools/"* -maxdepth 0 -type d -print -quit) -_LISA_PATH="$LISA_HOME/shell/:$LISA_HOME/tools/$LISA_HOST_ABI:$LISA_HOME/tools/:$ANDROID_HOME/tools/:$ANDROID_HOME/platform-tools/:$_ANDROID_BUILD_TOOLS" +_LISA_PATH="$LISA_HOME/shell/:$LISA_HOME/tools/$LISA_HOST_ABI:$LISA_HOME/tools/" + +if [[ -d "$ANDROID_HOME" ]]; then + _LISA_PATH="$_LISA_PATH:$ANDROID_HOME/tools/:$ANDROID_HOME/platform-tools" + + _ANDROID_BUILD_TOOLS="$ANDROID_HOME/build-tools/" + if [[ -d "$_ANDROID_BUILD_TOOLS" ]]; then + _ANDROID_BUILD_TOOLS=$(find "$_ANDROID_BUILD_TOOLS"* -maxdepth 0 -type d -print -quit || printf "%s" "$_ANDROID_BUILD_TOOLS") + _LISA_PATH="$_LISA_PATH:$_ANDROID_BUILD_TOOLS" + fi +fi + if [[ "$LISA_USE_SYSTEM_BIN" == 1 ]]; then export PATH=$PATH:$_LISA_PATH else -- GitLab From ef68c0296b4f82fe4c00d86e9e110403aae8aef4 Mon Sep 17 00:00:00 2001 From: Douglas Raillard Date: Fri, 2 Feb 2024 15:39:12 +0000 Subject: [PATCH 03/23] tools/batch-rebase: Remove dependency over lisa package Make batch-rebase independent of the lisa package. Also make it try to load the conf as JSON before trying as YAML. If it is indeed valid json, that also removes the dependency over ruamel.yaml. --- tools/batch-rebase | 147 +++++++++++++++++++++++++++++---------------- 1 file changed, 96 insertions(+), 51 deletions(-) diff --git a/tools/batch-rebase b/tools/batch-rebase index d5cc3a3d7..57188b448 100755 --- a/tools/batch-rebase +++ b/tools/batch-rebase @@ -33,8 +33,7 @@ import textwrap import logging import contextlib import typing - -from lisa.conf import SimpleMultiSrcConf, KeyDesc, LevelKeyDesc, TopLevelKeyDesc +import json def info(msg): logging.info(msg) @@ -42,36 +41,55 @@ def info(msg): def warn(msg): logging.warning(msg) -class BatchRebaseManifest(SimpleMultiSrcConf): - """ - Configuration of batch-rebase +def get_nested_key(conf, key): + for k in key: + conf = conf[k] + return conf + +def load_conf(path): + + def postprocess(conf): + return conf['rebase-conf'] + + try: + from ruamel.yaml import YAML + except ImportError: + try_yaml = False + else: + try_yaml = True + + def load_yaml(path): + yaml = YAML(typ='safe') + with open(path) as f: + return yaml.load(f) + + def load_json(path): + with open(path) as f: + return json.load(f) + + loaders = [ + load_json, + *([load_yaml] if try_yaml else []) + ] + + last_excep = ValueError('No loader selected') + for loader in loaders: + try: + conf = loader(path) + except Exception as e: + last_excep = e + error(e) + else: + return postprocess(conf) + + raise last_excep + + +def dump_conf(conf, path): + conf = {'rebase-conf': conf} + with open(path, 'w') as f: + json.dump(conf, f) - {generated_help} - """ - STRUCTURE = TopLevelKeyDesc('rebase-conf', 'Batch rebase configuration', ( - KeyDesc('topics', 'List of topics. Each topic is described by a mapping with "name", "remote", "base" (or "nr-commits) and "tip" git references keys. Also, a tag can be added between topics with "action: tag" key and "name: tag-name".', [list]), - KeyDesc('remotes', 'Git remotes. Keys are remote name, values are a mapping with an "url" key', [Mapping]), - KeyDesc('rr-cache', 'Path to git rr-cache. Relative paths are relative to that manifest file', [str, None]), - LevelKeyDesc('base', 'Base branche spec', [ - KeyDesc('remote', 'remote where the base branch is located', [str]), - KeyDesc('ref', 'Name of the base branch', [str]), - ]), - LevelKeyDesc('tip', 'New branche spec', [ - KeyDesc('ref', 'Name of the base branch. --refs-suffix can be used to tweak the name', [str]), - KeyDesc('tags', 'Default of --tags', [bool]), - ]), - LevelKeyDesc('resume', 'Internal state used for resuming after a conflict', [ - KeyDesc('conflict-topic', 'Topic where the conflict happened', [str]), - KeyDesc('persistent-refs', 'List of references that needs to be pushed back to the main repo', [typing.Sequence[str]]), - LevelKeyDesc('tags', 'Topic branch tags', [ - KeyDesc('persistent', 'whether tags should be pushed back or not', [bool]), - KeyDesc('suffix', 'suffix to use for tags, or None', [str]), - ]), - KeyDesc('repo', 'Path to the main repo', [str]), - KeyDesc('branch', 'Branch that is being created', [str]), - KeyDesc('rr-cache', 'Source of the rr-cache', [str, None]), - ]) - )) RESUME_MANIFEST_NAME = '.batch-rebase-state' @@ -216,13 +234,13 @@ def do_cherry_pick(repo, temp_repo, conf, persistent_tags, tags_suffix, branch, has_conflict, persistent_refs = _do_cherry_pick(temp_repo, conf, persistent_tags, tags_suffix) if has_conflict: - conf['resume'].add_src('conflict', { + conf['resume'] = { 'repo': str(repo), 'rr-cache': str(rr_cache) if rr_cache else rr_cache, 'branch': branch, - }) + } # Save the augmented manifest for later resumption - conf.to_yaml_map(temp_repo/RESUME_MANIFEST_NAME) + dump_conf(conf, temp_repo/RESUME_MANIFEST_NAME) else: # Copy back the rr-cache if rr_cache: @@ -282,7 +300,7 @@ def _do_cherry_pick(repo, conf, persistent_tags, tags_suffix): ) # If we are resuming after a conflict - if 'conflict-topic' in conf['resume']: + if 'conflict-topic' in conf.get('resume', {}): resume_topic = conf['resume']['conflict-topic'] persistent_refs.update(conf['resume']['persistent-refs']) @@ -365,17 +383,14 @@ def _do_cherry_pick(repo, conf, persistent_tags, tags_suffix): if not cherry_pick_ref(repo, range_ref): # Save the current state for later resumption - conf['resume'].add_src( - 'conflict', - { - 'conflict-topic': name, - 'persistent-refs': sorted(persistent_refs), - 'tags': { - 'persistent': persistent_tags, - 'suffix': tags_suffix, - } - }, - ) + conf['resume'] = { + 'conflict-topic': name, + 'persistent-refs': sorted(persistent_refs), + 'tags': { + 'persistent': persistent_tags, + 'suffix': tags_suffix, + } + } info(textwrap.dedent(""" A conflict occured while cherry-picking topic "{topic}" ({range_ref}) In order to fix it, please: @@ -518,7 +533,37 @@ def main(): The manifest is a YAML file following that structure: - {manifest_doc} + ├ rebase-conf: Batch rebase configuration + ├ topics (list): + List of topics. Each topic is described by a mapping with + "name", "remote", "base" (or "nr-commits) and "tip" git + references keys. Also, a tag can be added between topics + with "action: tag" key and "name: tag-name".. + ├ remotes (Mapping): + Git remotes. Keys are remote name, values are a mapping with + an "url" key. + ├ rr-cache (str or None): + Path to git rr-cache. Relative paths are relative to that + manifest file. + ├ base: Base branche spec + ├ remote (str): remote where the base branch is located. + └ ref (str): Name of the base branch. + ├ tip: New branche spec + ├ ref (str): + Name of the base branch. --refs-suffix can be used to tweak + the name. + └ tags (bool): Default of --tags. + └ resume: Internal state used for resuming after a conflict + ├ conflict-topic (str): Topic where the conflict happened. + ├ persistent-refs (typing.Sequence[str]): + List of references that needs to be pushed back to the main + repo. + ├ tags: Topic branch tags + ├ persistent (bool): whether tags should be pushed back or not. + └ suffix (str): suffix to use for tags, or None. + ├ repo (str): Path to the main repo. + ├ branch (str): Branch that is being created. + └ rr-cache (str or None): Source of the rr-cache. EXAMPLE @@ -555,7 +600,7 @@ def main(): url: git://linux-arm.org/linux-power.git - ''').format(manifest_doc=BatchRebaseManifest.get_help()), + '''), formatter_class=argparse.RawDescriptionHelpFormatter, ) @@ -635,10 +680,10 @@ def main(): temp_repo = None try: if args.subcommand == 'create': - conf = BatchRebaseManifest.from_yaml_map(args.manifest) + conf = load_conf(args.manifest) def default_from_conf(x, key): try: - default = conf.get_nested_key(key) + default = get_nested_key(conf, key) except KeyError: return x else: @@ -686,7 +731,7 @@ def main(): elif args.subcommand == 'resume': temp_repo = repo - conf = BatchRebaseManifest.from_yaml_map(temp_repo/RESUME_MANIFEST_NAME) + conf = load_conf(temp_repo/RESUME_MANIFEST_NAME) delete_temp_repo, ret = do_resume(temp_repo, conf) finally: -- GitLab From 8f5ad0a6b772d4685478bf2ad9a9a214274744cd Mon Sep 17 00:00:00 2001 From: Douglas Raillard Date: Fri, 2 Feb 2024 15:42:15 +0000 Subject: [PATCH 04/23] tools/batch-rebase: require subcommand in CLI FIX --- tools/batch-rebase | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/batch-rebase b/tools/batch-rebase index 57188b448..a87109031 100755 --- a/tools/batch-rebase +++ b/tools/batch-rebase @@ -605,7 +605,7 @@ def main(): ) - subparsers = parser.add_subparsers(title='subcommands', dest='subcommand') + subparsers = parser.add_subparsers(title='subcommands', dest='subcommand', required=True) create_parser = subparsers.add_parser('create', help='Create a branch' ) -- GitLab From b9ac47046a7fcadc5df5d7991483cec73be4e252 Mon Sep 17 00:00:00 2001 From: Douglas Raillard Date: Fri, 2 Feb 2024 14:18:30 +0000 Subject: [PATCH 05/23] doc/conf.py: Ensure we fetch tags when building doc FIX --- doc/conf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/conf.py b/doc/conf.py index b5a8622aa..132cae867 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -61,6 +61,7 @@ if RTD: # history to be able to generate breaking change list and any other # git-based documentation subprocess.run(['git', 'fetch', '--unshallow'], check=False) + subprocess.run(['git', 'fetch', '--tags'], check=False) # Ensure we have the changelog notes that supplement commit messages, as # sometimes the markers such as FEATURE were forgotten and later added -- GitLab From f3b9ef45e5685497213712eec3056e1a5b2f6539 Mon Sep 17 00:00:00 2001 From: Douglas Raillard Date: Thu, 1 Feb 2024 12:13:41 +0000 Subject: [PATCH 06/23] Update GitHub URLs to GitLab ones --- README.rst | 13 ++++++++----- doc/changes.rst | 2 +- doc/contributors_guide.rst | 2 +- doc/index.rst | 6 +++--- doc/setup.rst | 8 ++++---- doc/test_example.py | 1 - doc/workflows/notebook.rst | 2 +- lisa/energy_meter.py | 13 ------------- setup.py | 7 +++---- tools/bisector/doc/index.rst | 4 ++-- tools/exekall/README.rst | 4 ++-- tools/exekall/doc/index.rst | 4 ++-- tools/exekall/setup.py | 6 +++--- tools/kmodules/lisa-in-tree/fetch_lisa_module.py | 2 +- 14 files changed, 31 insertions(+), 43 deletions(-) delete mode 120000 doc/test_example.py diff --git a/README.rst b/README.rst index 0638fa123..2b95fabca 100644 --- a/README.rst +++ b/README.rst @@ -1,8 +1,11 @@ +⚠️ LISA has moved from GitHub to GitLab and is now available at: +https://gitlab.arm.com/tooling/lisa. Please update your clone URLs and note +that the GitHub repository will not be used for any pull requests or issue +management. ⚠️ ⚠️ ``lisa_tests`` package will be moved to another repository in January 2024. When more details are available, this pull request will be -updated: https://github.com/ARM-software/lisa/pull/2140 ⚠️ - +updated: https://gitlab.arm.com/tooling/lisa/-/merge_requests/2140/ ⚠️ Introduction |CI status| |Documentation Status| =================================================== @@ -76,7 +79,7 @@ Here are some noteworthy sections: How to reach us =============== -Bug reports should be raised against the `GitHub issue tracker `__. +Bug reports should be raised against the `GitLab issue tracker `__. External Links ============== @@ -113,7 +116,7 @@ where you have authored all of the code. If you do this on work time make sure your employer is cool with this. We also have a `Contributor Guide `__ -.. |CI status| image:: https://github.com/ARM-software/lisa/workflows/test/badge.svg?branch=main - :target: https://github.com/ARM-software/lisa/actions +.. |CI status| image:: https://gitlab.arm.com/tooling/lisa/badges/main/pipeline.svg + :target: https://gitlab.arm.com/tooling/lisa/-/commits/main .. |Documentation Status| image:: https://readthedocs.org/projects/lisa-linux-integrated-system-analysis/badge/?version=main :target: https://lisa-linux-integrated-system-analysis.readthedocs.io/en/main/ diff --git a/doc/changes.rst b/doc/changes.rst index 54ed9cc66..8fc678ed6 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -24,7 +24,7 @@ Everything else is private. important API laws. This is unfortunately not future-proof, as new versions can add new methods that would also require being overridden and kept in sync. If for some reason subclassing is required, please get in touch in the - `github issue tracker `_ + `GitLab issue tracker `_ before relying on that for production. .. note:: Instance attributes are considered public following the same diff --git a/doc/contributors_guide.rst b/doc/contributors_guide.rst index cd2d251c6..2909077d9 100644 --- a/doc/contributors_guide.rst +++ b/doc/contributors_guide.rst @@ -10,7 +10,7 @@ How to reach us =============== If you’re hitting an error/bug and need help, it’s best to raise an -issue on `GitHub `__. +issue on `GitLab `__. Coding style ============ diff --git a/doc/index.rst b/doc/index.rst index 2e2ef21ac..32b9a9dd0 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -11,14 +11,14 @@ LISA Documentation LISA - "Linux Integrated System Analysis" is a toolkit for interactive analysis and automated regression testing of Linux kernel behaviour. -- See the README on the project's `Github home page`__ for an overview. +- See the README on the project's `GitLab home page`__ for an overview. - Once you have LISA running, take a look at the tutorial and example notebooks included with the installation. -__ https://github.com/ARM-software/lisa +__ https://gitlab.arm.com/tooling/lisa Contributions to LISA and its documentation are very welcome, and handled -via Github pull requests. +via GitLab merge requests. .. _Readme: diff --git a/doc/setup.rst b/doc/setup.rst index 1ff986a4f..45ded3dd5 100644 --- a/doc/setup.rst +++ b/doc/setup.rst @@ -26,10 +26,10 @@ LISA is available on `PyPI `_: them. Alternatively, the installation from the git repository allows setting up a full environment. -From GitHub +From GitLab ----------- -LISA is hosted at `github `_. +LISA is hosted on `GitLab `_. The following references are available: * ``main`` branch: Main development branch where pull requests are merged as they @@ -76,7 +76,7 @@ need. .. code:: shell - git clone https://github.com/ARM-software/lisa.git + git clone https://gitlab.arm.com/tooling/lisa # Jump into the cloned repo directory cd lisa # This will provide a more accurate changelog when building the doc @@ -142,7 +142,7 @@ Once these two components are available on your machine, issue these commands: .. code:: shell - git clone https://github.com/ARM-software/lisa.git + git clone https://gitlab.arm.com/tooling/lisa cd lisa vagrant up diff --git a/doc/test_example.py b/doc/test_example.py deleted file mode 120000 index ccd801e51..000000000 --- a/doc/test_example.py +++ /dev/null @@ -1 +0,0 @@ -../lisa/test_example.py \ No newline at end of file diff --git a/doc/workflows/notebook.rst b/doc/workflows/notebook.rst index da4f76e89..71f306ce0 100644 --- a/doc/workflows/notebook.rst +++ b/doc/workflows/notebook.rst @@ -33,7 +33,7 @@ Once the server is started you can have a look at the provided tutorial notebooks are accessible by following this `link `__. This initial tutorial can be seen (but not executed) also on `github -`__. +`__. Notebooks as development environment ==================================== diff --git a/lisa/energy_meter.py b/lisa/energy_meter.py index 3807a6a2a..1fbb0e62d 100644 --- a/lisa/energy_meter.py +++ b/lisa/energy_meter.py @@ -389,18 +389,6 @@ class Monsoon(_DevlibContinuousEnergyMeter): self._instrument.reset() -_acme_install_instructions = ''' - - If you need to measure energy using an ACME EnergyProbe, - please do follow installation instructions available here: - https://github.com/ARM-software/lisa/wiki/Energy-Meters-Requirements#iiocapture---baylibre-acme-cape - - Othwerwise, please select a different energy meter in your - configuration file. - -''' - - class ACMEConf(SimpleMultiSrcConf, HideExekallID): """ Configuration class for :class:`ACME`. @@ -461,7 +449,6 @@ class ACME(EnergyMeter): subprocess.call([self._iiocapturebin, '-h'], stdout=PIPE, stderr=STDOUT) except FileNotFoundError as e: logger.error(f'iio-capture binary {self._iiocapturebin} not available') - logger.warning(_acme_install_instructions) raise FileNotFoundError('Missing iio-capture binary') from e def sample(self): diff --git a/setup.py b/setup.py index a6584c179..7d7aeea2a 100755 --- a/setup.py +++ b/setup.py @@ -81,7 +81,6 @@ extras_require={ "pytest", "build", "twine", - "github3.py", ], "wa": [ @@ -117,11 +116,11 @@ if __name__ == "__main__": version=lisa_version, maintainer='Arm Ltd.', packages=packages, - url='https://github.com/ARM-software/lisa', + url='https://gitlab.arm.com/tooling/lisa', project_urls={ - "Bug Tracker": "https://github.com/ARM-software/lisa/issues", "Documentation": "https://lisa-linux-integrated-system-analysis.readthedocs.io/", - "Source Code": "https://github.com/ARM-software/lisa", + "Bug Tracker": "https://gitlab.arm.com/tooling/lisa/-/issues", + "Source Code": "https://gitlab.arm.com/tooling/lisa", }, description='A stick to probe the kernel with', long_description=long_description, diff --git a/tools/bisector/doc/index.rst b/tools/bisector/doc/index.rst index 7d12d5883..9d5858f6e 100644 --- a/tools/bisector/doc/index.rst +++ b/tools/bisector/doc/index.rst @@ -13,7 +13,7 @@ Overview ======== Bisector is a ``git bisect run`` compatible tool used in LISA. Check out the -project's `Github`__ for some guides to installation and setup. +project's `GitLab`__ for some guides to installation and setup. ``bisector`` allows setting up the steps of a test iteration, repeating them an infinite number of times (by default). These steps can involve flashing @@ -32,7 +32,7 @@ that happen on long running sessions. ``bisector`` will never leave you with an inconsistent report, or worse, no report at all. A new report is saved after each iteration and can be inspected as the execution goes on. -__ https://github.com/ARM-software/lisa +__ https://gitlab.arm.com/tooling/lisa .. [#] https://git-scm.com/docs/git-bisect Contents diff --git a/tools/exekall/README.rst b/tools/exekall/README.rst index 878aebd79..e7b593863 100644 --- a/tools/exekall/README.rst +++ b/tools/exekall/README.rst @@ -1,7 +1,7 @@ -Exekall is the test runner of LISA. Check out the project's `Github`__ for some +Exekall is the test runner of LISA. Check out the project's `GitLab`__ for some guides to installation and setup. ``exekall`` runs a set of expressions that are discovered from Python sources, allowing to build test suites in a highly modular way. -__ https://github.com/ARM-software/lisa +__ https://gitlab.arm.com/tooling/lisa diff --git a/tools/exekall/doc/index.rst b/tools/exekall/doc/index.rst index eb40215ba..bb2ed8087 100644 --- a/tools/exekall/doc/index.rst +++ b/tools/exekall/doc/index.rst @@ -12,13 +12,13 @@ Exekall Documentation Overview ======== -Exekall is the test runner of LISA. Check out the project's `Github`__ for some +Exekall is the test runner of LISA. Check out the project's `GitLab`__ for some guides to installation and setup. ``exekall`` runs a set of expressions that are discovered from Python sources, allowing to build test suites in a highly modular way. -__ https://github.com/ARM-software/lisa +__ https://gitlab.arm.com/tooling/lisa Contents ======== diff --git a/tools/exekall/setup.py b/tools/exekall/setup.py index 374a4cae7..623daf33f 100755 --- a/tools/exekall/setup.py +++ b/tools/exekall/setup.py @@ -26,11 +26,11 @@ setup( version='1.0', maintainer='Arm Ltd.', packages=['exekall'], - url='https://github.com/ARM-software/lisa', + url='https://gitlab.arm.com/tooling/lisa', project_urls={ - "Bug Tracker": "https://github.com/ARM-software/lisa/issues", + "Bug Tracker": "https://gitlab.arm.com/tooling/lisa/-/issues", + "Source Code": "https://gitlab.arm.com/tooling/lisa", "Documentation": "https://lisa-linux-integrated-system-analysis.readthedocs.io/", - "Source Code": "https://github.com/ARM-software/lisa", }, license='Apache License 2.0', description='Python expression execution engine', diff --git a/tools/kmodules/lisa-in-tree/fetch_lisa_module.py b/tools/kmodules/lisa-in-tree/fetch_lisa_module.py index 8c20dca55..229090aca 100755 --- a/tools/kmodules/lisa-in-tree/fetch_lisa_module.py +++ b/tools/kmodules/lisa-in-tree/fetch_lisa_module.py @@ -30,7 +30,7 @@ def main(): parser.add_argument('--module-kernel-path', help='Path relative to the kernel tree root where the module will be stored') parser.add_argument('--git-remote', help='Git remote to pull the module from', - default='https://github.com/ARM-software/lisa.git') + default='https://gitlab.arm.com/tooling/lisa') args = parser.parse_args() module_kernel_path = Path(args.module_kernel_path).resolve() -- GitLab From 785ea25b3ad2eb179982e5f61e2b8c97be9993ac Mon Sep 17 00:00:00 2001 From: Douglas Raillard Date: Thu, 1 Feb 2024 19:38:28 +0000 Subject: [PATCH 07/23] .gitlab-ci.yml: Cleanup gitlab CI support * Merge back ci/autobranch-job-script.sh into .gitlab-ci.yml as it was done for GitHub * Remove a pip install and add a dev dependency in lisa package * Remove redundant apt commands --- .gitlab-ci.yml | 82 ++++++++++++++++++++++++++++--------- ci/autobranch-job-script.sh | 51 ----------------------- 2 files changed, 62 insertions(+), 71 deletions(-) delete mode 100755 ci/autobranch-job-script.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e68561a32..d865b9850 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -51,55 +51,91 @@ Test: matrix: - PYVERSION: [8, 11] before_script: + - sudo sh -c 'echo "APT::Acquire::Retries=3;" >> /etc/apt/apt.conf.d/99retry-downloads' - export DEBIAN_FRONTEND=noninteractive - - sudo apt update -y - - sudo apt install software-properties-common -y + - sudo apt-get update -y + - sudo apt-get upgrade -y + - sudo apt-get install software-properties-common -y - sudo add-apt-repository ppa:deadsnakes/ppa -y - sudo apt-get update -y - sudo apt-get -y install python3.${PYVERSION}-venv python3.${PYVERSION}-tk python3.${PYVERSION}-full + - echo "$(python3 --version)" + - sudo chmod o+rx /sys/kernel/debug script: - export LISA_PYTHON=python3.${PYVERSION} - - sudo sh -c 'echo "APT::Acquire::Retries=3;" >> /etc/apt/apt.conf.d/99retry-downloads' - sudo ./install_base.sh --install-all - - echo "$(python3 --version)" - - sudo chmod o+rx /sys/kernel/debug - bash ./tools/tests.sh Autobranch: stage: Deploy variables: - GIT_STRATEGY: none #Required to enable the autobranch run from a specific branch:$CI_DEFAULT_BRANCH + # Do not clone or fetch anything, as we will do it manually so that the job + # runs with the code from the default branch (main) rather than the merge + # request branch for safety reasons. + GIT_STRATEGY: none rules: - if: $CI_PIPELINE_SOURCE == "web" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $ENABLE_AUTOBRANCH == "True" - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_LABELS =~ /^autobranch$/ #regex to pickup 'autobranch' label alone + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_LABELS =~ /(^|,)autobranch($|,)/ resource_group: autobranch before_script: - - sudo apt update -y - - sudo apt install tzdata -y - - sudo apt install software-properties-common -y + - export DEBIAN_FRONTEND=noninteractive - sudo apt-get update -y - - sudo apt-get install python3.10 git python3.10-venv -y - - sudo apt-get install gcc python3-dev -y + - sudo apt-get upgrade -y + - sudo apt-get install -y tzdata git python3.10-full + - | - git clone -b $CI_DEFAULT_BRANCH $CI_REPOSITORY_URL origin-repo + origin_repo=$(mktemp -d) + git clone -b "$CI_DEFAULT_BRANCH" "$CI_REPOSITORY_URL" "$origin_repo" + + # This is critical for safety: We want to be executing the code from the + # main branch to avoid any manipulation from the MR branch + cd "$origin_repo" || exit 1 - cd origin-repo && + - | + git remote set-url origin "https://gitlab-ci:${GITLAB_REPO_TOKEN}@${CI_SERVER_HOST}/${CI_PROJECT_PATH}.git" + git remote -v + git branch --list --remotes git config --global user.name 'Gitlab CI' git config --global user.email 'gitlab-ci@arm.com' - git remote -v - git branch --list --remotes script: - | set -e - export LC_ALL=C + source init_env + + function update_branch() { + local label=$1 + local branch=$2 + local force_branch=$3 + + local worktree=$(mktemp -d --suffix "${branch}-repo") + local patch=$(mktemp --suffix "$branch.patch") + + lisa-combine-gitlab-mr --repo "$CI_PROJECT_PATH" --mr-label "$label" --branch "$force_branch" && - source init_env && - pip install requests # Required for calling Gitlab API - source ci/autobranch-job-script.sh + git fetch origin "$branch" && + + # Work in a separate worktree so that there is no risk of folders + # added to PATH by init_env being manipulated + git worktree add "$worktree" --checkout "$branch" && + + git -C "$worktree" diff --binary "HEAD..$force_branch" > "$patch" && + + if [[ -s "$patch" ]]; then + # Apply the patch to the index as well, so that any file created + # is automatically added to the commit we are about to create. + git -C "$worktree" apply --index "$patch" && + git -C "$worktree" commit --all -m "Autocommit to $branch branch on $(date) tracking $force_branch" + + git push --force origin "$force_branch" + git push origin "$branch" + else + echo "Empty patch, $branch and $force_branch branches are up to date." + fi + } ret=0 function keepgoing { @@ -110,3 +146,9 @@ Autobranch: keepgoing update_branch for-preview-autobranch preview preview-force exit $ret + +Autobranch-manual: + needs: ['Autobranch'] + when: manual + allow_failure: false # Set to false to return to the previous behavior. + script: exit 0 diff --git a/ci/autobranch-job-script.sh b/ci/autobranch-job-script.sh deleted file mode 100755 index de2930628..000000000 --- a/ci/autobranch-job-script.sh +++ /dev/null @@ -1,51 +0,0 @@ -# -# SPDX-License-Identifier: Apache-2.0 -# -# Copyright (C) 2024, Arm Limited and contributors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -function update_branch() { - local label=$1 - local branch=$2 - local force_branch=$3 - - local worktree=../${branch}-repo - local patch=${branch}-update.patch - - lisa-combine-gitlab-mr --repo "$CI_PROJECT_PATH" --mr-label "$label" --branch "$force_branch" && - - git fetch origin "$branch" && - - # Work in a separate worktree so that there is no risk of folders - # added to PATH by init_env being manipulated - git worktree add "$worktree" --checkout "$branch" && - - git -C "$worktree" diff --binary "HEAD..$force_branch" > "$patch" && - - if [[ -s "$patch" ]]; then - # Apply the patch to the index as well, so that any file created - # is automatically added to the commit we are about to create. - git -C "$worktree" apply --index "../origin-repo/$patch" && - git -C "$worktree" commit --all -m "Autocommit to $branch branch on $(date) tracking $force_branch" - - git remote set-url origin https://gitlab-ci:${GITLAB_REPO_TOKEN}@${CI_SERVER_HOST}/${CI_PROJECT_PATH}.git - - git push --force origin "$force_branch" - git push origin "$branch" - else - echo "Empty patch, $branch and $force_branch branches are up to date." - fi -} -- GitLab From 1a1aebafc44f684e458375353fde515832e9431a Mon Sep 17 00:00:00 2001 From: Douglas Raillard Date: Thu, 1 Feb 2024 20:46:20 +0000 Subject: [PATCH 08/23] tools/lisa-combine-gitlab-mr: Cleanup and remove github support --- .gitlab-ci.yml | 2 +- tools/lisa-combine-gitlab-mr | 111 +++++++++++++++++++------------ tools/lisa-combine-pr | 123 ----------------------------------- 3 files changed, 69 insertions(+), 167 deletions(-) delete mode 100755 tools/lisa-combine-pr diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d865b9850..973f1b236 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -114,7 +114,7 @@ Autobranch: local worktree=$(mktemp -d --suffix "${branch}-repo") local patch=$(mktemp --suffix "$branch.patch") - lisa-combine-gitlab-mr --repo "$CI_PROJECT_PATH" --mr-label "$label" --branch "$force_branch" && + lisa-combine-gitlab-mr --server "$CI_SERVER_HOST" --repo "$CI_PROJECT_PATH" --project-id "$CI_PROJECT_ID" --api-url "$CI_API_V4_URL" --mr-label "$label" --branch "$force_branch" && git fetch origin "$branch" && diff --git a/tools/lisa-combine-gitlab-mr b/tools/lisa-combine-gitlab-mr index 17d3b1fe5..98807c30b 100755 --- a/tools/lisa-combine-gitlab-mr +++ b/tools/lisa-combine-gitlab-mr @@ -20,50 +20,67 @@ import subprocess from itertools import chain from tempfile import NamedTemporaryFile -import json, os +import json from collections import ChainMap from operator import itemgetter import argparse import logging + import requests # Use HTTP APIs of GitLab to retrieve associated mrs # of this project. -def get_gitlab_mrs(state="opened", scope="all", labels=""): - def _call_gitlab_api(endpoint): - api_token = os.environ.get("GITLAB_REPO_TOKEN") - api_url = os.environ.get("CI_API_V4_URL") - headers = {"PRIVATE-TOKEN": api_token} - r = requests.get("/".join([api_url,endpoint]), headers=headers) - if r.status_code != 200: - raise Exception(r.text) +def get_gitlab_mrs(api_url, project_id, api_token=None, state="opened", scope="all", labels=None): + if labels: + labels = ','.join(labels) + labels = f'&labels={labels}' + else: + labels = '' + + def call_gitlab_api(endpoint): + headers = { + k: v + for k, v in [('PRIVATE-TOKEN', api_token)] + if v + } + r = requests.get(f'{api_url}/{endpoint}', headers=headers) + r.raise_for_status() return r - this_project_id = int(os.environ.get("CI_PROJECT_ID")) - page_number = 1 # obvioulsy at start 1 + def get_mr(mr): + # populate commits count - use another api + mr_commit_response = call_gitlab_api( + f"projects/{project_id}/merge_requests/{mr['iid']}/commits" + ) + mr_commit_response = mr_commit_response.json() + assert isinstance(mr_commit_response, list) + commits_count = len(mr_commit_response) + + # mr could be from a fork - use another api + project_response = call_gitlab_api( + f"projects/{mr['source_project_id']}" + ) + clone_url = project_response.json()['http_url_to_repo'] + + return dict( + sha=mr['sha'], + source_branch=mr['source_branch'], + commits_count=commits_count, + clone_url=clone_url, + ) results = [] + # obvioulsy at start 1 + page_number = 1 while True: - mr_response = _call_gitlab_api( - f"merge_requests?state={state}&scope={scope}&labels={labels}&page={page_number}" + mr_response = call_gitlab_api( + f"merge_requests?state={state}&scope={scope}{labels}&page={page_number}" + ) + results.extend( + get_mr(mr) + for mr in mr_response.json() + if mr.get("project_id") == project_id ) - for this_mr in mr_response.json(): - if this_mr.get("project_id") == this_project_id: - # populate commits count - use another api - mr_commit_response = _call_gitlab_api( - f"projects/{this_project_id}/merge_requests/{this_mr['iid']}/commits" - ) - this_mr["commits_count"] = len(mr_commit_response.json()) - - # this_mr could be from a fork - use another api - project_response = _call_gitlab_api( - f"projects/{this_mr['source_project_id']}" - ) - this_mr["http_url_to_repo"] = project_response.json()[ - "http_url_to_repo" - ] - - results.append(this_mr) # handle paging - only required for mr if not mr_response.headers["X-Next-Page"]: @@ -80,29 +97,32 @@ def main(): """, ) + parser.add_argument('--server', required=True, help='Gitlab server URL') + parser.add_argument('--api-url', required=True, help='Gitlab API URL') + parser.add_argument('--api-token', help='Gitlab API token. If omitted, anonymous requests will be used which may fail') + parser.add_argument('--project-id', required=True, help='Gitlab Project ID') parser.add_argument('--repo', required=True, help='Gitlab repository as owner/name') parser.add_argument('--mr-label', action='append', required=True, help='Merge request labels to look for') parser.add_argument('--branch', required=True, help='Name of the branch to be created. If the branch exists, it will be forcefully updated') args = parser.parse_args() - project = args.repo owner, repo = args.repo.split('/', 1) - labels = ','.join(args.mr_label) + labels = args.mr_label branch = args.branch + server = args.server + api_url = args.api_url + api_token = args.api_token + project_id = args.project_id logging.basicConfig(level=logging.INFO) - server_host_ssh = os.environ.get('CI_SERVER_SHELL_SSH_HOST') - - gl_mrs = get_gitlab_mrs(labels=labels) - def make_topic(mr): remote = f'remote_{mr["sha"]}' return ( { remote: { - 'url': mr["http_url_to_repo"] + 'url': mr["clone_url"] } }, { @@ -112,17 +132,22 @@ def main(): 'tip': mr["source_branch"], } ) - - topics = [] - - for mr in gl_mrs: - topics += [ make_topic(mr) ] + + topics = [ + make_topic(mr) + for mr in get_gitlab_mrs( + api_url=api_url, + api_token=api_token, + project_id=project_id, + labels=labels + ) + ] remotes, topics = zip(*topics) if topics else ([], []) remotes = dict(ChainMap(*chain( [{ 'gitlab': { - 'url': f'https://{server_host_ssh}/{owner}/{repo}.git' + 'url': f'https://{server}/{owner}/{repo}.git' } }], remotes diff --git a/tools/lisa-combine-pr b/tools/lisa-combine-pr deleted file mode 100755 index d1d1dca09..000000000 --- a/tools/lisa-combine-pr +++ /dev/null @@ -1,123 +0,0 @@ -#! /usr/bin/env python3 -# -# SPDX-License-Identifier: Apache-2.0 -# -# Copyright (C) 2023, Arm Limited and contributors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import subprocess -from itertools import starmap, chain -from tempfile import NamedTemporaryFile -import json -from collections import ChainMap -from operator import itemgetter -import argparse -import logging - -from github3 import GitHub - -def main(): - parser = argparse.ArgumentParser( - description=""" - Combine github pull requests with the given tag into a branch, rebasing all - PRs on top of each other. - """, - ) - - parser.add_argument('--repo', required=True, help='Github repository as owner/name') - parser.add_argument('--pr-label', action='append', required=True, help='Pull request labels to look for') - parser.add_argument('--branch', required=True, help='Name of the branch to be created. If the branch exists, it will be forcefully updated') - - args = parser.parse_args() - - owner, repo = args.repo.split('/', 1) - labels = args.pr_label - branch = args.branch - - logging.basicConfig(level=logging.INFO) - - gh = GitHub() - issues = gh.issues_on( - username=owner, - repository=repo, - state='open', - labels=labels, - ) - - prs = [ - ( - issue, - gh.pull_request( - owner=owner, - repository=repo, - number=issue.number - ) - ) - for issue in issues - if issue.pull_request_urls - ] - - def make_topic(issue, pr): - remote = f'remote_{pr.head.sha}' - return ( - { - remote: { - 'url': pr.head.repository.clone_url - } - }, - { - 'name': pr.head.ref, - 'remote': remote, - 'nr-commits': pr.commits_count, - 'tip': pr.head.ref, - } - ) - - topics = list(starmap(make_topic, prs)) - remotes, topics = zip(*topics) if topics else ([], []) - remotes = dict(ChainMap(*chain( - [{ - 'github': { - 'url': f'https://github.com/{owner}/{repo}.git' - } - }], - remotes - ))) - - conf = { - 'rebase-conf': { - 'rr-cache': './rr-cache', - 'remotes': remotes, - 'base': { - 'remote': 'github', - 'ref': 'main', - }, - 'topics': sorted(topics, key=itemgetter('name')) - } - } - conf = json.dumps(conf, indent=4) - logging.info(conf) - - with NamedTemporaryFile(mode='w+', suffix='.manifest.json') as f: - f.write(conf) - f.flush() - - manifest = f.name - - cmd = ['batch-rebase', 'create', '.', '--manifest', manifest, '--create-branch', branch] - logging.info(f'Running {" ".join(map(str, cmd))}') - subprocess.check_call(cmd) - -main() -- GitLab From 7c55360528da68a92b74bab9b7392743ade6874d Mon Sep 17 00:00:00 2001 From: Douglas Raillard Date: Thu, 1 Feb 2024 21:23:15 +0000 Subject: [PATCH 09/23] .gitlab-ci.yml: Use stageless pipeline Order jobs using the "needs" keyword instead of stages: https://docs.gitlab.com/ee/ci/yaml/index.html#needs --- .gitlab-ci.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 973f1b236..a9ddca56e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -37,12 +37,7 @@ variables: - "False" description: True to run Autobranch job. Defaults to false. Autobranch is allowed only on main. -stages: - - Test - - Deploy - Test: - stage: Test rules: - if: $CI_PIPELINE_SOURCE == "web" && $ENABLE_TEST == "True" - if: $CI_PIPELINE_SOURCE == "merge_request_event" @@ -67,7 +62,8 @@ Test: - bash ./tools/tests.sh Autobranch: - stage: Deploy + needs: ['Test'] + variables: # Do not clone or fetch anything, as we will do it manually so that the job # runs with the code from the default branch (main) rather than the merge -- GitLab From 9942c477d3e8bc6dec818d6a1e84349faf87525a Mon Sep 17 00:00:00 2001 From: Douglas Raillard Date: Thu, 1 Feb 2024 21:43:13 +0000 Subject: [PATCH 10/23] .gitlab-ci.yml: Avoid errors inside init_env to propagate We only care about the overall return code. --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a9ddca56e..7345fa353 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -98,9 +98,9 @@ Autobranch: script: - | - set -e export LC_ALL=C - source init_env + source init_env || exit 1 + set -e function update_branch() { local label=$1 -- GitLab From bcae812f3ef1164fa25ca992b7b719bda5fa93b2 Mon Sep 17 00:00:00 2001 From: Douglas Raillard Date: Fri, 2 Feb 2024 14:31:20 +0000 Subject: [PATCH 11/23] .gitlab-ci.yml: Use environment to restrict access to GITLAB_REPO_TOKEN --- .gitlab-ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7345fa353..bd9bebea9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -63,6 +63,9 @@ Test: Autobranch: needs: ['Test'] + environment: + # This grants the GITLAB_REPO_TOKEN variable allowing to push to the repo + name: repo_write_access variables: # Do not clone or fetch anything, as we will do it manually so that the job -- GitLab From 6472cffb2e60ec7bb6a571ec2d2baa756f541050 Mon Sep 17 00:00:00 2001 From: Douglas Raillard Date: Fri, 2 Feb 2024 16:31:43 +0000 Subject: [PATCH 12/23] tools/batch-rebase: Create a separate Python package --- devmode_requirements.txt | 1 + requirements.txt | 1 + tools/batch-rebase/LICENSE.txt | 1 + tools/batch-rebase/pyproject.toml | 54 +++++++++++++++++++ .../src/batch_rebase/main.py} | 0 vagrant_requirements.txt | 1 + 6 files changed, 58 insertions(+) create mode 120000 tools/batch-rebase/LICENSE.txt create mode 100644 tools/batch-rebase/pyproject.toml rename tools/{batch-rebase => batch-rebase/src/batch_rebase/main.py} (100%) diff --git a/devmode_requirements.txt b/devmode_requirements.txt index fe5224395..6fe85e53d 100644 --- a/devmode_requirements.txt +++ b/devmode_requirements.txt @@ -12,3 +12,4 @@ -e ./[all] -e ./tools/bisector +-e ./tools/batch-rebase diff --git a/requirements.txt b/requirements.txt index 16687201b..2d252703d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ ./tools/exekall ./[all] ./tools/bisector +./tools/batch-rebase diff --git a/tools/batch-rebase/LICENSE.txt b/tools/batch-rebase/LICENSE.txt new file mode 120000 index 000000000..1ef648f64 --- /dev/null +++ b/tools/batch-rebase/LICENSE.txt @@ -0,0 +1 @@ +../../LICENSE.txt \ No newline at end of file diff --git a/tools/batch-rebase/pyproject.toml b/tools/batch-rebase/pyproject.toml new file mode 100644 index 000000000..e9b06e07b --- /dev/null +++ b/tools/batch-rebase/pyproject.toml @@ -0,0 +1,54 @@ +[project] +name = "batch-rebase" # Required +version = "1.0.0" # Required +description = "Merge several branches by rebasing them on top of each other" +requires-python = ">=3.8" +license = {file = "LICENSE.txt"} +keywords = ["git", "development"] + +# readme = "README.md" +authors = [ + {name = "Douglas RAILLARD"} +] +maintainers = [ + {name = "Arm Ltd." } +] + +# Classifiers help users find your project by categorizing it. +# +# For a list of valid classifiers, see https://pypi.org/classifiers/ +classifiers = [ # Optional + # How mature is this project? Common values are + # 3 - Alpha + # 4 - Beta + # 5 - Production/Stable + "Development Status :: 4 - Beta", + + # Indicate who your project is intended for + "Intended Audience :: Developers", + "Topic :: Software Development :: Build Tools", + + # This is not a standard classifier, as there is nothing defined for + # Apache 2.0 yet: + # https://pypi.org/classifiers/ +] + +dependencies = [ + "ruamel.yaml", +] + +[project.urls] +"Homepage" = "https://gitlab.arm.com/tooling/lisa" +"Source" = "https://gitlab.arm.com/tooling/lisa" +"Documentation" = "https://lisa-linux-integrated-system-analysis.readthedocs.io/" +"Bug Tracker" = "https://gitlab.arm.com/tooling/lisa/-/issues" + +# The following would provide a command line executable called `sample` +# which executes the function `main` from this package when invoked. +[project.scripts] # Optional +batch-rebase = "batch_rebase.main:main" + +[build-system] +requires = ["pdm-backend"] +build-backend = "pdm.backend" + diff --git a/tools/batch-rebase b/tools/batch-rebase/src/batch_rebase/main.py similarity index 100% rename from tools/batch-rebase rename to tools/batch-rebase/src/batch_rebase/main.py diff --git a/vagrant_requirements.txt b/vagrant_requirements.txt index 701196a52..c1dabbbca 100644 --- a/vagrant_requirements.txt +++ b/vagrant_requirements.txt @@ -3,3 +3,4 @@ ./external/workload-automation/ -e ./[all] -e ./tools/bisector +-e ./tools/batch-rebase -- GitLab From 3d3f0a524c04a2cca977a39dfdec1d3d37a15a29 Mon Sep 17 00:00:00 2001 From: Douglas Raillard Date: Mon, 5 Feb 2024 10:27:52 +0000 Subject: [PATCH 13/23] tools/lisa-combine-gitlab-mr: Create a separate Python package --- devmode_requirements.txt | 2 +- requirements.txt | 2 +- tools/lisa-combine-gitlab-mr/LICENSE.txt | 1 + tools/lisa-combine-gitlab-mr/pyproject.toml | 55 +++++++++++++++++++ .../src/lisa_combine_gitlab_mr/main.py} | 0 vagrant_requirements.txt | 2 +- 6 files changed, 59 insertions(+), 3 deletions(-) create mode 120000 tools/lisa-combine-gitlab-mr/LICENSE.txt create mode 100644 tools/lisa-combine-gitlab-mr/pyproject.toml rename tools/{lisa-combine-gitlab-mr => lisa-combine-gitlab-mr/src/lisa_combine_gitlab_mr/main.py} (100%) diff --git a/devmode_requirements.txt b/devmode_requirements.txt index 6fe85e53d..c97069d6a 100644 --- a/devmode_requirements.txt +++ b/devmode_requirements.txt @@ -12,4 +12,4 @@ -e ./[all] -e ./tools/bisector --e ./tools/batch-rebase +-e ./tools/lisa-combine-gitlab-mr diff --git a/requirements.txt b/requirements.txt index 2d252703d..c1361b798 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ ./tools/exekall ./[all] ./tools/bisector -./tools/batch-rebase +./tools/lisa-combine-gitlab-mr diff --git a/tools/lisa-combine-gitlab-mr/LICENSE.txt b/tools/lisa-combine-gitlab-mr/LICENSE.txt new file mode 120000 index 000000000..1ef648f64 --- /dev/null +++ b/tools/lisa-combine-gitlab-mr/LICENSE.txt @@ -0,0 +1 @@ +../../LICENSE.txt \ No newline at end of file diff --git a/tools/lisa-combine-gitlab-mr/pyproject.toml b/tools/lisa-combine-gitlab-mr/pyproject.toml new file mode 100644 index 000000000..8b86939ed --- /dev/null +++ b/tools/lisa-combine-gitlab-mr/pyproject.toml @@ -0,0 +1,55 @@ +[project] +name = "lisa-combine-gitlab-mr" # Required +version = "1.0.0" # Required +description = "Combine GitLab merge requests with the given labels for the specified project" +requires-python = ">=3.8" +license = {file = "LICENSE.txt"} +keywords = ["git", "development"] + +# readme = "README.md" +authors = [ + {name = "Douglas RAILLARD"} +] +maintainers = [ + {name = "Arm Ltd." } +] + +# Classifiers help users find your project by categorizing it. +# +# For a list of valid classifiers, see https://pypi.org/classifiers/ +classifiers = [ # Optional + # How mature is this project? Common values are + # 3 - Alpha + # 4 - Beta + # 5 - Production/Stable + "Development Status :: 4 - Beta", + + # Indicate who your project is intended for + "Intended Audience :: Developers", + "Topic :: Software Development :: Build Tools", + + # This is not a standard classifier, as there is nothing defined for + # Apache 2.0 yet: + # https://pypi.org/classifiers/ +] + +dependencies = [ + # To query GitLab API + "requests", + "batch-rebase @ file:///${PROJECT_ROOT}/../batch-rebase/", +] + +[project.urls] +"Homepage" = "https://gitlab.arm.com/tooling/lisa" +"Source" = "https://gitlab.arm.com/tooling/lisa" +"Documentation" = "https://lisa-linux-integrated-system-analysis.readthedocs.io/" +"Bug Tracker" = "https://gitlab.arm.com/tooling/lisa/-/issues" + +# The following would provide a command line executable called `sample` +# which executes the function `main` from this package when invoked. +[project.scripts] # Optional +lisa-combine-gitlab-mr = "lisa_combine_gitlab_mr.main:main" + +[build-system] +requires = ["pdm-backend"] +build-backend = "pdm.backend" diff --git a/tools/lisa-combine-gitlab-mr b/tools/lisa-combine-gitlab-mr/src/lisa_combine_gitlab_mr/main.py similarity index 100% rename from tools/lisa-combine-gitlab-mr rename to tools/lisa-combine-gitlab-mr/src/lisa_combine_gitlab_mr/main.py diff --git a/vagrant_requirements.txt b/vagrant_requirements.txt index c1dabbbca..f92ef0d8b 100644 --- a/vagrant_requirements.txt +++ b/vagrant_requirements.txt @@ -3,4 +3,4 @@ ./external/workload-automation/ -e ./[all] -e ./tools/bisector --e ./tools/batch-rebase +-e ./tools/lisa-combine-gitlab-mr -- GitLab From 334e2ec3694ef7ac32125cbb41d8e65590079aa0 Mon Sep 17 00:00:00 2001 From: Douglas Raillard Date: Mon, 5 Feb 2024 11:31:19 +0000 Subject: [PATCH 14/23] tools/bisector: Use pyproject.toml --- tools/bisector/LICENSE.txt | 203 +----------------- tools/bisector/doc/api.rst | 2 +- tools/bisector/pyproject.toml | 61 ++++++ tools/bisector/setup.py | 73 ------- .../bisector.py => src/bisector/main.py} | 0 5 files changed, 63 insertions(+), 276 deletions(-) mode change 100644 => 120000 tools/bisector/LICENSE.txt create mode 100644 tools/bisector/pyproject.toml delete mode 100755 tools/bisector/setup.py rename tools/bisector/{bisector/bisector.py => src/bisector/main.py} (100%) diff --git a/tools/bisector/LICENSE.txt b/tools/bisector/LICENSE.txt deleted file mode 100644 index d64569567..000000000 --- a/tools/bisector/LICENSE.txt +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/tools/bisector/LICENSE.txt b/tools/bisector/LICENSE.txt new file mode 120000 index 000000000..1ef648f64 --- /dev/null +++ b/tools/bisector/LICENSE.txt @@ -0,0 +1 @@ +../../LICENSE.txt \ No newline at end of file diff --git a/tools/bisector/doc/api.rst b/tools/bisector/doc/api.rst index 4b6a29dd9..2c06c4bae 100644 --- a/tools/bisector/doc/api.rst +++ b/tools/bisector/doc/api.rst @@ -9,6 +9,6 @@ In order to support report edition (``bisector edit``) and more stable serialized format, new steps constructors must be written in a similar way to exisiting ones. -.. automodule:: bisector.bisector +.. automodule:: bisector.main :members: diff --git a/tools/bisector/pyproject.toml b/tools/bisector/pyproject.toml new file mode 100644 index 000000000..757299468 --- /dev/null +++ b/tools/bisector/pyproject.toml @@ -0,0 +1,61 @@ +[project] +name = "bisector" # Required +version = "1.0.0" # Required +description = "Command execution sequencer" +requires-python = ">=3.8" +license = {file = "LICENSE.txt"} +keywords = ["development"] + +readme = "README.rst" +authors = [ + {name = "Douglas RAILLARD"} +] +maintainers = [ + {name = "Arm Ltd." } +] + +# Classifiers help users find your project by categorizing it. +# +# For a list of valid classifiers, see https://pypi.org/classifiers/ +classifiers = [ # Optional + # How mature is this project? Common values are + # 3 - Alpha + # 4 - Beta + # 5 - Production/Stable + "Development Status :: 4 - Beta", + + "Programming Language :: Python :: 3 :: Only", + # This is not a standard classifier, as there is nothing defined for + # Apache 2.0 yet: + # https://pypi.org/classifiers/ + "License :: OSI Approved :: Apache 2.0", + # It has not been tested under any other OS + "Operating System :: POSIX :: Linux", + "Topic :: Software Development :: Testing", + "Intended Audience :: Developers", +] + +dependencies = [ + # Older versions will have troubles with serializing complex nested + # objects hierarchy implementing custom __getstate__ and __setstate__ + "ruamel.yaml >= 0.15.72", + "pandas", + "scipy", + "requests", +] + +[project.urls] +"Homepage" = "https://gitlab.arm.com/tooling/lisa" +"Source" = "https://gitlab.arm.com/tooling/lisa" +"Documentation" = "https://lisa-linux-integrated-system-analysis.readthedocs.io/" +"Bug Tracker" = "https://gitlab.arm.com/tooling/lisa/-/issues" + +# The following would provide a command line executable called `sample` +# which executes the function `main` from this package when invoked. +[project.scripts] # Optional +bisector = "bisector.main:main" + +[build-system] +requires = ["pdm-backend"] +build-backend = "pdm.backend" + diff --git a/tools/bisector/setup.py b/tools/bisector/setup.py deleted file mode 100755 index c9e7d07bd..000000000 --- a/tools/bisector/setup.py +++ /dev/null @@ -1,73 +0,0 @@ -#! /usr/bin/env python3 -# SPDX-License-Identifier: Apache-2.0 -# -# Copyright (C) 2018, Arm Limited and contributors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from setuptools import setup - -with open('README.rst') as fh: - long_description = fh.read() - -setup( - name='bisector', - version='1.0', - author='Arm Ltd', - # TODO: figure out which email to put here - # author_email= - packages=['bisector'], - # url='http://pypi.python.org/pypi/TowelStuff/', - license='LICENSE.txt', - description='Command execution sequencer', - long_description=long_description, - entry_points={ - 'console_scripts': [ - 'bisector=bisector.bisector:main', - ], - }, - python_requires='>= 3.5', - install_requires=[ - # Older versions will have troubles with serializing complex nested - # objects hierarchy implementing custom __getstate__ and __setstate__ - "ruamel.yaml >= 0.15.72", - "pandas", - "scipy", - "requests", - ], - - extras_require={ - 'dbus': [ - 'pydbus', - 'pygobject', - # You will also need gobject-introspection package from your - # distribution - ] - }, - - classifiers=[ - "Programming Language :: Python :: 3 :: Only", - # This is not a standard classifier, as there is nothing defined for - # Apache 2.0 yet: - # https://pypi.org/classifiers/ - "License :: OSI Approved :: Apache 2.0", - # It has not been tested under any other OS - "Operating System :: POSIX :: Linux", - - "Topic :: Software Development :: Testing", - "Intended Audience :: Developers", - ], -) - -# vim :set tabstop=4 shiftwidth=4 textwidth=80 expandtab diff --git a/tools/bisector/bisector/bisector.py b/tools/bisector/src/bisector/main.py similarity index 100% rename from tools/bisector/bisector/bisector.py rename to tools/bisector/src/bisector/main.py -- GitLab From 8de96cdf7e98007ba90028b81e7f6d1b4d212148 Mon Sep 17 00:00:00 2001 From: Douglas Raillard Date: Mon, 5 Feb 2024 11:35:40 +0000 Subject: [PATCH 15/23] tools/exekall: Use pyproject.toml --- tools/exekall/LICENSE.txt | 203 +----------------- tools/exekall/pyproject.toml | 53 +++++ tools/exekall/setup.py | 54 ----- tools/exekall/{ => src}/exekall/__init__.py | 0 .../exekall/_tests}/exekall_customize.py | 2 +- .../tests => src/exekall/_tests}/suite.py | 20 +- .../tests => src/exekall/_tests}/utils.py | 0 tools/exekall/{ => src}/exekall/_utils.py | 0 .../{ => src}/exekall/customization.py | 0 tools/exekall/{ => src}/exekall/engine.py | 0 tools/exekall/{ => src}/exekall/main.py | 0 tools/exekall/{ => src}/exekall/utils.py | 0 tools/exekall/{ => src}/exekall/valuedb.py | 0 tools/tests.sh | 2 +- 14 files changed, 66 insertions(+), 268 deletions(-) mode change 100644 => 120000 tools/exekall/LICENSE.txt create mode 100644 tools/exekall/pyproject.toml delete mode 100755 tools/exekall/setup.py rename tools/exekall/{ => src}/exekall/__init__.py (100%) rename tools/exekall/{exekall/tests => src/exekall/_tests}/exekall_customize.py (96%) rename tools/exekall/{exekall/tests => src/exekall/_tests}/suite.py (91%) rename tools/exekall/{exekall/tests => src/exekall/_tests}/utils.py (100%) rename tools/exekall/{ => src}/exekall/_utils.py (100%) rename tools/exekall/{ => src}/exekall/customization.py (100%) rename tools/exekall/{ => src}/exekall/engine.py (100%) rename tools/exekall/{ => src}/exekall/main.py (100%) rename tools/exekall/{ => src}/exekall/utils.py (100%) rename tools/exekall/{ => src}/exekall/valuedb.py (100%) diff --git a/tools/exekall/LICENSE.txt b/tools/exekall/LICENSE.txt deleted file mode 100644 index d64569567..000000000 --- a/tools/exekall/LICENSE.txt +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/tools/exekall/LICENSE.txt b/tools/exekall/LICENSE.txt new file mode 120000 index 000000000..1ef648f64 --- /dev/null +++ b/tools/exekall/LICENSE.txt @@ -0,0 +1 @@ +../../LICENSE.txt \ No newline at end of file diff --git a/tools/exekall/pyproject.toml b/tools/exekall/pyproject.toml new file mode 100644 index 000000000..da3d279d7 --- /dev/null +++ b/tools/exekall/pyproject.toml @@ -0,0 +1,53 @@ +[project] +name = "exekall" # Required +version = "1.0.0" # Required +description = "Python expression execution engine" +requires-python = ">=3.8" +license = {file = "LICENSE.txt"} +keywords = ["development"] + +readme = "README.rst" +authors = [ + {name = "Douglas RAILLARD"} +] +maintainers = [ + {name = "Arm Ltd." } +] + +# Classifiers help users find your project by categorizing it. +# +# For a list of valid classifiers, see https://pypi.org/classifiers/ +classifiers = [ # Optional + # How mature is this project? Common values are + # 3 - Alpha + # 4 - Beta + # 5 - Production/Stable + "Development Status :: 4 - Beta", + + "Programming Language :: Python :: 3 :: Only", + # This is not a standard classifier, as there is nothing defined for + # Apache 2.0 yet: + # https://pypi.org/classifiers/ + # It has not been tested under any other OS + "Operating System :: POSIX :: Linux", + + "Intended Audience :: Developers", +] + +dependencies = [] + +[project.urls] +"Homepage" = "https://gitlab.arm.com/tooling/lisa" +"Source" = "https://gitlab.arm.com/tooling/lisa" +"Documentation" = "https://lisa-linux-integrated-system-analysis.readthedocs.io/" +"Bug Tracker" = "https://gitlab.arm.com/tooling/lisa/-/issues" + +# The following would provide a command line executable called `sample` +# which executes the function `main` from this package when invoked. +[project.scripts] # Optional +exekall = "exekall.main:main" + +[build-system] +requires = ["pdm-backend"] +build-backend = "pdm.backend" + diff --git a/tools/exekall/setup.py b/tools/exekall/setup.py deleted file mode 100755 index 623daf33f..000000000 --- a/tools/exekall/setup.py +++ /dev/null @@ -1,54 +0,0 @@ -#! /usr/bin/env python3 -# SPDX-License-Identifier: Apache-2.0 -# -# Copyright (C) 2021, Arm Limited and contributors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from setuptools import setup - -with open('README.rst') as fh: - long_description = fh.read() - -setup( - name='exekall', - version='1.0', - maintainer='Arm Ltd.', - packages=['exekall'], - url='https://gitlab.arm.com/tooling/lisa', - project_urls={ - "Bug Tracker": "https://gitlab.arm.com/tooling/lisa/-/issues", - "Source Code": "https://gitlab.arm.com/tooling/lisa", - "Documentation": "https://lisa-linux-integrated-system-analysis.readthedocs.io/", - }, - license='Apache License 2.0', - description='Python expression execution engine', - long_description=long_description, - entry_points={ - 'console_scripts': ['exekall=exekall.main:main'], - }, - python_requires='>= 3.5', - classifiers=[ - "Programming Language :: Python :: 3 :: Only", - # This is not a standard classifier, as there is nothing defined for - # Apache 2.0 yet: - # https://pypi.org/classifiers/ - # It has not been tested under any other OS - "Operating System :: POSIX :: Linux", - - "Intended Audience :: Developers", - ], -) - -# vim :set tabstop=4 shiftwidth=4 textwidth=80 expandtab diff --git a/tools/exekall/exekall/__init__.py b/tools/exekall/src/exekall/__init__.py similarity index 100% rename from tools/exekall/exekall/__init__.py rename to tools/exekall/src/exekall/__init__.py diff --git a/tools/exekall/exekall/tests/exekall_customize.py b/tools/exekall/src/exekall/_tests/exekall_customize.py similarity index 96% rename from tools/exekall/exekall/tests/exekall_customize.py rename to tools/exekall/src/exekall/_tests/exekall_customize.py index 9234e0a2a..78f6967e3 100644 --- a/tools/exekall/exekall/tests/exekall_customize.py +++ b/tools/exekall/src/exekall/_tests/exekall_customize.py @@ -18,7 +18,7 @@ from exekall.utils import flatten_seq from exekall.customization import AdaptorBase -from exekall.tests.suite import TestCaseABC, TestResult, TestResultStatus +from exekall._tests.suite import TestCaseABC, TestResult, TestResultStatus class SelfTestAdaptor(AdaptorBase): diff --git a/tools/exekall/exekall/tests/suite.py b/tools/exekall/src/exekall/_tests/suite.py similarity index 91% rename from tools/exekall/exekall/tests/suite.py rename to tools/exekall/src/exekall/_tests/suite.py index 044fc50ed..bbaa6f225 100644 --- a/tools/exekall/exekall/tests/suite.py +++ b/tools/exekall/src/exekall/_tests/suite.py @@ -25,7 +25,7 @@ import shutil import exekall.utils as utils import exekall.engine as engine -from exekall.tests.utils import indent +from exekall._tests.utils import indent class TestResultStatus(enum.Enum): @@ -478,11 +478,11 @@ class SingleExprTestCase(NoExcepTestCase): # key: get_id() kwargs dict as a tuple to be passed to dict() # val: ID EXPR_VAL_ID = { - (('qual', True),): 'exekall.tests.suite.init:exekall.tests.suite.middle[tag1=val1][tag2=val2]:exekall.tests.suite.final(b2=exekall.tests.suite.init:exekall.tests.suite.middle2,b3=exekall.tests.suite.init:exekall.tests.suite.middle2:exekall.tests.suite.middle3)', + (('qual', True),): 'exekall._tests.suite.init:exekall._tests.suite.middle[tag1=val1][tag2=val2]:exekall._tests.suite.final(b2=exekall._tests.suite.init:exekall._tests.suite.middle2,b3=exekall._tests.suite.init:exekall._tests.suite.middle2:exekall._tests.suite.middle3)', (('qual', False),): 'init:middle[tag1=val1][tag2=val2]:final(b2=init:middle2,b3=init:middle2:middle3)', } EXPR_ID = { - (('qual', True),): 'exekall.tests.suite.init:exekall.tests.suite.middle:exekall.tests.suite.final(b2=exekall.tests.suite.init:exekall.tests.suite.middle2,b3=exekall.tests.suite.init:exekall.tests.suite.middle2:exekall.tests.suite.middle3)', + (('qual', True),): 'exekall._tests.suite.init:exekall._tests.suite.middle:exekall._tests.suite.final(b2=exekall._tests.suite.init:exekall._tests.suite.middle2,b3=exekall._tests.suite.init:exekall._tests.suite.middle2:exekall._tests.suite.middle3)', (('qual', False),): 'init:middle:final(b2=init:middle2,b3=init:middle2:middle3)', } @@ -530,7 +530,7 @@ def final_derived(b: B) -> Final: class InheritanceTestCase(NoExcepTestCase): CALLABLES = {init, middle_derived, final_derived} EXPR_ID = { - (('qual', True),): 'exekall.tests.suite.init:exekall.tests.suite.middle_derived:exekall.tests.suite.final_derived', + (('qual', True),): 'exekall._tests.suite.init:exekall._tests.suite.middle_derived:exekall._tests.suite.final_derived', (('qual', False),): 'init:middle_derived:final_derived', } # no tags used @@ -552,7 +552,7 @@ class ConsumerTestCase(NoExcepTestCase): ] EXPR_ID = { - (('qual', True),): 'exekall.tests.suite.init_consumer:exekall.tests.suite.middle:exekall.tests.suite.final(b2=exekall.tests.suite.init_consumer:exekall.tests.suite.middle2,b3=exekall.tests.suite.init_consumer:exekall.tests.suite.middle2:exekall.tests.suite.middle3)', + (('qual', True),): 'exekall._tests.suite.init_consumer:exekall._tests.suite.middle:exekall._tests.suite.final(b2=exekall._tests.suite.init_consumer:exekall._tests.suite.middle2,b3=exekall._tests.suite.init_consumer:exekall._tests.suite.middle2:exekall._tests.suite.middle3)', (('qual', False),): 'init_consumer:middle:final(b2=init_consumer:middle2,b3=init_consumer:middle2:middle3)', } # no tags used @@ -571,7 +571,7 @@ class ExceptionTestCase(TestCaseBase): CALLABLES = {init, middle_excep, middle2, final_excep} EXPR_ID = { - (('qual', True),): 'exekall.tests.suite.init:exekall.tests.suite.middle_excep:exekall.tests.suite.final_excep(b2=exekall.tests.suite.init:exekall.tests.suite.middle2)', + (('qual', True),): 'exekall._tests.suite.init:exekall._tests.suite.middle_excep:exekall._tests.suite.final_excep(b2=exekall._tests.suite.init:exekall._tests.suite.middle2)', (('qual', False),): 'init:middle_excep:final_excep(b2=init:middle2)', } # no tags used @@ -610,7 +610,7 @@ class CloneTestCase(NoExcepTestCase): ] EXPR_ID = { - (('qual', True),): 'exekall.tests.suite.init:exekall.tests.suite.middle:exekall.tests.suite.final(b2=exekall.tests.suite.init:exekall.tests.suite.middle2,b3=exekall.tests.suite.init:exekall.tests.suite.middle2:exekall.tests.suite.middle3)', + (('qual', True),): 'exekall._tests.suite.init:exekall._tests.suite.middle:exekall._tests.suite.final(b2=exekall._tests.suite.init:exekall._tests.suite.middle2,b3=exekall._tests.suite.init:exekall._tests.suite.middle2:exekall._tests.suite.middle3)', (('qual', False),): 'init:middle:final(b2=init:middle2,b3=init:middle2:middle3)', } # no tags used @@ -653,7 +653,7 @@ class AssociatedTypesTestCase(NoExcepTestCase): AssociatedDerived1_CLS.final, } EXPR_ID = { - (('qual', True),): 'exekall.tests.suite.AssociatedDerived1.make:exekall.tests.suite.AssociatedBase.make_associated:exekall.tests.suite.AssociatedDerived1_CLS.final', + (('qual', True),): 'exekall._tests.suite.AssociatedDerived1.make:exekall._tests.suite.AssociatedBase.make_associated:exekall._tests.suite.AssociatedDerived1_CLS.final', (('qual', False),): 'AssociatedDerived1.make:AssociatedBase.make_associated:AssociatedDerived1_CLS.final', } # no tags used @@ -686,7 +686,7 @@ def consume_HavingFactoryMethodDerived(x: HavingFactoryMethodDerived) -> Final: class BasicFactoryTestCase(NoExcepTestCase): CALLABLES = {HavingFactoryMethodDerived.factory_basic, consume_HavingFactoryMethodDerived} EXPR_ID = { - (('qual', True),): 'exekall.tests.suite.HavingFactoryMethodDerived.factory_basic:exekall.tests.suite.consume_HavingFactoryMethodDerived', + (('qual', True),): 'exekall._tests.suite.HavingFactoryMethodDerived.factory_basic:exekall._tests.suite.consume_HavingFactoryMethodDerived', (('qual', False),): 'HavingFactoryMethodDerived.factory_basic:consume_HavingFactoryMethodDerived', } # no tags used @@ -697,7 +697,7 @@ if test_typing_self: class SelfFactoryTestCase(NoExcepTestCase): CALLABLES = {HavingFactoryMethodDerived.factory_self, consume_HavingFactoryMethodDerived} EXPR_ID = { - (('qual', True),): 'exekall.tests.suite.HavingFactoryMethodDerived.factory_self:exekall.tests.suite.consume_HavingFactoryMethodDerived', + (('qual', True),): 'exekall._tests.suite.HavingFactoryMethodDerived.factory_self:exekall._tests.suite.consume_HavingFactoryMethodDerived', (('qual', False),): 'HavingFactoryMethodDerived.factory_self:consume_HavingFactoryMethodDerived', } # no tags used diff --git a/tools/exekall/exekall/tests/utils.py b/tools/exekall/src/exekall/_tests/utils.py similarity index 100% rename from tools/exekall/exekall/tests/utils.py rename to tools/exekall/src/exekall/_tests/utils.py diff --git a/tools/exekall/exekall/_utils.py b/tools/exekall/src/exekall/_utils.py similarity index 100% rename from tools/exekall/exekall/_utils.py rename to tools/exekall/src/exekall/_utils.py diff --git a/tools/exekall/exekall/customization.py b/tools/exekall/src/exekall/customization.py similarity index 100% rename from tools/exekall/exekall/customization.py rename to tools/exekall/src/exekall/customization.py diff --git a/tools/exekall/exekall/engine.py b/tools/exekall/src/exekall/engine.py similarity index 100% rename from tools/exekall/exekall/engine.py rename to tools/exekall/src/exekall/engine.py diff --git a/tools/exekall/exekall/main.py b/tools/exekall/src/exekall/main.py similarity index 100% rename from tools/exekall/exekall/main.py rename to tools/exekall/src/exekall/main.py diff --git a/tools/exekall/exekall/utils.py b/tools/exekall/src/exekall/utils.py similarity index 100% rename from tools/exekall/exekall/utils.py rename to tools/exekall/src/exekall/utils.py diff --git a/tools/exekall/exekall/valuedb.py b/tools/exekall/src/exekall/valuedb.py similarity index 100% rename from tools/exekall/exekall/valuedb.py rename to tools/exekall/src/exekall/valuedb.py diff --git a/tools/tests.sh b/tools/tests.sh index c125f08ef..3c86e393d 100755 --- a/tools/tests.sh +++ b/tools/tests.sh @@ -33,7 +33,7 @@ echo "Starting self tests ..." timeout -s INT 1h python3 -m pytest -vv echo "Starting exekall self tests" -exekall run "$LISA_HOME/tools/exekall/exekall/tests" +exekall run "$LISA_HOME/tools/exekall/src/exekall/_tests" echo "Available LISA tests:" lisa-test --list -- GitLab From 7d34400b625b7c1bf34adbddaf07bd03d8c1b26d Mon Sep 17 00:00:00 2001 From: Douglas Raillard Date: Mon, 5 Feb 2024 11:41:55 +0000 Subject: [PATCH 16/23] .gitlab-ci.yml: Minimal install for autobranch job Install the minimum amount of dependencies in the autobranch workflow. --- .gitlab-ci.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bd9bebea9..776056bf8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -101,9 +101,12 @@ Autobranch: script: - | - export LC_ALL=C - source init_env || exit 1 set -e + export LC_ALL=C + venv=$(mktemp -d) + python3 -m venv "$venv" && source "$venv/bin/activate" + pip install --upgrade pip + pip install ./tools/lisa-combine-gitlab-mr function update_branch() { local label=$1 -- GitLab From 01e4e887e075ac0113291c326f274abd711c7a56 Mon Sep 17 00:00:00 2001 From: Douglas Raillard Date: Mon, 5 Feb 2024 15:51:39 +0000 Subject: [PATCH 17/23] .gitlab-ci.yml: Remove useless manual job Remove manual job since it's not any easier to trigger from GitLab UI. --- .gitlab-ci.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 776056bf8..bfb738401 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -148,9 +148,3 @@ Autobranch: keepgoing update_branch for-preview-autobranch preview preview-force exit $ret - -Autobranch-manual: - needs: ['Autobranch'] - when: manual - allow_failure: false # Set to false to return to the previous behavior. - script: exit 0 -- GitLab From a1fc7988e566662eda5f265ae2c08ccd4a3c52f4 Mon Sep 17 00:00:00 2001 From: Douglas Raillard Date: Mon, 5 Feb 2024 15:55:01 +0000 Subject: [PATCH 18/23] .gitlab-ci.yml: Update the variable names --- .gitlab-ci.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bfb738401..4942ef000 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -24,22 +24,22 @@ default: # Required for Manual runs through UI variables: - ENABLE_TEST: - value: "False" + WEB_ENABLE_TEST: + value: "True" options: - "True" - "False" - description: True to run test job. Defaults to false. - ENABLE_AUTOBRANCH: - value: "False" + description: True to run test job when triggered via Web UI. + WEB_ENABLE_AUTOBRANCH: + value: "True" options: - "True" - "False" - description: True to run Autobranch job. Defaults to false. Autobranch is allowed only on main. + description: True to run test job when triggered via Web UI. Test: rules: - - if: $CI_PIPELINE_SOURCE == "web" && $ENABLE_TEST == "True" + - if: $CI_PIPELINE_SOURCE == "web" && $WEB_ENABLE_TEST == "True" - if: $CI_PIPELINE_SOURCE == "merge_request_event" - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH parallel: @@ -73,7 +73,7 @@ Autobranch: # request branch for safety reasons. GIT_STRATEGY: none rules: - - if: $CI_PIPELINE_SOURCE == "web" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $ENABLE_AUTOBRANCH == "True" + - if: $CI_PIPELINE_SOURCE == "web" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $WEB_ENABLE_AUTOBRANCH == "True" - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_LABELS =~ /(^|,)autobranch($|,)/ resource_group: autobranch -- GitLab From 0859e6508536f942c43e24a102a871d2ff4ac1f8 Mon Sep 17 00:00:00 2001 From: Douglas Raillard Date: Mon, 5 Feb 2024 16:45:07 +0000 Subject: [PATCH 19/23] .gitlab-ci.yml: Avoid cloning in tmpfs --- .gitlab-ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4942ef000..206587eb0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -84,6 +84,10 @@ Autobranch: - sudo apt-get install -y tzdata git python3.10-full - | + # Avoid using /tmp since it's probably in tmpfs and would consume RAM + export TMPDIR=$(readlink -f .)/tmp + mkdir -p "$TMPDIR" + origin_repo=$(mktemp -d) git clone -b "$CI_DEFAULT_BRANCH" "$CI_REPOSITORY_URL" "$origin_repo" -- GitLab From 5966ce5d5a9db911d533df00b708eb2893a97bf4 Mon Sep 17 00:00:00 2001 From: Douglas Raillard Date: Mon, 5 Feb 2024 16:55:51 +0000 Subject: [PATCH 20/23] .gitlab-ci.yml: Use reliable packages --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 206587eb0..e13b168d7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -81,7 +81,7 @@ Autobranch: - export DEBIAN_FRONTEND=noninteractive - sudo apt-get update -y - sudo apt-get upgrade -y - - sudo apt-get install -y tzdata git python3.10-full + - sudo apt-get install -y git python3-full - | # Avoid using /tmp since it's probably in tmpfs and would consume RAM -- GitLab From 04ed1c57af0c9dad08970396e28bbc9e1b696f70 Mon Sep 17 00:00:00 2001 From: Douglas Raillard Date: Mon, 5 Feb 2024 17:16:49 +0000 Subject: [PATCH 21/23] .gitlab-ci.yml: Add GitLab pages conf --- .gitlab-ci.yml | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e13b168d7..30e7168ed 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -152,3 +152,34 @@ Autobranch: keepgoing update_branch for-preview-autobranch preview preview-force exit $ret + + +pages: + needs: ['Autobranch', 'Test'] + resource_group: pages + + artifacts: + paths: + - doc/public + publish: doc/public + + rules: + - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + + before_script: + - PYVERSION=10 + - sudo sh -c 'echo "APT::Acquire::Retries=3;" >> /etc/apt/apt.conf.d/99retry-downloads' + - export DEBIAN_FRONTEND=noninteractive + - sudo apt-get update -y + - sudo apt-get upgrade -y + - sudo apt-get -y install python3.${PYVERSION}-full + - echo "$(python3 --version)" + + script: + - export LISA_PYTHON=python3.${PYVERSION} + - sudo ./install_base.sh --install-doc-extras + - source init_env + - | + set -e + cd doc/ + sphinx-build -b html . "public/$CI_COMMIT_BRANCH" -- GitLab From 1bb6bfd7bd6c41dfde2ed1d71d98535bcd3f74cc Mon Sep 17 00:00:00 2001 From: Douglas Raillard Date: Mon, 5 Feb 2024 17:41:33 +0000 Subject: [PATCH 22/23] .gitlab-ci.yml: Use CI_COMMIT_REF_NAME for doc namespace Use CI_COMMIT_REF_NAME instead of CI_COMMIT_BRANCH since the latter is not always available depending on how the job was triggered. --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 30e7168ed..15e701c56 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -182,4 +182,4 @@ pages: - | set -e cd doc/ - sphinx-build -b html . "public/$CI_COMMIT_BRANCH" + sphinx-build -b html . "public/$CI_COMMIT_REF_NAME" -- GitLab From acd867629c09a8662ad17210b0ce74942b7f9761 Mon Sep 17 00:00:00 2001 From: Douglas Raillard Date: Mon, 5 Feb 2024 18:00:10 +0000 Subject: [PATCH 23/23] .gitlab-ci.yml: Always use Python from PPA as distro package is broken Fix the following issue by installing the package from deadsnakes PPA: The following packages have unmet dependencies: idle-python3.10 : Depends: libjs-mathjax but it is not installable libpython3.10-testsuite : Depends: net-tools but it is not installable python3-tk : Depends: blt (>= 2.4z-9) but it is not installable Depends: libtk8.6 (>= 8.6.0) but it is not installable Depends: tk8.6-blt2.5 (>= 2.5.3) but it is not installable E: Unable to correct problems, you have held broken packages. --- .gitlab-ci.yml | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 15e701c56..78a83a561 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -24,6 +24,10 @@ default: # Required for Manual runs through UI variables: + PYVERSION: + value: 11 + description: Default version of Python to use for non-test jobs + WEB_ENABLE_TEST: value: "True" options: @@ -53,8 +57,7 @@ Test: - sudo apt-get install software-properties-common -y - sudo add-apt-repository ppa:deadsnakes/ppa -y - sudo apt-get update -y - - sudo apt-get -y install python3.${PYVERSION}-venv python3.${PYVERSION}-tk python3.${PYVERSION}-full - - echo "$(python3 --version)" + - sudo apt-get -y install python3.${PYVERSION}-full - sudo chmod o+rx /sys/kernel/debug script: - export LISA_PYTHON=python3.${PYVERSION} @@ -78,10 +81,15 @@ Autobranch: - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_LABELS =~ /(^|,)autobranch($|,)/ resource_group: autobranch before_script: + - sudo sh -c 'echo "APT::Acquire::Retries=3;" >> /etc/apt/apt.conf.d/99retry-downloads' - export DEBIAN_FRONTEND=noninteractive - sudo apt-get update -y - sudo apt-get upgrade -y - - sudo apt-get install -y git python3-full + - sudo apt-get install software-properties-common -y + - sudo add-apt-repository ppa:deadsnakes/ppa -y + - sudo apt-get update -y + - sudo apt-get -y install python3.${PYVERSION}-full + - export LISA_PYTHON=python3.${PYVERSION} - | # Avoid using /tmp since it's probably in tmpfs and would consume RAM @@ -109,8 +117,8 @@ Autobranch: export LC_ALL=C venv=$(mktemp -d) python3 -m venv "$venv" && source "$venv/bin/activate" - pip install --upgrade pip - pip install ./tools/lisa-combine-gitlab-mr + $LISA_PYTHON -m pip install --upgrade pip + $LISA_PYTHON -m pip install ./tools/lisa-combine-gitlab-mr function update_branch() { local label=$1 @@ -167,16 +175,17 @@ pages: - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH before_script: - - PYVERSION=10 - sudo sh -c 'echo "APT::Acquire::Retries=3;" >> /etc/apt/apt.conf.d/99retry-downloads' - export DEBIAN_FRONTEND=noninteractive - sudo apt-get update -y - sudo apt-get upgrade -y + - sudo apt-get install software-properties-common -y + - sudo add-apt-repository ppa:deadsnakes/ppa -y + - sudo apt-get update -y - sudo apt-get -y install python3.${PYVERSION}-full - - echo "$(python3 --version)" + - export LISA_PYTHON=python3.${PYVERSION} script: - - export LISA_PYTHON=python3.${PYVERSION} - sudo ./install_base.sh --install-doc-extras - source init_env - | -- GitLab