[neon-notifications] Changes in repo-metadata
Neon CI
noreply at kde.org
Sun May 5 19:46:23 BST 2024
commit f32f4f4d98563694833a69bb3ab0ad1946492db3
Author: Andrew Shark <ashark at linuxcomp.ru>
Date: Sun May 5 20:02:49 2024 +0300
Individual lint rules for regenerate_dependencies.py
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 40e274af..789fbd03 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -24,8 +24,9 @@ lint:
extends: .python
script:
- pip3 install pylint mypy networkx types-PyYAML types-regex
- - FILES="*.py git-helpers/* dependencies/tools/*"
+ - FILES="*.py git-helpers/* dependencies/tools/build_order dependencies/tools/list_dependencies dependencies/tools/list_preferred_repo_branch"
- pylint $FILES
+ - pylint dependencies/tools/regenerate_dependencies.py --disable=C0103,C0114,C0115,C0116,C0301,W0621,W0640,W1514
- mypy --strict verify-repo-metadata.py
- mypy --strict git-helpers/git-kclone
- mypy --strict git-helpers/git-kpull
diff --git a/dependencies/tools/regenerate_dependencies.py b/dependencies/tools/regenerate_dependencies.py
index e5ac8ce6..06910abe 100755
--- a/dependencies/tools/regenerate_dependencies.py
+++ b/dependencies/tools/regenerate_dependencies.py
@@ -12,6 +12,7 @@
# Use `--list` to get a list of branches used for projects.
+import sys
import argparse
import os
import pathlib
@@ -209,7 +210,7 @@ if __name__ == "__main__":
branch = "master"
print(f"{project.repopath} {branch}")
- exit()
+ sys.exit()
qt6_ignored_projects = [
"frameworks/khtml",
commit 3e867e51f0da66ea60dd54baaed26c22cad15e1c
Author: Andrew Shark <ashark at linuxcomp.ru>
Date: Sun May 5 19:20:31 2024 +0300
Add regenerate_dependencies.py script
diff --git a/dependencies/tools/regenerate_dependencies.py b/dependencies/tools/regenerate_dependencies.py
new file mode 100755
index 00000000..e5ac8ce6
--- /dev/null
+++ b/dependencies/tools/regenerate_dependencies.py
@@ -0,0 +1,272 @@
+#!/usr/bin/env python3
+
+# This script is used to bring (update) the project's dependencies info from their .kde-ci.yaml files, so it is available to kde-builder.
+
+# Usage:
+# export REPOS=/home/nico/kde/src
+# export REPO_METADATA=/home/nico/kde/src/sysadmin-repo-metadata
+#
+# regenerate-dependency-data --branch-group kf6-qt6
+
+# `--branch-group` is a mandatory argument. Supported values: `kf6-qt6`, `stable-kf6-qt6`, `kf5-qt5`, `stable-kf5-qt5`.
+# Use `--list` to get a list of branches used for projects.
+
+
+import argparse
+import os
+import pathlib
+import re
+from dataclasses import dataclass
+import yaml
+
+
+ at dataclass
+class MetaData:
+ repopath: str
+ projectpath: str
+ repoactive: bool
+
+
+ at dataclass
+class Dependencies:
+ on: list[str]
+ require: dict
+
+
+ at dataclass
+class KdeCi:
+ dependencies: list[Dependencies] | None
+ runtime_dependencies: list[Dependencies] | None
+
+
+ at dataclass
+class BranchGroups:
+ kf5_qt5: str | None
+ kf6_qt6: str | None
+ stable_kf5_qt5: str | None
+ stable_kf6_qt6: str | None
+
+
+class RepoMetadata:
+ def __repr__(self):
+ return f"{{path: {self.path}, projects: {self.projects}}}"
+
+ def __init__(self, path: str):
+ self.path: str = path
+ self.projects: list[MetaData]
+
+ ignored_categories = [
+ "websites",
+ "wikitolearn",
+ "webapps",
+ "historical",
+ "documentation",
+ "sysadmin",
+ "neon",
+ "packaging",
+ "unmaintained",
+ ]
+
+ entries: list[MetaData] = []
+ projects_invent = pathlib.Path(f"{path}/projects-invent")
+ projects_invent = projects_invent.rglob("*.yaml")
+ projects_invent = [x for x in projects_invent if x.parts[-1 - 2] not in ignored_categories]
+ for project in projects_invent:
+ with open(project, "r") as f:
+ content = yaml.safe_load(f)
+ entries.append(MetaData(content["repopath"], content["projectpath"], content["repoactive"]))
+ self.projects = entries
+
+ def logical_module_structure(self) -> dict[str, BranchGroups]:
+ with open(f"{self.path}/dependencies/logical-module-structure.json", "r") as f:
+ json = yaml.safe_load(f)
+
+ groups = json["groups"]
+ weighted_keys = get_project_keys(groups)
+ result: dict[str, BranchGroups] = {}
+
+ for project in self.projects:
+ for key in weighted_keys:
+ if key.glob.endswith("*"):
+ fixed_regexp = key.glob.replace("*", ".*")
+ pattern = re.compile(rf"{fixed_regexp}")
+ else:
+ pattern = re.compile(rf"{key.glob}$") # this prevents matching line with the extra symbols, for example for "extragear/office/kbibtex-testset" to not match for "extragear/office/kbibtex"
+
+ if re.match(pattern, project.projectpath):
+ b: BranchGroups = groups[key.glob]
+ result[project.projectpath] = b
+ break
+
+ return result
+
+
+ at dataclass
+class ProjectKey:
+ glob: str
+ weight: int
+
+
+def get_project_keys(groups: dict) -> list[ProjectKey]:
+ # We need the most specific glob to win, e.g. kde/workspace/* must win over kde/*,
+ # and kde/workspace/foo must win over kde/workspace/*
+
+ weighted_keys: list[ProjectKey] = []
+ for key in groups.keys():
+ weight = None
+
+ if "*" not in key:
+ # Exact match is the highest weight
+ weight = 10
+ else:
+ # Prefer more specific matches
+ weight = key.count("/")
+
+ weighted_keys.append(ProjectKey(key, weight))
+
+ weighted_keys.sort(key=lambda x: x.weight, reverse=True)
+ return weighted_keys
+
+
+def get_dependencies(input_dependencies, branch_group: str) -> list[str]:
+ result: list[str] = []
+
+ for dep in input_dependencies:
+ def should_include() -> bool:
+ if "@all" in dep["on"]:
+ return True
+
+ if "Linux" in dep["on"]:
+ return True
+
+ if "qt6" not in branch_group and "Linux/Qt5" in dep["on"]:
+ return True
+
+ if "qt6" in branch_group and "Linux/Qt6" in dep["on"]:
+ return True
+
+ return False
+
+ if not should_include():
+ continue
+
+ for key in dep["require"]:
+ result.append(key)
+ return result
+
+
+def list_dependencies(project: MetaData, branch_group: str) -> list[str]:
+ base_name = project.repopath.split("/")[1]
+
+ kdeci_file = f"""{os.environ["REPOS"]}/{base_name}/.kde-ci.yml"""
+
+ try:
+ with open(kdeci_file, "r"):
+ pass
+ except FileNotFoundError:
+ return []
+
+ with open(kdeci_file, "r") as f:
+ content = yaml.safe_load(f)
+ kdeci = KdeCi(content.get("Dependencies", []), content.get("RuntimeDependencies", []))
+
+ dependencies = get_dependencies(kdeci.dependencies, branch_group)
+ runtime_dependencies = get_dependencies(kdeci.runtime_dependencies, branch_group)
+
+ dependencies.extend(runtime_dependencies)
+ return dependencies
+
+
+if __name__ == "__main__":
+ repo_metadata = RepoMetadata(os.environ["REPO_METADATA"])
+ lms = repo_metadata.logical_module_structure()
+
+ arg_parser = argparse.ArgumentParser()
+ arg_parser.add_argument("--branch-group", required=True)
+ arg_parser.add_argument("--list", action="store_true")
+ args = arg_parser.parse_args()
+
+ if args.list:
+ for project in repo_metadata.projects:
+ if not project.repoactive:
+ continue
+
+ pr_groups = lms.get(project.projectpath, {})
+
+ match args.branch_group:
+ case "kf5-qt5":
+ branch = pr_groups.get("kf5-qt5")
+ case "kf6-qt6":
+ branch = pr_groups.get("kf6-qt6")
+ case "stable-kf5-qt5":
+ branch = pr_groups.get("stable-kf5-qt5")
+ case "stable-kf6-qt6":
+ branch = pr_groups.get("stable-kf6-qt6")
+ case _:
+ raise ValueError
+
+ if branch is None:
+ branch = "master"
+
+ print(f"{project.repopath} {branch}")
+ exit()
+
+ qt6_ignored_projects = [
+ "frameworks/khtml",
+ "frameworks/kdelibs4support",
+ "frameworks/kdewebkit",
+ "frameworks/kemoticons",
+ "frameworks/kinit",
+ "frameworks/kjs",
+ "frameworks/kjsembed",
+ "frameworks/kmediaplayer",
+ "frameworks/kross",
+ "frameworks/kdesignerplugin",
+ "frameworks/kxmlrpcclient",
+ "libraries/kross-interpreters",
+ "libraries/kwebkitpart",
+ ]
+
+ ignored_deps = {
+ "plasma/kwin": ["plasma/plasma-workspace"],
+ "plasma/kscreenlocker": ["plasma/plasma-workspace"],
+ "plasma/plasma-workspace": ["plasma/powerdevil"],
+ "sdk/selenium-webdriver-at-spi": ["plasma/kwin"],
+ }
+
+ print("# This file has been auto-generated from .kde-ci.yml files")
+ print("# Instead of editing it manually re-run the generation")
+
+ for project in repo_metadata.projects:
+ if "qt6" in args.branch_group and project.repopath in qt6_ignored_projects:
+ continue
+
+ if not project.repoactive:
+ continue
+
+ deps = list_dependencies(project, args.branch_group)
+
+ print()
+ print(f"# {project.repopath}")
+ print("# This data was autogenerated from the project's .kde-ci.yml file. DO NOT MODIFY. Add new dependencies to the .kde-ci.yml files instead.")
+
+ for dep in deps:
+ projectpath = None
+
+ if "qt6" in args.branch_group and dep in qt6_ignored_projects:
+ continue
+
+ if project.repopath in ignored_deps and dep in ignored_deps.get(project.repopath):
+ continue
+
+ if "third-party" in dep:
+ projectpath = dep
+ else:
+ maybe_project = next((p for p in repo_metadata.projects if p.repopath == dep), None)
+
+ if maybe_project is not None:
+ projectpath = maybe_project.projectpath
+ else:
+ raise ValueError(f"{project.projectpath} requests non-existent dependency {dep}")
+
+ print(f"{project.projectpath}: {projectpath}")
More information about the neon-notifications
mailing list