diff --git a/.github/actions/features_parse/action.yml b/.github/actions/features_parse/action.yml index dac7b586..99c79a2f 100644 --- a/.github/actions/features_parse/action.yml +++ b/.github/actions/features_parse/action.yml @@ -11,7 +11,7 @@ outputs: runs: using: composite steps: - - uses: gardenlinux/python-gardenlinux-lib/.github/actions/setup@0.10.10 + - uses: gardenlinux/python-gardenlinux-lib/.github/actions/setup@0.10.11 - id: result shell: bash run: | diff --git a/.github/actions/flavors_parse/action.yml b/.github/actions/flavors_parse/action.yml index 9bb8ff79..b31b58a2 100644 --- a/.github/actions/flavors_parse/action.yml +++ b/.github/actions/flavors_parse/action.yml @@ -13,7 +13,7 @@ outputs: runs: using: composite steps: - - uses: gardenlinux/python-gardenlinux-lib/.github/actions/setup@0.10.10 + - uses: gardenlinux/python-gardenlinux-lib/.github/actions/setup@0.10.11 - id: matrix shell: bash run: | diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 7021f953..375a67fe 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -4,7 +4,7 @@ description: Installs the given GardenLinux Python library inputs: version: description: GardenLinux Python library version - default: "0.10.10" + default: "0.10.11" python_version: description: Python version to setup default: "3.13" diff --git a/.github/workflows/pytests.yml b/.github/workflows/pytests.yml index a6538b5c..b10d5aa7 100644 --- a/.github/workflows/pytests.yml +++ b/.github/workflows/pytests.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python_version: ["3.13"] + python_version: ["3.13", "3.14"] steps: - name: Checkout uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 diff --git a/poetry.lock b/poetry.lock index e0e6712c..21846aee 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.1.4 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.2.0 and should not be changed by hand. [[package]] name = "alabaster" @@ -2514,5 +2514,5 @@ test = ["pytest", "pytest-cov"] [metadata] lock-version = "2.1" -python-versions = ">=3.13, <3.14" -content-hash = "ecda6b78f55473432a4125c4b7c74b03e26abf18dee0c87f242fc164410a82c0" +python-versions = ">=3.13, !=3.14.1" +content-hash = "cfd1fe0f7ea50e1d87ef53df17ca9b21f1dce9654b6003b08e89eeff3cbcfc6a" diff --git a/pyproject.toml b/pyproject.toml index 9a17b7ab..b7b0b7b6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "gardenlinux" -version = "0.10.10" +version = "0.10.11" description = "Contains tools to work with the features directory of gardenlinux, for example deducting dependencies from feature sets or validating cnames" authors = ["Garden Linux Maintainers "] license = "Apache-2.0" @@ -8,7 +8,7 @@ readme = "README.md" packages = [{ include = "gardenlinux", from = "src" }] [tool.poetry.dependencies] -python = ">=3.13, <3.14" +python = ">=3.13, !=3.14.1" apt-repo = "^0.5" boto3 = "^1.42.10" click = "^8.3.1" diff --git a/src/gardenlinux/features/cname.py b/src/gardenlinux/features/cname.py index 27c8d064..712385ba 100644 --- a/src/gardenlinux/features/cname.py +++ b/src/gardenlinux/features/cname.py @@ -69,17 +69,46 @@ def __init__( commit_id_or_hash = None - re_match = re.match( - "([a-zA-Z0-9]+([\\_\\-][a-zA-Z0-9]+)*?)(-([a-z0-9]+)(-([a-z0-9.]+)-([a-z0-9]+))*)?$", - cname, + if version is not None: + # Support version values formatted as - + if commit_hash is None: + re_match = re.match("([a-z0-9.]+)(-([a-z0-9]+))?$", version) + assert re_match, f"Not a valid version {version}" + + commit_id_or_hash = re_match[3] + version = re_match[1] + else: + commit_id_or_hash = commit_hash + + re_object = re.compile( + "([a-zA-Z0-9]+([\\_\\-][a-zA-Z0-9]+)*?)(-([a-z0-9]+)(-([a-z0-9.]+)-([a-z0-9]+))*)?$" ) + re_match = re_object.match(cname) + + # Workaround Garden Linux canonical names without mandatory final commit hash + if ( + not re_match + and commit_id_or_hash + and re.match( + "([a-zA-Z0-9]+([\\_\\-][a-zA-Z0-9]+)*?)(-([a-z0-9]+)(-([a-z0-9.]+))*)?$", + cname, + ) + ): + re_match = re_object.match(f"{cname}-{commit_id_or_hash}") + assert re_match, f"Not a valid Garden Linux canonical name {cname}" if re_match.lastindex == 1: self._flavor = re_match[1] else: - commit_id_or_hash = re_match[7] + if commit_id_or_hash is None: + commit_id_or_hash = re_match[7] + elif re_match.group(7) is not None: + assert commit_id_or_hash.startswith(re_match[7]), ( + f"Mismatch between Garden Linux canonical name '{cname}' and given commit ID '{commit_id_or_hash}' detected" + ) + self._flavor = re_match[1] self._version = re_match[6] @@ -91,17 +120,13 @@ def __init__( if self._arch is None and arch is not None: self._arch = arch - if self._version is None and version is not None: - # Support version values formatted as - - if commit_hash is None: - re_match = re.match("([a-z0-9.]+)(-([a-z0-9]+))?$", version) - assert re_match, f"Not a valid version {version}" - - commit_id_or_hash = re_match[3] - self._version = re_match[1] - else: - commit_id_or_hash = commit_hash + if version is not None: + if self._version is None: self._version = version + else: + assert version == self._version, ( + f"Mismatch between Garden Linux canonical name '{cname}' and given version '{version}' detected" + ) if commit_id_or_hash is not None: self._commit_id = commit_id_or_hash[:8] @@ -312,21 +337,36 @@ def release_metadata_string(self) -> str: assert len(features["platform"]) < 2 "Only one platform is supported" + commit_hash = self.commit_hash + commit_id = self.commit_id elements = ",".join(features["element"]) flags = ",".join(features["flag"]) platform = features["platform"][0] platforms = ",".join(features["platform"]) platform_variant = self.platform_variant + version = self.version + + if commit_id is None: + commit_id = "" + + if commit_hash is None: + commit_hash = commit_id if platform_variant is None: platform_variant = "" + if version is None: + pretty_name = f"{GL_DISTRIBUTION_NAME} unsupported version" + version = "" + else: + pretty_name = f"{GL_DISTRIBUTION_NAME} {version}" + metadata = f""" ID={GL_RELEASE_ID} ID_LIKE=debian NAME="{GL_DISTRIBUTION_NAME}" -PRETTY_NAME="{GL_DISTRIBUTION_NAME} {self.version}" -IMAGE_VERSION={self.version} +PRETTY_NAME="{pretty_name}" +IMAGE_VERSION={version} VARIANT_ID="{self.flavor}-{self.arch}" HOME_URL="{GL_HOME_URL}" SUPPORT_URL="{GL_SUPPORT_URL}" @@ -338,9 +378,9 @@ def release_metadata_string(self) -> str: GARDENLINUX_FEATURES_FLAGS="{flags}" GARDENLINUX_PLATFORM="{platform}" GARDENLINUX_PLATFORM_VARIANT="{platform_variant}" -GARDENLINUX_VERSION="{self.version}" -GARDENLINUX_COMMIT_ID="{self.commit_id}" -GARDENLINUX_COMMIT_ID_LONG="{self.commit_hash}" +GARDENLINUX_VERSION="{version}" +GARDENLINUX_COMMIT_ID="{commit_id}" +GARDENLINUX_COMMIT_ID_LONG="{commit_hash}" """.strip() return metadata @@ -365,7 +405,7 @@ def version_and_commit_id(self) -> Optional[str]: :since: 0.7.0 """ - if self._commit_id is None: + if self._version is None or self._commit_id is None: return None return f"{self._version}-{self._commit_id}" diff --git a/src/gardenlinux/s3/__main__.py b/src/gardenlinux/s3/__main__.py index 3d82b972..1c2da70d 100644 --- a/src/gardenlinux/s3/__main__.py +++ b/src/gardenlinux/s3/__main__.py @@ -9,11 +9,6 @@ from .s3_artifacts import S3Artifacts -_ARGS_ACTION_ALLOWED = [ - "download-artifacts-from-bucket", - "upload-artifacts-to-bucket", -] - def main() -> None: """ @@ -25,11 +20,16 @@ def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--bucket", dest="bucket") - parser.add_argument("--cname", required=False, dest="cname") parser.add_argument("--path", required=False, dest="path") parser.add_argument("--dry-run", action="store_true") - parser.add_argument("action", nargs="?", choices=_ARGS_ACTION_ALLOWED) + subparsers = parser.add_subparsers(dest="action") + + download_parser = subparsers.add_parser("download-artifacts-from-bucket") + download_parser.add_argument("--cname", required=False, dest="cname") + + upload_parser = subparsers.add_parser("upload-artifacts-to-bucket") + upload_parser.add_argument("--artifact-name", required=False, dest="artifact_name") args = parser.parse_args() @@ -37,5 +37,5 @@ def main() -> None: S3Artifacts(args.bucket).download_to_directory(args.cname, args.path) elif args.action == "upload-artifacts-to-bucket": S3Artifacts(args.bucket).upload_from_directory( - args.cname, args.path, dry_run=args.dry_run + args.artifact_name, args.path, dry_run=args.dry_run ) diff --git a/src/gardenlinux/s3/s3_artifacts.py b/src/gardenlinux/s3/s3_artifacts.py index 30a181a9..a777067f 100644 --- a/src/gardenlinux/s3/s3_artifacts.py +++ b/src/gardenlinux/s3/s3_artifacts.py @@ -98,7 +98,7 @@ def download_to_directory( def upload_from_directory( self, - cname: str, + base_name: str, artifacts_dir: PathLike[str] | str, delete_before_push: bool = False, dry_run: bool = False, @@ -106,7 +106,7 @@ def upload_from_directory( """ Pushes S3 artifacts to the underlying bucket. - :param cname: Canonical name of the GardenLinux S3 artifacts + :param base_name: Base name of the GardenLinux S3 artifacts :param artifacts_dir: Path of the image artifacts :param delete_before_push: True to delete objects before upload @@ -115,28 +115,22 @@ def upload_from_directory( artifacts_dir = Path(artifacts_dir) - cname_object = CName(cname) - if not artifacts_dir.is_dir(): raise RuntimeError(f"Artifacts directory given is invalid: {artifacts_dir}") - release_file = artifacts_dir.joinpath(f"{cname}.release") - release_timestamp = stat(release_file).st_ctime - - cname_object.load_from_release_file(release_file) + release_file = artifacts_dir.joinpath(f"{base_name}.release") - if cname_object.arch is None: - raise RuntimeError( - "Architecture could not be determined from GardenLinux canonical name or release file" - ) + cname_object = CName.new_from_release_file(release_file) if cname_object.version_and_commit_id is None: raise RuntimeError( - "Version information could not be determined from GardenLinux canonical name or release file" + "Version information could not be determined from release file" ) + arch = cname_object.arch feature_list = cname_object.feature_set - requirements_file = artifacts_dir.joinpath(f"{cname}.requirements") + release_timestamp = stat(release_file).st_ctime + requirements_file = artifacts_dir.joinpath(f"{base_name}.requirements") require_uefi = None secureboot = None @@ -144,6 +138,9 @@ def upload_from_directory( requirements_config = ConfigParser(allow_unnamed_section=True) requirements_config.read(requirements_file) + if requirements_config.has_option(UNNAMED_SECTION, "arch"): + arch = requirements_config.get(UNNAMED_SECTION, "arch") + if requirements_config.has_option(UNNAMED_SECTION, "uefi"): require_uefi = requirements_config.getboolean(UNNAMED_SECTION, "uefi") @@ -152,16 +149,25 @@ def upload_from_directory( UNNAMED_SECTION, "secureboot" ) + if arch is None: + raise RuntimeError( + "Architecture could not be determined from release or requirements file" + ) + if require_uefi is None: require_uefi = "_usi" in feature_list if secureboot is None: secureboot = "_trustedboot" in feature_list - commit_hash = cname_object.commit_hash + # RegEx for S3 supported characters + re_object = re.compile("[^a-zA-Z0-9\\s+\\-=.\\_:/@]") + + arch = re_object.sub("+", arch) + commit_id_or_hash = cname_object.commit_hash - if commit_hash is None: - commit_hash = "" + if commit_id_or_hash is None: + commit_id_or_hash = cname_object.commit_id version_epoch = str(cname_object.version_epoch) @@ -170,18 +176,18 @@ def upload_from_directory( metadata = { "platform": cname_object.feature_set_platform, - "architecture": cname_object.arch, + "architecture": arch, "base_image": None, - "build_committish": commit_hash, + "build_committish": commit_id_or_hash, "build_timestamp": datetime.fromtimestamp(release_timestamp).isoformat(), - "gardenlinux_epoch": {version_epoch}, + "gardenlinux_epoch": version_epoch, "logs": None, "modifiers": cname_object.feature_set, "require_uefi": require_uefi, "secureboot": secureboot, "published_image_metadata": None, "s3_bucket": self._bucket.name, - "s3_key": f"meta/singles/{cname}", + "s3_key": f"meta/singles/{base_name}", "test_result": None, "version": cname_object.version, "paths": [], @@ -192,39 +198,34 @@ def upload_from_directory( if platform_variant is not None: metadata["platform_variant"] = platform_variant - re_object = re.compile("[^a-zA-Z0-9\\s+\\-=.\\_:/@]") + base_name_length = len(base_name) for artifact in artifacts_dir.iterdir(): - if not artifact.match(f"{cname}*"): + if not artifact.match(f"{base_name}*"): continue - if not artifact.name.startswith(cname): - raise RuntimeError( - f"Artifact name '{artifact.name}' does not start with cname '{cname}'" - ) - - s3_key = f"objects/{cname}/{artifact.name}" + s3_key = f"objects/{base_name}/{artifact.name}" with artifact.open("rb") as fp: md5sum = file_digest(fp, "md5").hexdigest() sha256sum = file_digest(fp, "sha256").hexdigest() - suffix = artifact.name[len(cname) :] + suffixes = "".join(artifact.name)[1 + base_name_length :] artifact_metadata = { "name": artifact.name, "s3_bucket_name": self._bucket.name, "s3_key": s3_key, - "suffix": suffix, + "suffix": re_object.sub("+", suffixes), "md5sum": md5sum, "sha256sum": sha256sum, } s3_tags = { - "architecture": re_object.sub("+", cname_object.arch), + "architecture": arch, "platform": re_object.sub("+", cname_object.platform), "version": re_object.sub("+", cname_object.version), # type: ignore[arg-type] - "committish": commit_hash, + "committish": commit_id_or_hash, "md5sum": md5sum, "sha256sum": sha256sum, } @@ -246,7 +247,7 @@ def upload_from_directory( else: if delete_before_push: self._bucket.delete_objects( - Delete={"Objects": [{"Key": f"meta/singles/{cname}"}]} + Delete={"Objects": [{"Key": f"meta/singles/{base_name}"}]} ) with TemporaryFile(mode="wb+") as fp: @@ -254,5 +255,7 @@ def upload_from_directory( fp.seek(0) self._bucket.upload_fileobj( - fp, f"meta/singles/{cname}", ExtraArgs={"ContentType": "text/yaml"} + fp, + f"meta/singles/{base_name}", + ExtraArgs={"ContentType": "text/yaml"}, ) diff --git a/tests/s3/conftest.py b/tests/s3/conftest.py index 50fa01b4..792843cd 100644 --- a/tests/s3/conftest.py +++ b/tests/s3/conftest.py @@ -27,12 +27,12 @@ def make_cname( flavor: str = "container", arch: str = "amd64", version: str = "1234.1", - commit: str = "abc123", + commit: str = "abc123long", ) -> str: """ Helper function to build cname. Can be used to customized the cname. """ - return f"{flavor}-{arch}-{version}-{commit}" + return f"{flavor}-{arch}-{version}-{commit[:8]}" # Helpers to compute digests for fake files diff --git a/tests/s3/constants.py b/tests/s3/constants.py new file mode 100644 index 00000000..2f127e00 --- /dev/null +++ b/tests/s3/constants.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- + +RELEASE_DATA = """ +GARDENLINUX_CNAME="container-amd64-1234.1" +GARDENLINUX_VERSION=1234.1 +GARDENLINUX_COMMIT_ID="abc123lo" +GARDENLINUX_COMMIT_ID_LONG="abc123long" +GARDENLINUX_FEATURES="_usi,_trustedboot" +GARDENLINUX_FEATURES_ELEMENTS= +GARDENLINUX_FEATURES_FLAGS="_usi,_trustedboot" +GARDENLINUX_FEATURES_PLATFORMS="container" +""" + +S3_METADATA = """ +platform: container +architecture: amd64 +base_image: null +build_committish: abc123lo +build_timestamp: '{build_timestamp}' +gardenlinux_epoch: '1234' +logs: null +modifiers: _usi,_trustedboot +require_uefi: true +secureboot: true +published_image_metadata: null +s3_bucket: test-bucket +s3_key: meta/singles/container-amd64-1234.1-abc123lo +test_result: null +version: '1234.1' +paths: +- name: container-amd64-1234.1-abc123lo.release + s3_bucket_name: test-bucket + s3_key: objects/container-amd64-1234.1-abc123lo/container-amd64-1234.1-abc123lo.release + suffix: release + md5sum: {md5sum} + sha256sum: {sha256sum} +""".strip() diff --git a/tests/s3/test_main.py b/tests/s3/test_main.py index e1b44c69..a939f0ca 100644 --- a/tests/s3/test_main.py +++ b/tests/s3/test_main.py @@ -1,3 +1,4 @@ +import re import sys from typing import Any, Dict, List from unittest.mock import MagicMock, patch @@ -6,6 +7,9 @@ import gardenlinux.s3.__main__ as s3m +from .conftest import S3Env +from .constants import RELEASE_DATA, S3_METADATA + @pytest.mark.parametrize( "argv, expected_method, expected_args, expected_kwargs", @@ -15,11 +19,11 @@ "__main__.py", "--bucket", "test-bucket", - "--cname", - "test-cname", "--path", "some/path", "download-artifacts-from-bucket", + "--cname", + "test-cname", ], "download_to_directory", ["test-cname", "some/path"], @@ -30,11 +34,11 @@ "__main__.py", "--bucket", "test-bucket", - "--cname", - "test-cname", "--path", "some/path", "upload-artifacts-to-bucket", + "--artifact-name", + "test-cname", ], "upload_from_directory", ["test-cname", "some/path"], @@ -48,14 +52,58 @@ def test_main_calls_correct_artifacts( expected_args: List[Any], expected_kwargs: Dict[str, Any], ) -> None: - with patch.object(sys, "argv", argv): - with patch.object(s3m, "S3Artifacts") as mock_s3_cls: - mock_instance = MagicMock() - mock_s3_cls.return_value = mock_instance + with ( + patch.object(sys, "argv", argv), + patch.object(s3m, "S3Artifacts") as mock_s3_cls, + ): + mock_instance = MagicMock() + mock_s3_cls.return_value = mock_instance + + s3m.main() + + method = getattr(mock_instance, expected_method) + method.assert_called_once_with(*expected_args, **expected_kwargs) + + mock_s3_cls.assert_called_once_with("test-bucket") + + +def test_main_with_expected_result( + s3_setup: S3Env, capsys: pytest.CaptureFixture[str] +) -> None: + env = s3_setup + + # Arrange + with patch.object( + sys, + "argv", + [ + "__main__.py", + "--dry-run", + "--bucket", + env.bucket_name, + "--path", + str(env.tmp_path), + "upload-artifacts-to-bucket", + "--artifact-name", + env.cname, + ], + ): + release_path = env.tmp_path / f"{env.cname}.release" + release_path.write_text(RELEASE_DATA) + + s3m.main() + + result = capsys.readouterr().out.strip() - s3m.main() + result = re.sub( + "^(.*)build_timestamp\\: '.+'$", + "\\1build_timestamp: '{build_timestamp}'", + result, + flags=re.M, + ) - method = getattr(mock_instance, expected_method) - method.assert_called_once_with(*expected_args, **expected_kwargs) + result = re.sub( + "^(.*)(md5sum|sha256sum)\\: .+$", "\\1\\2: {\\2}", result, flags=re.M + ) - mock_s3_cls.assert_called_once_with("test-bucket") + assert result == S3_METADATA diff --git a/tests/s3/test_s3_artifacts.py b/tests/s3/test_s3_artifacts.py index 3c48d282..1fb4f867 100644 --- a/tests/s3/test_s3_artifacts.py +++ b/tests/s3/test_s3_artifacts.py @@ -9,17 +9,7 @@ from gardenlinux.s3.s3_artifacts import S3Artifacts from .conftest import S3Env - -RELEASE_DATA = """ -GARDENLINUX_CNAME="container-amd64-1234.1-abc123" -GARDENLINUX_VERSION=1234.1 -GARDENLINUX_COMMIT_ID="abc123" -GARDENLINUX_COMMIT_ID_LONG="abc123long" -GARDENLINUX_FEATURES="_usi,_trustedboot" -GARDENLINUX_FEATURES_ELEMENTS= -GARDENLINUX_FEATURES_FLAGS="_usi,_trustedboot" -GARDENLINUX_FEATURES_PLATFORMS="container" -""" +from .constants import RELEASE_DATA def test_s3artifacts_init_success(s3_setup: S3Env) -> None: @@ -174,9 +164,9 @@ def test_upload_from_directory_invalid_dir_raises(s3_setup: S3Env) -> None: artifacts.upload_from_directory(env.cname, "/invalid/path") -def test_upload_from_directory_version_mismatch_raises(s3_setup: S3Env) -> None: +def test_upload_from_directory_version_mismatch(s3_setup: S3Env) -> None: """ - RuntimeError if version in release file does not match cname. + Validate that the release file may contain a different version not matching the artifact name. """ # Arrange env = s3_setup @@ -186,8 +176,7 @@ def test_upload_from_directory_version_mismatch_raises(s3_setup: S3Env) -> None: artifacts = S3Artifacts(env.bucket_name) # Act / Assert - with pytest.raises(RuntimeError, match="failed consistency check"): - artifacts.upload_from_directory(env.cname, env.tmp_path) + artifacts.upload_from_directory(env.cname, env.tmp_path) def test_upload_from_directory_succeeds_because_of_release_file( @@ -226,8 +215,10 @@ def test_upload_from_directory_invalid_artifact_name(s3_setup: S3Env) -> None: assert len(list(bucket.objects.filter(Prefix=f"meta/singles/{env.cname}"))) == 1 -def test_upload_from_directory_commit_mismatch_raises(s3_setup: S3Env) -> None: - """Raise RuntimeError when commit ID is not matching with cname.""" +def test_upload_from_directory_commit_mismatch(s3_setup: S3Env) -> None: + """ + Validate that the release file may contain a different commit hash not matching the artifact name. + """ # Arrange env = s3_setup release_path = env.tmp_path / f"{env.cname}.release" @@ -236,8 +227,7 @@ def test_upload_from_directory_commit_mismatch_raises(s3_setup: S3Env) -> None: artifacts = S3Artifacts(env.bucket_name) # Act / Assert - with pytest.raises(RuntimeError, match="failed consistency check"): - artifacts.upload_from_directory(env.cname, env.tmp_path) + artifacts.upload_from_directory(env.cname, env.tmp_path) def test_upload_from_directory_with_platform_variant(s3_setup: S3Env) -> None: