Add GitHub Packages (GHCR) support with full container image download

README.rst

@@ -193,6 +193,16 @@ keys:
                 under which the release's assets will be saved.  If this is not
                 specified, no release assets will be downloaded.
 
+            ``packages``
+                A template string that will be instantiated for each package
+                version (from GitHub Container Registry) to produce the path
+                for the directory (relative to the current working directory)
+                under which the package version metadata and manifest will be
+                saved as JSON files.  For container packages, this includes
+                both the package metadata (tags, version info) and the
+                OCI/Docker manifest.  If this is not specified, no package
+                data will be saved.
+
         ``workflows``
             A specification of the workflows for which to retrieve assets.
             This can be either a list of workflow basenames, including the file
@@ -221,6 +231,33 @@ keys:
             When ``workflows`` is not specified, assets are retrieved for all
             workflows in the repository.
 
+        ``packages``
+            A specification of the packages for which to retrieve metadata
+            and manifests.  This can be either a list of package names or a
+            mapping containing the following fields:
+
+            ``include``
+                A list of packages to retrieve, given as either names or
+                (when ``regex`` is true) `Python regular expressions`_ to
+                match against package names.  If ``include`` is omitted, it
+                defaults to including all packages.
+
+            ``exclude``
+                A list of packages to not retrieve, given as either names or
+                (when ``regex`` is true) `Python regular expressions`_ to
+                match against package names.  If ``exclude`` is omitted, no
+                packages are excluded.  Packages that match both ``include``
+                and ``exclude`` are excluded.
+
+            ``regex``
+                A boolean.  If true (default false), the elements of the
+                ``include`` and ``exclude`` fields are treated as `Python
+                regular expressions`_ that are matched (unanchored) against
+                package names; if false, they are used as exact names
+
+            When ``packages`` is not specified, metadata and manifests are
+            retrieved for all packages in the repository.
+
     ``travis``
         Configuration for retrieving logs from Travis-CI.com.  Subfield:
 
@@ -399,10 +436,13 @@ A sample config file:
           logs: '{build_prefix}/{wf_name}/{number}/logs/'
           artifacts: '{build_prefix}/{wf_name}/{number}/artifacts/'
           releases: '{path_prefix}/{release_tag}/'
+          packages: '{year}//{month}/{ci}/packages/{package_name}/{tag}/'
         workflows:
           - test_crippled.yml
           - test_extensions.yml
           - test_macos.yml
+        packages:
+          - tinuous-inception
       travis:
         paths:
           logs: '{build_prefix}/{number}/{job}.txt'
@@ -432,9 +472,10 @@ A sample config file:
 Path Templates
 --------------
 
-The path at which assets for a given workflow run, build job, or release are
-saved is determined by instantiating the appropriate path template string given
-in the configuration file for the corresponding CI system.  A template string
+The path at which assets for a given workflow run, build job, release, or
+package version are saved is determined by instantiating the appropriate path
+template string given in the configuration file for the corresponding CI system.
+A template string
 is a filepath containing placeholders of the form ``{field}``, where the
 available placeholders are:
 
@@ -468,7 +509,7 @@ Placeholder             Definition
                         ``appveyor``, or ``circleci``)
 ``{type}``              The event type that triggered the build (``cron``,
                         ``manual``, ``pr``, or ``push``), or ``release`` for
-                        GitHub releases
+                        GitHub releases, or ``package`` for GitHub Packages
 ``{type_id}``           Further information on the triggering event; for
                         ``cron`` and ``manual``, this is a timestamp for the
                         start of the build; for ``pr``, this is the number of
@@ -519,6 +560,18 @@ Placeholder             Definition
 ``{step_name}``         *(CircleCI only)* The escaped [1]_ name of the step [2]_
 ``{index}``             *(CircleCI only)* The index of the parallel container
                         that the step ran on [2]_
+``{package_name}``      *(``packages`` path only)* The escaped [1]_ name of the
+                        package [3]_
+``{package_type}``      *(``packages`` path only)* The type of the package
+                        (e.g., ``container``) [3]_
+``{version_id}``        *(``packages`` path only)* The unique ID of the package
+                        version [3]_
+``{version_name}``      *(``packages`` path only)* The escaped [1]_ name/digest
+                        of the package version [3]_
+``{tag}``               *(``packages`` path only)* The primary tag of the
+                        package version, or version_name if no tags [3]_
+``{tags}``              *(``packages`` path only)* Comma-separated list of all
+                        tags for the package version [3]_
 ======================  =======================================================
 
 .. _datetime: https://docs.python.org/3/library/datetime.html#datetime-objects
@@ -529,7 +582,9 @@ Placeholder             Definition
        replacing each whitespace character with a space.
 
 .. [2] These placeholders are only available for ``path`` and
-       ``artifacts_path``, not ``releases_path``
+       ``artifacts_path``, not ``releases_path`` or ``packages``
+
+.. [3] These placeholders are only available for ``packages`` path
 
 A placeholder's value may be truncated to the first ``n`` characters by writing
 ``{placeholder[:n]}``, e.g., ``{commit[:7]}``.

src/tinuous/__main__.py

@@ -104,8 +104,13 @@ def fetch(config_file: str, state_path: Optional[str], sanitize_secrets: bool) -
     logs_added = 0
     artifacts_added = 0
     relassets_added = 0
+    packages_added = 0
     for name, cicfg in cfg.ci.items():
-        if not cicfg.gets_builds() and not cicfg.gets_releases():
+        if (
+            not cicfg.gets_builds()
+            and not cicfg.gets_releases()
+            and not cicfg.gets_packages()
+        ):
             log.info("No paths configured for %s; skipping", name)
             continue
         log.info("Fetching resources from %s", name)
@@ -143,23 +148,37 @@ def fetch(config_file: str, state_path: Optional[str], sanitize_secrets: bool) -
             assert isinstance(cicfg.paths, GHPathsDict)
             releases_path = cicfg.paths.releases
             assert releases_path is not None
-            for asset in ci.get_release_assets():
-                path = asset.expand_path(releases_path, cfg.vars)
+            for rel_asset in ci.get_release_assets():
+                path = rel_asset.expand_path(releases_path, cfg.vars)
                 if cfg.datalad.enabled:
                     ensure_datalad(ds, path, cfg.datalad.cfg_proc)
-                paths = asset.download(Path(path))
+                paths = rel_asset.download(Path(path))
                 relassets_added += len(paths)
+        if cicfg.gets_packages():
+            assert isinstance(ci, GitHubActions)
+            assert isinstance(cicfg.paths, GHPathsDict)
+            packages_path = cicfg.paths.packages
+            assert packages_path is not None
+            for pkg_asset in ci.get_package_assets():
+                path = pkg_asset.expand_path(packages_path, cfg.vars)
+                if cfg.datalad.enabled:
+                    ensure_datalad(ds, path, cfg.datalad.cfg_proc)
+                paths = pkg_asset.download(Path(path))
+                packages_added += len(paths)
         statefile.set_since(name, ci.new_since())
     log.info("%d logs downloaded", logs_added)
     log.info("%d artifacts downloaded", artifacts_added)
     log.info("%d release assets downloaded", relassets_added)
+    log.info("%d package versions saved", packages_added)
     if cfg.datalad.enabled:
-        if logs_added or artifacts_added or relassets_added:
+        if logs_added or artifacts_added or relassets_added or packages_added:
             msg = f"[tinuous] {logs_added} logs added"
             if artifacts_added:
                 msg += f", {artifacts_added} artifacts added"
             if relassets_added:
                 msg += f", {relassets_added} release assets added"
+            if packages_added:
+                msg += f", {packages_added} package versions added"
             msg += f"\n\nProduced by tinuous {__version__}"
             ds.save(recursive=True, message=msg)
         elif statefile.modified:

src/tinuous/config.py

@@ -25,10 +25,14 @@ def gets_builds(self) -> bool:
     def gets_releases(self) -> bool:
         return False
 
+    def gets_packages(self) -> bool:
+        return False
+
 
 class GHPathsDict(PathsDict):
     artifacts: Optional[str] = None
     releases: Optional[str] = None
+    packages: Optional[str] = None
 
     def gets_builds(self) -> bool:
         # <https://github.com/pydantic/pydantic/issues/8052>
@@ -37,6 +41,9 @@ def gets_builds(self) -> bool:
     def gets_releases(self) -> bool:
         return self.releases is not None
 
+    def gets_packages(self) -> bool:
+        return self.packages is not None
+
 
 class CCIPathsDict(PathsDict):
     artifacts: Optional[str] = None
@@ -70,10 +77,14 @@ def gets_builds(self) -> bool:
     def gets_releases(self) -> bool:
         return self.paths.gets_releases()
 
+    def gets_packages(self) -> bool:
+        return self.paths.gets_packages()
+
 
 class GitHubConfig(CIConfig):
     paths: GHPathsDict = Field(default_factory=GHPathsDict)
     workflows: GHWorkflowSpec = Field(default_factory=GHWorkflowSpec)
+    packages: WorkflowSpec = Field(default_factory=WorkflowSpec)
 
     @field_validator("workflows", mode="before")
     @classmethod
@@ -83,6 +94,14 @@ def _workflow_list(cls, v: Any) -> Any:
         else:
             return v
 
+    @field_validator("packages", mode="before")
+    @classmethod
+    def _package_list(cls, v: Any) -> Any:
+        if isinstance(v, list):
+            return {"include": v}
+        else:
+            return v
+
     @staticmethod
     def get_auth_tokens() -> dict[str, str]:
         return GitHubActions.get_auth_tokens()
@@ -100,6 +119,7 @@ def get_system(
             until=until,
             token=tokens["github"],
             workflow_spec=self.workflows,
+            package_spec=self.packages,
         )