prathish D
6cf5d80be8
* feat: embed build metadata in generated dotslash files - Add build metadata collection from GitHub Actions environment - Include source config path, CI information, and generation timestamp - Add CI job URL for traceability back to the generating workflow - Support disabling metadata via include-build-metadata input - Update action.yml to expose new input parameter - Add comprehensive documentation with examples Fixes #2 * feat: pass include-build-metadata as argument instead of environment variable - Remove environment variable approach from action.yml - Pass include-build-metadata as command-line argument - Update argument parsing to handle string values ('true'/'false') - Maintain backward compatibility with environment variable fallback Addresses review feedback from @bigfootjon in PR #13
544 lines
18 KiB
Python
Executable File
544 lines
18 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
|
#
|
|
# This source code is licensed under the MIT license found in the
|
|
# LICENSE file in the root directory of this source tree.
|
|
|
|
import argparse
|
|
import json
|
|
import logging
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import datetime
|
|
|
|
from functools import cache
|
|
from typing import Any, Dict, Literal, Optional, Tuple, Union
|
|
|
|
|
|
HashAlgorithm = Literal["blake3", "sha256"]
|
|
ArtifactFormat = Literal["gz", "tar", "tar.gz", "tar.zst", "zst", "tar.xz", "xz", "zip"]
|
|
|
|
# Recognized properties in the JSON config.
|
|
OUTPUTS_PARAM = "outputs"
|
|
EXCLUDE_HTTP_PROVIDER_PARAM = "exclude-http-provider"
|
|
EXCLUDE_GITHUB_PROVIDER_PARAM = "exclude-github-release-provider"
|
|
|
|
|
|
def collect_build_metadata(config_path: str) -> Dict[str, Any]:
|
|
"""Collect build metadata from GitHub Actions environment variables."""
|
|
metadata = {}
|
|
|
|
# Source file information
|
|
if config_path:
|
|
metadata["source_config"] = config_path
|
|
|
|
# CI/GitHub Actions information
|
|
github_env_vars = [
|
|
("github_repository", "GITHUB_REPOSITORY"),
|
|
("github_ref", "GITHUB_REF"),
|
|
("github_sha", "GITHUB_SHA"),
|
|
("github_run_id", "GITHUB_RUN_ID"),
|
|
("github_run_number", "GITHUB_RUN_NUMBER"),
|
|
("github_workflow", "GITHUB_WORKFLOW"),
|
|
("github_actor", "GITHUB_ACTOR"),
|
|
("github_event_name", "GITHUB_EVENT_NAME"),
|
|
("github_server_url", "GITHUB_SERVER_URL"),
|
|
]
|
|
|
|
ci_info = {}
|
|
for key, env_var in github_env_vars:
|
|
value = os.getenv(env_var)
|
|
if value:
|
|
ci_info[key] = value
|
|
|
|
if ci_info:
|
|
metadata["ci"] = ci_info
|
|
|
|
# Build timestamp
|
|
metadata["generated_at"] = datetime.datetime.utcnow().isoformat() + "Z"
|
|
|
|
# Add link to the CI job if we have the necessary info
|
|
if all(os.getenv(var) for var in ["GITHUB_SERVER_URL", "GITHUB_REPOSITORY", "GITHUB_RUN_ID"]):
|
|
job_url = f"{os.getenv('GITHUB_SERVER_URL')}/{os.getenv('GITHUB_REPOSITORY')}/actions/runs/{os.getenv('GITHUB_RUN_ID')}"
|
|
metadata["ci_job_url"] = job_url
|
|
|
|
return metadata
|
|
|
|
|
|
def main() -> None:
|
|
exit_code = _main()
|
|
sys.exit(exit_code)
|
|
|
|
|
|
def _main() -> int:
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format="%(asctime)s [%(levelname)s] %(message)s",
|
|
handlers=[logging.StreamHandler()],
|
|
)
|
|
|
|
args = parse_args()
|
|
repo: str = args.repo
|
|
if not repo:
|
|
raise ValueError(
|
|
"no repo specified: must specify --repo or set the GITHUB_REPOSITORY environment variable"
|
|
)
|
|
|
|
output_folder = args.output
|
|
if not output_folder:
|
|
output_folder = tempfile.mkdtemp(prefix=f"{repo.replace('/', '_')}_dotslash")
|
|
logging.info(f"DotSlash files will be written to `{output_folder}")
|
|
|
|
tag = args.tag
|
|
github_server_url = args.server
|
|
api_server_url = args.api_server
|
|
gh_repo_arg = f"{github_server_url}/{repo}"
|
|
|
|
if args.local_config:
|
|
print(args.config)
|
|
with open(args.config, "r") as f:
|
|
config = json.load(f)
|
|
else:
|
|
config = get_config(
|
|
path_to_config=args.config,
|
|
config_ref=args.config_ref,
|
|
github_repository=repo,
|
|
api_url=api_server_url,
|
|
)
|
|
if not isinstance(config, dict):
|
|
logging.error(f"config should be a dict, but was:")
|
|
logging.error(json.dumps(config, indent=2))
|
|
return 1
|
|
|
|
outputs = config.get(OUTPUTS_PARAM)
|
|
if not outputs:
|
|
logging.error(f"no {OUTPUTS_PARAM} specified in config:")
|
|
logging.error(json.dumps(config, indent=2))
|
|
return 1
|
|
|
|
exclude_http_provider = config.get(EXCLUDE_HTTP_PROVIDER_PARAM, False)
|
|
if not isinstance(exclude_http_provider, bool):
|
|
logging.error(
|
|
f'"{EXCLUDE_HTTP_PROVIDER_PARAM}" field must be a boolean, but was `{exclude_http_provider}`'
|
|
)
|
|
return 1
|
|
exclude_github_release_provider = config.get(EXCLUDE_GITHUB_PROVIDER_PARAM, False)
|
|
if not isinstance(exclude_github_release_provider, bool):
|
|
logging.error(
|
|
f'"{EXCLUDE_GITHUB_PROVIDER_PARAM}" field must be a boolean, but was `{exclude_github_release_provider}`'
|
|
)
|
|
return 1
|
|
|
|
logging.info("using config:")
|
|
logging.info(json.dumps(config, indent=2))
|
|
|
|
# Collect build metadata from the environment
|
|
build_metadata = None
|
|
|
|
# Convert string argument to boolean
|
|
include_metadata_str = str(args.include_build_metadata).lower()
|
|
include_metadata = include_metadata_str not in ["false", "0", "no"] and not args.exclude_build_metadata
|
|
|
|
# Also check for environment variable (for Docker action support)
|
|
include_metadata_env = os.getenv("INCLUDE_BUILD_METADATA", "true").lower()
|
|
if include_metadata_env in ["false", "0", "no"]:
|
|
include_metadata = False
|
|
|
|
if include_metadata:
|
|
build_metadata = collect_build_metadata(args.config)
|
|
logging.info("build metadata:")
|
|
logging.info(json.dumps(build_metadata, indent=2))
|
|
|
|
name_to_asset = get_release_assets(tag=tag, github_repository=repo)
|
|
logging.info(json.dumps(name_to_asset, indent=2))
|
|
|
|
for output_filename, output_config in outputs.items():
|
|
platform_entries = map_platforms(output_config, name_to_asset)
|
|
if not isinstance(platform_entries, dict):
|
|
logging.error(f"failed with error type {platform_entries}")
|
|
return 1
|
|
|
|
logging.info(json.dumps(platform_entries, indent=2))
|
|
|
|
manifest_file_contents = generate_manifest_file(
|
|
output_filename,
|
|
gh_repo_arg,
|
|
tag,
|
|
platform_entries,
|
|
include_http_provider=not exclude_http_provider,
|
|
include_github_release_provider=not exclude_github_release_provider,
|
|
build_metadata=build_metadata,
|
|
)
|
|
logging.info(manifest_file_contents)
|
|
|
|
output_file = os.path.join(output_folder, output_filename)
|
|
with open(output_file, "w") as f:
|
|
f.write(manifest_file_contents)
|
|
|
|
# `chmod +x` if not on Windows.
|
|
if not sys.platform.startswith('win'):
|
|
fd = f.fileno()
|
|
os.fchmod(fd, 0o755)
|
|
logging.info(f"wrote manifest to {output_file}")
|
|
|
|
if args.upload:
|
|
# Upload manifest to release, but do not clobber. Note that this may
|
|
# fail if this action has been called more than once for the same config.
|
|
subprocess.run(
|
|
[
|
|
"gh",
|
|
"release",
|
|
"upload",
|
|
tag,
|
|
output_file,
|
|
"--repo",
|
|
gh_repo_arg,
|
|
],
|
|
check=True
|
|
)
|
|
|
|
return 0
|
|
|
|
|
|
def generate_manifest_file(
|
|
name: str,
|
|
gh_repo_arg: str,
|
|
tag: str,
|
|
platform_entries,
|
|
include_http_provider: bool,
|
|
include_github_release_provider: bool,
|
|
build_metadata: Optional[Dict[str, Any]] = None,
|
|
) -> str:
|
|
platforms = {}
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
for platform_name, platform_entry in platform_entries.items():
|
|
asset, platform_config = platform_entry
|
|
hash_algo = platform_config.get("hash", "blake3")
|
|
size = asset.get("size")
|
|
if size is None:
|
|
logging.error(f"missing 'size' field in asset: {asset}")
|
|
return 1
|
|
|
|
asset_name = asset.get("name")
|
|
if asset_name is None:
|
|
logging.error(f"missing 'name' field in asset: {asset}")
|
|
return 1
|
|
|
|
path = platform_config.get("path")
|
|
if not path:
|
|
logging.error(f"missing `path` field in asset: {asset}")
|
|
return 1
|
|
|
|
if "format" in platform_config:
|
|
# If the user is knowingly not using any sort of compression,
|
|
# then `"format": null` must be explicitly specified in the JSON.
|
|
asset_format = platform_config["format"]
|
|
else:
|
|
asset_format = guess_artifact_format_from_asset_name(asset_name)
|
|
if not asset_format:
|
|
logging.error(
|
|
f'"format" could not be inferred from asset name: {asset_name} in {asset}, must specify explicitly'
|
|
)
|
|
return 1
|
|
|
|
hash_hex = compute_hash(
|
|
gh_repo_arg, temp_dir, tag, asset_name, hash_algo, size
|
|
)
|
|
|
|
providers = []
|
|
if include_http_provider:
|
|
providers.append(
|
|
{
|
|
"url": asset["url"],
|
|
}
|
|
)
|
|
if include_github_release_provider:
|
|
providers.append(
|
|
{
|
|
"type": "github-release",
|
|
"repo": gh_repo_arg,
|
|
"tag": tag,
|
|
"name": asset_name,
|
|
}
|
|
)
|
|
|
|
artifact_entry = {
|
|
"size": size,
|
|
"hash": hash_algo,
|
|
"digest": hash_hex,
|
|
"format": asset_format,
|
|
"path": path,
|
|
"providers": providers,
|
|
}
|
|
|
|
# If `"format": null` was specified, there should not be a "format"
|
|
# field in the arifact entry.
|
|
if not asset_format:
|
|
del artifact_entry["format"]
|
|
|
|
platforms[platform_name] = artifact_entry
|
|
|
|
manifest = {
|
|
"name": name,
|
|
"platforms": platforms,
|
|
}
|
|
|
|
# Add build metadata if available
|
|
if build_metadata:
|
|
manifest["build_metadata"] = build_metadata
|
|
|
|
return f"""#!/usr/bin/env dotslash
|
|
|
|
{json.dumps(manifest, indent=2)}
|
|
"""
|
|
|
|
|
|
def map_platforms(
|
|
config, name_to_asset: Dict[str, Any]
|
|
) -> Union[
|
|
Dict[str, Tuple[Any, Any]],
|
|
Literal["BothNameAndRegex", "NeitherNameNorRegex", "NoMatchForAsset", "ParseError"],
|
|
]:
|
|
"""Attempts to take every platform specified in the config and return a map
|
|
of platform names to their corresponding asset information. If successful,
|
|
each value in the dict will be a tuple of (asset, platform_config).
|
|
|
|
Note that it is possible that not all assets have been uploaded yet, in
|
|
which case "NoMatchForAsset" will be returned.
|
|
"""
|
|
platforms = config.get("platforms")
|
|
if platforms is None:
|
|
logging.error("'platforms' field missing from config: {config}")
|
|
return "ParseError"
|
|
|
|
platform_entries = {}
|
|
for platform, platform_config in platforms.items():
|
|
name = platform_config.get("name")
|
|
name_regex = platform_config.get("regex")
|
|
if name and name_regex:
|
|
logging.error(
|
|
f"only one of 'name' and 'regex' should be specified for {platform}"
|
|
)
|
|
return "BothNameAndRegex"
|
|
elif not name and not name_regex:
|
|
logging.error(
|
|
f"exactly one of 'name' and 'regex' should be specified for {platform}"
|
|
)
|
|
return "NeitherNameNorRegex"
|
|
|
|
if name:
|
|
# Try to match the name exactly:
|
|
for asset_name, asset in name_to_asset.items():
|
|
if asset_name == name:
|
|
platform_entries[platform] = (asset, platform_config)
|
|
break
|
|
if platform in platform_entries:
|
|
continue
|
|
else:
|
|
logging.error(f"could not find asset with name '{name}'")
|
|
return "NoMatchForAsset"
|
|
else:
|
|
# Try to match the name using a regular expression.
|
|
regex = re.compile(name_regex)
|
|
for asset_name, asset in name_to_asset.items():
|
|
if regex.match(asset_name):
|
|
platform_entries[platform] = (asset, platform_config)
|
|
break
|
|
if platform in platform_entries:
|
|
continue
|
|
else:
|
|
logging.error(f"could not find asset matching regex '{name_regex}'")
|
|
return "NoMatchForAsset"
|
|
|
|
return platform_entries
|
|
|
|
|
|
@cache
|
|
def compute_hash(
|
|
gh_repo_arg: str,
|
|
temp_dir: str,
|
|
tag: str,
|
|
name: str,
|
|
hash_algo: HashAlgorithm,
|
|
size: int,
|
|
) -> str:
|
|
"""Fetches the release entry corresponding to the specified (tag, name) tuple,
|
|
fetches the contents, verifies the size matches, and computes the hash.
|
|
|
|
Return value is a hex string representing the hash.
|
|
"""
|
|
output_filename = os.path.join(temp_dir, name)
|
|
|
|
# Fetch the url using the gh CLI to ensure authentication is handled correctly.
|
|
args = [
|
|
"gh",
|
|
"release",
|
|
"download",
|
|
tag,
|
|
"--repo",
|
|
gh_repo_arg,
|
|
# --pattern takes a "glob pattern", though we want to match an exact
|
|
# filename. Using re.escape() seems to do the right thing, though adding
|
|
# ^ and $ appears to break things.
|
|
"--pattern",
|
|
re.escape(name),
|
|
"--output",
|
|
output_filename,
|
|
]
|
|
subprocess.run(args, check=True)
|
|
stats = os.stat(output_filename)
|
|
if stats.st_size != size:
|
|
raise Exception(f"expected size {size} for {name} but got {stats.st_size}")
|
|
|
|
if hash_algo == "blake3":
|
|
import blake3
|
|
|
|
hasher = blake3.blake3()
|
|
with open(output_filename, "rb") as f:
|
|
for chunk in iter(lambda: f.read(4096), b""):
|
|
hasher.update(chunk)
|
|
digest = hasher.digest()
|
|
return digest.hex()
|
|
elif hash_algo == "sha256":
|
|
import hashlib
|
|
|
|
hasher = hashlib.sha256()
|
|
with open(output_filename, "rb") as f:
|
|
for chunk in iter(lambda: f.read(4096), b""):
|
|
hasher.update(chunk)
|
|
return hasher.hexdigest()
|
|
|
|
|
|
def get_config(
|
|
*, path_to_config: str, config_ref: str, github_repository: str, api_url: str
|
|
) -> Any:
|
|
args = [
|
|
"gh",
|
|
"api",
|
|
"-X",
|
|
"GET",
|
|
f"{api_url}/repos/{github_repository}/contents/{path_to_config}",
|
|
"-H",
|
|
"Accept: application/vnd.github.raw",
|
|
"-f",
|
|
f"ref={config_ref}",
|
|
]
|
|
output = subprocess.check_output(args)
|
|
return json.loads(output.decode("utf-8"))
|
|
|
|
|
|
def get_release_assets(*, tag: str, github_repository) -> Dict[str, Any]:
|
|
args = [
|
|
"gh",
|
|
"release",
|
|
"view",
|
|
tag,
|
|
"--repo",
|
|
github_repository,
|
|
"--json",
|
|
"assets",
|
|
]
|
|
output = subprocess.check_output(args)
|
|
release_data = json.loads(output.decode("utf-8"))
|
|
assets = release_data.get("assets")
|
|
if not assets:
|
|
raise Exception(f"no assets found for release '{tag}'")
|
|
return {asset["name"]: asset for asset in assets if asset["state"] == "uploaded"}
|
|
|
|
|
|
def guess_artifact_format_from_asset_name(asset_name: str) -> Optional[ArtifactFormat]:
|
|
if asset_name.endswith(".tar.gz") or asset_name.endswith(".tgz"):
|
|
return "tar.gz"
|
|
elif asset_name.endswith(".tar.zst") or asset_name.endswith(".tzst"):
|
|
return "tar.zst"
|
|
elif asset_name.endswith(".tar.xz"):
|
|
return "tar.xz"
|
|
elif asset_name.endswith(".tar"):
|
|
return "tar"
|
|
elif asset_name.endswith(".gz"):
|
|
return "gz"
|
|
elif asset_name.endswith(".zst"):
|
|
return "zst"
|
|
elif asset_name.endswith(".xz"):
|
|
return "xz"
|
|
elif asset_name.endswith(".zip"):
|
|
return "zip"
|
|
|
|
else:
|
|
return None
|
|
|
|
|
|
def parse_args():
|
|
parser = argparse.ArgumentParser(
|
|
description="Generate DotSlash files for a GitHub release"
|
|
)
|
|
|
|
parser.add_argument("--tag", required=True, help="tag identifying the release")
|
|
parser.add_argument("--config", required=True, help="path to JSON config file")
|
|
parser.add_argument(
|
|
"--local-config",
|
|
action="store_true",
|
|
help="if specified, --config is treated as a local path and --config-ref is ignored",
|
|
)
|
|
parser.add_argument(
|
|
"--repo",
|
|
help="github repo specified in `ORG/REPO` format",
|
|
default=os.getenv("GITHUB_REPOSITORY"),
|
|
)
|
|
parser.add_argument(
|
|
"--upload",
|
|
action="store_true",
|
|
help="if specified, upload the generated DotSlash files to the release",
|
|
)
|
|
|
|
# It would make things slightly easier for the user to default to the
|
|
# default branch of the repo, which might not be main.
|
|
default_config_ref = "main"
|
|
parser.add_argument(
|
|
"--config-ref",
|
|
help=f"SHA of Git commit to look up the config, defaults to {default_config_ref}",
|
|
default=os.getenv("GITHUB_SHA", default_config_ref),
|
|
)
|
|
|
|
default_server = "https://github.com"
|
|
parser.add_argument(
|
|
"--server",
|
|
help=f"URL for the GitHub server, defaults to {default_server}",
|
|
default=os.getenv("GITHUB_SERVER_URL", default_server),
|
|
)
|
|
|
|
default_api_server = "https://api.github.com"
|
|
parser.add_argument(
|
|
"--api-server",
|
|
help=f"URL for the GitHub API server, defaults to {default_api_server}",
|
|
default=os.getenv("GITHUB_API_URL", default_api_server),
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--output",
|
|
help=f"folder where DotSlash files should be written, defaults to $GITHUB_WORKSPACE",
|
|
default=os.getenv("GITHUB_WORKSPACE"),
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--include-build-metadata",
|
|
type=str,
|
|
default="true",
|
|
help="include build metadata in the generated DotSlash files (default: true)",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--exclude-build-metadata",
|
|
action="store_true",
|
|
help="exclude build metadata from the generated DotSlash files",
|
|
)
|
|
|
|
return parser.parse_args()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|