SDL/build-scripts/build-release.py
Anonymous Maarten e4126d8d6f Add parametrized release script
[ci skip]
2024-10-04 23:42:29 +02:00

975 lines
45 KiB
Python
Executable file

#!/usr/bin/env python
"""
This script is shared between SDL2, SDL2_image, SDL2_mixer and SDL2_ttf.
Don't specialize this script for doing project-specific modifications.
Rather, modify release-info.json.
"""
import argparse
import collections
from collections.abc import Callable
import contextlib
import datetime
import fnmatch
import glob
import io
import json
import logging
import multiprocessing
import os
from pathlib import Path
import platform
import re
import shutil
import subprocess
import sys
import tarfile
import tempfile
import textwrap
import typing
import zipfile
logger = logging.getLogger(__name__)
GIT_HASH_FILENAME = ".git-hash"
def safe_isotime_to_datetime(str_isotime: str) -> datetime.datetime:
try:
return datetime.datetime.fromisoformat(str_isotime)
except ValueError:
pass
logger.warning("Invalid iso time: %s", str_isotime)
if str_isotime[-6:-5] in ("+", "-"):
# Commits can have isotime with invalid timezone offset (e.g. "2021-07-04T20:01:40+32:00")
modified_str_isotime = str_isotime[:-6] + "+00:00"
try:
return datetime.datetime.fromisoformat(modified_str_isotime)
except ValueError:
pass
raise ValueError(f"Invalid isotime: {str_isotime}")
class VsArchPlatformConfig:
def __init__(self, arch: str, platform: str, configuration: str):
self.arch = arch
self.platform = platform
self.configuration = configuration
def configure(self, s: str) -> str:
return s.replace("@ARCH@", self.arch).replace("@PLATFORM@", self.platform).replace("@CONFIGURATION@", self.configuration)
@contextlib.contextmanager
def chdir(path):
original_cwd = os.getcwd()
try:
os.chdir(path)
yield
finally:
os.chdir(original_cwd)
class Executer:
def __init__(self, root: Path, dry: bool=False):
self.root = root
self.dry = dry
def run(self, cmd, cwd=None, env=None):
logger.info("Executing args=%r", cmd)
sys.stdout.flush()
if not self.dry:
subprocess.run(cmd, check=True, cwd=cwd or self.root, env=env, text=True)
def check_output(self, cmd, cwd=None, dry_out=None, env=None, text=True):
logger.info("Executing args=%r", cmd)
sys.stdout.flush()
if self.dry:
return dry_out
return subprocess.check_output(cmd, cwd=cwd or self.root, env=env, text=text)
class SectionPrinter:
@contextlib.contextmanager
def group(self, title: str):
print(f"{title}:")
yield
class GitHubSectionPrinter(SectionPrinter):
def __init__(self):
super().__init__()
self.in_group = False
@contextlib.contextmanager
def group(self, title: str):
print(f"::group::{title}")
assert not self.in_group, "Can enter a group only once"
self.in_group = True
yield
self.in_group = False
print("::endgroup::")
class VisualStudio:
def __init__(self, executer: Executer, year: typing.Optional[str]=None):
self.executer = executer
self.vsdevcmd = self.find_vsdevcmd(year)
self.msbuild = self.find_msbuild()
@property
def dry(self) -> bool:
return self.executer.dry
VS_YEAR_TO_VERSION = {
"2022": 17,
"2019": 16,
"2017": 15,
"2015": 14,
"2013": 12,
}
def find_vsdevcmd(self, year: typing.Optional[str]=None) -> typing.Optional[Path]:
vswhere_spec = ["-latest"]
if year is not None:
try:
version = self.VS_YEAR_TO_VERSION[year]
except KeyError:
logger.error("Invalid Visual Studio year")
return None
vswhere_spec.extend(["-version", f"[{version},{version+1})"])
vswhere_cmd = ["vswhere"] + vswhere_spec + ["-property", "installationPath"]
vs_install_path = Path(self.executer.check_output(vswhere_cmd, dry_out="/tmp").strip())
logger.info("VS install_path = %s", vs_install_path)
assert vs_install_path.is_dir(), "VS installation path does not exist"
vsdevcmd_path = vs_install_path / "Common7/Tools/vsdevcmd.bat"
logger.info("vsdevcmd path = %s", vsdevcmd_path)
if self.dry:
vsdevcmd_path.parent.mkdir(parents=True, exist_ok=True)
vsdevcmd_path.touch(exist_ok=True)
assert vsdevcmd_path.is_file(), "vsdevcmd.bat batch file does not exist"
return vsdevcmd_path
def find_msbuild(self) -> typing.Optional[Path]:
vswhere_cmd = ["vswhere", "-latest", "-requires", "Microsoft.Component.MSBuild", "-find", r"MSBuild\**\Bin\MSBuild.exe"]
msbuild_path = Path(self.executer.check_output(vswhere_cmd, dry_out="/tmp/MSBuild.exe").strip())
logger.info("MSBuild path = %s", msbuild_path)
if self.dry:
msbuild_path.parent.mkdir(parents=True, exist_ok=True)
msbuild_path.touch(exist_ok=True)
assert msbuild_path.is_file(), "MSBuild.exe does not exist"
return msbuild_path
def build(self, arch_platform: VsArchPlatformConfig, projects: list[Path]):
assert projects, "Need at least one project to build"
vsdev_cmd_str = f"\"{self.vsdevcmd}\" -arch={arch_platform.arch}"
msbuild_cmd_str = " && ".join([f"\"{self.msbuild}\" \"{project}\" /m /p:BuildInParallel=true /p:Platform={arch_platform.platform} /p:Configuration={arch_platform.configuration}" for project in projects])
bat_contents = f"{vsdev_cmd_str} && {msbuild_cmd_str}\n"
bat_path = Path(tempfile.gettempdir()) / "cmd.bat"
with bat_path.open("w") as f:
f.write(bat_contents)
logger.info("Running cmd.exe script (%s): %s", bat_path, bat_contents)
cmd = ["cmd.exe", "/D", "/E:ON", "/V:OFF", "/S", "/C", f"CALL {str(bat_path)}"]
self.executer.run(cmd)
class Archiver:
def __init__(self, zip_path: typing.Optional[Path]=None, tgz_path: typing.Optional[Path]=None, txz_path: typing.Optional[Path]=None):
self._zip_files = []
self._tar_files = []
self._added_files = set()
if zip_path:
self._zip_files.append(zipfile.ZipFile(zip_path, "w", compression=zipfile.ZIP_DEFLATED))
if tgz_path:
self._tar_files.append(tarfile.open(tgz_path, "w:gz"))
if txz_path:
self._tar_files.append(tarfile.open(txz_path, "w:xz"))
@property
def added_files(self) -> set[str]:
return self._added_files
def add_file_data(self, arcpath: str, data: bytes, mode: int, time: datetime.datetime):
for zf in self._zip_files:
file_data_time = (time.year, time.month, time.day, time.hour, time.minute, time.second)
zip_info = zipfile.ZipInfo(filename=arcpath, date_time=file_data_time)
zip_info.external_attr = mode << 16
zip_info.compress_type = zipfile.ZIP_DEFLATED
zf.writestr(zip_info, data=data)
for tf in self._tar_files:
tar_info = tarfile.TarInfo(arcpath)
tar_info.type = tarfile.REGTYPE
tar_info.mode = mode
tar_info.size = len(data)
tar_info.mtime = int(time.timestamp())
tf.addfile(tar_info, fileobj=io.BytesIO(data))
self._added_files.add(arcpath)
def add_symlink(self, arcpath: str, target: str, time: datetime.datetime, files_for_zip):
for zf in self._zip_files:
file_data_time = (time.year, time.month, time.day, time.hour, time.minute, time.second)
for f in files_for_zip:
zip_info = zipfile.ZipInfo(filename=f["arcpath"], date_time=file_data_time)
zip_info.external_attr = f["mode"] << 16
zip_info.compress_type = zipfile.ZIP_DEFLATED
zf.writestr(zip_info, data=f["data"])
for tf in self._tar_files:
tar_info = tarfile.TarInfo(arcpath)
tar_info.type = tarfile.SYMTYPE
tar_info.mode = 0o777
tar_info.mtime = int(time.timestamp())
tar_info.linkname = target
tf.addfile(tar_info)
self._added_files.update(f["arcpath"] for f in files_for_zip)
def add_git_hash(self, commit: str, arcdir: typing.Optional[str]=None, time: typing.Optional[datetime.datetime]=None):
arcpath = GIT_HASH_FILENAME
if arcdir and arcdir[-1:] != "/":
arcpath = f"{arcdir}/{arcpath}"
if not time:
time = datetime.datetime(year=2024, month=4, day=1)
data = f"{commit}\n".encode()
self.add_file_data(arcpath=arcpath, data=data, mode=0o100644, time=time)
def add_file_path(self, arcpath: str, path: Path):
assert path.is_file(), f"{path} should be a file"
for zf in self._zip_files:
zf.write(path, arcname=arcpath)
for tf in self._tar_files:
tf.add(path, arcname=arcpath)
def add_file_directory(self, arcdirpath: str, dirpath: Path):
assert dirpath.is_dir()
if arcdirpath and arcdirpath[-1:] != "/":
arcdirpath += "/"
for f in dirpath.iterdir():
if f.is_file():
arcpath = f"{arcdirpath}{f.name}"
logger.debug("Adding %s to %s", f, arcpath)
self.add_file_path(arcpath=arcpath, path=f)
def close(self):
# Archiver is intentionally made invalid after this function
del self._zip_files
self._zip_files = None
del self._tar_files
self._tar_files = None
def __enter__(self):
return self
def __exit__(self, type, value, traceback):
self.close()
class SourceCollector:
TreeItem = collections.namedtuple("TreeItem", ("path", "mode", "data", "symtarget", "directory", "time"))
def __init__(self, root: Path, commit: str, filter: typing.Optional[Callable[[str], bool]], executer: Executer):
self.root = root
self.commit = commit
self.filter = filter
self.executer = executer
self._git_contents: typing.Optional[dict[str, SourceCollector.TreeItem]] = None
def _get_git_contents(self) -> dict[str, TreeItem]:
contents_tgz = subprocess.check_output(["git", "archive", "--format=tar.gz", self.commit, "-o", "/dev/stdout"], cwd=self.root, text=False)
tar_archive = tarfile.open(fileobj=io.BytesIO(contents_tgz), mode="r:gz")
filenames = tuple(m.name for m in tar_archive if (m.isfile() or m.issym()))
file_times = self._get_file_times(paths=filenames)
git_contents = {}
for ti in tar_archive:
if self.filter and not self.filter(ti.name):
continue
data = None
symtarget = None
directory = False
file_time = None
if ti.isfile():
contents_file = tar_archive.extractfile(ti.name)
data = contents_file.read()
file_time = file_times[ti.name]
elif ti.issym():
symtarget = ti.linkname
file_time = file_times[ti.name]
elif ti.isdir():
directory = True
else:
raise ValueError(f"{ti.name}: unknown type")
git_contents[ti.name] = self.TreeItem(path=ti.name, mode=ti.mode, data=data, symtarget=symtarget, directory=directory, time=file_time)
return git_contents
@property
def git_contents(self) -> dict[str, TreeItem]:
if self._git_contents is None:
self._git_contents = self._get_git_contents()
return self._git_contents
def _get_file_times(self, paths: tuple[str, ...]) -> dict[str, datetime.datetime]:
dry_out = textwrap.dedent("""\
time=2024-03-14T15:40:25-07:00
M\tCMakeLists.txt
""")
git_log_out = self.executer.check_output(["git", "log", "--name-status", '--pretty=time=%cI', self.commit], dry_out=dry_out, cwd=self.root).splitlines(keepends=False)
current_time = None
set_paths = set(paths)
path_times: dict[str, datetime.datetime] = {}
for line in git_log_out:
if not line:
continue
if line.startswith("time="):
current_time = safe_isotime_to_datetime(line.removeprefix("time="))
continue
mod_type, file_paths = line.split(maxsplit=1)
assert current_time is not None
for file_path in file_paths.split("\t"):
if file_path in set_paths and file_path not in path_times:
path_times[file_path] = current_time
# FIXME: find out why some files are not shown in "git log"
# assert set(path_times.keys()) == set_paths
if set(path_times.keys()) != set_paths:
found_times = set(path_times.keys())
paths_without_times = set_paths.difference(found_times)
logger.warning("No times found for these paths: %s", paths_without_times)
max_time = max(time for time in path_times.values())
for path in paths_without_times:
path_times[path] = max_time
return path_times
def add_to_archiver(self, archive_base: str, archiver: Archiver):
remaining_symlinks = set()
added_files = dict()
def calculate_symlink_target(s: SourceCollector.TreeItem) -> str:
dest_dir = os.path.dirname(s.path)
if dest_dir:
dest_dir += "/"
target = dest_dir + s.symtarget
while True:
new_target, n = re.subn(r"([^/]+/+[.]{2}/)", "", target)
print(f"{target=} {new_target=}")
target = new_target
if not n:
break
return target
# Add files in first pass
for git_file in self.git_contents.values():
if git_file.data is not None:
archiver.add_file_data(arcpath=f"{archive_base}/{git_file.path}", data=git_file.data, time=git_file.time, mode=git_file.mode)
added_files[git_file.path] = git_file
elif git_file.symtarget is not None:
remaining_symlinks.add(git_file)
# Resolve symlinks in second pass: zipfile does not support symlinks, so add files to zip archive
while True:
if not remaining_symlinks:
break
symlinks_this_time = set()
extra_added_files = {}
for symlink in remaining_symlinks:
symlink_files_for_zip = {}
symlink_target_path = calculate_symlink_target(symlink)
if symlink_target_path in added_files:
symlink_files_for_zip[symlink.path] = added_files[symlink_target_path]
else:
symlink_target_path_slash = symlink_target_path + "/"
for added_file in added_files:
if added_file.startswith(symlink_target_path_slash):
path_in_symlink = symlink.path + "/" + added_file.removeprefix(symlink_target_path_slash)
symlink_files_for_zip[path_in_symlink] = added_files[added_file]
if symlink_files_for_zip:
symlinks_this_time.add(symlink)
extra_added_files.update(symlink_files_for_zip)
files_for_zip = [{"arcpath": f"{archive_base}/{sym_path}", "data": sym_info.data, "mode": sym_info.mode} for sym_path, sym_info in symlink_files_for_zip.items()]
archiver.add_symlink(arcpath=f"{archive_base}/{symlink.path}", target=symlink.symtarget, time=symlink.time, files_for_zip=files_for_zip)
# if not symlinks_this_time:
# logger.info("files added: %r", set(path for path in added_files.keys()))
assert symlinks_this_time, f"No targets found for symlinks: {remaining_symlinks}"
remaining_symlinks.difference_update(symlinks_this_time)
added_files.update(extra_added_files)
class Releaser:
def __init__(self, release_info: dict, commit: str, root: Path, dist_path: Path, section_printer: SectionPrinter, executer: Executer, cmake_generator: str, deps_path: Path, overwrite: bool, github: bool, fast: bool):
self.release_info = release_info
self.project = release_info["name"]
self.version = self.extract_sdl_version(root=root, release_info=release_info)
self.root = root
self.commit = commit
self.dist_path = dist_path
self.section_printer = section_printer
self.executer = executer
self.cmake_generator = cmake_generator
self.cpu_count = multiprocessing.cpu_count()
self.deps_path = deps_path
self.overwrite = overwrite
self.github = github
self.fast = fast
self.artifacts: dict[str, Path] = {}
@property
def dry(self) -> bool:
return self.executer.dry
def prepare(self):
logger.debug("Creating dist folder")
self.dist_path.mkdir(parents=True, exist_ok=True)
@classmethod
def _path_filter(cls, path: str) -> bool:
if ".gitmodules" in path:
return True
if path.startswith(".git"):
return False
return True
@classmethod
def _external_repo_path_filter(cls, path: str) -> bool:
if not cls._path_filter(path):
return False
if path.startswith("test/") or path.startswith("tests/"):
return False
return True
def create_source_archives(self) -> None:
archive_base = f"{self.project}-{self.version}"
project_souce_collector = SourceCollector(root=self.root, commit=self.commit, executer=self.executer, filter=self._path_filter)
latest_mod_time = max(item.time for item in project_souce_collector.git_contents.values() if item.time)
zip_path = self.dist_path / f"{archive_base}.zip"
tgz_path = self.dist_path / f"{archive_base}.tar.gz"
txz_path = self.dist_path / f"{archive_base}.tar.xz"
logger.info("Creating zip/tgz/txz source archives ...")
if self.dry:
zip_path.touch()
tgz_path.touch()
txz_path.touch()
else:
with Archiver(zip_path=zip_path, tgz_path=tgz_path, txz_path=txz_path) as archiver:
archiver.add_file_data(arcpath=f"{archive_base}/VERSION.txt", data=f"{self.version}\n".encode(), mode=0o100644, time=latest_mod_time)
archiver.add_file_data(arcpath=f"{archive_base}/{GIT_HASH_FILENAME}", data=f"{self.commit}\n".encode(), mode=0o100644, time=latest_mod_time)
print(f"Adding source files of main project ...")
project_souce_collector.add_to_archiver(archive_base=archive_base, archiver=archiver)
for extra_repo in self.release_info["source"].get("extra-repos", []):
extra_repo_root = self.root / extra_repo
assert (extra_repo_root / ".git").exists(), f"{extra_repo_root} must be a git repo"
extra_repo_commit = self.executer.check_output(["git", "rev-parse", "HEAD"], dry_out=f"gitsha-extra-repo-{extra_repo}", cwd=extra_repo_root).strip()
extra_repo_source_collector = SourceCollector(root=extra_repo_root, commit=extra_repo_commit, executer=self.executer, filter=self._external_repo_path_filter)
print(f"Adding source files of {extra_repo} ...")
extra_repo_source_collector.add_to_archiver(archive_base=f"{archive_base}/{extra_repo}", archiver=archiver)
for file in self.release_info["source"]["checks"]:
assert f"{archive_base}/{file}" in archiver.added_files, f"'{archive_base}/{file}' must exist"
logger.info("... done")
self.artifacts["src-zip"] = zip_path
self.artifacts["src-tar-gz"] = tgz_path
self.artifacts["src-tar-xz"] = txz_path
if not self.dry:
with tgz_path.open("r+b") as f:
# Zero the embedded timestamp in the gzip'ed tarball
f.seek(4, 0)
f.write(b"\x00\x00\x00\x00")
def create_dmg(self, configuration: str="Release") -> None:
dmg_in = self.root / self.release_info["dmg"]["path"]
xcode_project = self.root / self.release_info["dmg"]["project"]
assert xcode_project.is_dir(), f"{xcode_project} must be a directory"
assert (xcode_project / "project.pbxproj").is_file, f"{xcode_project} must contain project.pbxproj"
dmg_in.unlink(missing_ok=True)
build_xcconfig = self.release_info["dmg"].get("build-xcconfig")
if build_xcconfig:
shutil.copy(self.root / build_xcconfig, xcode_project.parent / "build.xcconfig")
xcode_scheme = self.release_info["dmg"].get("scheme")
xcode_target = self.release_info["dmg"].get("target")
assert xcode_scheme or xcode_target, "dmg needs scheme or target"
assert not (xcode_scheme and xcode_target), "dmg cannot have both scheme and target set"
if xcode_scheme:
scheme_or_target = "-scheme"
target_like = xcode_scheme
else:
scheme_or_target = "-target"
target_like = xcode_target
self.executer.run(["xcodebuild", "ONLY_ACTIVE_ARCH=NO", "-project", xcode_project, scheme_or_target, target_like, "-configuration", configuration])
if self.dry:
dmg_in.parent.mkdir(parents=True, exist_ok=True)
dmg_in.touch()
assert dmg_in.is_file(), f"{self.project}.dmg was not created by xcodebuild"
dmg_out = self.dist_path / f"{self.project}-{self.version}.dmg"
shutil.copy(dmg_in, dmg_out)
self.artifacts["dmg"] = dmg_out
@property
def git_hash_data(self) -> bytes:
return f"{self.commit}\n".encode()
def _tar_add_git_hash(self, tar_object: tarfile.TarFile, root: typing.Optional[str]=None, time: typing.Optional[datetime.datetime]=None):
if not time:
time = datetime.datetime(year=2024, month=4, day=1)
path = GIT_HASH_FILENAME
if root:
path = f"{root}/{path}"
tar_info = tarfile.TarInfo(path)
tar_info.mode = 0o100644
tar_info.size = len(self.git_hash_data)
tar_info.mtime = int(time.timestamp())
tar_object.addfile(tar_info, fileobj=io.BytesIO(self.git_hash_data))
def create_mingw_archives(self) -> None:
build_type = "Release"
build_parent_dir = self.root / "build-mingw"
assert "autotools" in self.release_info["mingw"]
assert "cmake" not in self.release_info["mingw"]
mingw_archs = self.release_info["mingw"]["autotools"]["archs"]
ARCH_TO_TRIPLET = {
"x86": "i686-w64-mingw32",
"x64": "x86_64-w64-mingw32",
}
new_env = dict(os.environ)
if "dependencies" in self.release_info["mingw"]:
mingw_deps_path = self.deps_path / "mingw-deps"
shutil.rmtree(mingw_deps_path, ignore_errors=True)
mingw_deps_path.mkdir()
for triplet in ARCH_TO_TRIPLET.values():
(mingw_deps_path / triplet).mkdir()
def extract_filter(member: tarfile.TarInfo, path: str, /):
if member.name.startswith("SDL"):
member.name = "/".join(Path(member.name).parts[1:])
return member
for dep in self.release_info["dependencies"].keys():
extract_dir = mingw_deps_path / f"extract-{dep}"
extract_dir.mkdir()
with chdir(extract_dir):
tar_path = glob.glob(self.release_info["mingw"]["dependencies"][dep]["artifact"], root_dir=self.deps_path)[0]
logger.info("Extracting %s to %s", tar_path, mingw_deps_path)
with tarfile.open(self.deps_path / tar_path, mode="r:gz") as tarf:
tarf.extractall(filter=extract_filter)
for triplet in ARCH_TO_TRIPLET.values():
self.executer.run(["make", f"-j{os.cpu_count()}", "-C", str(extract_dir), "install-package", f"arch={triplet}", f"prefix={str(mingw_deps_path / triplet)}"])
dep_binpath = mingw_deps_path / triplet / "bin"
assert dep_binpath.is_dir(), f"{dep_binpath} for PATH should exist"
dep_pkgconfig = mingw_deps_path / triplet / "lib/pkgconfig"
assert dep_pkgconfig.is_dir(), f"{dep_pkgconfig} for PKG_CONFIG_PATH should exist"
new_env["PATH"] = os.pathsep.join([str(dep_binpath), new_env["PATH"]])
new_env["PKG_CONFIG_PATH"] = str(dep_pkgconfig)
new_env["CFLAGS"] = f"-O2 -ffile-prefix-map={self.root}=/src/{self.project}"
new_env["CXXFLAGS"] = f"-O2 -ffile-prefix-map={self.root}=/src/{self.project}"
arch_install_paths = {}
arch_files = {}
for arch in mingw_archs:
triplet = ARCH_TO_TRIPLET[arch]
new_env["CC"] = f"{triplet}-gcc"
new_env["CXX"] = f"{triplet}-g++"
new_env["RC"] = f"{triplet}-windres"
build_path = build_parent_dir / f"build-{triplet}"
install_path = build_parent_dir / f"install-{triplet}"
arch_install_paths[arch] = install_path
shutil.rmtree(install_path, ignore_errors=True)
build_path.mkdir(parents=True, exist_ok=True)
with self.section_printer.group(f"Configuring MinGW {triplet}"):
extra_args = [arg.replace("@DEP_PREFIX@", str(mingw_deps_path / triplet)) for arg in self.release_info["mingw"]["autotools"]["args"]]
assert "@" not in " ".join(extra_args), f"@ should not be present in extra arguments ({extra_args})"
self.executer.run([
self.root / "configure",
f"--prefix={install_path}",
f"--includedir={install_path}/include",
f"--libdir={install_path}/lib",
f"--bindir={install_path}/bin",
f"--host={triplet}",
f"--build=x86_64-none-linux-gnu",
] + extra_args, cwd=build_path, env=new_env)
with self.section_printer.group(f"Build MinGW {triplet}"):
self.executer.run(["make", f"-j{self.cpu_count}"], cwd=build_path, env=new_env)
with self.section_printer.group(f"Install MinGW {triplet}"):
self.executer.run(["make", "install"], cwd=build_path, env=new_env)
arch_files[arch] = list(Path(r) / f for r, _, files in os.walk(install_path) for f in files)
print("Collecting files for MinGW development archive ...")
archived_files = {}
arc_root = f"{self.project}-{self.version}"
for arch in mingw_archs:
triplet = ARCH_TO_TRIPLET[arch]
install_path = arch_install_paths[arch]
arcname_parent = f"{arc_root}/{triplet}"
for file in arch_files[arch]:
arcname = os.path.join(arcname_parent, file.relative_to(install_path))
logger.debug("Adding %s as %s", file, arcname)
archived_files[arcname] = file
for meta_destdir, file_globs in self.release_info["mingw"]["files"].items():
assert meta_destdir[0] == "/" and meta_destdir[-1] == "/", f"'{meta_destdir}' must begin and end with '/'"
if "@" in meta_destdir:
destdirs = list(meta_destdir.replace("@TRIPLET@", triplet) for triplet in ARCH_TO_TRIPLET.values())
assert not any("A" in d for d in destdirs)
else:
destdirs = [meta_destdir]
assert isinstance(file_globs, list), f"'{file_globs}' in release_info.json must be a list of globs instead"
for file_glob in file_globs:
file_paths = glob.glob(file_glob, root_dir=self.root)
assert file_paths, f"glob '{file_glob}' does not match any file"
for file_path in file_paths:
file_path = self.root / file_path
for destdir in destdirs:
arcname = f"{arc_root}{destdir}{file_path.name}"
logger.debug("Adding %s as %s", file_path, arcname)
archived_files[arcname] = file_path
print("... done")
print("Creating zip/tgz/txz development archives ...")
zip_path = self.dist_path / f"{self.project}-devel-{self.version}-mingw.zip"
tgz_path = self.dist_path / f"{self.project}-devel-{self.version}-mingw.tar.gz"
txz_path = self.dist_path / f"{self.project}-devel-{self.version}-mingw.tar.xz"
with Archiver(zip_path=zip_path, tgz_path=tgz_path, txz_path=txz_path) as archiver:
for arcpath, path in archived_files.items():
archiver.add_file_path(arcpath=arcpath, path=path)
print("... done")
self.artifacts["mingw-devel-zip"] = zip_path
self.artifacts["mingw-devel-tar-gz"] = tgz_path
self.artifacts["mingw-devel-tar-xz"] = txz_path
def download_dependencies(self):
shutil.rmtree(self.deps_path, ignore_errors=True)
self.deps_path.mkdir(parents=True)
if self.github:
with open(os.environ["GITHUB_OUTPUT"], "a") as f:
f.write(f"dep-path={self.deps_path.absolute()}\n")
for dep, depinfo in self.release_info["dependencies"].items():
startswith = depinfo["startswith"]
dep_repo = depinfo["repo"]
dep_string_data = self.executer.check_output(["gh", "-R", dep_repo, "release", "list", "--exclude-drafts", "--exclude-pre-releases", "--json", "name,createdAt,tagName", "--jq", f'[.[]|select(.name|startswith("{startswith}"))]|max_by(.createdAt)']).strip()
dep_data = json.loads(dep_string_data)
dep_tag = dep_data["tagName"]
dep_version = dep_data["name"]
logger.info("Download dependency %s version %s (tag=%s) ", dep, dep_version, dep_tag)
self.executer.run(["gh", "-R", dep_repo, "release", "download", dep_tag], cwd=self.deps_path)
if self.github:
with open(os.environ["GITHUB_OUTPUT"], "a") as f:
f.write(f"dep-{dep.lower()}-version={dep_version}\n")
def verify_dependencies(self):
for dep, depinfo in self.release_info.get("dependencies", {}).items():
mingw_matches = glob.glob(self.release_info["mingw"]["dependencies"][dep]["artifact"], root_dir=self.deps_path)
assert len(mingw_matches) == 1, f"Exactly one archive matches mingw {dep} dependency: {mingw_matches}"
dmg_matches = glob.glob(self.release_info["dmg"]["dependencies"][dep]["artifact"], root_dir=self.deps_path)
assert len(dmg_matches) == 1, f"Exactly one archive matches dmg {dep} dependency: {dmg_matches}"
msvc_matches = glob.glob(self.release_info["msvc"]["dependencies"][dep]["artifact"], root_dir=self.deps_path)
assert len(msvc_matches) == 1, f"Exactly one archive matches msvc {dep} dependency: {msvc_matches}"
def build_vs(self, arch_platform: VsArchPlatformConfig, vs: VisualStudio):
msvc_deps_path = self.deps_path / "msvc-deps"
shutil.rmtree(msvc_deps_path, ignore_errors=True)
if "dependencies" in self.release_info["msvc"]:
for dep, depinfo in self.release_info["msvc"]["dependencies"].items():
msvc_zip = self.deps_path / glob.glob(depinfo["artifact"], root_dir=self.deps_path)[0]
src_globs = [arch_platform.configure(instr["src"]) for instr in depinfo["copy"]]
with zipfile.ZipFile(msvc_zip, "r") as zf:
for member in zf.namelist():
member_path = "/".join(Path(member).parts[1:])
for src_i, src_glob in enumerate(src_globs):
if fnmatch.fnmatch(member_path, src_glob):
dst = (self.root / arch_platform.configure(depinfo["copy"][src_i]["dst"])).resolve() / Path(member_path).name
zip_data = zf.read(member)
if dst.exists():
identical = False
if dst.is_file():
orig_bytes = dst.read_bytes()
if orig_bytes == zip_data:
identical = True
if not identical:
logger.warning("Extracting dependency %s, will cause %s to be overwritten", dep, dst)
if not self.overwrite:
raise RuntimeError("Run with --overwrite to allow overwriting")
logger.debug("Extracting %s -> %s", member, dst)
dst.parent.mkdir(exist_ok=True, parents=True)
dst.write_bytes(zip_data)
assert "msbuild" in self.release_info["msvc"]
assert "cmake" not in self.release_info["msvc"]
built_paths = [
self.root / arch_platform.configure(f) for msbuild_files in self.release_info["msvc"]["msbuild"]["files"] for f in msbuild_files["paths"]
]
for b in built_paths:
b.unlink(missing_ok=True)
projects = self.release_info["msvc"]["msbuild"]["projects"]
with self.section_printer.group(f"Build {arch_platform.arch} VS binary"):
vs.build(arch_platform=arch_platform, projects=projects)
if self.dry:
for b in built_paths:
b.parent.mkdir(parents=True, exist_ok=True)
b.touch()
for b in built_paths:
assert b.is_file(), f"{b} has not been created"
b.parent.mkdir(parents=True, exist_ok=True)
b.touch()
zip_path = self.dist_path / f"{self.project}-{self.version}-win32-{arch_platform.arch}.zip"
zip_path.unlink(missing_ok=True)
logger.info("Creating %s", zip_path)
with Archiver(zip_path=zip_path) as archiver:
for msbuild_files in self.release_info["msvc"]["msbuild"]["files"]:
if "lib" in msbuild_files:
arcdir = arch_platform.configure(msbuild_files["lib"])
for p in msbuild_files["paths"]:
p = arch_platform.configure(p)
archiver.add_file_path(path=self.root / p, arcpath=f"{arcdir}/{Path(p).name}")
for extra_files in self.release_info["msvc"]["files"]:
if "lib" in extra_files:
arcdir = arch_platform.configure(extra_files["lib"])
for p in extra_files["paths"]:
p = arch_platform.configure(p)
archiver.add_file_path(path=self.root / p, arcpath=f"{arcdir}/{Path(p).name}")
archiver.add_git_hash(commit=self.commit)
self.artifacts[f"VC-{arch_platform.arch}"] = zip_path
for p in built_paths:
assert p.is_file(), f"{p} should exist"
def build_vs_devel(self, arch_platforms: list[VsArchPlatformConfig]) -> None:
zip_path = self.dist_path / f"{self.project}-devel-{self.version}-VC.zip"
archive_prefix = f"{self.project}-{self.version}"
with Archiver(zip_path=zip_path) as archiver:
for msbuild_files in self.release_info["msvc"]["msbuild"]["files"]:
if "devel" in msbuild_files:
for meta_glob_path in msbuild_files["paths"]:
if "@" in meta_glob_path or "@" in msbuild_files["devel"]:
for arch_platform in arch_platforms:
glob_path = arch_platform.configure(meta_glob_path)
paths = glob.glob(glob_path, root_dir=self.root)
dst_subdirpath = arch_platform.configure(msbuild_files['devel'])
for path in paths:
path = self.root / path
arcpath = f"{archive_prefix}/{dst_subdirpath}/{Path(path).name}"
archiver.add_file_path(path=path, arcpath=arcpath)
else:
paths = glob.glob(meta_glob_path, root_dir=self.root)
for path in paths:
path = self.root / path
arcpath = f"{archive_prefix}/{msbuild_files['devel']}/{Path(path).name}"
archiver.add_file_path(path=path, arcpath=arcpath)
for extra_files in self.release_info["msvc"]["files"]:
if "devel" in extra_files:
for meta_glob_path in extra_files["paths"]:
if "@" in meta_glob_path or "@" in extra_files["devel"]:
for arch_platform in arch_platforms:
glob_path = arch_platform.configure(meta_glob_path)
paths = glob.glob(glob_path, root_dir=self.root)
dst_subdirpath = arch_platform.configure(extra_files['devel'])
for path in paths:
path = self.root / path
arcpath = f"{archive_prefix}/{dst_subdirpath}/{Path(path).name}"
archiver.add_file_path(path=path, arcpath=arcpath)
else:
paths = glob.glob(meta_glob_path, root_dir=self.root)
for path in paths:
path = self.root / path
arcpath = f"{archive_prefix}/{extra_files['devel']}/{Path(path).name}"
archiver.add_file_path(path=path, arcpath=arcpath)
archiver.add_git_hash(commit=self.commit, arcdir=archive_prefix)
self.artifacts["VC-devel"] = zip_path
@classmethod
def extract_sdl_version(cls, root: Path, release_info: dict) -> str:
with open(root / release_info["version"]["file"], "r") as f:
text = f.read()
major = next(re.finditer(release_info["version"]["re_major"], text, flags=re.M)).group(1)
minor = next(re.finditer(release_info["version"]["re_minor"], text, flags=re.M)).group(1)
micro = next(re.finditer(release_info["version"]["re_micro"], text, flags=re.M)).group(1)
return f"{major}.{minor}.{micro}"
def main(argv=None) -> int:
if sys.version_info < (3, 11):
logger.error("This script needs at least python 3.11")
return 1
parser = argparse.ArgumentParser(allow_abbrev=False, description="Create SDL release artifacts")
parser.add_argument("--root", metavar="DIR", type=Path, default=Path(__file__).absolute().parents[1], help="Root of project")
parser.add_argument("--release-info", metavar="JSON", dest="path_release_info", type=Path, default=Path(__file__).absolute().parent / "release-info.json", help="Path of release-info.json")
parser.add_argument("--dependency-folder", metavar="FOLDER", dest="deps_path", type=Path, default="deps", help="Directory containing pre-built archives of dependencies (will be removed when downloading archives)")
parser.add_argument("--out", "-o", metavar="DIR", dest="dist_path", type=Path, default="dist", help="Output directory")
parser.add_argument("--github", action="store_true", help="Script is running on a GitHub runner")
parser.add_argument("--commit", default="HEAD", help="Git commit/tag of which a release should be created")
parser.add_argument("--actions", choices=["download", "source", "mingw", "msvc", "dmg"], required=True, nargs="+", dest="actions", help="What to do?")
parser.set_defaults(loglevel=logging.INFO)
parser.add_argument('--vs-year', dest="vs_year", help="Visual Studio year")
parser.add_argument('--cmake-generator', dest="cmake_generator", default="Ninja", help="CMake Generator")
parser.add_argument('--debug', action='store_const', const=logging.DEBUG, dest="loglevel", help="Print script debug information")
parser.add_argument('--dry-run', action='store_true', dest="dry", help="Don't execute anything")
parser.add_argument('--force', action='store_true', dest="force", help="Ignore a non-clean git tree")
parser.add_argument('--overwrite', action='store_true', dest="overwrite", help="Allow potentially overwriting other projects")
parser.add_argument('--fast', action='store_true', dest="fast", help="Don't do a rebuild")
args = parser.parse_args(argv)
logging.basicConfig(level=args.loglevel, format='[%(levelname)s] %(message)s')
args.deps_path = args.deps_path.absolute()
args.dist_path = args.dist_path.absolute()
args.root = args.root.absolute()
args.dist_path = args.dist_path.absolute()
if args.dry:
args.dist_path = args.dist_path / "dry"
if args.github:
section_printer: SectionPrinter = GitHubSectionPrinter()
else:
section_printer = SectionPrinter()
if args.github and "GITHUB_OUTPUT" not in os.environ:
os.environ["GITHUB_OUTPUT"] = "/tmp/github_output.txt"
executer = Executer(root=args.root, dry=args.dry)
root_git_hash_path = args.root / GIT_HASH_FILENAME
root_is_maybe_archive = root_git_hash_path.is_file()
if root_is_maybe_archive:
logger.warning("%s detected: Building from archive", GIT_HASH_FILENAME)
archive_commit = root_git_hash_path.read_text().strip()
if args.commit != archive_commit:
logger.warning("Commit argument is %s, but archive commit is %s. Using %s.", args.commit, archive_commit, archive_commit)
args.commit = archive_commit
else:
args.commit = executer.check_output(["git", "rev-parse", args.commit], dry_out="e5812a9fd2cda317b503325a702ba3c1c37861d9").strip()
logger.info("Using commit %s", args.commit)
try:
with args.path_release_info.open() as f:
release_info = json.load(f)
except FileNotFoundError:
logger.error(f"Could not find {args.path_release_info}")
releaser = Releaser(
release_info=release_info,
commit=args.commit,
root=args.root,
dist_path=args.dist_path,
executer=executer,
section_printer=section_printer,
cmake_generator=args.cmake_generator,
deps_path=args.deps_path,
overwrite=args.overwrite,
github=args.github,
fast=args.fast,
)
if root_is_maybe_archive:
logger.warning("Building from archive. Skipping clean git tree check.")
else:
porcelain_status = executer.check_output(["git", "status", "--ignored", "--porcelain"], dry_out="\n").strip()
if porcelain_status:
print(porcelain_status)
logger.warning("The tree is dirty! Do not publish any generated artifacts!")
if not args.force:
raise Exception("The git repo contains modified and/or non-committed files. Run with --force to ignore.")
if args.fast:
logger.warning("Doing fast build! Do not publish generated artifacts!")
with section_printer.group("Arguments"):
print(f"project = {releaser.project}")
print(f"version = {releaser.version}")
print(f"commit = {args.commit}")
print(f"out = {args.dist_path}")
print(f"actions = {args.actions}")
print(f"dry = {args.dry}")
print(f"force = {args.force}")
print(f"overwrite = {args.overwrite}")
print(f"cmake_generator = {args.cmake_generator}")
releaser.prepare()
if "download" in args.actions:
releaser.download_dependencies()
if set(args.actions).intersection({"msvc", "mingw"}):
print("Verifying presence of dependencies (run 'download' action to download) ...")
releaser.verify_dependencies()
print("... done")
if "source" in args.actions:
if root_is_maybe_archive:
raise Exception("Cannot build source archive from source archive")
with section_printer.group("Create source archives"):
releaser.create_source_archives()
if "dmg" in args.actions:
if platform.system() != "Darwin" and not args.dry:
parser.error("framework artifact(s) can only be built on Darwin")
releaser.create_dmg()
if "msvc" in args.actions:
if platform.system() != "Windows" and not args.dry:
parser.error("msvc artifact(s) can only be built on Windows")
with section_printer.group("Find Visual Studio"):
vs = VisualStudio(executer=executer)
arch_platforms = [
VsArchPlatformConfig(arch="x86", platform="Win32", configuration="Release"),
VsArchPlatformConfig(arch="x64", platform="x64", configuration="Release"),
]
for arch_platform in arch_platforms:
releaser.build_vs(arch_platform=arch_platform, vs=vs)
with section_printer.group("Create SDL VC development zip"):
releaser.build_vs_devel(arch_platforms)
if "mingw" in args.actions:
releaser.create_mingw_archives()
with section_printer.group("Summary"):
print(f"artifacts = {releaser.artifacts}")
if args.github:
with open(os.environ["GITHUB_OUTPUT"], "a") as f:
f.write(f"project={releaser.project}\n")
f.write(f"version={releaser.version}\n")
for k, v in releaser.artifacts.items():
f.write(f"{k}={v.name}\n")
return 0
if __name__ == "__main__":
raise SystemExit(main())