Initial commit: AirCoding V1.0.0 Alpha architecture baseline
Complete architecture document set with multi-model review remediation: - Frozen interface contracts, runtime semantics, DB schemas - Event/tool/error/provider registries - Scheduler and main agent state machines - C4 module/code views, solution architecture, baseline V1 - Multi-model review reports and joint assessment - Phase-gate remediation complete (P0/P1/P2/UX resolved) - Implementation plan with T-000A through T-045 - Reference folders kept as placeholders only
This commit is contained in:
AirCoding
172
reference/openai-codex/.github/scripts/build-codex-package-archive.sh
vendored
Normal file
172
reference/openai-codex/.github/scripts/build-codex-package-archive.sh
vendored
Normal file
@@ -0,0 +1,172 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
Usage: build-codex-package-archive.sh \
|
||||
--target <rust-target> \
|
||||
--bundle <primary|app-server> \
|
||||
--entrypoint-dir <dir> \
|
||||
--archive-dir <dir> \
|
||||
[--bwrap-bin <path>] \
|
||||
[--codex-command-runner-bin <path>] \
|
||||
[--codex-windows-sandbox-setup-bin <path>] \
|
||||
[--target-suffixed-entrypoint]
|
||||
EOF
|
||||
}
|
||||
|
||||
target=""
|
||||
bundle=""
|
||||
entrypoint_dir=""
|
||||
archive_dir=""
|
||||
target_suffixed_entrypoint="false"
|
||||
resource_args=()
|
||||
bwrap_bin_provided="false"
|
||||
command_runner_bin_provided="false"
|
||||
sandbox_setup_bin_provided="false"
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--target)
|
||||
target="${2:?--target requires a value}"
|
||||
shift 2
|
||||
;;
|
||||
--bundle)
|
||||
bundle="${2:?--bundle requires a value}"
|
||||
shift 2
|
||||
;;
|
||||
--entrypoint-dir)
|
||||
entrypoint_dir="${2:?--entrypoint-dir requires a value}"
|
||||
shift 2
|
||||
;;
|
||||
--archive-dir)
|
||||
archive_dir="${2:?--archive-dir requires a value}"
|
||||
shift 2
|
||||
;;
|
||||
--bwrap-bin)
|
||||
resource_args+=(--bwrap-bin "${2:?--bwrap-bin requires a value}")
|
||||
bwrap_bin_provided="true"
|
||||
shift 2
|
||||
;;
|
||||
--codex-command-runner-bin)
|
||||
resource_args+=(
|
||||
--codex-command-runner-bin
|
||||
"${2:?--codex-command-runner-bin requires a value}"
|
||||
)
|
||||
command_runner_bin_provided="true"
|
||||
shift 2
|
||||
;;
|
||||
--codex-windows-sandbox-setup-bin)
|
||||
resource_args+=(
|
||||
--codex-windows-sandbox-setup-bin
|
||||
"${2:?--codex-windows-sandbox-setup-bin requires a value}"
|
||||
)
|
||||
sandbox_setup_bin_provided="true"
|
||||
shift 2
|
||||
;;
|
||||
--target-suffixed-entrypoint)
|
||||
target_suffixed_entrypoint="true"
|
||||
shift
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "Unexpected argument: $1" >&2
|
||||
usage >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ -z "$target" || -z "$bundle" || -z "$entrypoint_dir" || -z "$archive_dir" ]]; then
|
||||
usage >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
case "$bundle" in
|
||||
primary)
|
||||
variant="codex"
|
||||
entrypoint="codex"
|
||||
archive_stem="codex-package"
|
||||
;;
|
||||
app-server)
|
||||
variant="codex-app-server"
|
||||
entrypoint="codex-app-server"
|
||||
archive_stem="codex-app-server-package"
|
||||
;;
|
||||
*)
|
||||
echo "No Codex package variant for bundle: $bundle" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
exe_suffix=""
|
||||
case "$target" in
|
||||
*windows*)
|
||||
exe_suffix=".exe"
|
||||
;;
|
||||
esac
|
||||
|
||||
entrypoint_name="$entrypoint"
|
||||
if [[ "$target_suffixed_entrypoint" == "true" ]]; then
|
||||
entrypoint_name="${entrypoint_name}-${target}"
|
||||
fi
|
||||
|
||||
case "$target" in
|
||||
*linux*)
|
||||
bwrap_bin="${entrypoint_dir%/}/bwrap"
|
||||
if [[ "$bwrap_bin_provided" == "false" && -f "$bwrap_bin" ]]; then
|
||||
resource_args+=(--bwrap-bin "$bwrap_bin")
|
||||
fi
|
||||
;;
|
||||
*windows*)
|
||||
command_runner_bin="${entrypoint_dir%/}/codex-command-runner.exe"
|
||||
sandbox_setup_bin="${entrypoint_dir%/}/codex-windows-sandbox-setup.exe"
|
||||
if [[ "$command_runner_bin_provided" == "false" && -f "$command_runner_bin" ]]; then
|
||||
resource_args+=(--codex-command-runner-bin "$command_runner_bin")
|
||||
fi
|
||||
if [[ "$sandbox_setup_bin_provided" == "false" && -f "$sandbox_setup_bin" ]]; then
|
||||
resource_args+=(--codex-windows-sandbox-setup-bin "$sandbox_setup_bin")
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
|
||||
repo_root="${GITHUB_WORKSPACE:-}"
|
||||
if [[ -z "$repo_root" ]]; then
|
||||
repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
fi
|
||||
|
||||
if command -v python3 >/dev/null 2>&1; then
|
||||
python_bin="python3"
|
||||
else
|
||||
python_bin="python"
|
||||
fi
|
||||
|
||||
if ! command -v zstd >/dev/null 2>&1 && [[ -x "${repo_root}/.github/workflows/zstd" ]]; then
|
||||
export PATH="${repo_root}/.github/workflows:${PATH}"
|
||||
fi
|
||||
|
||||
mkdir -p "$archive_dir"
|
||||
package_dir="${RUNNER_TEMP:-/tmp}/${archive_stem}-${target}"
|
||||
gzip_archive_path="${archive_dir}/${archive_stem}-${target}.tar.gz"
|
||||
zstd_archive_path="${archive_dir}/${archive_stem}-${target}.tar.zst"
|
||||
rm -rf "$package_dir"
|
||||
|
||||
python_args=(
|
||||
"${repo_root}/scripts/build_codex_package.py"
|
||||
--target "$target"
|
||||
--variant "$variant"
|
||||
--entrypoint-bin "${entrypoint_dir%/}/${entrypoint_name}${exe_suffix}"
|
||||
--cargo-profile release
|
||||
--package-dir "$package_dir"
|
||||
--archive-output "$gzip_archive_path"
|
||||
--archive-output "$zstd_archive_path"
|
||||
)
|
||||
if ((${#resource_args[@]} > 0)); then
|
||||
python_args+=("${resource_args[@]}")
|
||||
fi
|
||||
python_args+=(--force)
|
||||
|
||||
"$python_bin" "${python_args[@]}"
|
||||
61
reference/openai-codex/.github/scripts/build-zsh-release-artifact.sh
vendored
Executable file
61
reference/openai-codex/.github/scripts/build-zsh-release-artifact.sh
vendored
Executable file
@@ -0,0 +1,61 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
if [[ "$#" -ne 1 ]]; then
|
||||
echo "usage: $0 <archive-path>" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
archive_path="$1"
|
||||
workspace="${GITHUB_WORKSPACE:?missing GITHUB_WORKSPACE}"
|
||||
zsh_commit="${ZSH_COMMIT:?missing ZSH_COMMIT}"
|
||||
zsh_patch="${ZSH_PATCH:?missing ZSH_PATCH}"
|
||||
temp_root="${RUNNER_TEMP:-/tmp}"
|
||||
work_root="$(mktemp -d "${temp_root%/}/codex-zsh-release.XXXXXX")"
|
||||
trap 'rm -rf "$work_root"' EXIT
|
||||
|
||||
source_root="${work_root}/zsh"
|
||||
package_root="${work_root}/codex-zsh"
|
||||
wrapper_path="${work_root}/exec-wrapper"
|
||||
stdout_path="${work_root}/stdout.txt"
|
||||
wrapper_log_path="${work_root}/wrapper.log"
|
||||
|
||||
git clone https://git.code.sf.net/p/zsh/code "$source_root"
|
||||
cd "$source_root"
|
||||
git checkout "$zsh_commit"
|
||||
git apply "${workspace}/${zsh_patch}"
|
||||
./Util/preconfig
|
||||
./configure
|
||||
|
||||
cores="$(command -v nproc >/dev/null 2>&1 && nproc || getconf _NPROCESSORS_ONLN)"
|
||||
make -j"${cores}"
|
||||
|
||||
cat > "$wrapper_path" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
: "${CODEX_WRAPPER_LOG:?missing CODEX_WRAPPER_LOG}"
|
||||
printf '%s\n' "$@" > "$CODEX_WRAPPER_LOG"
|
||||
file="$1"
|
||||
shift
|
||||
if [[ "$#" -eq 0 ]]; then
|
||||
exec "$file"
|
||||
fi
|
||||
arg0="$1"
|
||||
shift
|
||||
exec -a "$arg0" "$file" "$@"
|
||||
EOF
|
||||
chmod +x "$wrapper_path"
|
||||
|
||||
CODEX_WRAPPER_LOG="$wrapper_log_path" \
|
||||
EXEC_WRAPPER="$wrapper_path" \
|
||||
"${source_root}/Src/zsh" -fc '/bin/echo smoke-zsh' > "$stdout_path"
|
||||
|
||||
grep -Fx "smoke-zsh" "$stdout_path"
|
||||
grep -Fx "/bin/echo" "$wrapper_log_path"
|
||||
|
||||
mkdir -p "$package_root/bin" "$(dirname "${workspace}/${archive_path}")"
|
||||
cp "${source_root}/Src/zsh" "$package_root/bin/zsh"
|
||||
chmod +x "$package_root/bin/zsh"
|
||||
|
||||
(cd "$work_root" && tar -czf "${workspace}/${archive_path}" codex-zsh)
|
||||
113
reference/openai-codex/.github/scripts/compute-bazel-windows-path.ps1
vendored
Normal file
113
reference/openai-codex/.github/scripts/compute-bazel-windows-path.ps1
vendored
Normal file
@@ -0,0 +1,113 @@
|
||||
<#
|
||||
BuildBuddy cache keys include the action and test environment, so Bazel should
|
||||
not inherit the full hosted-runner PATH on Windows. That PATH includes volatile
|
||||
tool entries, such as Maven, that can change independently of this repo and
|
||||
cause avoidable cache misses.
|
||||
|
||||
This script derives a smaller, cache-stable PATH that keeps the Windows
|
||||
toolchain entries Bazel-backed CI tasks need: MSVC and Windows SDK paths,
|
||||
MinGW runtime DLL paths for gnullvm-built tests, Git, PowerShell, Node, Python,
|
||||
DotSlash, and the standard Windows system directories.
|
||||
`setup-bazel-ci` runs this after exporting the MSVC environment, and the script
|
||||
publishes the result via `GITHUB_ENV` as `CODEX_BAZEL_WINDOWS_PATH` so later
|
||||
steps can pass that explicit PATH to Bazel.
|
||||
#>
|
||||
|
||||
$stablePathEntries = New-Object System.Collections.Generic.List[string]
|
||||
$seenEntries = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::OrdinalIgnoreCase)
|
||||
$windowsAppsPath = if ([string]::IsNullOrWhiteSpace($env:LOCALAPPDATA)) {
|
||||
$null
|
||||
} else {
|
||||
"$($env:LOCALAPPDATA)\Microsoft\WindowsApps"
|
||||
}
|
||||
$windowsDir = if ($env:WINDIR) {
|
||||
$env:WINDIR
|
||||
} elseif ($env:SystemRoot) {
|
||||
$env:SystemRoot
|
||||
} else {
|
||||
$null
|
||||
}
|
||||
|
||||
function Add-StablePathEntry {
|
||||
param([string]$PathEntry)
|
||||
|
||||
if ([string]::IsNullOrWhiteSpace($PathEntry)) {
|
||||
return
|
||||
}
|
||||
|
||||
if ($seenEntries.Add($PathEntry)) {
|
||||
[void]$stablePathEntries.Add($PathEntry)
|
||||
}
|
||||
}
|
||||
|
||||
foreach ($pathEntry in ($env:PATH -split ';')) {
|
||||
if ([string]::IsNullOrWhiteSpace($pathEntry)) {
|
||||
continue
|
||||
}
|
||||
|
||||
if (
|
||||
$pathEntry -like '*Microsoft Visual Studio*' -or
|
||||
$pathEntry -like '*Windows Kits*' -or
|
||||
$pathEntry -like '*Microsoft SDKs*' -or
|
||||
$pathEntry -eq 'C:\mingw64\bin' -or
|
||||
$pathEntry -like 'C:\msys64\*\bin' -or
|
||||
$pathEntry -like 'C:\Program Files\Git\*' -or
|
||||
$pathEntry -like 'C:\Program Files\PowerShell\*' -or
|
||||
$pathEntry -like 'C:\hostedtoolcache\windows\node\*' -or
|
||||
$pathEntry -like 'C:\hostedtoolcache\windows\Python\*' -or
|
||||
$pathEntry -eq 'D:\a\_temp\install-dotslash\bin' -or
|
||||
($windowsDir -and ($pathEntry -eq $windowsDir -or $pathEntry -like "${windowsDir}\*"))
|
||||
) {
|
||||
Add-StablePathEntry $pathEntry
|
||||
}
|
||||
}
|
||||
|
||||
$gitCommand = Get-Command git -ErrorAction SilentlyContinue
|
||||
if ($gitCommand) {
|
||||
Add-StablePathEntry (Split-Path $gitCommand.Source -Parent)
|
||||
}
|
||||
|
||||
$nodeCommand = Get-Command node -ErrorAction SilentlyContinue
|
||||
if ($nodeCommand) {
|
||||
Add-StablePathEntry (Split-Path $nodeCommand.Source -Parent)
|
||||
}
|
||||
|
||||
$python3Command = Get-Command python3 -ErrorAction SilentlyContinue
|
||||
if ($python3Command) {
|
||||
Add-StablePathEntry (Split-Path $python3Command.Source -Parent)
|
||||
}
|
||||
|
||||
$pythonCommand = Get-Command python -ErrorAction SilentlyContinue
|
||||
if ($pythonCommand) {
|
||||
Add-StablePathEntry (Split-Path $pythonCommand.Source -Parent)
|
||||
}
|
||||
|
||||
$pwshCommand = Get-Command pwsh -ErrorAction SilentlyContinue
|
||||
if ($pwshCommand) {
|
||||
Add-StablePathEntry (Split-Path $pwshCommand.Source -Parent)
|
||||
}
|
||||
|
||||
foreach ($mingwPath in @('C:\mingw64\bin', 'C:\msys64\mingw64\bin', 'C:\msys64\ucrt64\bin')) {
|
||||
if (Test-Path $mingwPath) {
|
||||
Add-StablePathEntry $mingwPath
|
||||
}
|
||||
}
|
||||
|
||||
if ($windowsAppsPath) {
|
||||
Add-StablePathEntry $windowsAppsPath
|
||||
}
|
||||
|
||||
if ($stablePathEntries.Count -eq 0) {
|
||||
throw 'Failed to derive cache-stable Windows PATH.'
|
||||
}
|
||||
|
||||
if ([string]::IsNullOrWhiteSpace($env:GITHUB_ENV)) {
|
||||
throw 'GITHUB_ENV must be set.'
|
||||
}
|
||||
|
||||
$stablePath = $stablePathEntries -join ';'
|
||||
Write-Host 'Derived CODEX_BAZEL_WINDOWS_PATH entries:'
|
||||
foreach ($pathEntry in $stablePathEntries) {
|
||||
Write-Host " $pathEntry"
|
||||
}
|
||||
"CODEX_BAZEL_WINDOWS_PATH=$stablePath" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
|
||||
279
reference/openai-codex/.github/scripts/install-musl-build-tools.sh
vendored
Normal file
279
reference/openai-codex/.github/scripts/install-musl-build-tools.sh
vendored
Normal file
@@ -0,0 +1,279 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
: "${TARGET:?TARGET environment variable is required}"
|
||||
: "${GITHUB_ENV:?GITHUB_ENV environment variable is required}"
|
||||
|
||||
apt_update_args=()
|
||||
if [[ -n "${APT_UPDATE_ARGS:-}" ]]; then
|
||||
# shellcheck disable=SC2206
|
||||
apt_update_args=(${APT_UPDATE_ARGS})
|
||||
fi
|
||||
|
||||
apt_install_args=()
|
||||
if [[ -n "${APT_INSTALL_ARGS:-}" ]]; then
|
||||
# shellcheck disable=SC2206
|
||||
apt_install_args=(${APT_INSTALL_ARGS})
|
||||
fi
|
||||
|
||||
sudo apt-get update "${apt_update_args[@]}"
|
||||
sudo apt-get install -y "${apt_install_args[@]}" ca-certificates curl musl-tools pkg-config libcap-dev g++ clang libc++-dev libc++abi-dev lld xz-utils
|
||||
|
||||
case "${TARGET}" in
|
||||
x86_64-unknown-linux-musl)
|
||||
arch="x86_64"
|
||||
;;
|
||||
aarch64-unknown-linux-musl)
|
||||
arch="aarch64"
|
||||
;;
|
||||
*)
|
||||
echo "Unexpected musl target: ${TARGET}" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
libcap_version="2.75"
|
||||
libcap_sha256="de4e7e064c9ba451d5234dd46e897d7c71c96a9ebf9a0c445bc04f4742d83632"
|
||||
libcap_tarball_name="libcap-${libcap_version}.tar.xz"
|
||||
libcap_download_url="https://mirrors.edge.kernel.org/pub/linux/libs/security/linux-privs/libcap2/${libcap_tarball_name}"
|
||||
|
||||
# Use the musl toolchain as the Rust linker to avoid Zig injecting its own CRT.
|
||||
if command -v "${arch}-linux-musl-gcc" >/dev/null; then
|
||||
musl_linker="$(command -v "${arch}-linux-musl-gcc")"
|
||||
elif command -v musl-gcc >/dev/null; then
|
||||
musl_linker="$(command -v musl-gcc)"
|
||||
else
|
||||
echo "musl gcc not found after install; arch=${arch}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
zig_target="${TARGET/-unknown-linux-musl/-linux-musl}"
|
||||
runner_temp="${RUNNER_TEMP:-/tmp}"
|
||||
tool_root="${runner_temp}/codex-musl-tools-${TARGET}"
|
||||
mkdir -p "${tool_root}"
|
||||
|
||||
libcap_root="${tool_root}/libcap-${libcap_version}"
|
||||
libcap_src_root="${libcap_root}/src"
|
||||
libcap_prefix="${libcap_root}/prefix"
|
||||
libcap_pkgconfig_dir="${libcap_prefix}/lib/pkgconfig"
|
||||
|
||||
if [[ ! -f "${libcap_prefix}/lib/libcap.a" ]]; then
|
||||
mkdir -p "${libcap_src_root}" "${libcap_prefix}/lib" "${libcap_prefix}/include/sys" "${libcap_prefix}/include/linux" "${libcap_pkgconfig_dir}"
|
||||
libcap_tarball="${libcap_root}/${libcap_tarball_name}"
|
||||
|
||||
curl -fsSL "${libcap_download_url}" -o "${libcap_tarball}"
|
||||
echo "${libcap_sha256} ${libcap_tarball}" | sha256sum -c -
|
||||
|
||||
tar -xJf "${libcap_tarball}" -C "${libcap_src_root}"
|
||||
libcap_source_dir="${libcap_src_root}/libcap-${libcap_version}"
|
||||
make -C "${libcap_source_dir}/libcap" -j"$(nproc)" \
|
||||
CC="${musl_linker}" \
|
||||
AR=ar \
|
||||
RANLIB=ranlib
|
||||
|
||||
cp "${libcap_source_dir}/libcap/libcap.a" "${libcap_prefix}/lib/libcap.a"
|
||||
cp "${libcap_source_dir}/libcap/include/uapi/linux/capability.h" "${libcap_prefix}/include/linux/capability.h"
|
||||
cp "${libcap_source_dir}/libcap/../libcap/include/sys/capability.h" "${libcap_prefix}/include/sys/capability.h"
|
||||
|
||||
cat > "${libcap_pkgconfig_dir}/libcap.pc" <<EOF
|
||||
prefix=${libcap_prefix}
|
||||
exec_prefix=\${prefix}
|
||||
libdir=\${prefix}/lib
|
||||
includedir=\${prefix}/include
|
||||
|
||||
Name: libcap
|
||||
Description: Linux capabilities
|
||||
Version: ${libcap_version}
|
||||
Libs: -L\${libdir} -lcap
|
||||
Cflags: -I\${includedir}
|
||||
EOF
|
||||
fi
|
||||
|
||||
sysroot=""
|
||||
if command -v zig >/dev/null; then
|
||||
zig_bin="$(command -v zig)"
|
||||
cc="${tool_root}/zigcc"
|
||||
cxx="${tool_root}/zigcxx"
|
||||
|
||||
cat >"${cc}" <<EOF
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
args=()
|
||||
skip_next=0
|
||||
pending_include=0
|
||||
for arg in "\$@"; do
|
||||
if [[ "\${pending_include}" -eq 1 ]]; then
|
||||
pending_include=0
|
||||
if [[ "\${arg}" == /usr/include || "\${arg}" == /usr/include/* ]]; then
|
||||
# Keep host-only headers available, but after the target sysroot headers.
|
||||
args+=("-idirafter" "\${arg}")
|
||||
else
|
||||
args+=("-I" "\${arg}")
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ "\${skip_next}" -eq 1 ]]; then
|
||||
skip_next=0
|
||||
continue
|
||||
fi
|
||||
case "\${arg}" in
|
||||
--target)
|
||||
skip_next=1
|
||||
continue
|
||||
;;
|
||||
--target=*|-target=*|-target)
|
||||
# Drop any explicit --target/-target flags. Zig expects -target and
|
||||
# rejects Rust triples like *-unknown-linux-musl.
|
||||
if [[ "\${arg}" == "-target" ]]; then
|
||||
skip_next=1
|
||||
fi
|
||||
continue
|
||||
;;
|
||||
-I)
|
||||
pending_include=1
|
||||
continue
|
||||
;;
|
||||
-I/usr/include|-I/usr/include/*)
|
||||
# Avoid making glibc headers win over musl headers.
|
||||
args+=("-idirafter" "\${arg#-I}")
|
||||
continue
|
||||
;;
|
||||
-Wp,-U_FORTIFY_SOURCE)
|
||||
# aws-lc-sys emits this GCC preprocessor forwarding form in debug
|
||||
# builds, but zig cc expects the define flag directly.
|
||||
args+=("-U_FORTIFY_SOURCE")
|
||||
continue
|
||||
;;
|
||||
esac
|
||||
args+=("\${arg}")
|
||||
done
|
||||
|
||||
exec "${zig_bin}" cc -target "${zig_target}" "\${args[@]}"
|
||||
EOF
|
||||
cat >"${cxx}" <<EOF
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
args=()
|
||||
skip_next=0
|
||||
pending_include=0
|
||||
for arg in "\$@"; do
|
||||
if [[ "\${pending_include}" -eq 1 ]]; then
|
||||
pending_include=0
|
||||
if [[ "\${arg}" == /usr/include || "\${arg}" == /usr/include/* ]]; then
|
||||
# Keep host-only headers available, but after the target sysroot headers.
|
||||
args+=("-idirafter" "\${arg}")
|
||||
else
|
||||
args+=("-I" "\${arg}")
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ "\${skip_next}" -eq 1 ]]; then
|
||||
skip_next=0
|
||||
continue
|
||||
fi
|
||||
case "\${arg}" in
|
||||
--target)
|
||||
# Drop explicit --target and its value: we always pass zig's -target below.
|
||||
skip_next=1
|
||||
continue
|
||||
;;
|
||||
--target=*|-target=*|-target)
|
||||
# Zig expects -target and rejects Rust triples like *-unknown-linux-musl.
|
||||
if [[ "\${arg}" == "-target" ]]; then
|
||||
skip_next=1
|
||||
fi
|
||||
continue
|
||||
;;
|
||||
-I)
|
||||
pending_include=1
|
||||
continue
|
||||
;;
|
||||
-I/usr/include|-I/usr/include/*)
|
||||
# Avoid making glibc headers win over musl headers.
|
||||
args+=("-idirafter" "\${arg#-I}")
|
||||
continue
|
||||
;;
|
||||
-Wp,-U_FORTIFY_SOURCE)
|
||||
# aws-lc-sys emits this GCC forwarding form in debug builds; zig c++
|
||||
# expects the define flag directly.
|
||||
args+=("-U_FORTIFY_SOURCE")
|
||||
continue
|
||||
;;
|
||||
esac
|
||||
args+=("\${arg}")
|
||||
done
|
||||
|
||||
exec "${zig_bin}" c++ -target "${zig_target}" "\${args[@]}"
|
||||
EOF
|
||||
chmod +x "${cc}" "${cxx}"
|
||||
|
||||
sysroot="$("${zig_bin}" cc -target "${zig_target}" -print-sysroot 2>/dev/null || true)"
|
||||
else
|
||||
cc="${musl_linker}"
|
||||
|
||||
if command -v "${arch}-linux-musl-g++" >/dev/null; then
|
||||
cxx="$(command -v "${arch}-linux-musl-g++")"
|
||||
elif command -v musl-g++ >/dev/null; then
|
||||
cxx="$(command -v musl-g++)"
|
||||
else
|
||||
cxx="${cc}"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ -n "${sysroot}" && "${sysroot}" != "/" ]]; then
|
||||
echo "BORING_BSSL_SYSROOT=${sysroot}" >> "$GITHUB_ENV"
|
||||
boring_sysroot_var="BORING_BSSL_SYSROOT_${TARGET}"
|
||||
boring_sysroot_var="${boring_sysroot_var//-/_}"
|
||||
echo "${boring_sysroot_var}=${sysroot}" >> "$GITHUB_ENV"
|
||||
fi
|
||||
|
||||
cflags="-pthread"
|
||||
cxxflags="-pthread"
|
||||
if [[ "${TARGET}" == "aarch64-unknown-linux-musl" ]]; then
|
||||
# BoringSSL enables -Wframe-larger-than=25344 under clang and treats warnings as errors.
|
||||
cflags="${cflags} -Wno-error=frame-larger-than"
|
||||
cxxflags="${cxxflags} -Wno-error=frame-larger-than"
|
||||
fi
|
||||
|
||||
echo "CFLAGS=${cflags}" >> "$GITHUB_ENV"
|
||||
echo "CXXFLAGS=${cxxflags}" >> "$GITHUB_ENV"
|
||||
echo "CC=${cc}" >> "$GITHUB_ENV"
|
||||
echo "TARGET_CC=${cc}" >> "$GITHUB_ENV"
|
||||
target_cc_var="CC_${TARGET}"
|
||||
target_cc_var="${target_cc_var//-/_}"
|
||||
echo "${target_cc_var}=${cc}" >> "$GITHUB_ENV"
|
||||
echo "CXX=${cxx}" >> "$GITHUB_ENV"
|
||||
echo "TARGET_CXX=${cxx}" >> "$GITHUB_ENV"
|
||||
target_cxx_var="CXX_${TARGET}"
|
||||
target_cxx_var="${target_cxx_var//-/_}"
|
||||
echo "${target_cxx_var}=${cxx}" >> "$GITHUB_ENV"
|
||||
|
||||
cargo_linker_var="CARGO_TARGET_${TARGET^^}_LINKER"
|
||||
cargo_linker_var="${cargo_linker_var//-/_}"
|
||||
echo "${cargo_linker_var}=${musl_linker}" >> "$GITHUB_ENV"
|
||||
|
||||
echo "CMAKE_C_COMPILER=${cc}" >> "$GITHUB_ENV"
|
||||
echo "CMAKE_CXX_COMPILER=${cxx}" >> "$GITHUB_ENV"
|
||||
echo "CMAKE_ARGS=-DCMAKE_HAVE_THREADS_LIBRARY=1 -DCMAKE_USE_PTHREADS_INIT=1 -DCMAKE_THREAD_LIBS_INIT=-pthread -DTHREADS_PREFER_PTHREAD_FLAG=ON" >> "$GITHUB_ENV"
|
||||
|
||||
# Allow pkg-config resolution during cross-compilation.
|
||||
echo "PKG_CONFIG_ALLOW_CROSS=1" >> "$GITHUB_ENV"
|
||||
pkg_config_path="${libcap_pkgconfig_dir}"
|
||||
if [[ -n "${PKG_CONFIG_PATH:-}" ]]; then
|
||||
pkg_config_path="${pkg_config_path}:${PKG_CONFIG_PATH}"
|
||||
fi
|
||||
echo "PKG_CONFIG_PATH=${pkg_config_path}" >> "$GITHUB_ENV"
|
||||
pkg_config_path_var="PKG_CONFIG_PATH_${TARGET}"
|
||||
pkg_config_path_var="${pkg_config_path_var//-/_}"
|
||||
echo "${pkg_config_path_var}=${libcap_pkgconfig_dir}" >> "$GITHUB_ENV"
|
||||
|
||||
if [[ -n "${sysroot}" && "${sysroot}" != "/" ]]; then
|
||||
echo "PKG_CONFIG_SYSROOT_DIR=${sysroot}" >> "$GITHUB_ENV"
|
||||
pkg_config_sysroot_var="PKG_CONFIG_SYSROOT_DIR_${TARGET}"
|
||||
pkg_config_sysroot_var="${pkg_config_sysroot_var//-/_}"
|
||||
echo "${pkg_config_sysroot_var}=${sysroot}" >> "$GITHUB_ENV"
|
||||
fi
|
||||
80
reference/openai-codex/.github/scripts/run-argument-comment-lint-bazel.sh
vendored
Executable file
80
reference/openai-codex/.github/scripts/run-argument-comment-lint-bazel.sh
vendored
Executable file
@@ -0,0 +1,80 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
bazel_lint_args=("$@")
|
||||
if [[ "${RUNNER_OS:-}" == "Windows" ]]; then
|
||||
has_host_platform_override=0
|
||||
for arg in "${bazel_lint_args[@]}"; do
|
||||
if [[ "$arg" == --host_platform=* ]]; then
|
||||
has_host_platform_override=1
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ $has_host_platform_override -eq 0 ]]; then
|
||||
# The nightly Windows lint toolchain is registered with an MSVC exec
|
||||
# platform even though the lint target platform stays on `windows-gnullvm`.
|
||||
# Override the host platform here so the exec-side helper binaries actually
|
||||
# match the registered toolchain set.
|
||||
bazel_lint_args+=("--host_platform=//:local_windows_msvc")
|
||||
fi
|
||||
|
||||
# Native Windows lint runs need exec-side Rust helper binaries and proc-macros
|
||||
# to use rust-lld instead of the C++ linker path. The default `none`
|
||||
# preference resolves to `cc` when a cc_toolchain is present, which currently
|
||||
# routes these exec actions through clang++ with an argument shape it cannot
|
||||
# consume.
|
||||
bazel_lint_args+=("--@rules_rust//rust/settings:toolchain_linker_preference=rust")
|
||||
|
||||
# Some Rust top-level targets are still intentionally incompatible with the
|
||||
# local Windows MSVC exec platform. Skip those explicit targets so the native
|
||||
# lint aspect can run across the compatible crate graph instead of failing the
|
||||
# whole build after analysis.
|
||||
bazel_lint_args+=("--skip_incompatible_explicit_targets")
|
||||
fi
|
||||
|
||||
read_query_labels() {
|
||||
local query="$1"
|
||||
local query_stdout
|
||||
local query_stderr
|
||||
query_stdout="$(mktemp)"
|
||||
query_stderr="$(mktemp)"
|
||||
|
||||
if ! ./.github/scripts/run-bazel-query-ci.sh \
|
||||
--keep_going \
|
||||
--output=label \
|
||||
-- "$query" >"$query_stdout" 2>"$query_stderr"; then
|
||||
cat "$query_stderr" >&2
|
||||
rm -f "$query_stdout" "$query_stderr"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
cat "$query_stdout"
|
||||
rm -f "$query_stdout" "$query_stderr"
|
||||
}
|
||||
|
||||
final_build_targets=(//codex-rs/...)
|
||||
if [[ "${RUNNER_OS:-}" == "Windows" ]]; then
|
||||
# Bazel's local Windows platform currently lacks a default test toolchain for
|
||||
# `rust_test`, so target the concrete Rust crate rules directly. The lint
|
||||
# aspect still walks their crate graph, which preserves incremental reuse for
|
||||
# non-test code while avoiding non-Rust wrapper targets such as platform_data.
|
||||
final_build_targets=()
|
||||
while IFS= read -r label; do
|
||||
[[ -n "$label" ]] || continue
|
||||
final_build_targets+=("$label")
|
||||
done < <(read_query_labels 'kind("rust_(library|binary|proc_macro) rule", //codex-rs/...)')
|
||||
|
||||
if [[ ${#final_build_targets[@]} -eq 0 ]]; then
|
||||
echo "Failed to discover Windows Bazel lint targets." >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
./.github/scripts/run-bazel-ci.sh \
|
||||
-- \
|
||||
build \
|
||||
"${bazel_lint_args[@]}" \
|
||||
-- \
|
||||
"${final_build_targets[@]}"
|
||||
453
reference/openai-codex/.github/scripts/run-bazel-ci.sh
vendored
Executable file
453
reference/openai-codex/.github/scripts/run-bazel-ci.sh
vendored
Executable file
@@ -0,0 +1,453 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
print_failed_bazel_test_logs=0
|
||||
print_failed_bazel_action_summary=0
|
||||
remote_download_toplevel=0
|
||||
windows_msvc_host_platform=0
|
||||
windows_cross_compile=0
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--print-failed-test-logs)
|
||||
print_failed_bazel_test_logs=1
|
||||
shift
|
||||
;;
|
||||
--print-failed-action-summary)
|
||||
print_failed_bazel_action_summary=1
|
||||
shift
|
||||
;;
|
||||
--remote-download-toplevel)
|
||||
remote_download_toplevel=1
|
||||
shift
|
||||
;;
|
||||
--windows-msvc-host-platform)
|
||||
windows_msvc_host_platform=1
|
||||
shift
|
||||
;;
|
||||
--windows-cross-compile)
|
||||
windows_cross_compile=1
|
||||
shift
|
||||
;;
|
||||
--)
|
||||
shift
|
||||
break
|
||||
;;
|
||||
*)
|
||||
echo "Unknown option: $1" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ $# -eq 0 ]]; then
|
||||
echo "Usage: $0 [--print-failed-test-logs] [--print-failed-action-summary] [--remote-download-toplevel] [--windows-msvc-host-platform] [--windows-cross-compile] -- <bazel args> -- <targets>" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
bazel_startup_args=()
|
||||
if [[ -n "${BAZEL_OUTPUT_USER_ROOT:-}" ]]; then
|
||||
bazel_startup_args+=("--output_user_root=${BAZEL_OUTPUT_USER_ROOT}")
|
||||
fi
|
||||
|
||||
run_bazel() {
|
||||
if [[ "${RUNNER_OS:-}" == "Windows" ]]; then
|
||||
MSYS2_ARG_CONV_EXCL='*' bazel "$@"
|
||||
return
|
||||
fi
|
||||
|
||||
bazel "$@"
|
||||
}
|
||||
|
||||
ci_config=ci-linux
|
||||
case "${RUNNER_OS:-}" in
|
||||
macOS)
|
||||
ci_config=ci-macos
|
||||
;;
|
||||
Windows)
|
||||
if [[ $windows_cross_compile -eq 1 ]]; then
|
||||
ci_config=ci-windows-cross
|
||||
else
|
||||
ci_config=ci-windows
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
|
||||
print_bazel_test_log_tails() {
|
||||
local console_log="$1"
|
||||
local testlogs_dir
|
||||
local -a bazel_info_cmd=(bazel)
|
||||
local -a bazel_info_args=(info)
|
||||
|
||||
if (( ${#bazel_startup_args[@]} > 0 )); then
|
||||
bazel_info_cmd+=("${bazel_startup_args[@]}")
|
||||
fi
|
||||
|
||||
# `bazel info` needs the same CI config as the failed test invocation so
|
||||
# platform-specific output roots match. On Windows, omitting `ci-windows`
|
||||
# would point at `local_windows-fastbuild` even when the test ran with the
|
||||
# MSVC host platform under `local_windows_msvc-fastbuild`.
|
||||
if [[ -n "${BUILDBUDDY_API_KEY:-}" ]]; then
|
||||
bazel_info_args+=(
|
||||
"--config=${ci_config}"
|
||||
"--remote_header=x-buildbuddy-api-key=${BUILDBUDDY_API_KEY}"
|
||||
)
|
||||
fi
|
||||
# Only pass flags that affect Bazel's output-root selection or repository
|
||||
# lookup. Test/build-only flags such as execution logs or remote download
|
||||
# mode can make `bazel info` fail, which would hide the real test log path.
|
||||
for arg in "${post_config_bazel_args[@]}"; do
|
||||
case "$arg" in
|
||||
--host_platform=* | --repo_contents_cache=* | --repository_cache=*)
|
||||
bazel_info_args+=("$arg")
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
testlogs_dir="$(run_bazel "${bazel_info_cmd[@]:1}" \
|
||||
--noexperimental_remote_repo_contents_cache \
|
||||
"${bazel_info_args[@]}" \
|
||||
bazel-testlogs 2>/dev/null || echo bazel-testlogs)"
|
||||
|
||||
local failed_targets=()
|
||||
while IFS= read -r target; do
|
||||
failed_targets+=("$target")
|
||||
done < <(
|
||||
grep -E '^(FAIL: //|ERROR: .* Testing //)' "$console_log" \
|
||||
| sed -E 's#^FAIL: (//[^ ]+).*#\1#; s#^ERROR: .* Testing (//[^ ]+) failed:.*#\1#' \
|
||||
| sort -u
|
||||
)
|
||||
|
||||
if [[ ${#failed_targets[@]} -eq 0 ]]; then
|
||||
echo "No failed Bazel test targets were found in console output."
|
||||
return
|
||||
fi
|
||||
|
||||
for target in "${failed_targets[@]}"; do
|
||||
local rel_path="${target#//}"
|
||||
rel_path="${rel_path/://}"
|
||||
local test_log="${testlogs_dir}/${rel_path}/test.log"
|
||||
local reported_test_log
|
||||
reported_test_log="$(grep -F "FAIL: ${target} " "$console_log" | sed -nE 's#.* \(see (.*[\\/]test\.log)\).*#\1#p' | head -n 1 || true)"
|
||||
if [[ -n "$reported_test_log" ]]; then
|
||||
reported_test_log="${reported_test_log//\\//}"
|
||||
test_log="$reported_test_log"
|
||||
fi
|
||||
|
||||
echo "::group::Bazel test log tail for ${target}"
|
||||
if [[ -f "$test_log" ]]; then
|
||||
tail -n 200 "$test_log"
|
||||
else
|
||||
echo "Missing test log: $test_log"
|
||||
fi
|
||||
echo "::endgroup::"
|
||||
done
|
||||
}
|
||||
|
||||
print_bazel_action_failure_summary() {
|
||||
local console_log="$1"
|
||||
local escaped_summary
|
||||
local summary
|
||||
|
||||
summary="$(
|
||||
awk '
|
||||
function clean(line) {
|
||||
gsub(sprintf("%c", 27) "\\[[0-9;]*m", "", line)
|
||||
sub(/^.*\t[^\t]*\t[0-9TZ:._-]+ /, "", line)
|
||||
return line
|
||||
}
|
||||
|
||||
function is_diagnostic(line) {
|
||||
return line ~ /^(error(\[[^]]+\])?:|warning:|note:|help:)/ ||
|
||||
line ~ /^[[:space:]]+-->/ ||
|
||||
line ~ /^[[:space:]]*[0-9]+[[:space:]]+\|/ ||
|
||||
line ~ /^[[:space:]]*\|/ ||
|
||||
line ~ /^[[:space:]]+= (note|help):/ ||
|
||||
line ~ /^[[:space:]]*\^[[:space:]^~-]*$/ ||
|
||||
line ~ /^For more information/ ||
|
||||
line ~ /^error: aborting/
|
||||
}
|
||||
|
||||
{
|
||||
line = clean($0)
|
||||
}
|
||||
|
||||
line ~ /^ERROR: .* failed:/ {
|
||||
if (printed) {
|
||||
print ""
|
||||
}
|
||||
print line
|
||||
in_failure = 1
|
||||
seen_diagnostic = 0
|
||||
printed = 1
|
||||
next
|
||||
}
|
||||
|
||||
in_failure && is_diagnostic(line) {
|
||||
print line
|
||||
seen_diagnostic = 1
|
||||
next
|
||||
}
|
||||
|
||||
in_failure && seen_diagnostic && line == "" {
|
||||
print ""
|
||||
next
|
||||
}
|
||||
|
||||
in_failure && seen_diagnostic {
|
||||
in_failure = 0
|
||||
seen_diagnostic = 0
|
||||
next
|
||||
}
|
||||
' "$console_log"
|
||||
)"
|
||||
|
||||
if [[ -z "$summary" ]]; then
|
||||
summary="$(grep -E '^ERROR: |^FAILED: ' "$console_log" | tail -n 50 || true)"
|
||||
fi
|
||||
|
||||
if [[ -z "$summary" ]]; then
|
||||
echo "No Bazel action failures were found in the captured console output."
|
||||
return
|
||||
fi
|
||||
|
||||
if [[ "${GITHUB_ACTIONS:-}" == "true" ]]; then
|
||||
escaped_summary="$(
|
||||
printf '%s' "$summary" \
|
||||
| awk 'BEGIN { ORS = "" } {
|
||||
gsub(/%/, "%25")
|
||||
gsub(/\r/, "%0D")
|
||||
print sep $0
|
||||
sep = "%0A"
|
||||
}'
|
||||
)"
|
||||
echo "::error title=Bazel failed action diagnostics::${escaped_summary}"
|
||||
fi
|
||||
|
||||
echo
|
||||
echo "Bazel failed action diagnostics:"
|
||||
echo "--------------------------------"
|
||||
printf '%s\n' "$summary"
|
||||
echo "--------------------------------"
|
||||
}
|
||||
|
||||
bazel_args=()
|
||||
bazel_targets=()
|
||||
found_target_separator=0
|
||||
for arg in "$@"; do
|
||||
if [[ "$arg" == "--" && $found_target_separator -eq 0 ]]; then
|
||||
found_target_separator=1
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ $found_target_separator -eq 0 ]]; then
|
||||
bazel_args+=("$arg")
|
||||
else
|
||||
bazel_targets+=("$arg")
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ ${#bazel_args[@]} -eq 0 || ${#bazel_targets[@]} -eq 0 ]]; then
|
||||
echo "Expected Bazel args and targets separated by --" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ "${RUNNER_OS:-}" == "Windows" && $windows_cross_compile -eq 1 && -z "${BUILDBUDDY_API_KEY:-}" ]]; then
|
||||
# Fork PRs do not receive the BuildBuddy secret needed for the remote
|
||||
# cross-compile config. Preserve the previous local Windows build shape.
|
||||
windows_msvc_host_platform=1
|
||||
fi
|
||||
|
||||
post_config_bazel_args=()
|
||||
if [[ "${RUNNER_OS:-}" == "Windows" && $windows_msvc_host_platform -eq 1 ]]; then
|
||||
has_host_platform_override=0
|
||||
for arg in "${bazel_args[@]}"; do
|
||||
if [[ "$arg" == --host_platform=* ]]; then
|
||||
has_host_platform_override=1
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ $has_host_platform_override -eq 0 ]]; then
|
||||
# Use the MSVC Windows platform for jobs that need helper binaries like
|
||||
# Rust test wrappers and V8 generators to resolve a compatible toolchain.
|
||||
# Callers that need a different Windows target platform should pass an
|
||||
# explicit `--platforms=...` flag.
|
||||
post_config_bazel_args+=("--host_platform=//:local_windows_msvc")
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ $remote_download_toplevel -eq 1 ]]; then
|
||||
# Override the CI config's remote_download_minimal setting when callers need
|
||||
# the built artifact to exist on disk after the command completes.
|
||||
post_config_bazel_args+=(--remote_download_toplevel)
|
||||
fi
|
||||
|
||||
if [[ "${RUNNER_OS:-}" == "Windows" && $windows_cross_compile -eq 1 && -n "${BUILDBUDDY_API_KEY:-}" ]]; then
|
||||
# `--enable_platform_specific_config` expands `common:windows` on Windows
|
||||
# hosts after ordinary rc configs, which can override `ci-windows-cross`'s
|
||||
# RBE host platform. Repeat the host platform on the command line so V8 and
|
||||
# other genrules execute on Linux RBE workers instead of Git Bash locally.
|
||||
#
|
||||
# Bazel also derives the default genrule shell from the client host. Without
|
||||
# an explicit shell executable, remote Linux actions can be asked to run
|
||||
# `C:\Program Files\Git\usr\bin\bash.exe`.
|
||||
post_config_bazel_args+=(--host_platform=//:rbe --shell_executable=/bin/bash)
|
||||
fi
|
||||
|
||||
if [[ "${RUNNER_OS:-}" == "Windows" && $windows_cross_compile -eq 1 && -z "${BUILDBUDDY_API_KEY:-}" ]]; then
|
||||
# The Windows cross-compile config depends on remote execution. Fork PRs do
|
||||
# not receive the BuildBuddy secret, so fall back to the existing local build
|
||||
# shape and keep its lower concurrency cap.
|
||||
post_config_bazel_args+=(--jobs=8)
|
||||
fi
|
||||
|
||||
if [[ -n "${BAZEL_REPO_CONTENTS_CACHE:-}" ]]; then
|
||||
# Windows self-hosted runners can run multiple Bazel jobs concurrently. Give
|
||||
# each job its own repo contents cache so they do not fight over the shared
|
||||
# path configured in `ci-windows`.
|
||||
post_config_bazel_args+=("--repo_contents_cache=${BAZEL_REPO_CONTENTS_CACHE}")
|
||||
fi
|
||||
|
||||
if [[ -n "${BAZEL_REPOSITORY_CACHE:-}" ]]; then
|
||||
post_config_bazel_args+=("--repository_cache=${BAZEL_REPOSITORY_CACHE}")
|
||||
fi
|
||||
|
||||
if [[ -n "${CODEX_BAZEL_EXECUTION_LOG_COMPACT_DIR:-}" ]]; then
|
||||
post_config_bazel_args+=(
|
||||
"--execution_log_compact_file=${CODEX_BAZEL_EXECUTION_LOG_COMPACT_DIR}/execution-log-${bazel_args[0]}-${GITHUB_JOB:-local}-$$.zst"
|
||||
)
|
||||
fi
|
||||
|
||||
if [[ "${RUNNER_OS:-}" == "Windows" ]]; then
|
||||
pass_windows_build_env=1
|
||||
if [[ $windows_cross_compile -eq 1 && -n "${BUILDBUDDY_API_KEY:-}" ]]; then
|
||||
# Remote build actions execute on Linux RBE workers. Passing the Windows
|
||||
# runner's build environment there makes Bazel genrules try to execute
|
||||
# C:\Program Files\Git\usr\bin\bash.exe on Linux.
|
||||
pass_windows_build_env=0
|
||||
fi
|
||||
|
||||
if [[ $pass_windows_build_env -eq 1 ]]; then
|
||||
windows_action_env_vars=(
|
||||
INCLUDE
|
||||
LIB
|
||||
LIBPATH
|
||||
UCRTVersion
|
||||
UniversalCRTSdkDir
|
||||
VCINSTALLDIR
|
||||
VCToolsInstallDir
|
||||
WindowsLibPath
|
||||
WindowsSdkBinPath
|
||||
WindowsSdkDir
|
||||
WindowsSDKLibVersion
|
||||
WindowsSDKVersion
|
||||
)
|
||||
|
||||
for env_var in "${windows_action_env_vars[@]}"; do
|
||||
if [[ -n "${!env_var:-}" ]]; then
|
||||
post_config_bazel_args+=("--action_env=${env_var}" "--host_action_env=${env_var}")
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
if [[ -z "${CODEX_BAZEL_WINDOWS_PATH:-}" ]]; then
|
||||
echo "CODEX_BAZEL_WINDOWS_PATH must be set for Windows Bazel CI." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ $pass_windows_build_env -eq 1 ]]; then
|
||||
post_config_bazel_args+=(
|
||||
"--action_env=PATH=${CODEX_BAZEL_WINDOWS_PATH}"
|
||||
"--host_action_env=PATH=${CODEX_BAZEL_WINDOWS_PATH}"
|
||||
)
|
||||
elif [[ $windows_cross_compile -eq 1 ]]; then
|
||||
# Remote build actions run on Linux RBE workers. Give their shell snippets
|
||||
# a Linux PATH while preserving CODEX_BAZEL_WINDOWS_PATH below for local
|
||||
# Windows test execution.
|
||||
post_config_bazel_args+=(
|
||||
"--action_env=PATH=/usr/bin:/bin"
|
||||
"--host_action_env=PATH=/usr/bin:/bin"
|
||||
)
|
||||
fi
|
||||
post_config_bazel_args+=("--test_env=PATH=${CODEX_BAZEL_WINDOWS_PATH}")
|
||||
fi
|
||||
|
||||
bazel_console_log="$(mktemp)"
|
||||
trap 'rm -f "$bazel_console_log"' EXIT
|
||||
|
||||
bazel_cmd=(bazel)
|
||||
if (( ${#bazel_startup_args[@]} > 0 )); then
|
||||
bazel_cmd+=("${bazel_startup_args[@]}")
|
||||
fi
|
||||
|
||||
if [[ -n "${BUILDBUDDY_API_KEY:-}" ]]; then
|
||||
echo "BuildBuddy API key is available; using remote Bazel configuration."
|
||||
# Work around Bazel 9 remote repo contents cache / overlay materialization failures
|
||||
# seen in CI (for example "is not a symlink" or permission errors while
|
||||
# materializing external repos such as rules_perl). We still use BuildBuddy for
|
||||
# remote execution/cache; this only disables the startup-level repo contents cache.
|
||||
bazel_run_args=(
|
||||
"${bazel_args[@]}"
|
||||
"--config=${ci_config}"
|
||||
"--remote_header=x-buildbuddy-api-key=${BUILDBUDDY_API_KEY}"
|
||||
)
|
||||
if (( ${#post_config_bazel_args[@]} > 0 )); then
|
||||
bazel_run_args+=("${post_config_bazel_args[@]}")
|
||||
fi
|
||||
set +e
|
||||
run_bazel "${bazel_cmd[@]:1}" \
|
||||
--noexperimental_remote_repo_contents_cache \
|
||||
"${bazel_run_args[@]}" \
|
||||
-- \
|
||||
"${bazel_targets[@]}" \
|
||||
2>&1 | tee "$bazel_console_log"
|
||||
bazel_status=${PIPESTATUS[0]}
|
||||
set -e
|
||||
else
|
||||
echo "BuildBuddy API key is not available; using local Bazel configuration."
|
||||
# Keep fork/community PRs on Bazel but disable remote services that are
|
||||
# configured in .bazelrc and require auth.
|
||||
#
|
||||
# Flag docs:
|
||||
# - Command-line reference: https://bazel.build/reference/command-line-reference
|
||||
# - Remote caching overview: https://bazel.build/remote/caching
|
||||
# - Remote execution overview: https://bazel.build/remote/rbe
|
||||
# - Build Event Protocol overview: https://bazel.build/remote/bep
|
||||
#
|
||||
# --noexperimental_remote_repo_contents_cache:
|
||||
# disable remote repo contents cache enabled in .bazelrc startup options.
|
||||
# https://bazel.build/reference/command-line-reference#startup_options-flag--experimental_remote_repo_contents_cache
|
||||
# --remote_cache= and --remote_executor=:
|
||||
# clear remote cache/execution endpoints configured in .bazelrc.
|
||||
# https://bazel.build/reference/command-line-reference#common_options-flag--remote_cache
|
||||
# https://bazel.build/reference/command-line-reference#common_options-flag--remote_executor
|
||||
bazel_run_args=(
|
||||
"${bazel_args[@]}"
|
||||
--remote_cache=
|
||||
--remote_executor=
|
||||
)
|
||||
if (( ${#post_config_bazel_args[@]} > 0 )); then
|
||||
bazel_run_args+=("${post_config_bazel_args[@]}")
|
||||
fi
|
||||
set +e
|
||||
run_bazel "${bazel_cmd[@]:1}" \
|
||||
--noexperimental_remote_repo_contents_cache \
|
||||
"${bazel_run_args[@]}" \
|
||||
-- \
|
||||
"${bazel_targets[@]}" \
|
||||
2>&1 | tee "$bazel_console_log"
|
||||
bazel_status=${PIPESTATUS[0]}
|
||||
set -e
|
||||
fi
|
||||
|
||||
if [[ ${bazel_status:-0} -ne 0 ]]; then
|
||||
if [[ $print_failed_bazel_action_summary -eq 1 ]]; then
|
||||
print_bazel_action_failure_summary "$bazel_console_log"
|
||||
fi
|
||||
if [[ $print_failed_bazel_test_logs -eq 1 ]]; then
|
||||
print_bazel_test_log_tails "$bazel_console_log"
|
||||
fi
|
||||
exit "$bazel_status"
|
||||
fi
|
||||
84
reference/openai-codex/.github/scripts/run-bazel-query-ci.sh
vendored
Executable file
84
reference/openai-codex/.github/scripts/run-bazel-query-ci.sh
vendored
Executable file
@@ -0,0 +1,84 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Run Bazel queries with the same CI startup settings as the main build/test
|
||||
# invocation so target-discovery queries can reuse the same Bazel server.
|
||||
|
||||
query_args=()
|
||||
windows_cross_compile=0
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--windows-cross-compile)
|
||||
windows_cross_compile=1
|
||||
shift
|
||||
;;
|
||||
--)
|
||||
shift
|
||||
break
|
||||
;;
|
||||
*)
|
||||
query_args+=("$1")
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ $# -ne 1 ]]; then
|
||||
echo "Usage: $0 [--windows-cross-compile] [<bazel query args>...] -- <query expression>" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
query_expression="$1"
|
||||
|
||||
ci_config=ci-linux
|
||||
case "${RUNNER_OS:-}" in
|
||||
macOS)
|
||||
ci_config=ci-macos
|
||||
;;
|
||||
Windows)
|
||||
if [[ $windows_cross_compile -eq 1 ]]; then
|
||||
ci_config=ci-windows-cross
|
||||
else
|
||||
ci_config=ci-windows
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
|
||||
bazel_startup_args=()
|
||||
if [[ -n "${BAZEL_OUTPUT_USER_ROOT:-}" ]]; then
|
||||
bazel_startup_args+=("--output_user_root=${BAZEL_OUTPUT_USER_ROOT}")
|
||||
fi
|
||||
|
||||
run_bazel() {
|
||||
if [[ "${RUNNER_OS:-}" == "Windows" ]]; then
|
||||
MSYS2_ARG_CONV_EXCL='*' bazel "$@"
|
||||
return
|
||||
fi
|
||||
|
||||
bazel "$@"
|
||||
}
|
||||
|
||||
bazel_query_args=(--noexperimental_remote_repo_contents_cache query)
|
||||
if [[ -n "${BUILDBUDDY_API_KEY:-}" ]]; then
|
||||
bazel_query_args+=(
|
||||
"--config=${ci_config}"
|
||||
"--remote_header=x-buildbuddy-api-key=${BUILDBUDDY_API_KEY}"
|
||||
)
|
||||
fi
|
||||
|
||||
if [[ -n "${BAZEL_REPO_CONTENTS_CACHE:-}" ]]; then
|
||||
bazel_query_args+=("--repo_contents_cache=${BAZEL_REPO_CONTENTS_CACHE}")
|
||||
fi
|
||||
|
||||
if [[ -n "${BAZEL_REPOSITORY_CACHE:-}" ]]; then
|
||||
bazel_query_args+=("--repository_cache=${BAZEL_REPOSITORY_CACHE}")
|
||||
fi
|
||||
|
||||
bazel_query_args+=("${query_args[@]}" "$query_expression")
|
||||
|
||||
if (( ${#bazel_startup_args[@]} > 0 )); then
|
||||
run_bazel "${bazel_startup_args[@]}" "${bazel_query_args[@]}"
|
||||
else
|
||||
run_bazel "${bazel_query_args[@]}"
|
||||
fi
|
||||
412
reference/openai-codex/.github/scripts/rusty_v8_bazel.py
vendored
Normal file
412
reference/openai-codex/.github/scripts/rusty_v8_bazel.py
vendored
Normal file
@@ -0,0 +1,412 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import gzip
|
||||
import hashlib
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
|
||||
from rusty_v8_module_bazel import (
|
||||
RustyV8ChecksumError,
|
||||
check_module_bazel,
|
||||
rusty_v8_http_file_versions,
|
||||
update_module_bazel,
|
||||
)
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
MODULE_BAZEL = ROOT / "MODULE.bazel"
|
||||
RUSTY_V8_CHECKSUMS_DIR = ROOT / "third_party" / "v8"
|
||||
RELEASE_ARTIFACT_PROFILE = "release"
|
||||
SANDBOX_ARTIFACT_PROFILE = "ptrcomp_sandbox_release"
|
||||
ARTIFACT_BAZEL_CONFIGS = ["rusty-v8-upstream-libcxx"]
|
||||
|
||||
|
||||
def bazel_remote_args() -> list[str]:
|
||||
buildbuddy_api_key = os.environ.get("BUILDBUDDY_API_KEY")
|
||||
if not buildbuddy_api_key:
|
||||
return []
|
||||
return [f"--remote_header=x-buildbuddy-api-key={buildbuddy_api_key}"]
|
||||
|
||||
|
||||
def bazel_execroot() -> Path:
|
||||
result = subprocess.run(
|
||||
["bazel", "info", "execution_root"],
|
||||
cwd=ROOT,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
return Path(result.stdout.strip())
|
||||
|
||||
|
||||
def bazel_output_base() -> Path:
|
||||
result = subprocess.run(
|
||||
["bazel", "info", "output_base"],
|
||||
cwd=ROOT,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
return Path(result.stdout.strip())
|
||||
|
||||
|
||||
def bazel_output_path(path: str) -> Path:
|
||||
if path.startswith("external/"):
|
||||
return bazel_output_base() / path
|
||||
return bazel_execroot() / path
|
||||
|
||||
|
||||
def bazel_output_files(
|
||||
platform: str,
|
||||
labels: list[str],
|
||||
compilation_mode: str = "fastbuild",
|
||||
bazel_configs: list[str] | None = None,
|
||||
) -> list[Path]:
|
||||
expression = "set(" + " ".join(labels) + ")"
|
||||
bazel_configs = bazel_configs or []
|
||||
result = subprocess.run(
|
||||
[
|
||||
"bazel",
|
||||
"cquery",
|
||||
"-c",
|
||||
compilation_mode,
|
||||
f"--platforms=@llvm//platforms:{platform}",
|
||||
*[f"--config={config}" for config in bazel_configs],
|
||||
*bazel_remote_args(),
|
||||
"--output=files",
|
||||
expression,
|
||||
],
|
||||
cwd=ROOT,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
return [bazel_output_path(line.strip()) for line in result.stdout.splitlines() if line.strip()]
|
||||
|
||||
|
||||
def bazel_build(
|
||||
platform: str,
|
||||
labels: list[str],
|
||||
compilation_mode: str = "fastbuild",
|
||||
bazel_configs: list[str] | None = None,
|
||||
download_toplevel: bool = False,
|
||||
) -> None:
|
||||
bazel_configs = bazel_configs or []
|
||||
download_args = ["--remote_download_toplevel"] if download_toplevel else []
|
||||
subprocess.run(
|
||||
[
|
||||
"bazel",
|
||||
"build",
|
||||
"-c",
|
||||
compilation_mode,
|
||||
f"--platforms=@llvm//platforms:{platform}",
|
||||
*[f"--config={config}" for config in bazel_configs],
|
||||
*bazel_remote_args(),
|
||||
*download_args,
|
||||
*labels,
|
||||
],
|
||||
cwd=ROOT,
|
||||
check=True,
|
||||
)
|
||||
|
||||
|
||||
def ensure_bazel_output_files(
|
||||
platform: str,
|
||||
labels: list[str],
|
||||
compilation_mode: str = "fastbuild",
|
||||
bazel_configs: list[str] | None = None,
|
||||
) -> list[Path]:
|
||||
# Bazel output paths can be reused across config flips, so existence alone
|
||||
# does not prove the files match the requested flags.
|
||||
bazel_build(
|
||||
platform,
|
||||
labels,
|
||||
compilation_mode,
|
||||
bazel_configs,
|
||||
download_toplevel=True,
|
||||
)
|
||||
outputs = bazel_output_files(platform, labels, compilation_mode, bazel_configs)
|
||||
missing = [str(path) for path in outputs if not path.exists()]
|
||||
if missing:
|
||||
raise SystemExit(f"missing built outputs for {labels}: {missing}")
|
||||
return outputs
|
||||
|
||||
|
||||
def artifact_bazel_configs(bazel_configs: list[str] | None = None) -> list[str]:
|
||||
configured = list(ARTIFACT_BAZEL_CONFIGS)
|
||||
for config in bazel_configs or []:
|
||||
if config not in configured:
|
||||
configured.append(config)
|
||||
return configured
|
||||
|
||||
|
||||
def release_pair_label(target: str, sandbox: bool = False) -> str:
|
||||
target_suffix = target.replace("-", "_")
|
||||
pair_kind = "sandbox_release_pair" if sandbox else "release_pair"
|
||||
return f"//third_party/v8:rusty_v8_{pair_kind}_{target_suffix}"
|
||||
|
||||
|
||||
def resolved_v8_crate_version() -> str:
|
||||
cargo_lock = tomllib.loads((ROOT / "codex-rs" / "Cargo.lock").read_text())
|
||||
versions = sorted(
|
||||
{
|
||||
package["version"]
|
||||
for package in cargo_lock["package"]
|
||||
if package["name"] == "v8"
|
||||
}
|
||||
)
|
||||
if len(versions) == 1:
|
||||
return versions[0]
|
||||
if len(versions) > 1:
|
||||
raise SystemExit(f"expected exactly one resolved v8 version, found: {versions}")
|
||||
|
||||
module_bazel = (ROOT / "MODULE.bazel").read_text()
|
||||
matches = sorted(
|
||||
set(
|
||||
re.findall(
|
||||
r'https://static\.crates\.io/crates/v8/v8-([0-9]+\.[0-9]+\.[0-9]+)\.crate',
|
||||
module_bazel,
|
||||
)
|
||||
)
|
||||
)
|
||||
if len(matches) != 1:
|
||||
raise SystemExit(
|
||||
"expected exactly one pinned v8 crate version in MODULE.bazel, "
|
||||
f"found: {matches}"
|
||||
)
|
||||
return matches[0]
|
||||
|
||||
|
||||
def rusty_v8_checksum_manifest_path(version: str) -> Path:
|
||||
return RUSTY_V8_CHECKSUMS_DIR / f"rusty_v8_{version.replace('.', '_')}.sha256"
|
||||
|
||||
|
||||
def command_version(version: str | None) -> str:
|
||||
if version is not None:
|
||||
return version
|
||||
|
||||
manifest_versions = rusty_v8_http_file_versions(MODULE_BAZEL.read_text())
|
||||
if len(manifest_versions) == 1:
|
||||
return manifest_versions[0]
|
||||
if len(manifest_versions) > 1:
|
||||
raise SystemExit(
|
||||
"expected at most one rusty_v8 http_file version in MODULE.bazel, "
|
||||
f"found: {manifest_versions}; pass --version explicitly"
|
||||
)
|
||||
|
||||
return resolved_v8_crate_version()
|
||||
|
||||
|
||||
def command_manifest_path(manifest: Path | None, version: str) -> Path:
|
||||
if manifest is None:
|
||||
return rusty_v8_checksum_manifest_path(version)
|
||||
if manifest.is_absolute():
|
||||
return manifest
|
||||
return ROOT / manifest
|
||||
|
||||
|
||||
def staged_archive_name(target: str, source_path: Path, artifact_profile: str) -> str:
|
||||
if target.endswith("-pc-windows-msvc"):
|
||||
return f"rusty_v8_{artifact_profile}_{target}.lib.gz"
|
||||
return f"librusty_v8_{artifact_profile}_{target}.a.gz"
|
||||
|
||||
|
||||
def staged_binding_name(target: str, artifact_profile: str) -> str:
|
||||
return f"src_binding_{artifact_profile}_{target}.rs"
|
||||
|
||||
|
||||
def staged_checksums_name(target: str, artifact_profile: str) -> str:
|
||||
return f"rusty_v8_{artifact_profile}_{target}.sha256"
|
||||
|
||||
|
||||
def stage_artifacts(
|
||||
target: str,
|
||||
lib_path: Path,
|
||||
binding_path: Path,
|
||||
output_dir: Path,
|
||||
sandbox: bool,
|
||||
) -> None:
|
||||
missing_paths = [str(path) for path in [lib_path, binding_path] if not path.exists()]
|
||||
if missing_paths:
|
||||
raise SystemExit(f"missing release outputs for {target}: {missing_paths}")
|
||||
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
artifact_profile = SANDBOX_ARTIFACT_PROFILE if sandbox else RELEASE_ARTIFACT_PROFILE
|
||||
staged_library = output_dir / staged_archive_name(target, lib_path, artifact_profile)
|
||||
staged_binding = output_dir / staged_binding_name(target, artifact_profile)
|
||||
|
||||
with lib_path.open("rb") as src, staged_library.open("wb") as dst:
|
||||
with gzip.GzipFile(
|
||||
filename="",
|
||||
mode="wb",
|
||||
fileobj=dst,
|
||||
compresslevel=6,
|
||||
mtime=0,
|
||||
) as gz:
|
||||
shutil.copyfileobj(src, gz)
|
||||
|
||||
shutil.copyfile(binding_path, staged_binding)
|
||||
|
||||
staged_checksums = output_dir / staged_checksums_name(target, artifact_profile)
|
||||
with staged_checksums.open("w", encoding="utf-8") as checksums:
|
||||
for path in [staged_library, staged_binding]:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as artifact:
|
||||
for chunk in iter(lambda: artifact.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
checksums.write(f"{digest.hexdigest()} {path.name}\n")
|
||||
|
||||
print(staged_library)
|
||||
print(staged_binding)
|
||||
print(staged_checksums)
|
||||
|
||||
|
||||
def upstream_release_pair_paths(source_root: Path, target: str) -> tuple[Path, Path]:
|
||||
lib_name = "rusty_v8.lib" if target.endswith("-pc-windows-msvc") else "librusty_v8.a"
|
||||
gn_out = source_root / "target" / target / "release" / "gn_out"
|
||||
return gn_out / "obj" / lib_name, gn_out / "src_binding.rs"
|
||||
|
||||
|
||||
def stage_upstream_release_pair(
|
||||
source_root: Path,
|
||||
target: str,
|
||||
output_dir: Path,
|
||||
sandbox: bool = False,
|
||||
) -> None:
|
||||
lib_path, binding_path = upstream_release_pair_paths(source_root, target)
|
||||
stage_artifacts(target, lib_path, binding_path, output_dir, sandbox)
|
||||
|
||||
|
||||
def stage_release_pair(
|
||||
platform: str,
|
||||
target: str,
|
||||
output_dir: Path,
|
||||
compilation_mode: str = "fastbuild",
|
||||
bazel_configs: list[str] | None = None,
|
||||
sandbox: bool = False,
|
||||
) -> None:
|
||||
bazel_configs = artifact_bazel_configs(bazel_configs)
|
||||
outputs = ensure_bazel_output_files(
|
||||
platform,
|
||||
[release_pair_label(target, sandbox)],
|
||||
compilation_mode,
|
||||
bazel_configs,
|
||||
)
|
||||
|
||||
try:
|
||||
lib_path = next(path for path in outputs if path.suffix in {".a", ".lib"})
|
||||
except StopIteration as exc:
|
||||
raise SystemExit(f"missing static library output for {target}") from exc
|
||||
|
||||
try:
|
||||
binding_path = next(path for path in outputs if path.suffix == ".rs")
|
||||
except StopIteration as exc:
|
||||
raise SystemExit(f"missing Rust binding output for {target}") from exc
|
||||
|
||||
stage_artifacts(target, lib_path, binding_path, output_dir, sandbox)
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
subparsers = parser.add_subparsers(dest="command", required=True)
|
||||
|
||||
stage_release_pair_parser = subparsers.add_parser("stage-release-pair")
|
||||
stage_release_pair_parser.add_argument("--platform", required=True)
|
||||
stage_release_pair_parser.add_argument("--target", required=True)
|
||||
stage_release_pair_parser.add_argument("--output-dir", required=True)
|
||||
stage_release_pair_parser.add_argument("--sandbox", action="store_true")
|
||||
stage_release_pair_parser.add_argument(
|
||||
"--bazel-config",
|
||||
action="append",
|
||||
default=[],
|
||||
dest="bazel_configs",
|
||||
)
|
||||
stage_release_pair_parser.add_argument(
|
||||
"--compilation-mode",
|
||||
default="fastbuild",
|
||||
choices=["fastbuild", "opt", "dbg"],
|
||||
)
|
||||
|
||||
stage_upstream_release_pair_parser = subparsers.add_parser(
|
||||
"stage-upstream-release-pair"
|
||||
)
|
||||
stage_upstream_release_pair_parser.add_argument("--source-root", type=Path, required=True)
|
||||
stage_upstream_release_pair_parser.add_argument("--target", required=True)
|
||||
stage_upstream_release_pair_parser.add_argument("--output-dir", required=True)
|
||||
stage_upstream_release_pair_parser.add_argument("--sandbox", action="store_true")
|
||||
|
||||
subparsers.add_parser("resolved-v8-crate-version")
|
||||
|
||||
check_module_bazel_parser = subparsers.add_parser("check-module-bazel")
|
||||
check_module_bazel_parser.add_argument("--version")
|
||||
check_module_bazel_parser.add_argument("--manifest", type=Path)
|
||||
check_module_bazel_parser.add_argument(
|
||||
"--module-bazel",
|
||||
type=Path,
|
||||
default=MODULE_BAZEL,
|
||||
)
|
||||
|
||||
update_module_bazel_parser = subparsers.add_parser("update-module-bazel")
|
||||
update_module_bazel_parser.add_argument("--version")
|
||||
update_module_bazel_parser.add_argument("--manifest", type=Path)
|
||||
update_module_bazel_parser.add_argument(
|
||||
"--module-bazel",
|
||||
type=Path,
|
||||
default=MODULE_BAZEL,
|
||||
)
|
||||
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
if args.command == "stage-release-pair":
|
||||
stage_release_pair(
|
||||
platform=args.platform,
|
||||
target=args.target,
|
||||
output_dir=Path(args.output_dir),
|
||||
compilation_mode=args.compilation_mode,
|
||||
bazel_configs=args.bazel_configs,
|
||||
sandbox=args.sandbox,
|
||||
)
|
||||
return 0
|
||||
if args.command == "stage-upstream-release-pair":
|
||||
stage_upstream_release_pair(
|
||||
source_root=args.source_root,
|
||||
target=args.target,
|
||||
output_dir=Path(args.output_dir),
|
||||
sandbox=args.sandbox,
|
||||
)
|
||||
return 0
|
||||
if args.command == "resolved-v8-crate-version":
|
||||
print(resolved_v8_crate_version())
|
||||
return 0
|
||||
if args.command == "check-module-bazel":
|
||||
version = command_version(args.version)
|
||||
manifest_path = command_manifest_path(args.manifest, version)
|
||||
try:
|
||||
check_module_bazel(args.module_bazel, manifest_path, version)
|
||||
except RustyV8ChecksumError as exc:
|
||||
raise SystemExit(str(exc)) from exc
|
||||
return 0
|
||||
if args.command == "update-module-bazel":
|
||||
version = command_version(args.version)
|
||||
manifest_path = command_manifest_path(args.manifest, version)
|
||||
try:
|
||||
update_module_bazel(args.module_bazel, manifest_path, version)
|
||||
except RustyV8ChecksumError as exc:
|
||||
raise SystemExit(str(exc)) from exc
|
||||
return 0
|
||||
raise SystemExit(f"unsupported command: {args.command}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
243
reference/openai-codex/.github/scripts/rusty_v8_module_bazel.py
vendored
Normal file
243
reference/openai-codex/.github/scripts/rusty_v8_module_bazel.py
vendored
Normal file
@@ -0,0 +1,243 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
SHA256_RE = re.compile(r"[0-9a-f]{64}")
|
||||
HTTP_FILE_BLOCK_RE = re.compile(r"(?ms)^http_file\(\n.*?^\)\n?")
|
||||
HTTP_FILE_VERSION_RE = re.compile(r"^rusty_v8_([0-9]+)_([0-9]+)_([0-9]+)_")
|
||||
|
||||
|
||||
class RustyV8ChecksumError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RustyV8HttpFile:
|
||||
start: int
|
||||
end: int
|
||||
block: str
|
||||
name: str
|
||||
downloaded_file_path: str
|
||||
sha256: str | None
|
||||
|
||||
|
||||
def parse_checksum_manifest(path: Path) -> dict[str, str]:
|
||||
try:
|
||||
lines = path.read_text(encoding="utf-8").splitlines()
|
||||
except FileNotFoundError as exc:
|
||||
raise RustyV8ChecksumError(f"missing checksum manifest: {path}") from exc
|
||||
|
||||
checksums: dict[str, str] = {}
|
||||
for line_number, line in enumerate(lines, 1):
|
||||
if not line.strip():
|
||||
continue
|
||||
parts = line.split()
|
||||
if len(parts) != 2:
|
||||
raise RustyV8ChecksumError(
|
||||
f"{path}:{line_number}: expected '<sha256> <filename>'"
|
||||
)
|
||||
checksum, filename = parts
|
||||
if not SHA256_RE.fullmatch(checksum):
|
||||
raise RustyV8ChecksumError(
|
||||
f"{path}:{line_number}: invalid SHA-256 digest for {filename}"
|
||||
)
|
||||
if not filename or filename in {".", ".."} or "/" in filename:
|
||||
raise RustyV8ChecksumError(
|
||||
f"{path}:{line_number}: expected a bare artifact filename"
|
||||
)
|
||||
if filename in checksums:
|
||||
raise RustyV8ChecksumError(
|
||||
f"{path}:{line_number}: duplicate checksum for {filename}"
|
||||
)
|
||||
checksums[filename] = checksum
|
||||
|
||||
if not checksums:
|
||||
raise RustyV8ChecksumError(f"empty checksum manifest: {path}")
|
||||
return checksums
|
||||
|
||||
|
||||
def string_field(block: str, field: str) -> str | None:
|
||||
# Matches one-line string fields inside http_file blocks, e.g. `sha256 = "...",`.
|
||||
match = re.search(rf'^\s*{re.escape(field)}\s*=\s*"([^"]+)",\s*$', block, re.M)
|
||||
if match:
|
||||
return match.group(1)
|
||||
return None
|
||||
|
||||
|
||||
def rusty_v8_http_files(module_bazel: str, version: str) -> list[RustyV8HttpFile]:
|
||||
version_slug = version.replace(".", "_")
|
||||
name_prefix = f"rusty_v8_{version_slug}_"
|
||||
entries = []
|
||||
for match in HTTP_FILE_BLOCK_RE.finditer(module_bazel):
|
||||
block = match.group(0)
|
||||
name = string_field(block, "name")
|
||||
if not name or not name.startswith(name_prefix):
|
||||
continue
|
||||
downloaded_file_path = string_field(block, "downloaded_file_path")
|
||||
if not downloaded_file_path:
|
||||
raise RustyV8ChecksumError(
|
||||
f"MODULE.bazel {name} is missing downloaded_file_path"
|
||||
)
|
||||
entries.append(
|
||||
RustyV8HttpFile(
|
||||
start=match.start(),
|
||||
end=match.end(),
|
||||
block=block,
|
||||
name=name,
|
||||
downloaded_file_path=downloaded_file_path,
|
||||
sha256=string_field(block, "sha256"),
|
||||
)
|
||||
)
|
||||
return entries
|
||||
|
||||
|
||||
def rusty_v8_http_file_versions(module_bazel: str) -> list[str]:
|
||||
versions = set()
|
||||
for match in HTTP_FILE_BLOCK_RE.finditer(module_bazel):
|
||||
name = string_field(match.group(0), "name")
|
||||
if not name:
|
||||
continue
|
||||
version_match = HTTP_FILE_VERSION_RE.match(name)
|
||||
if version_match:
|
||||
versions.add(".".join(version_match.groups()))
|
||||
return sorted(versions)
|
||||
|
||||
|
||||
def module_entry_set_errors(
|
||||
entries: list[RustyV8HttpFile],
|
||||
checksums: dict[str, str],
|
||||
version: str,
|
||||
) -> list[str]:
|
||||
errors = []
|
||||
if not entries:
|
||||
errors.append(f"MODULE.bazel has no rusty_v8 http_file entries for {version}")
|
||||
return errors
|
||||
|
||||
module_files: dict[str, RustyV8HttpFile] = {}
|
||||
duplicate_files = set()
|
||||
for entry in entries:
|
||||
if entry.downloaded_file_path in module_files:
|
||||
duplicate_files.add(entry.downloaded_file_path)
|
||||
module_files[entry.downloaded_file_path] = entry
|
||||
|
||||
for filename in sorted(duplicate_files):
|
||||
errors.append(f"MODULE.bazel has duplicate http_file entries for {filename}")
|
||||
|
||||
for filename in sorted(set(module_files) - set(checksums)):
|
||||
entry = module_files[filename]
|
||||
errors.append(f"MODULE.bazel {entry.name} has no checksum in the manifest")
|
||||
|
||||
for filename in sorted(set(checksums) - set(module_files)):
|
||||
errors.append(f"manifest has {filename}, but MODULE.bazel has no http_file")
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
def module_checksum_errors(
|
||||
entries: list[RustyV8HttpFile],
|
||||
checksums: dict[str, str],
|
||||
) -> list[str]:
|
||||
errors = []
|
||||
for entry in entries:
|
||||
expected = checksums.get(entry.downloaded_file_path)
|
||||
if expected is None:
|
||||
continue
|
||||
if entry.sha256 is None:
|
||||
errors.append(f"MODULE.bazel {entry.name} is missing sha256")
|
||||
elif entry.sha256 != expected:
|
||||
errors.append(
|
||||
f"MODULE.bazel {entry.name} has sha256 {entry.sha256}, "
|
||||
f"expected {expected}"
|
||||
)
|
||||
return errors
|
||||
|
||||
|
||||
def raise_checksum_errors(message: str, errors: list[str]) -> None:
|
||||
if errors:
|
||||
formatted_errors = "\n".join(f"- {error}" for error in errors)
|
||||
raise RustyV8ChecksumError(f"{message}:\n{formatted_errors}")
|
||||
|
||||
|
||||
def check_module_bazel_text(
|
||||
module_bazel: str,
|
||||
checksums: dict[str, str],
|
||||
version: str,
|
||||
) -> None:
|
||||
entries = rusty_v8_http_files(module_bazel, version)
|
||||
errors = [
|
||||
*module_entry_set_errors(entries, checksums, version),
|
||||
*module_checksum_errors(entries, checksums),
|
||||
]
|
||||
raise_checksum_errors("rusty_v8 MODULE.bazel checksum drift", errors)
|
||||
|
||||
|
||||
def block_with_sha256(block: str, checksum: str) -> str:
|
||||
sha256_line_re = re.compile(r'(?m)^(\s*)sha256\s*=\s*"[0-9a-f]+",\s*$')
|
||||
if sha256_line_re.search(block):
|
||||
return sha256_line_re.sub(
|
||||
lambda match: f'{match.group(1)}sha256 = "{checksum}",',
|
||||
block,
|
||||
count=1,
|
||||
)
|
||||
|
||||
downloaded_file_path_match = re.search(
|
||||
r'(?m)^(\s*)downloaded_file_path\s*=\s*"[^"]+",\n',
|
||||
block,
|
||||
)
|
||||
if not downloaded_file_path_match:
|
||||
raise RustyV8ChecksumError("http_file block is missing downloaded_file_path")
|
||||
insert_at = downloaded_file_path_match.end()
|
||||
indent = downloaded_file_path_match.group(1)
|
||||
return f'{block[:insert_at]}{indent}sha256 = "{checksum}",\n{block[insert_at:]}'
|
||||
|
||||
|
||||
def update_module_bazel_text(
|
||||
module_bazel: str,
|
||||
checksums: dict[str, str],
|
||||
version: str,
|
||||
) -> str:
|
||||
entries = rusty_v8_http_files(module_bazel, version)
|
||||
errors = module_entry_set_errors(entries, checksums, version)
|
||||
raise_checksum_errors("cannot update rusty_v8 MODULE.bazel checksums", errors)
|
||||
|
||||
updated = []
|
||||
previous_end = 0
|
||||
for entry in entries:
|
||||
updated.append(module_bazel[previous_end : entry.start])
|
||||
updated.append(
|
||||
block_with_sha256(entry.block, checksums[entry.downloaded_file_path])
|
||||
)
|
||||
previous_end = entry.end
|
||||
updated.append(module_bazel[previous_end:])
|
||||
return "".join(updated)
|
||||
|
||||
|
||||
def check_module_bazel(
|
||||
module_bazel_path: Path,
|
||||
manifest_path: Path,
|
||||
version: str,
|
||||
) -> None:
|
||||
checksums = parse_checksum_manifest(manifest_path)
|
||||
module_bazel = module_bazel_path.read_text(encoding="utf-8")
|
||||
check_module_bazel_text(module_bazel, checksums, version)
|
||||
print(f"{module_bazel_path} rusty_v8 {version} checksums match {manifest_path}")
|
||||
|
||||
|
||||
def update_module_bazel(
|
||||
module_bazel_path: Path,
|
||||
manifest_path: Path,
|
||||
version: str,
|
||||
) -> None:
|
||||
checksums = parse_checksum_manifest(manifest_path)
|
||||
module_bazel = module_bazel_path.read_text(encoding="utf-8")
|
||||
updated_module_bazel = update_module_bazel_text(module_bazel, checksums, version)
|
||||
if updated_module_bazel == module_bazel:
|
||||
print(f"{module_bazel_path} rusty_v8 {version} checksums are already current")
|
||||
return
|
||||
module_bazel_path.write_text(updated_module_bazel, encoding="utf-8")
|
||||
print(f"updated {module_bazel_path} rusty_v8 {version} checksums")
|
||||
62
reference/openai-codex/.github/scripts/setup-dev-drive.ps1
vendored
Normal file
62
reference/openai-codex/.github/scripts/setup-dev-drive.ps1
vendored
Normal file
@@ -0,0 +1,62 @@
|
||||
# Configure a fast drive for Windows CI jobs.
|
||||
#
|
||||
# GitHub-hosted Windows runners do not always expose a secondary D: volume. When
|
||||
# they do not, try to create a Dev Drive VHD and fall back to C: if the runner
|
||||
# image does not allow that provisioning path.
|
||||
|
||||
function Use-FallbackDrive {
|
||||
param([string]$Reason)
|
||||
|
||||
Write-Warning "$Reason Falling back to C:"
|
||||
return "C:"
|
||||
}
|
||||
|
||||
function Invoke-BestEffort {
|
||||
param([scriptblock]$Script, [string]$Description)
|
||||
|
||||
try {
|
||||
& $Script
|
||||
} catch {
|
||||
Write-Warning "$Description failed: $($_.Exception.Message)"
|
||||
}
|
||||
}
|
||||
|
||||
if (Test-Path "D:\") {
|
||||
Write-Output "Using existing drive at D:"
|
||||
$Drive = "D:"
|
||||
} else {
|
||||
try {
|
||||
$VhdPath = Join-Path $env:RUNNER_TEMP "codex-dev-drive.vhdx"
|
||||
$SizeBytes = 64GB
|
||||
|
||||
if (Test-Path $VhdPath) {
|
||||
Remove-Item -Path $VhdPath -Force
|
||||
}
|
||||
|
||||
New-VHD -Path $VhdPath -SizeBytes $SizeBytes -Dynamic -ErrorAction Stop | Out-Null
|
||||
$Mounted = Mount-VHD -Path $VhdPath -Passthru -ErrorAction Stop
|
||||
$Disk = $Mounted | Get-Disk -ErrorAction Stop
|
||||
$Disk | Initialize-Disk -PartitionStyle GPT -ErrorAction Stop
|
||||
$Partition = $Disk | New-Partition -AssignDriveLetter -UseMaximumSize -ErrorAction Stop
|
||||
$Volume = $Partition | Format-Volume -FileSystem ReFS -NewFileSystemLabel "CodexDevDrive" -DevDrive -Confirm:$false -Force -ErrorAction Stop
|
||||
|
||||
$Drive = "$($Volume.DriveLetter):"
|
||||
|
||||
Invoke-BestEffort { fsutil devdrv trust $Drive } "Trusting Dev Drive $Drive"
|
||||
Invoke-BestEffort { fsutil devdrv enable /disallowAv } "Disabling AV filter attachment for Dev Drives"
|
||||
Invoke-BestEffort { fsutil devdrv query $Drive } "Querying Dev Drive $Drive"
|
||||
|
||||
Write-Output "Using Dev Drive at $Drive"
|
||||
} catch {
|
||||
$Drive = Use-FallbackDrive "Failed to create Dev Drive: $($_.Exception.Message)"
|
||||
}
|
||||
}
|
||||
|
||||
$Tmp = "$Drive\codex-tmp"
|
||||
New-Item -Path $Tmp -ItemType Directory -Force | Out-Null
|
||||
|
||||
@(
|
||||
"DEV_DRIVE=$Drive"
|
||||
"TMP=$Tmp"
|
||||
"TEMP=$Tmp"
|
||||
) | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
|
||||
413
reference/openai-codex/.github/scripts/test_rusty_v8_bazel.py
vendored
Normal file
413
reference/openai-codex/.github/scripts/test_rusty_v8_bazel.py
vendored
Normal file
@@ -0,0 +1,413 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import textwrap
|
||||
import unittest
|
||||
from os import environ
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
from unittest.mock import patch
|
||||
|
||||
import rusty_v8_bazel
|
||||
import rusty_v8_module_bazel
|
||||
|
||||
|
||||
class RustyV8BazelTest(unittest.TestCase):
|
||||
def test_consumer_selectors_track_resolved_crate_version(self) -> None:
|
||||
build_bazel = (
|
||||
rusty_v8_bazel.ROOT / "third_party" / "v8" / "BUILD.bazel"
|
||||
).read_text()
|
||||
version_suffix = rusty_v8_bazel.resolved_v8_crate_version().replace(".", "_")
|
||||
|
||||
for selector in [
|
||||
"aarch64_apple_darwin_bazel",
|
||||
"aarch64_pc_windows_gnullvm",
|
||||
"aarch64_pc_windows_msvc",
|
||||
"aarch64_unknown_linux_gnu_bazel",
|
||||
"aarch64_unknown_linux_musl_release_base",
|
||||
"x86_64_apple_darwin_bazel",
|
||||
"x86_64_pc_windows_gnullvm",
|
||||
"x86_64_pc_windows_msvc",
|
||||
"x86_64_unknown_linux_gnu_bazel",
|
||||
"x86_64_unknown_linux_musl_release",
|
||||
]:
|
||||
self.assertIn(
|
||||
f":v8_{version_suffix}_{selector}",
|
||||
build_bazel,
|
||||
)
|
||||
|
||||
for selector in [
|
||||
"aarch64_apple_darwin",
|
||||
"aarch64_pc_windows_gnullvm",
|
||||
"aarch64_pc_windows_msvc",
|
||||
"aarch64_unknown_linux_gnu",
|
||||
"aarch64_unknown_linux_musl",
|
||||
"x86_64_apple_darwin",
|
||||
"x86_64_pc_windows_gnullvm",
|
||||
"x86_64_pc_windows_msvc",
|
||||
"x86_64_unknown_linux_gnu",
|
||||
"x86_64_unknown_linux_musl",
|
||||
]:
|
||||
self.assertIn(
|
||||
f":src_binding_release_{selector}_{version_suffix}_release",
|
||||
build_bazel,
|
||||
)
|
||||
|
||||
def test_command_version_tracks_remaining_http_file_assets(self) -> None:
|
||||
with TemporaryDirectory() as temp_dir:
|
||||
module_bazel = Path(temp_dir) / "MODULE.bazel"
|
||||
module_bazel.write_text(
|
||||
textwrap.dedent(
|
||||
"""\
|
||||
http_file(
|
||||
name = "rusty_v8_146_4_0_x86_64_unknown_linux_gnu_archive",
|
||||
downloaded_file_path = "librusty_v8_release_x86_64-unknown-linux-gnu.a.gz",
|
||||
urls = ["https://example.test/archive.gz"],
|
||||
)
|
||||
"""
|
||||
)
|
||||
)
|
||||
|
||||
with patch.object(rusty_v8_bazel, "MODULE_BAZEL", module_bazel):
|
||||
self.assertEqual("146.4.0", rusty_v8_bazel.command_version(None))
|
||||
|
||||
def test_artifact_bazel_configs_always_enable_upstream_libcxx(self) -> None:
|
||||
self.assertEqual(
|
||||
["rusty-v8-upstream-libcxx"],
|
||||
rusty_v8_bazel.artifact_bazel_configs(),
|
||||
)
|
||||
self.assertEqual(
|
||||
["rusty-v8-upstream-libcxx", "v8-release-compat"],
|
||||
rusty_v8_bazel.artifact_bazel_configs(["v8-release-compat"]),
|
||||
)
|
||||
self.assertEqual(
|
||||
["rusty-v8-upstream-libcxx", "v8-release-compat"],
|
||||
rusty_v8_bazel.artifact_bazel_configs(
|
||||
["rusty-v8-upstream-libcxx", "v8-release-compat"]
|
||||
),
|
||||
)
|
||||
|
||||
def test_bazel_remote_args_include_buildbuddy_header_when_present(self) -> None:
|
||||
with patch.dict(environ, {"BUILDBUDDY_API_KEY": "token"}, clear=False):
|
||||
self.assertEqual(
|
||||
["--remote_header=x-buildbuddy-api-key=token"],
|
||||
rusty_v8_bazel.bazel_remote_args(),
|
||||
)
|
||||
|
||||
with patch.dict(environ, {}, clear=True):
|
||||
self.assertEqual([], rusty_v8_bazel.bazel_remote_args())
|
||||
|
||||
def test_release_pair_labels_and_staged_names_distinguish_sandbox_artifacts(self) -> None:
|
||||
self.assertEqual(
|
||||
"//third_party/v8:rusty_v8_release_pair_x86_64_unknown_linux_musl",
|
||||
rusty_v8_bazel.release_pair_label("x86_64-unknown-linux-musl"),
|
||||
)
|
||||
self.assertEqual(
|
||||
"//third_party/v8:rusty_v8_sandbox_release_pair_x86_64_unknown_linux_musl",
|
||||
rusty_v8_bazel.release_pair_label("x86_64-unknown-linux-musl", sandbox=True),
|
||||
)
|
||||
self.assertEqual(
|
||||
"//third_party/v8:rusty_v8_sandbox_release_pair_x86_64_apple_darwin",
|
||||
rusty_v8_bazel.release_pair_label("x86_64-apple-darwin", sandbox=True),
|
||||
)
|
||||
self.assertEqual(
|
||||
"librusty_v8_release_x86_64-unknown-linux-musl.a.gz",
|
||||
rusty_v8_bazel.staged_archive_name(
|
||||
"x86_64-unknown-linux-musl",
|
||||
Path("libv8.a"),
|
||||
rusty_v8_bazel.RELEASE_ARTIFACT_PROFILE,
|
||||
),
|
||||
)
|
||||
self.assertEqual(
|
||||
"rusty_v8_ptrcomp_sandbox_release_x86_64-pc-windows-msvc.lib.gz",
|
||||
rusty_v8_bazel.staged_archive_name(
|
||||
"x86_64-pc-windows-msvc",
|
||||
Path("v8.a"),
|
||||
rusty_v8_bazel.SANDBOX_ARTIFACT_PROFILE,
|
||||
),
|
||||
)
|
||||
self.assertEqual(
|
||||
"src_binding_ptrcomp_sandbox_release_x86_64-unknown-linux-musl.rs",
|
||||
rusty_v8_bazel.staged_binding_name(
|
||||
"x86_64-unknown-linux-musl",
|
||||
rusty_v8_bazel.SANDBOX_ARTIFACT_PROFILE,
|
||||
),
|
||||
)
|
||||
self.assertEqual(
|
||||
"rusty_v8_ptrcomp_sandbox_release_x86_64-unknown-linux-musl.sha256",
|
||||
rusty_v8_bazel.staged_checksums_name(
|
||||
"x86_64-unknown-linux-musl",
|
||||
rusty_v8_bazel.SANDBOX_ARTIFACT_PROFILE,
|
||||
),
|
||||
)
|
||||
|
||||
def test_stage_artifacts(self) -> None:
|
||||
with TemporaryDirectory() as source_dir, TemporaryDirectory() as output_dir:
|
||||
source_root = Path(source_dir)
|
||||
archive = source_root / "librusty_v8.a"
|
||||
binding = source_root / "src_binding.rs"
|
||||
archive.write_bytes(b"archive")
|
||||
binding.write_text("binding")
|
||||
|
||||
rusty_v8_bazel.stage_artifacts(
|
||||
"aarch64-apple-darwin",
|
||||
archive,
|
||||
binding,
|
||||
Path(output_dir),
|
||||
sandbox=True,
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
{
|
||||
"librusty_v8_ptrcomp_sandbox_release_aarch64-apple-darwin.a.gz",
|
||||
"src_binding_ptrcomp_sandbox_release_aarch64-apple-darwin.rs",
|
||||
"rusty_v8_ptrcomp_sandbox_release_aarch64-apple-darwin.sha256",
|
||||
},
|
||||
{path.name for path in Path(output_dir).iterdir()},
|
||||
)
|
||||
|
||||
def test_upstream_release_pair_paths(self) -> None:
|
||||
self.assertEqual(
|
||||
(
|
||||
Path(
|
||||
"/tmp/rusty_v8/target/x86_64-apple-darwin/release/gn_out/obj/"
|
||||
"librusty_v8.a"
|
||||
),
|
||||
Path(
|
||||
"/tmp/rusty_v8/target/x86_64-apple-darwin/release/gn_out/"
|
||||
"src_binding.rs"
|
||||
),
|
||||
),
|
||||
rusty_v8_bazel.upstream_release_pair_paths(
|
||||
Path("/tmp/rusty_v8"),
|
||||
"x86_64-apple-darwin",
|
||||
),
|
||||
)
|
||||
self.assertEqual(
|
||||
(
|
||||
Path(
|
||||
"/tmp/rusty_v8/target/x86_64-pc-windows-msvc/release/gn_out/"
|
||||
"obj/rusty_v8.lib"
|
||||
),
|
||||
Path(
|
||||
"/tmp/rusty_v8/target/x86_64-pc-windows-msvc/release/gn_out/"
|
||||
"src_binding.rs"
|
||||
),
|
||||
),
|
||||
rusty_v8_bazel.upstream_release_pair_paths(
|
||||
Path("/tmp/rusty_v8"),
|
||||
"x86_64-pc-windows-msvc",
|
||||
),
|
||||
)
|
||||
|
||||
def test_stage_upstream_release_pair(self) -> None:
|
||||
with TemporaryDirectory() as source_dir, TemporaryDirectory() as output_dir:
|
||||
source_root = Path(source_dir)
|
||||
gn_out = (
|
||||
source_root
|
||||
/ "target"
|
||||
/ "x86_64-pc-windows-msvc"
|
||||
/ "release"
|
||||
/ "gn_out"
|
||||
)
|
||||
(gn_out / "obj").mkdir(parents=True)
|
||||
(gn_out / "obj" / "rusty_v8.lib").write_bytes(b"archive")
|
||||
(gn_out / "src_binding.rs").write_text("binding")
|
||||
|
||||
rusty_v8_bazel.stage_upstream_release_pair(
|
||||
source_root,
|
||||
"x86_64-pc-windows-msvc",
|
||||
Path(output_dir),
|
||||
sandbox=True,
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
{
|
||||
"rusty_v8_ptrcomp_sandbox_release_x86_64-pc-windows-msvc.lib.gz",
|
||||
"src_binding_ptrcomp_sandbox_release_x86_64-pc-windows-msvc.rs",
|
||||
"rusty_v8_ptrcomp_sandbox_release_x86_64-pc-windows-msvc.sha256",
|
||||
},
|
||||
{path.name for path in Path(output_dir).iterdir()},
|
||||
)
|
||||
|
||||
def test_ensure_bazel_output_files_rebuilds_existing_outputs(self) -> None:
|
||||
with TemporaryDirectory() as output_dir:
|
||||
output = Path(output_dir) / "libv8.a"
|
||||
output.write_bytes(b"archive")
|
||||
|
||||
with (
|
||||
patch.object(rusty_v8_bazel, "bazel_build") as bazel_build,
|
||||
patch.object(
|
||||
rusty_v8_bazel,
|
||||
"bazel_output_files",
|
||||
return_value=[output],
|
||||
) as bazel_output_files,
|
||||
):
|
||||
self.assertEqual(
|
||||
[output],
|
||||
rusty_v8_bazel.ensure_bazel_output_files(
|
||||
"macos_arm64",
|
||||
["//third_party/v8:pair"],
|
||||
"opt",
|
||||
["rusty-v8-upstream-libcxx"],
|
||||
),
|
||||
)
|
||||
|
||||
bazel_build.assert_called_once_with(
|
||||
"macos_arm64",
|
||||
["//third_party/v8:pair"],
|
||||
"opt",
|
||||
["rusty-v8-upstream-libcxx"],
|
||||
download_toplevel=True,
|
||||
)
|
||||
bazel_output_files.assert_called_once_with(
|
||||
"macos_arm64",
|
||||
["//third_party/v8:pair"],
|
||||
"opt",
|
||||
["rusty-v8-upstream-libcxx"],
|
||||
)
|
||||
|
||||
def test_update_module_bazel_replaces_and_inserts_sha256(self) -> None:
|
||||
module_bazel = textwrap.dedent(
|
||||
"""\
|
||||
http_file(
|
||||
name = "rusty_v8_146_4_0_x86_64_unknown_linux_gnu_archive",
|
||||
downloaded_file_path = "librusty_v8_release_x86_64-unknown-linux-gnu.a.gz",
|
||||
sha256 = "0000000000000000000000000000000000000000000000000000000000000000",
|
||||
urls = [
|
||||
"https://example.test/librusty_v8_release_x86_64-unknown-linux-gnu.a.gz",
|
||||
],
|
||||
)
|
||||
|
||||
http_file(
|
||||
name = "rusty_v8_146_4_0_x86_64_unknown_linux_musl_binding",
|
||||
downloaded_file_path = "src_binding_release_x86_64-unknown-linux-musl.rs",
|
||||
urls = [
|
||||
"https://example.test/src_binding_release_x86_64-unknown-linux-musl.rs",
|
||||
],
|
||||
)
|
||||
|
||||
http_file(
|
||||
name = "rusty_v8_145_0_0_x86_64_unknown_linux_gnu_archive",
|
||||
downloaded_file_path = "librusty_v8_release_x86_64-unknown-linux-gnu.a.gz",
|
||||
sha256 = "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff",
|
||||
urls = [
|
||||
"https://example.test/old.gz",
|
||||
],
|
||||
)
|
||||
"""
|
||||
)
|
||||
checksums = {
|
||||
"librusty_v8_release_x86_64-unknown-linux-gnu.a.gz": (
|
||||
"1111111111111111111111111111111111111111111111111111111111111111"
|
||||
),
|
||||
"src_binding_release_x86_64-unknown-linux-musl.rs": (
|
||||
"2222222222222222222222222222222222222222222222222222222222222222"
|
||||
),
|
||||
}
|
||||
|
||||
updated = rusty_v8_module_bazel.update_module_bazel_text(
|
||||
module_bazel,
|
||||
checksums,
|
||||
"146.4.0",
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
textwrap.dedent(
|
||||
"""\
|
||||
http_file(
|
||||
name = "rusty_v8_146_4_0_x86_64_unknown_linux_gnu_archive",
|
||||
downloaded_file_path = "librusty_v8_release_x86_64-unknown-linux-gnu.a.gz",
|
||||
sha256 = "1111111111111111111111111111111111111111111111111111111111111111",
|
||||
urls = [
|
||||
"https://example.test/librusty_v8_release_x86_64-unknown-linux-gnu.a.gz",
|
||||
],
|
||||
)
|
||||
|
||||
http_file(
|
||||
name = "rusty_v8_146_4_0_x86_64_unknown_linux_musl_binding",
|
||||
downloaded_file_path = "src_binding_release_x86_64-unknown-linux-musl.rs",
|
||||
sha256 = "2222222222222222222222222222222222222222222222222222222222222222",
|
||||
urls = [
|
||||
"https://example.test/src_binding_release_x86_64-unknown-linux-musl.rs",
|
||||
],
|
||||
)
|
||||
|
||||
http_file(
|
||||
name = "rusty_v8_145_0_0_x86_64_unknown_linux_gnu_archive",
|
||||
downloaded_file_path = "librusty_v8_release_x86_64-unknown-linux-gnu.a.gz",
|
||||
sha256 = "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff",
|
||||
urls = [
|
||||
"https://example.test/old.gz",
|
||||
],
|
||||
)
|
||||
"""
|
||||
),
|
||||
updated,
|
||||
)
|
||||
rusty_v8_module_bazel.check_module_bazel_text(updated, checksums, "146.4.0")
|
||||
|
||||
def test_check_module_bazel_rejects_manifest_drift(self) -> None:
|
||||
module_bazel = textwrap.dedent(
|
||||
"""\
|
||||
http_file(
|
||||
name = "rusty_v8_146_4_0_x86_64_unknown_linux_gnu_archive",
|
||||
downloaded_file_path = "librusty_v8_release_x86_64-unknown-linux-gnu.a.gz",
|
||||
sha256 = "1111111111111111111111111111111111111111111111111111111111111111",
|
||||
urls = [
|
||||
"https://example.test/librusty_v8_release_x86_64-unknown-linux-gnu.a.gz",
|
||||
],
|
||||
)
|
||||
"""
|
||||
)
|
||||
checksums = {
|
||||
"librusty_v8_release_x86_64-unknown-linux-gnu.a.gz": (
|
||||
"1111111111111111111111111111111111111111111111111111111111111111"
|
||||
),
|
||||
"orphan.gz": (
|
||||
"2222222222222222222222222222222222222222222222222222222222222222"
|
||||
),
|
||||
}
|
||||
|
||||
with self.assertRaisesRegex(
|
||||
rusty_v8_module_bazel.RustyV8ChecksumError,
|
||||
"manifest has orphan.gz",
|
||||
):
|
||||
rusty_v8_module_bazel.check_module_bazel_text(
|
||||
module_bazel,
|
||||
checksums,
|
||||
"146.4.0",
|
||||
)
|
||||
|
||||
def test_rusty_v8_http_file_versions(self) -> None:
|
||||
module_bazel = textwrap.dedent(
|
||||
"""\
|
||||
http_file(
|
||||
name = "rusty_v8_146_4_0_x86_64_unknown_linux_gnu_archive",
|
||||
downloaded_file_path = "archive.gz",
|
||||
urls = ["https://example.test/archive.gz"],
|
||||
)
|
||||
|
||||
http_file(
|
||||
name = "rusty_v8_147_4_0_x86_64_unknown_linux_gnu_archive",
|
||||
downloaded_file_path = "new-archive.gz",
|
||||
urls = ["https://example.test/new-archive.gz"],
|
||||
)
|
||||
|
||||
http_file(
|
||||
name = "unrelated_archive",
|
||||
downloaded_file_path = "other.gz",
|
||||
urls = ["https://example.test/other.gz"],
|
||||
)
|
||||
"""
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
["146.4.0", "147.4.0"],
|
||||
rusty_v8_module_bazel.rusty_v8_http_file_versions(module_bazel),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
234
reference/openai-codex/.github/scripts/verify_bazel_clippy_lints.py
vendored
Normal file
234
reference/openai-codex/.github/scripts/verify_bazel_clippy_lints.py
vendored
Normal file
@@ -0,0 +1,234 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import re
|
||||
import sys
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
DEFAULT_CARGO_TOML = ROOT / "codex-rs" / "Cargo.toml"
|
||||
DEFAULT_BAZELRC = ROOT / ".bazelrc"
|
||||
BAZEL_CLIPPY_FLAG_PREFIX = "build:clippy --@rules_rust//rust/settings:clippy_flag="
|
||||
BAZEL_SPECIAL_FLAGS = {"-Dwarnings"}
|
||||
VALID_LEVELS = {"allow", "warn", "deny", "forbid"}
|
||||
LONG_FLAG_RE = re.compile(
|
||||
r"^--(?P<level>allow|warn|deny|forbid)=clippy::(?P<lint>[a-z0-9_]+)$"
|
||||
)
|
||||
SHORT_FLAG_RE = re.compile(r"^-(?P<level>[AWDF])clippy::(?P<lint>[a-z0-9_]+)$")
|
||||
SHORT_LEVEL_NAMES = {
|
||||
"A": "allow",
|
||||
"W": "warn",
|
||||
"D": "deny",
|
||||
"F": "forbid",
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description=(
|
||||
"Verify that Bazel clippy flags in .bazelrc stay in sync with "
|
||||
"codex-rs/Cargo.toml [workspace.lints.clippy]."
|
||||
)
|
||||
)
|
||||
parser.add_argument(
|
||||
"--cargo-toml",
|
||||
type=Path,
|
||||
default=DEFAULT_CARGO_TOML,
|
||||
help="Path to the workspace Cargo.toml to inspect.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--bazelrc",
|
||||
type=Path,
|
||||
default=DEFAULT_BAZELRC,
|
||||
help="Path to the .bazelrc file to inspect.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
cargo_toml = args.cargo_toml.resolve()
|
||||
bazelrc = args.bazelrc.resolve()
|
||||
|
||||
cargo_lints = load_workspace_clippy_lints(cargo_toml)
|
||||
bazel_lints = load_bazel_clippy_lints(bazelrc)
|
||||
|
||||
missing = sorted(cargo_lints.keys() - bazel_lints.keys())
|
||||
extra = sorted(bazel_lints.keys() - cargo_lints.keys())
|
||||
mismatched = sorted(
|
||||
lint
|
||||
for lint in cargo_lints.keys() & bazel_lints.keys()
|
||||
if cargo_lints[lint] != bazel_lints[lint]
|
||||
)
|
||||
|
||||
if missing or extra or mismatched:
|
||||
print_sync_error(
|
||||
cargo_toml=cargo_toml,
|
||||
bazelrc=bazelrc,
|
||||
cargo_lints=cargo_lints,
|
||||
bazel_lints=bazel_lints,
|
||||
missing=missing,
|
||||
extra=extra,
|
||||
mismatched=mismatched,
|
||||
)
|
||||
return 1
|
||||
|
||||
print(
|
||||
"Bazel clippy flags in "
|
||||
f"{display_path(bazelrc)} match "
|
||||
f"{display_path(cargo_toml)} [workspace.lints.clippy]."
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
def load_workspace_clippy_lints(cargo_toml: Path) -> dict[str, str]:
|
||||
workspace = tomllib.loads(cargo_toml.read_text())["workspace"]
|
||||
clippy_lints = workspace["lints"]["clippy"]
|
||||
parsed: dict[str, str] = {}
|
||||
for lint, level in clippy_lints.items():
|
||||
if not isinstance(level, str):
|
||||
raise SystemExit(
|
||||
f"expected string lint level for clippy::{lint} in {cargo_toml}, got {level!r}"
|
||||
)
|
||||
normalized = level.strip().lower()
|
||||
if normalized not in VALID_LEVELS:
|
||||
raise SystemExit(
|
||||
f"unsupported lint level {level!r} for clippy::{lint} in {cargo_toml}"
|
||||
)
|
||||
parsed[lint] = normalized
|
||||
return parsed
|
||||
|
||||
|
||||
def load_bazel_clippy_lints(bazelrc: Path) -> dict[str, str]:
|
||||
parsed: dict[str, str] = {}
|
||||
line_numbers: dict[str, int] = {}
|
||||
|
||||
for lineno, line in enumerate(bazelrc.read_text().splitlines(), start=1):
|
||||
if not line.startswith(BAZEL_CLIPPY_FLAG_PREFIX):
|
||||
continue
|
||||
|
||||
flag = line.removeprefix(BAZEL_CLIPPY_FLAG_PREFIX).strip()
|
||||
if flag in BAZEL_SPECIAL_FLAGS:
|
||||
continue
|
||||
|
||||
parsed_flag = parse_bazel_lint_flag(flag)
|
||||
if parsed_flag is None:
|
||||
continue
|
||||
|
||||
lint, level = parsed_flag
|
||||
if lint in parsed:
|
||||
raise SystemExit(
|
||||
f"duplicate Bazel clippy entry for clippy::{lint} at "
|
||||
f"{bazelrc}:{line_numbers[lint]} and {bazelrc}:{lineno}"
|
||||
)
|
||||
parsed[lint] = level
|
||||
line_numbers[lint] = lineno
|
||||
|
||||
return parsed
|
||||
|
||||
|
||||
def parse_bazel_lint_flag(flag: str) -> tuple[str, str] | None:
|
||||
long_match = LONG_FLAG_RE.match(flag)
|
||||
if long_match:
|
||||
return long_match["lint"], long_match["level"]
|
||||
|
||||
short_match = SHORT_FLAG_RE.match(flag)
|
||||
if short_match:
|
||||
return short_match["lint"], SHORT_LEVEL_NAMES[short_match["level"]]
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def print_sync_error(
|
||||
*,
|
||||
cargo_toml: Path,
|
||||
bazelrc: Path,
|
||||
cargo_lints: dict[str, str],
|
||||
bazel_lints: dict[str, str],
|
||||
missing: list[str],
|
||||
extra: list[str],
|
||||
mismatched: list[str],
|
||||
) -> None:
|
||||
cargo_toml_display = display_path(cargo_toml)
|
||||
bazelrc_display = display_path(bazelrc)
|
||||
example_manifest = find_workspace_lints_example_manifest()
|
||||
|
||||
print(
|
||||
"ERROR: Bazel clippy flags are out of sync with Cargo workspace clippy lints.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
print(file=sys.stderr)
|
||||
print(
|
||||
f"Cargo defines the source of truth in {cargo_toml_display} "
|
||||
"[workspace.lints.clippy].",
|
||||
file=sys.stderr,
|
||||
)
|
||||
if example_manifest is not None:
|
||||
print(
|
||||
"Cargo applies those lint levels to member crates that opt into "
|
||||
f"`[lints] workspace = true`, for example {example_manifest}.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
print(
|
||||
"Bazel clippy does not ingest Cargo lint levels automatically, and "
|
||||
"`clippy.toml` can configure lint behavior but cannot set allow/warn/deny/forbid.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
print(
|
||||
f"Update {bazelrc_display} so its `build:clippy` "
|
||||
"`clippy_flag` entries match Cargo.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
|
||||
if missing:
|
||||
print(file=sys.stderr)
|
||||
print("Missing Bazel entries:", file=sys.stderr)
|
||||
for lint in missing:
|
||||
print(f" {render_bazelrc_line(lint, cargo_lints[lint])}", file=sys.stderr)
|
||||
|
||||
if mismatched:
|
||||
print(file=sys.stderr)
|
||||
print("Mismatched lint levels:", file=sys.stderr)
|
||||
for lint in mismatched:
|
||||
cargo_level = cargo_lints[lint]
|
||||
bazel_level = bazel_lints[lint]
|
||||
print(
|
||||
f" clippy::{lint}: Cargo has {cargo_level}, Bazel has {bazel_level}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
print(
|
||||
f" expected: {render_bazelrc_line(lint, cargo_level)}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
|
||||
if extra:
|
||||
print(file=sys.stderr)
|
||||
print("Extra Bazel entries with no Cargo counterpart:", file=sys.stderr)
|
||||
for lint in extra:
|
||||
print(f" {render_bazelrc_line(lint, bazel_lints[lint])}", file=sys.stderr)
|
||||
|
||||
|
||||
def render_bazelrc_line(lint: str, level: str) -> str:
|
||||
return f"{BAZEL_CLIPPY_FLAG_PREFIX}--{level}=clippy::{lint}"
|
||||
|
||||
|
||||
def display_path(path: Path) -> str:
|
||||
try:
|
||||
return str(path.relative_to(ROOT))
|
||||
except ValueError:
|
||||
return str(path)
|
||||
|
||||
|
||||
def find_workspace_lints_example_manifest() -> str | None:
|
||||
for cargo_toml in sorted((ROOT / "codex-rs").glob("**/Cargo.toml")):
|
||||
if cargo_toml == DEFAULT_CARGO_TOML:
|
||||
continue
|
||||
data = tomllib.loads(cargo_toml.read_text())
|
||||
if data.get("lints", {}).get("workspace") is True:
|
||||
return str(cargo_toml.relative_to(ROOT))
|
||||
return None
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
391
reference/openai-codex/.github/scripts/verify_cargo_workspace_manifests.py
vendored
Normal file
391
reference/openai-codex/.github/scripts/verify_cargo_workspace_manifests.py
vendored
Normal file
@@ -0,0 +1,391 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""Verify that codex-rs Cargo manifests follow workspace manifest policy.
|
||||
|
||||
Checks:
|
||||
- Crates inherit `[workspace.package]` metadata.
|
||||
- Crates opt into `[lints] workspace = true`.
|
||||
- Crate names follow the codex-rs directory naming conventions.
|
||||
- Workspace manifests do not introduce workspace crate feature toggles.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
CARGO_RS_ROOT = ROOT / "codex-rs"
|
||||
WORKSPACE_PACKAGE_FIELDS = ("version", "edition", "license")
|
||||
TOP_LEVEL_NAME_EXCEPTIONS = {
|
||||
"windows-sandbox-rs": "codex-windows-sandbox",
|
||||
}
|
||||
UTILITY_NAME_EXCEPTIONS = {
|
||||
"path-utils": "codex-utils-path",
|
||||
}
|
||||
MANIFEST_FEATURE_EXCEPTIONS = {
|
||||
"codex-rs/code-mode/Cargo.toml": {"sandbox": ("v8/v8_enable_sandbox",)},
|
||||
"codex-rs/v8-poc/Cargo.toml": {"sandbox": ("v8/v8_enable_sandbox",)},
|
||||
}
|
||||
OPTIONAL_DEPENDENCY_EXCEPTIONS = set()
|
||||
INTERNAL_DEPENDENCY_FEATURE_EXCEPTIONS = {}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
internal_package_names = workspace_package_names()
|
||||
used_manifest_feature_exceptions: set[str] = set()
|
||||
used_optional_dependency_exceptions: set[tuple[str, str, str]] = set()
|
||||
used_internal_dependency_feature_exceptions: set[tuple[str, str, str]] = set()
|
||||
failures_by_path: dict[str, list[str]] = {}
|
||||
|
||||
for path in manifests_to_verify():
|
||||
if errors := manifest_errors(
|
||||
path,
|
||||
internal_package_names,
|
||||
used_manifest_feature_exceptions,
|
||||
used_optional_dependency_exceptions,
|
||||
used_internal_dependency_feature_exceptions,
|
||||
):
|
||||
failures_by_path[manifest_key(path)] = errors
|
||||
|
||||
add_unused_exception_errors(
|
||||
failures_by_path,
|
||||
used_manifest_feature_exceptions,
|
||||
used_optional_dependency_exceptions,
|
||||
used_internal_dependency_feature_exceptions,
|
||||
)
|
||||
|
||||
if not failures_by_path:
|
||||
return 0
|
||||
|
||||
print(
|
||||
"Cargo manifests under codex-rs must inherit workspace package metadata, "
|
||||
"opt into workspace lints, and avoid introducing new workspace crate "
|
||||
"features."
|
||||
)
|
||||
print(
|
||||
"Workspace crate features are disallowed because our Bazel build setup "
|
||||
"does not honor them today, which can let issues hidden behind feature "
|
||||
"gates go unnoticed, and because they add extra crate build "
|
||||
"permutations we want to avoid."
|
||||
)
|
||||
print(
|
||||
"Cargo only applies `codex-rs/Cargo.toml` `[workspace.lints.clippy]` "
|
||||
"entries to a crate when that crate declares:"
|
||||
)
|
||||
print()
|
||||
print("[lints]")
|
||||
print("workspace = true")
|
||||
print()
|
||||
print(
|
||||
"Without that opt-in, `cargo clippy` can miss violations that Bazel clippy "
|
||||
"catches."
|
||||
)
|
||||
print()
|
||||
print(
|
||||
"Package-name checks apply to `codex-rs/<crate>/Cargo.toml` and "
|
||||
"`codex-rs/utils/<crate>/Cargo.toml`."
|
||||
)
|
||||
print(
|
||||
"Workspace crate features are forbidden; add a targeted exception here "
|
||||
"only if there is a deliberate temporary migration in flight."
|
||||
)
|
||||
print()
|
||||
for path in sorted(failures_by_path):
|
||||
errors = failures_by_path[path]
|
||||
print(f"{path}:")
|
||||
for error in errors:
|
||||
print(f" - {error}")
|
||||
|
||||
return 1
|
||||
|
||||
|
||||
def manifest_errors(
|
||||
path: Path,
|
||||
internal_package_names: set[str],
|
||||
used_manifest_feature_exceptions: set[str],
|
||||
used_optional_dependency_exceptions: set[tuple[str, str, str]],
|
||||
used_internal_dependency_feature_exceptions: set[tuple[str, str, str]],
|
||||
) -> list[str]:
|
||||
manifest = load_manifest(path)
|
||||
package = manifest.get("package")
|
||||
if not isinstance(package, dict) and path != CARGO_RS_ROOT / "Cargo.toml":
|
||||
return []
|
||||
|
||||
errors = []
|
||||
if isinstance(package, dict):
|
||||
for field in WORKSPACE_PACKAGE_FIELDS:
|
||||
if not is_workspace_reference(package.get(field)):
|
||||
errors.append(f"set `{field}.workspace = true` in `[package]`")
|
||||
|
||||
lints = manifest.get("lints")
|
||||
if not (isinstance(lints, dict) and lints.get("workspace") is True):
|
||||
errors.append("add `[lints]` with `workspace = true`")
|
||||
|
||||
expected_name = expected_package_name(path)
|
||||
if expected_name is not None:
|
||||
actual_name = package.get("name")
|
||||
if actual_name != expected_name:
|
||||
errors.append(
|
||||
f"set `[package].name` to `{expected_name}` (found `{actual_name}`)"
|
||||
)
|
||||
|
||||
path_key = manifest_key(path)
|
||||
features = manifest.get("features")
|
||||
if features is not None:
|
||||
normalized_features = normalize_feature_mapping(features)
|
||||
expected_features = MANIFEST_FEATURE_EXCEPTIONS.get(path_key)
|
||||
if expected_features is None:
|
||||
errors.append(
|
||||
"remove `[features]`; new workspace crate features are not allowed"
|
||||
)
|
||||
else:
|
||||
used_manifest_feature_exceptions.add(path_key)
|
||||
if normalized_features != expected_features:
|
||||
errors.append(
|
||||
"limit `[features]` to the existing exception list while "
|
||||
"workspace crate features are being removed "
|
||||
f"(expected {render_feature_mapping(expected_features)})"
|
||||
)
|
||||
|
||||
for section_name, dependencies in dependency_sections(manifest):
|
||||
for dependency_name, dependency in dependencies.items():
|
||||
if not isinstance(dependency, dict):
|
||||
continue
|
||||
|
||||
if dependency.get("optional") is True:
|
||||
exception_key = (path_key, section_name, dependency_name)
|
||||
if exception_key in OPTIONAL_DEPENDENCY_EXCEPTIONS:
|
||||
used_optional_dependency_exceptions.add(exception_key)
|
||||
else:
|
||||
errors.append(
|
||||
"remove `optional = true` from "
|
||||
f"`{dependency_entry_label(section_name, dependency_name)}`; "
|
||||
"new optional dependencies are not allowed because they "
|
||||
"create crate features"
|
||||
)
|
||||
|
||||
if not is_internal_dependency(path, dependency_name, dependency, internal_package_names):
|
||||
continue
|
||||
|
||||
dependency_features = dependency.get("features")
|
||||
if dependency_features is not None:
|
||||
normalized_dependency_features = normalize_string_list(
|
||||
dependency_features
|
||||
)
|
||||
exception_key = (path_key, section_name, dependency_name)
|
||||
expected_dependency_features = (
|
||||
INTERNAL_DEPENDENCY_FEATURE_EXCEPTIONS.get(exception_key)
|
||||
)
|
||||
if expected_dependency_features is None:
|
||||
errors.append(
|
||||
"remove `features = [...]` from workspace dependency "
|
||||
f"`{dependency_entry_label(section_name, dependency_name)}`; "
|
||||
"new workspace crate feature activations are not allowed"
|
||||
)
|
||||
else:
|
||||
used_internal_dependency_feature_exceptions.add(exception_key)
|
||||
if normalized_dependency_features != expected_dependency_features:
|
||||
errors.append(
|
||||
"limit workspace dependency features on "
|
||||
f"`{dependency_entry_label(section_name, dependency_name)}` "
|
||||
"to the existing exception list while workspace crate "
|
||||
"features are being removed "
|
||||
f"(expected {render_string_list(expected_dependency_features)})"
|
||||
)
|
||||
|
||||
if dependency.get("default-features") is False:
|
||||
errors.append(
|
||||
"remove `default-features = false` from workspace dependency "
|
||||
f"`{dependency_entry_label(section_name, dependency_name)}`; "
|
||||
"new workspace crate feature toggles are not allowed"
|
||||
)
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
def expected_package_name(path: Path) -> str | None:
|
||||
parts = path.relative_to(CARGO_RS_ROOT).parts
|
||||
if len(parts) == 2 and parts[1] == "Cargo.toml":
|
||||
directory = parts[0]
|
||||
return TOP_LEVEL_NAME_EXCEPTIONS.get(
|
||||
directory,
|
||||
directory if directory.startswith("codex-") else f"codex-{directory}",
|
||||
)
|
||||
if len(parts) == 3 and parts[0] == "utils" and parts[2] == "Cargo.toml":
|
||||
directory = parts[1]
|
||||
return UTILITY_NAME_EXCEPTIONS.get(directory, f"codex-utils-{directory}")
|
||||
return None
|
||||
|
||||
|
||||
def is_workspace_reference(value: object) -> bool:
|
||||
return isinstance(value, dict) and value.get("workspace") is True
|
||||
|
||||
|
||||
def manifest_key(path: Path) -> str:
|
||||
return str(path.relative_to(ROOT))
|
||||
|
||||
|
||||
def normalize_feature_mapping(value: object) -> dict[str, tuple[str, ...]] | None:
|
||||
if not isinstance(value, dict):
|
||||
return None
|
||||
|
||||
normalized = {}
|
||||
for key, features in value.items():
|
||||
if not isinstance(key, str):
|
||||
return None
|
||||
normalized_features = normalize_string_list(features)
|
||||
if normalized_features is None:
|
||||
return None
|
||||
normalized[key] = normalized_features
|
||||
return normalized
|
||||
|
||||
|
||||
def normalize_string_list(value: object) -> tuple[str, ...] | None:
|
||||
if not isinstance(value, list) or not all(isinstance(item, str) for item in value):
|
||||
return None
|
||||
return tuple(value)
|
||||
|
||||
|
||||
def render_feature_mapping(features: dict[str, tuple[str, ...]]) -> str:
|
||||
entries = [
|
||||
f"{name} = {render_string_list(items)}" for name, items in features.items()
|
||||
]
|
||||
return ", ".join(entries)
|
||||
|
||||
|
||||
def render_string_list(items: tuple[str, ...]) -> str:
|
||||
return "[" + ", ".join(f'"{item}"' for item in items) + "]"
|
||||
|
||||
|
||||
def dependency_sections(manifest: dict) -> list[tuple[str, dict]]:
|
||||
sections = []
|
||||
for section_name in ("dependencies", "dev-dependencies", "build-dependencies"):
|
||||
dependencies = manifest.get(section_name)
|
||||
if isinstance(dependencies, dict):
|
||||
sections.append((section_name, dependencies))
|
||||
|
||||
workspace = manifest.get("workspace")
|
||||
if isinstance(workspace, dict):
|
||||
workspace_dependencies = workspace.get("dependencies")
|
||||
if isinstance(workspace_dependencies, dict):
|
||||
sections.append(("workspace.dependencies", workspace_dependencies))
|
||||
|
||||
target = manifest.get("target")
|
||||
if not isinstance(target, dict):
|
||||
return sections
|
||||
|
||||
for target_name, tables in target.items():
|
||||
if not isinstance(tables, dict):
|
||||
continue
|
||||
for section_name in ("dependencies", "dev-dependencies", "build-dependencies"):
|
||||
dependencies = tables.get(section_name)
|
||||
if isinstance(dependencies, dict):
|
||||
sections.append((f"target.{target_name}.{section_name}", dependencies))
|
||||
|
||||
return sections
|
||||
|
||||
|
||||
def dependency_entry_label(section_name: str, dependency_name: str) -> str:
|
||||
return f"[{section_name}].{dependency_name}"
|
||||
|
||||
|
||||
def is_internal_dependency(
|
||||
manifest_path: Path,
|
||||
dependency_name: str,
|
||||
dependency: dict,
|
||||
internal_package_names: set[str],
|
||||
) -> bool:
|
||||
package_name = dependency.get("package", dependency_name)
|
||||
if isinstance(package_name, str) and package_name in internal_package_names:
|
||||
return True
|
||||
|
||||
dependency_path = dependency.get("path")
|
||||
if not isinstance(dependency_path, str):
|
||||
return False
|
||||
|
||||
resolved_dependency_path = (manifest_path.parent / dependency_path).resolve()
|
||||
try:
|
||||
resolved_dependency_path.relative_to(CARGO_RS_ROOT)
|
||||
except ValueError:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def add_unused_exception_errors(
|
||||
failures_by_path: dict[str, list[str]],
|
||||
used_manifest_feature_exceptions: set[str],
|
||||
used_optional_dependency_exceptions: set[tuple[str, str, str]],
|
||||
used_internal_dependency_feature_exceptions: set[tuple[str, str, str]],
|
||||
) -> None:
|
||||
for path_key in sorted(
|
||||
set(MANIFEST_FEATURE_EXCEPTIONS) - used_manifest_feature_exceptions
|
||||
):
|
||||
add_failure(
|
||||
failures_by_path,
|
||||
path_key,
|
||||
"remove the stale `[features]` exception from "
|
||||
"`MANIFEST_FEATURE_EXCEPTIONS`",
|
||||
)
|
||||
|
||||
for path_key, section_name, dependency_name in sorted(
|
||||
OPTIONAL_DEPENDENCY_EXCEPTIONS - used_optional_dependency_exceptions
|
||||
):
|
||||
add_failure(
|
||||
failures_by_path,
|
||||
path_key,
|
||||
"remove the stale optional-dependency exception for "
|
||||
f"`{dependency_entry_label(section_name, dependency_name)}` from "
|
||||
"`OPTIONAL_DEPENDENCY_EXCEPTIONS`",
|
||||
)
|
||||
|
||||
for path_key, section_name, dependency_name in sorted(
|
||||
set(INTERNAL_DEPENDENCY_FEATURE_EXCEPTIONS)
|
||||
- used_internal_dependency_feature_exceptions
|
||||
):
|
||||
add_failure(
|
||||
failures_by_path,
|
||||
path_key,
|
||||
"remove the stale internal dependency feature exception for "
|
||||
f"`{dependency_entry_label(section_name, dependency_name)}` from "
|
||||
"`INTERNAL_DEPENDENCY_FEATURE_EXCEPTIONS`",
|
||||
)
|
||||
|
||||
|
||||
def add_failure(failures_by_path: dict[str, list[str]], path_key: str, error: str) -> None:
|
||||
failures_by_path.setdefault(path_key, []).append(error)
|
||||
|
||||
|
||||
def workspace_package_names() -> set[str]:
|
||||
package_names = set()
|
||||
for path in cargo_manifests():
|
||||
manifest = load_manifest(path)
|
||||
package = manifest.get("package")
|
||||
if not isinstance(package, dict):
|
||||
continue
|
||||
package_name = package.get("name")
|
||||
if isinstance(package_name, str):
|
||||
package_names.add(package_name)
|
||||
return package_names
|
||||
|
||||
|
||||
def load_manifest(path: Path) -> dict:
|
||||
return tomllib.loads(path.read_text())
|
||||
|
||||
|
||||
def cargo_manifests() -> list[Path]:
|
||||
return sorted(
|
||||
path
|
||||
for path in CARGO_RS_ROOT.rglob("Cargo.toml")
|
||||
if path != CARGO_RS_ROOT / "Cargo.toml"
|
||||
)
|
||||
|
||||
|
||||
def manifests_to_verify() -> list[Path]:
|
||||
return [CARGO_RS_ROOT / "Cargo.toml", *cargo_manifests()]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
89
reference/openai-codex/.github/scripts/verify_tui_core_boundary.py
vendored
Normal file
89
reference/openai-codex/.github/scripts/verify_tui_core_boundary.py
vendored
Normal file
@@ -0,0 +1,89 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""Verify codex-tui does not depend on or import codex-core directly."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import sys
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
TUI_ROOT = ROOT / "codex-rs" / "tui"
|
||||
TUI_MANIFEST = TUI_ROOT / "Cargo.toml"
|
||||
FORBIDDEN_PACKAGE = "codex-core"
|
||||
FORBIDDEN_SOURCE_PATTERNS = (
|
||||
re.compile(r"\bcodex_core::"),
|
||||
re.compile(r"\buse\s+codex_core\b"),
|
||||
re.compile(r"\bextern\s+crate\s+codex_core\b"),
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
failures = []
|
||||
failures.extend(manifest_failures())
|
||||
failures.extend(source_failures())
|
||||
|
||||
if not failures:
|
||||
return 0
|
||||
|
||||
print("codex-tui must not depend on or import codex-core directly.")
|
||||
print(
|
||||
"Use the app-server protocol/client boundary instead; temporary embedded "
|
||||
"startup gaps belong behind codex_app_server_client::legacy_core."
|
||||
)
|
||||
print()
|
||||
for failure in failures:
|
||||
print(f"- {failure}")
|
||||
|
||||
return 1
|
||||
|
||||
|
||||
def manifest_failures() -> list[str]:
|
||||
manifest = tomllib.loads(TUI_MANIFEST.read_text())
|
||||
failures = []
|
||||
for section_name, dependencies in dependency_sections(manifest):
|
||||
if FORBIDDEN_PACKAGE in dependencies:
|
||||
failures.append(
|
||||
f"{relative_path(TUI_MANIFEST)} declares `{FORBIDDEN_PACKAGE}` "
|
||||
f"in `[{section_name}]`"
|
||||
)
|
||||
return failures
|
||||
|
||||
|
||||
def dependency_sections(manifest: dict) -> list[tuple[str, dict]]:
|
||||
sections: list[tuple[str, dict]] = []
|
||||
for section_name in ("dependencies", "dev-dependencies", "build-dependencies"):
|
||||
dependencies = manifest.get(section_name)
|
||||
if isinstance(dependencies, dict):
|
||||
sections.append((section_name, dependencies))
|
||||
|
||||
for target_name, target in manifest.get("target", {}).items():
|
||||
if not isinstance(target, dict):
|
||||
continue
|
||||
for section_name in ("dependencies", "dev-dependencies", "build-dependencies"):
|
||||
dependencies = target.get(section_name)
|
||||
if isinstance(dependencies, dict):
|
||||
sections.append((f'target.{target_name}.{section_name}', dependencies))
|
||||
|
||||
return sections
|
||||
|
||||
|
||||
def source_failures() -> list[str]:
|
||||
failures = []
|
||||
for path in sorted(TUI_ROOT.glob("**/*.rs")):
|
||||
text = path.read_text()
|
||||
for line_number, line in enumerate(text.splitlines(), start=1):
|
||||
if any(pattern.search(line) for pattern in FORBIDDEN_SOURCE_PATTERNS):
|
||||
failures.append(f"{relative_path(path)}:{line_number} imports `codex_core`")
|
||||
return failures
|
||||
|
||||
|
||||
def relative_path(path: Path) -> str:
|
||||
return str(path.relative_to(ROOT))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user