From 19d5534187173263ef3b09ec56a59a14a0b542af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Kara=C5=9B?= Date: Thu, 30 Oct 2025 11:20:10 +0100 Subject: [PATCH 1/5] CLOUDP-339241 - re-enable multi arch smoke tests --- .evergreen-functions.yml | 14 +- .evergreen-release.yml | 175 ++++++++++++++---- .evergreen.yml | 65 +++---- build_info.json | 74 +++++++- scripts/dev/contexts/e2e_smoke_arm_release | 17 ++ .../dev/contexts/e2e_smoke_ibm_power_release | 15 ++ scripts/dev/contexts/e2e_smoke_ibm_z_release | 15 ++ .../contexts/{e2e_smoke => e2e_smoke_release} | 3 + .../dev/contexts/e2e_static_smoke_arm_release | 21 +++ .../e2e_static_smoke_ibm_power_release | 19 ++ .../contexts/e2e_static_smoke_ibm_z_release | 19 ++ ..._static_smoke => e2e_static_smoke_release} | 3 + scripts/dev/recreate_python_venv.sh | 4 +- .../evergreen/e2e/build_tests_image_ibm.sh | 14 -- scripts/evergreen/e2e/single_e2e.sh | 2 +- scripts/evergreen/setup_minikube_host.sh | 2 +- scripts/release/atomic_pipeline.py | 71 +++++-- scripts/release/build/build_info.py | 7 + .../build/image_build_configuration.py | 1 + scripts/release/check_changelog.py | 4 +- scripts/release/pipeline.py | 13 +- scripts/release/tests/build_info_test.py | 66 ++++++- 22 files changed, 508 insertions(+), 116 deletions(-) create mode 100644 scripts/dev/contexts/e2e_smoke_arm_release create mode 100644 scripts/dev/contexts/e2e_smoke_ibm_power_release create mode 100644 scripts/dev/contexts/e2e_smoke_ibm_z_release rename scripts/dev/contexts/{e2e_smoke => e2e_smoke_release} (50%) create mode 100644 scripts/dev/contexts/e2e_static_smoke_arm_release create mode 100644 scripts/dev/contexts/e2e_static_smoke_ibm_power_release create mode 100644 scripts/dev/contexts/e2e_static_smoke_ibm_z_release rename scripts/dev/contexts/{e2e_static_smoke => e2e_static_smoke_release} (63%) delete mode 100755 scripts/evergreen/e2e/build_tests_image_ibm.sh diff --git a/.evergreen-functions.yml b/.evergreen-functions.yml index 7bddaa315..bd01d15d6 100644 --- a/.evergreen-functions.yml +++ b/.evergreen-functions.yml @@ -294,6 +294,9 @@ functions: - command: subprocess.exec type: setup params: + env: + SKIP_MINIKUBE_SETUP: ${skip_minikube_setup!|false} + SKIP_INSTALL_REQUIREMENTS: ${skip_install_python_requirements!|true} working_dir: src/github.com/mongodb/mongodb-kubernetes add_to_path: - ${workdir}/bin @@ -523,15 +526,6 @@ functions: working_dir: src/github.com/mongodb/mongodb-kubernetes binary: scripts/release/kubectl_mongodb/download_kubectl_plugin.sh - build_test_image_ibm: - - command: subprocess.exec - params: - shell: bash - working_dir: src/github.com/mongodb/mongodb-kubernetes - add_to_path: - - ${workdir}/bin - binary: scripts/evergreen/e2e/build_tests_image_ibm.sh - pipeline: - command: subprocess.exec retry_on_failure: true @@ -540,6 +534,8 @@ functions: shell: bash <<: *e2e_include_expansions_in_env working_dir: src/github.com/mongodb/mongodb-kubernetes + add_to_path: + - ${workdir}/bin binary: scripts/release/pipeline.sh env: IMAGE_NAME: ${image_name} diff --git a/.evergreen-release.yml b/.evergreen-release.yml index b16960a21..beeb22193 100644 --- a/.evergreen-release.yml +++ b/.evergreen-release.yml @@ -2,18 +2,6 @@ include: - filename: .evergreen-functions.yml tasks: - - name: build_test_image_for_smoke_tests - display_name: build_test_image_for_smoke_tests - tags: [ "image_release" ] - allowed_requesters: [ "patch", "github_tag" ] - commands: - - func: clone - - func: setup_building_host - - func: download_multi_cluster_binary - - func: pipeline - vars: - image_name: meko-tests - - name: release_operator tags: [ "image_release" ] allowed_requesters: [ "patch", "github_tag" ] @@ -119,8 +107,8 @@ tasks: task: prepare_and_upload_openshift_bundles - name: release_kubectl_mongodb_plugin - allowed_requesters: [ "patch", "github_tag" ] tags: [ "binary_release" ] + allowed_requesters: [ "patch", "github_tag" ] commands: - func: clone - func: python_venv @@ -150,10 +138,10 @@ buildvariants: - name: release_images display_name: release_images tags: [ "release" ] - allowed_requesters: [ "patch", "github_tag" ] - max_hosts: -1 run_on: - release-ubuntu2404-small # This is required for CISA attestation https://jira.mongodb.org/browse/DEVPROD-17780 + allowed_requesters: [ "patch", "github_tag" ] + max_hosts: -1 tasks: - name: release_operator - name: release_init_appdb @@ -166,12 +154,12 @@ buildvariants: - name: preflight_release_images display_name: preflight_release_images tags: [ "release" ] + run_on: + - rhel90-large allowed_requesters: [ "patch", "github_tag" ] depends_on: - name: "*" variant: release_images - run_on: - - rhel90-large expansions: preflight_submit: true tasks: @@ -180,56 +168,91 @@ buildvariants: - name: prepare_openshift_bundles display_name: prepare_openshift_bundles tags: [ "release" ] + run_on: + - ubuntu2404-small allowed_requesters: [ "patch", "github_tag" ] depends_on: - name: "*" variant: release_images - name: "*" variant: preflight_release_images - run_on: - - ubuntu2404-small tasks: - name: run_conditionally_prepare_and_upload_openshift_bundles - name: prerelease_kind_code_snippets display_name: prerelease_kind_code_snippets tags: [ "release", "prerelease_code_snippets" ] + run_on: + - ubuntu2404-large allowed_requesters: [ "patch", "github_tag" ] depends_on: - name: "*" variant: release_images patch_optional: true - run_on: - - ubuntu2404-large tasks: - name: kind_code_snippets_task_group - name: prerelease_gke_code_snippets display_name: prerelease_gke_code_snippets tags: [ "release", "prerelease_code_snippets" ] + run_on: + - ubuntu2404-small allowed_requesters: ["patch", "github_tag"] depends_on: - variant: release_images name: '*' patch_optional: true - run_on: - - ubuntu2404-small tasks: - name: gke_code_snippets_task_group - - name: init_smoke_tests - display_name: init_smoke_tests + - name: init_test_run_release + display_name: init_test_run tags: [ "release", "e2e_smoke_release_test_suite" ] + run_on: + - release-ubuntu2404-small # This is required for CISA attestation https://jira.mongodb.org/browse/DEVPROD-17780 allowed_requesters: [ "patch", "github_tag" ] + tasks: + - name: build_test_image + + - name: init_smoke_tests_ibm_power_release + display_name: init_smoke_tests_ibm_power + max_hosts: -1 + tags: [ "release", "e2e_smoke_release_test_suite" ] + run_on: + - release-rhel9-power-small # This is required for CISA attestation https://jira.mongodb.org/browse/DEVPROD-17780 + - release-rhel9-power-large # This is required for CISA attestation https://jira.mongodb.org/browse/DEVPROD-17780 + allowed_requesters: [ "patch", "github_tag" ] + tasks: + - name: build_test_image_ibm_power + + - name: init_smoke_tests_ibm_z_release + display_name: init_smoke_tests_ibm_z + max_hosts: -1 + tags: [ "release", "e2e_smoke_release_test_suite" ] + # TODO: Re-enable when ibm_z series is stable + # https://jira.mongodb.org/browse/DEVPROD-23283 + disable: true + run_on: + - release-rhel9-zseries-small # This is required for CISA attestation https://jira.mongodb.org/browse/DEVPROD-17780 + - release-rhel9-zseries-large # This is required for CISA attestation https://jira.mongodb.org/browse/DEVPROD-17780 + allowed_requesters: [ "patch", "github_tag" ] + tasks: + - name: build_test_image_ibm_z + + - name: init_smoke_tests_arm_release + display_name: init_smoke_tests_arm + max_hosts: -1 + tags: [ "release", "e2e_smoke_release_test_suite" ] run_on: - release-ubuntu2404-small # This is required for CISA attestation https://jira.mongodb.org/browse/DEVPROD-17780 + allowed_requesters: [ "patch", "github_tag" ] depends_on: - name: release_kubectl_mongodb_plugin variant: release_kubectl_mongodb_plugin tasks: - - name: build_test_image_for_smoke_tests + - name: build_test_image_arm - - name: e2e_smoke + - name: e2e_smoke_release display_name: e2e_smoke tags: [ "release", "e2e_smoke_release_test_suite" ] run_on: @@ -239,13 +262,13 @@ buildvariants: - name: "*" variant: release_images - name: "*" - variant: init_smoke_tests + variant: init_test_run_release tasks: - name: e2e_smoke_task_group - - name: e2e_static_smoke + - name: e2e_static_smoke_release display_name: e2e_static_smoke - tags: [ "release", "e2e_smoke_release_test_suite" ] + tags: [ "release", "e2e_smoke_release_test_suite", "static" ] run_on: - ubuntu2404-large allowed_requesters: [ "patch", "github_tag" ] @@ -253,10 +276,100 @@ buildvariants: - name: "*" variant: release_images - name: "*" - variant: init_smoke_tests + variant: init_test_run_release tasks: - name: e2e_smoke_task_group + - name: e2e_smoke_ibm_power_release + display_name: e2e_smoke_ibm_power + tags: [ "release", "e2e_smoke_release_test_suite" ] + run_on: + - rhel9-power-small + - rhel9-power-large + allowed_requesters: [ "patch", "github_tag" ] + depends_on: + - name: "*" + variant: release_images + - name: "*" + variant: init_smoke_tests_ibm_power_release + tasks: + - name: e2e_smoke_ibm_task_group + + - name: e2e_static_smoke_ibm_power_release + display_name: e2e_static_smoke_ibm_power + tags: [ "release", "e2e_smoke_release_test_suite", "static" ] + run_on: + - rhel9-power-small + - rhel9-power-large + allowed_requesters: [ "patch", "github_tag" ] + depends_on: + - name: "*" + variant: release_images + - name: "*" + variant: init_smoke_tests_ibm_power_release + tasks: + - name: e2e_smoke_ibm_task_group + + - name: e2e_smoke_ibm_z_release + display_name: e2e_smoke_ibm_z + tags: [ "release", "e2e_smoke_release_test_suite" ] + disable: true + run_on: + - rhel9-zseries-small + - rhel9-zseries-large + allowed_requesters: [ "patch", "github_tag" ] + depends_on: + - name: "*" + variant: release_images + - name: "*" + variant: init_smoke_tests_ibm_z_release + tasks: + - name: e2e_smoke_ibm_task_group + + - name: e2e_static_smoke_ibm_z_release + display_name: e2e_static_smoke_ibm_z + tags: [ "release", "e2e_smoke_release_test_suite", "static" ] + disable: true + run_on: + - rhel9-zseries-small + - rhel9-zseries-large + allowed_requesters: [ "patch", "github_tag" ] + depends_on: + - name: "*" + variant: release_images + - name: "*" + variant: init_smoke_tests_ibm_z_release + tasks: + - name: e2e_smoke_ibm_task_group + + - name: e2e_smoke_arm_release + display_name: e2e_smoke_arm + tags: [ "release", "e2e_smoke_release_test_suite" ] + run_on: + - ubuntu2404-arm64-large + allowed_requesters: [ "patch", "github_tag" ] + depends_on: + - name: "*" + variant: release_images + - name: "*" + variant: init_smoke_tests_arm_release + tasks: + - name: e2e_smoke_arm_task_group + + - name: e2e_static_smoke_arm_release + display_name: e2e_static_smoke_arm + tags: [ "release", "e2e_smoke_release_test_suite", "static" ] + run_on: + - ubuntu2404-arm64-large + allowed_requesters: [ "patch", "github_tag" ] + depends_on: + - name: "*" + variant: release_images + - name: "*" + variant: init_smoke_tests_arm_release + tasks: + - name: e2e_smoke_arm_task_group + - name: release_kubectl_mongodb_plugin display_name: release_kubectl_mongodb_plugin tags: [ "release" ] diff --git a/.evergreen.yml b/.evergreen.yml index 1732b36b6..6ce411661 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -212,6 +212,9 @@ patch_aliases: - alias: "smoke_test_release" variant_tags: [ "e2e_smoke_release_test_suite" ] task_tags: [ "patch-run" ] + - alias: "smoke_multi_arch_test" + variant_tags: [ "e2e_smoke_multi_arch" ] + task_tags: [ "patch-run" ] - alias: "patch-run-cloudqa" variant_tags: [ "cloudqa_non_static" ] task: ".*" @@ -368,20 +371,29 @@ tasks: vars: image_name: meko-tests-arm64 - - name: build_test_image_ibm + - name: build_test_image_ibm_z commands: - func: clone - - command: subprocess.exec - type: setup - params: - env: - SKIP_MINIKUBE_SETUP: "true" - working_dir: src/github.com/mongodb/mongodb-kubernetes - add_to_path: - - ${workdir}/bin - command: scripts/evergreen/setup_minikube_host.sh + - func: setup_building_host_minikube + vars: + skip_minikube_setup: true + skip_install_python_requirements: false + - func: download_multi_cluster_binary + - func: pipeline + vars: + image_name: meko-tests-ibm-z + + - name: build_test_image_ibm_power + commands: + - func: clone + - func: setup_building_host_minikube + vars: + skip_minikube_setup: true + skip_install_python_requirements: false - func: download_multi_cluster_binary - - func: build_test_image_ibm + - func: pipeline + vars: + image_name: meko-tests-ibm-power - name: build_mco_test_image commands: @@ -1411,9 +1423,6 @@ buildvariants: - rhel9-power-small - rhel9-power-large allowed_requesters: [ "patch", "commit" ] - # TODO: Re-enable when staging is added to pipeline - # https://jira.mongodb.org/browse/CLOUDP-349096 - disable: true depends_on: - name: build_operator_ubi variant: init_test_run @@ -1421,7 +1430,7 @@ buildvariants: variant: init_test_run - name: build_database_image_ubi variant: init_test_run - - name: build_test_image_ibm + - name: build_test_image_ibm_power variant: init_test_run_ibm_power tasks: - name: e2e_smoke_ibm_task_group @@ -1433,8 +1442,6 @@ buildvariants: - rhel9-power-small - rhel9-power-large allowed_requesters: [ "patch", "commit" ] - # TODO: Re-enable when staging is added to pipeline - # https://jira.mongodb.org/browse/CLOUDP-349096 disable: true depends_on: - name: build_operator_ubi @@ -1443,7 +1450,7 @@ buildvariants: variant: init_test_run - name: build_database_image_ubi variant: init_test_run - - name: build_test_image_ibm + - name: build_test_image_ibm_power variant: init_test_run_ibm_power tasks: - name: e2e_smoke_ibm_task_group @@ -1455,8 +1462,8 @@ buildvariants: - rhel9-zseries-small - rhel9-zseries-large allowed_requesters: [ "patch", "commit" ] - # TODO: Re-enable when staging is added to pipeline - # https://jira.mongodb.org/browse/CLOUDP-349096 + # TODO: Re-enable when ibm_z series is stable + # https://jira.mongodb.org/browse/DEVPROD-23283 disable: true depends_on: - name: build_operator_ubi @@ -1465,7 +1472,7 @@ buildvariants: variant: init_test_run - name: build_database_image_ubi variant: init_test_run - - name: build_test_image_ibm + - name: build_test_image_ibm_z variant: init_test_run_ibm_z tasks: - name: e2e_smoke_ibm_task_group @@ -1473,8 +1480,8 @@ buildvariants: - name: e2e_static_smoke_ibm_z display_name: e2e_static_smoke_ibm_z tags: [ "staging", "e2e_smoke_test_suite", "static" ] - # TODO: Re-enable when staging is added to pipeline - # https://jira.mongodb.org/browse/CLOUDP-349096 + # TODO: Re-enable when ibm_z series is stable + # https://jira.mongodb.org/browse/DEVPROD-23283 disable: true run_on: - rhel9-zseries-small @@ -1485,7 +1492,7 @@ buildvariants: variant: init_test_run - name: build_init_database_image_ubi variant: init_test_run - - name: build_test_image_ibm + - name: build_test_image_ibm_z variant: init_test_run_ibm_z tasks: - name: e2e_smoke_ibm_task_group @@ -1734,9 +1741,6 @@ buildvariants: display_name: init_test_run_ibm_power max_hosts: -1 tags: [ "staging" ] - # TODO: Re-enable when staging is added to pipeline - # https://jira.mongodb.org/browse/CLOUDP-349096 - disable: true run_on: - rhel9-power-small - rhel9-power-large @@ -1744,15 +1748,12 @@ buildvariants: - name: build_kubectl_mongodb_plugin variant: init_test_run tasks: - - name: build_test_image_ibm + - name: build_test_image_ibm_power - name: init_test_run_ibm_z display_name: init_test_run_ibm_z max_hosts: -1 tags: [ "staging" ] - # TODO: Re-enable when staging is added to pipeline - # https://jira.mongodb.org/browse/CLOUDP-349096 - disable: true run_on: - rhel9-zseries-small - rhel9-zseries-large @@ -1760,7 +1761,7 @@ buildvariants: - name: build_kubectl_mongodb_plugin variant: init_test_run tasks: - - name: build_test_image_ibm + - name: build_test_image_ibm_z - name: init_test_run_arm display_name: init_test_run_arm diff --git a/build_info.json b/build_info.json index 3bc34cff2..21efc0bd3 100644 --- a/build_info.json +++ b/build_info.json @@ -185,8 +185,7 @@ ] }, "release": { - "skip-if-exists": true, - "repositories": ["quay.io/mongodb/mongodb-kubernetes-tests"], + "repositories": ["268558157000.dkr.ecr.us-east-1.amazonaws.com/staging/mongodb-kubernetes-tests"], "platforms": [ "linux/amd64" ] @@ -211,6 +210,77 @@ "linux/arm64" ], "architecture_suffix": true + }, + "release": { + "repositories": [ + "268558157000.dkr.ecr.us-east-1.amazonaws.com/staging/mongodb-kubernetes-tests" + ], + "platforms": [ + "linux/arm64" + ], + "architecture_suffix": true + } + }, + "meko-tests-ibm-z": { + "dockerfile-path": "docker/mongodb-kubernetes-tests/Dockerfile", + "builder": "podman", + "patch": { + "repositories": [ + "268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/mongodb-kubernetes-tests" + ], + "platforms": [ + "linux/s390x" + ], + "architecture_suffix": true + }, + "staging": { + "repositories": [ + "268558157000.dkr.ecr.us-east-1.amazonaws.com/staging/mongodb-kubernetes-tests" + ], + "platforms": [ + "linux/s390x" + ], + "architecture_suffix": true + }, + "release": { + "repositories": [ + "268558157000.dkr.ecr.us-east-1.amazonaws.com/staging/mongodb-kubernetes-tests" + ], + "platforms": [ + "linux/s390x" + ], + "architecture_suffix": true + } + }, + "meko-tests-ibm-power": { + "dockerfile-path": "docker/mongodb-kubernetes-tests/Dockerfile", + "builder": "podman", + "patch": { + "repositories": [ + "268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/mongodb-kubernetes-tests" + ], + "platforms": [ + "linux/ppc64le" + ], + "architecture_suffix": true + }, + "staging": { + "repositories": [ + "268558157000.dkr.ecr.us-east-1.amazonaws.com/staging/mongodb-kubernetes-tests" + ], + "platforms": [ + "linux/ppc64le" + ], + "architecture_suffix": true + }, + "release": { + "repositories": [ + "268558157000.dkr.ecr.us-east-1.amazonaws.com/staging/mongodb-kubernetes-tests" + ], + "platforms": [ + "linux/ppc64le" + ], + "architecture_suffix": true } }, "mco-tests": { diff --git a/scripts/dev/contexts/e2e_smoke_arm_release b/scripts/dev/contexts/e2e_smoke_arm_release new file mode 100644 index 000000000..bfbd7be48 --- /dev/null +++ b/scripts/dev/contexts/e2e_smoke_arm_release @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/root-context" + +export ops_manager_version="cloud_qa" + +export CUSTOM_MDB_VERSION=8.0.12-ent +export KUBE_ENVIRONMENT_NAME=kind +export CLUSTER_TYPE=kind + +# During release the meko-tests image is pushed to a staging registry. We don't want to push it to production registry +export MEKO_TESTS_REGISTRY="268558157000.dkr.ecr.us-east-1.amazonaws.com/staging" diff --git a/scripts/dev/contexts/e2e_smoke_ibm_power_release b/scripts/dev/contexts/e2e_smoke_ibm_power_release new file mode 100644 index 000000000..580283f82 --- /dev/null +++ b/scripts/dev/contexts/e2e_smoke_ibm_power_release @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/root-context" +source "${script_dir}/variables/e2e_ibm_smoke_base" + +CUSTOM_OM_VERSION=$(grep -E "^\s*-\s*&ops_manager_70_latest\s+(\S+)\s+#" <"${PROJECT_DIR}"/.evergreen.yml | awk '{print $3}') +export CUSTOM_OM_VERSION + +# During release the meko-tests image is pushed to a staging registry. We don't want to push it to production registry +export MEKO_TESTS_REGISTRY="268558157000.dkr.ecr.us-east-1.amazonaws.com/staging" diff --git a/scripts/dev/contexts/e2e_smoke_ibm_z_release b/scripts/dev/contexts/e2e_smoke_ibm_z_release new file mode 100644 index 000000000..580283f82 --- /dev/null +++ b/scripts/dev/contexts/e2e_smoke_ibm_z_release @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/root-context" +source "${script_dir}/variables/e2e_ibm_smoke_base" + +CUSTOM_OM_VERSION=$(grep -E "^\s*-\s*&ops_manager_70_latest\s+(\S+)\s+#" <"${PROJECT_DIR}"/.evergreen.yml | awk '{print $3}') +export CUSTOM_OM_VERSION + +# During release the meko-tests image is pushed to a staging registry. We don't want to push it to production registry +export MEKO_TESTS_REGISTRY="268558157000.dkr.ecr.us-east-1.amazonaws.com/staging" diff --git a/scripts/dev/contexts/e2e_smoke b/scripts/dev/contexts/e2e_smoke_release similarity index 50% rename from scripts/dev/contexts/e2e_smoke rename to scripts/dev/contexts/e2e_smoke_release index 8c5438743..39b079c4c 100644 --- a/scripts/dev/contexts/e2e_smoke +++ b/scripts/dev/contexts/e2e_smoke_release @@ -7,3 +7,6 @@ script_dir=$(dirname "${script_name}") source "${script_dir}/root-context" source "${script_dir}/variables/om70" + +# During release the meko-tests image is pushed to a staging registry. We don't want to push it to production registry +export MEKO_TESTS_REGISTRY="268558157000.dkr.ecr.us-east-1.amazonaws.com/staging" diff --git a/scripts/dev/contexts/e2e_static_smoke_arm_release b/scripts/dev/contexts/e2e_static_smoke_arm_release new file mode 100644 index 000000000..569c11b48 --- /dev/null +++ b/scripts/dev/contexts/e2e_static_smoke_arm_release @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/root-context" + +export ops_manager_version="cloud_qa" + +CUSTOM_OM_VERSION=$(grep -E "^\s*-\s*&ops_manager_70_latest\s+(\S+)\s+#" <"${PROJECT_DIR}"/.evergreen.yml | awk '{print $3}') +export CUSTOM_OM_VERSION + +export CUSTOM_MDB_VERSION=7.0.22 # we only have ibm static images starting with 7.0.22 +export KUBE_ENVIRONMENT_NAME=kind +export CLUSTER_TYPE=kind +export MDB_DEFAULT_ARCHITECTURE=static + +# During release the meko-tests image is pushed to a staging registry. We don't want to push it to production registry +export MEKO_TESTS_REGISTRY="268558157000.dkr.ecr.us-east-1.amazonaws.com/staging" diff --git a/scripts/dev/contexts/e2e_static_smoke_ibm_power_release b/scripts/dev/contexts/e2e_static_smoke_ibm_power_release new file mode 100644 index 000000000..3da53dbda --- /dev/null +++ b/scripts/dev/contexts/e2e_static_smoke_ibm_power_release @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/root-context" +source "${script_dir}/variables/e2e_ibm_smoke_base" + +CUSTOM_OM_VERSION=$(grep -E "^\s*-\s*&ops_manager_70_latest\s+(\S+)\s+#" <"${PROJECT_DIR}"/.evergreen.yml | awk '{print $3}') +export CUSTOM_OM_VERSION + +export MDB_DEFAULT_ARCHITECTURE=static +# MDB supports ppc64le on RHEL9 only from 8.0.12 onwards +export CUSTOM_MDB_VERSION=8.0.12-ent + +# During release the meko-tests image is pushed to a staging registry. We don't want to push it to production registry +export MEKO_TESTS_REGISTRY="268558157000.dkr.ecr.us-east-1.amazonaws.com/staging" diff --git a/scripts/dev/contexts/e2e_static_smoke_ibm_z_release b/scripts/dev/contexts/e2e_static_smoke_ibm_z_release new file mode 100644 index 000000000..88e277b57 --- /dev/null +++ b/scripts/dev/contexts/e2e_static_smoke_ibm_z_release @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/root-context" +source "${script_dir}/variables/e2e_ibm_smoke_base" + +CUSTOM_OM_VERSION=$(grep -E "^\s*-\s*&ops_manager_70_latest\s+(\S+)\s+#" <"${PROJECT_DIR}"/.evergreen.yml | awk '{print $3}') +export CUSTOM_OM_VERSION + +export MDB_DEFAULT_ARCHITECTURE=static +# MDB supports s390x on RHEL9 only from 8.0.12 onwards +export CUSTOM_MDB_VERSION=8.0.12-ent + +# During release the meko-tests image is pushed to a staging registry. We don't want to push it to production registry +export MEKO_TESTS_REGISTRY="268558157000.dkr.ecr.us-east-1.amazonaws.com/staging" diff --git a/scripts/dev/contexts/e2e_static_smoke b/scripts/dev/contexts/e2e_static_smoke_release similarity index 63% rename from scripts/dev/contexts/e2e_static_smoke rename to scripts/dev/contexts/e2e_static_smoke_release index 2d73c4376..f97e80790 100644 --- a/scripts/dev/contexts/e2e_static_smoke +++ b/scripts/dev/contexts/e2e_static_smoke_release @@ -11,3 +11,6 @@ source "${script_dir}/variables/om70" export MDB_DEFAULT_ARCHITECTURE=static # For static smoke tests we need the previous MDB version to have a ubi9 binary export CUSTOM_MDB_PREV_VERSION=6.0.5 + +# During release the meko-tests image is pushed to a staging registry. We don't want to push it to production registry +export MEKO_TESTS_REGISTRY="268558157000.dkr.ecr.us-east-1.amazonaws.com/staging" diff --git a/scripts/dev/recreate_python_venv.sh b/scripts/dev/recreate_python_venv.sh index 0cee2b6fc..bc86bd455 100755 --- a/scripts/dev/recreate_python_venv.sh +++ b/scripts/dev/recreate_python_venv.sh @@ -113,8 +113,8 @@ fi # Ensure required Python version is available ensure_required_python -echo "Using Python: $(which python) ($(python --version))" >&2 -python -m venv venv +# Make sure we are using the correct Python version when setting up venv +PYENV_VERSION="${PYTHON_VERSION}" python -m venv venv source venv/bin/activate pip install --upgrade pip diff --git a/scripts/evergreen/e2e/build_tests_image_ibm.sh b/scripts/evergreen/e2e/build_tests_image_ibm.sh deleted file mode 100755 index d82ae9661..000000000 --- a/scripts/evergreen/e2e/build_tests_image_ibm.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash -source scripts/dev/set_env_context.sh - -# we need to use podman here as ibm machines don't have docker, therefore our current pipeline can't build these images - -cp -rf public docker/mongodb-kubernetes-tests/public -cp release.json docker/mongodb-kubernetes-tests/release.json -cp requirements.txt docker/mongodb-kubernetes-tests/requirements.txt -cp -rf helm_chart docker/mongodb-kubernetes-tests/helm_chart - -echo "Building mongodb-kubernetes-tests image with tag: ${REGISTRY}/mongodb-kubernetes-tests:${OPERATOR_VERSION}-$(arch)" -cd docker/mongodb-kubernetes-tests || exit -sudo podman buildx build --progress plain . -f Dockerfile -t "${REGISTRY}/mongodb-kubernetes-tests:${OPERATOR_VERSION}-$(arch)" --build-arg PYTHON_VERSION="${PYTHON_VERSION}" -sudo podman push --authfile="/root/.config/containers/auth.json" "${REGISTRY}/mongodb-kubernetes-tests:${OPERATOR_VERSION}-$(arch)" diff --git a/scripts/evergreen/e2e/single_e2e.sh b/scripts/evergreen/e2e/single_e2e.sh index 05277f914..766a1c146 100755 --- a/scripts/evergreen/e2e/single_e2e.sh +++ b/scripts/evergreen/e2e/single_e2e.sh @@ -209,7 +209,7 @@ run_tests() { wait_until_pod_is_running_or_failed_or_succeeded "${test_pod_context}" - title "Running e2e test ${task_name} (tag: ${TEST_IMAGE_TAG})" + title "Running e2e test ${task_name}" # we don't output logs to file when running tests locally if [[ "${MODE-}" == "dev" ]]; then diff --git a/scripts/evergreen/setup_minikube_host.sh b/scripts/evergreen/setup_minikube_host.sh index 2fd2368d7..0e0e02427 100755 --- a/scripts/evergreen/setup_minikube_host.sh +++ b/scripts/evergreen/setup_minikube_host.sh @@ -40,7 +40,7 @@ run_setup_step() { # Setup Python environment (needed for AWS CLI pip installation) export GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=1 -export SKIP_INSTALL_REQUIREMENTS=true +export SKIP_INSTALL_REQUIREMENTS=${SKIP_INSTALL_REQUIREMENTS:-true} run_setup_step "Python Virtual Environment" "scripts/dev/recreate_python_venv.sh" run_setup_step "AWS CLI Setup" "scripts/evergreen/setup_aws.sh" diff --git a/scripts/release/atomic_pipeline.py b/scripts/release/atomic_pipeline.py index 453a1bc1c..3d3a65d75 100755 --- a/scripts/release/atomic_pipeline.py +++ b/scripts/release/atomic_pipeline.py @@ -6,6 +6,7 @@ import json import os import shutil +import subprocess from concurrent.futures import ProcessPoolExecutor from copy import copy from queue import Queue @@ -24,6 +25,7 @@ generate_agent_build_args, generate_tools_build_args, ) +from scripts.release.build.build_info import BUILDER_DOCKER, BUILDER_PODMAN from scripts.release.build.image_build_configuration import ImageBuildConfiguration from scripts.release.build.image_build_process import ( check_if_image_exists, @@ -96,26 +98,59 @@ def build_image( logger.info("All specified image tags already exist. Skipping build.") return - logger.info( - f"Building image with tags {tags} for platforms={build_configuration.platforms}, dockerfile args: {build_args}" - ) + if build_configuration.builder == BUILDER_PODMAN: + logger.info( + f"Building image with podman, tags {tags} for platforms={build_configuration.platforms}, dockerfile args: {build_args}" + ) + try: + build_command = [ + "sudo", + "podman", + "buildx", + "build", + "--progress", + "plain", + build_path, + "-f", + build_configuration.dockerfile_path, + ] + for tag in tags: + build_command.extend(["-t", tag]) + for key, value in build_args.items(): + build_command.extend(["--build-arg", f"{key}={value}"]) + + result = subprocess.run(build_command, capture_output=True, text=True, check=True) + logger.debug(result.stdout) + + for tag in tags: + push_command = ["sudo", "podman", "push", "--authfile=/root/.config/containers/auth.json", tag] + result = subprocess.run(push_command, capture_output=True, text=True, check=True) + logger.debug(result.stdout) + except subprocess.CalledProcessError as e: + raise Exception(f"Podman command failed with code {e.returncode}, output: {e.stdout}: {e.stderr}") + elif build_configuration.builder == BUILDER_DOCKER: + logger.info( + f"Building image with docker, tags {tags} for platforms={build_configuration.platforms}, dockerfile args: {build_args}" + ) - execute_docker_build( - tags=tags, - dockerfile=build_configuration.dockerfile_path, - path=build_path, - args=build_args, - push=True, - platforms=build_configuration.platforms, - ) + execute_docker_build( + tags=tags, + dockerfile=build_configuration.dockerfile_path, + path=build_path, + args=build_args, + push=True, + platforms=build_configuration.platforms, + ) - if build_configuration.sign: - logger.info("Logging in MongoDB Artifactory for Garasign image") - mongodb_artifactory_login() - logger.info("Signing image") - for registry in registries: - sign_image(registry, build_configuration.version) - verify_signature(registry, build_configuration.version) + if build_configuration.sign: + logger.info("Logging in MongoDB Artifactory for Garasign image") + mongodb_artifactory_login() + logger.info("Signing image") + for registry in registries: + sign_image(registry, build_configuration.version) + verify_signature(registry, build_configuration.version) + else: + raise ValueError(f"Unsupported builder type: {build_configuration.builder}") def build_meko_tests_image(build_configuration: ImageBuildConfiguration): diff --git a/scripts/release/build/build_info.py b/scripts/release/build/build_info.py index 45db5b290..9b2b51091 100644 --- a/scripts/release/build/build_info.py +++ b/scripts/release/build/build_info.py @@ -6,6 +6,8 @@ MEKO_TESTS_IMAGE = "meko-tests" MEKO_TESTS_ARM64_IMAGE = "meko-tests-arm64" +MEKO_TESTS_IBM_Z_IMAGE = "meko-tests-ibm-z" +MEKO_TESTS_IBM_POWER_IMAGE = "meko-tests-ibm-power" OPERATOR_IMAGE = "operator" OPERATOR_RACE_IMAGE = "operator-race" MCO_TESTS_IMAGE = "mco-tests" @@ -20,12 +22,16 @@ KUBECTL_PLUGIN_BINARY = "kubectl-mongodb" +BUILDER_DOCKER = "docker" +BUILDER_PODMAN = "podman" + @dataclass class ImageInfo: repositories: List[str] platforms: list[str] dockerfile_path: str + builder: str = BUILDER_DOCKER sign: bool = False latest_tag: bool = False olm_tag: bool = False @@ -82,6 +88,7 @@ def load_build_info(scenario: BuildScenario) -> BuildInfo: repositories=scenario_data["repositories"], platforms=scenario_data["platforms"], dockerfile_path=data["dockerfile-path"], + builder=data.get("builder", BUILDER_DOCKER), sign=scenario_data.get("sign", False), latest_tag=scenario_data.get("latest-tag", False), olm_tag=scenario_data.get("olm-tag", False), diff --git a/scripts/release/build/image_build_configuration.py b/scripts/release/build/image_build_configuration.py index f17feb2a3..ff45e82c1 100644 --- a/scripts/release/build/image_build_configuration.py +++ b/scripts/release/build/image_build_configuration.py @@ -15,6 +15,7 @@ class ImageBuildConfiguration: olm_tag: bool registries: List[str] dockerfile_path: str + builder: str platforms: Optional[List[str]] = None sign: bool = False skip_if_exists: bool = False diff --git a/scripts/release/check_changelog.py b/scripts/release/check_changelog.py index 27680128b..78977a44c 100644 --- a/scripts/release/check_changelog.py +++ b/scripts/release/check_changelog.py @@ -41,7 +41,7 @@ ) parser.add_argument( "-f", - "--fail-on-no-changes", + "--fail-if-no-changes", default=True, metavar="", action="store", @@ -62,7 +62,7 @@ if not changelog: print("No changelog entries found.") - if args.fail_on_no_changes: + if args.fail_if_no_changes: print("Exiting with error due to no changelog entries found.") exit(1) else: diff --git a/scripts/release/pipeline.py b/scripts/release/pipeline.py index 3af1bb164..db4a6664c 100644 --- a/scripts/release/pipeline.py +++ b/scripts/release/pipeline.py @@ -36,12 +36,15 @@ ) from scripts.release.build.build_info import ( AGENT_IMAGE, + BUILDER_DOCKER, DATABASE_IMAGE, INIT_APPDB_IMAGE, INIT_DATABASE_IMAGE, INIT_OPS_MANAGER_IMAGE, MCO_TESTS_IMAGE, MEKO_TESTS_ARM64_IMAGE, + MEKO_TESTS_IBM_POWER_IMAGE, + MEKO_TESTS_IBM_Z_IMAGE, MEKO_TESTS_IMAGE, OPERATOR_IMAGE, OPERATOR_RACE_IMAGE, @@ -75,6 +78,8 @@ def get_builder_function_for_image_name() -> Dict[str, Callable]: image_builders = { MEKO_TESTS_IMAGE: build_meko_tests_image, MEKO_TESTS_ARM64_IMAGE: build_meko_tests_image, + MEKO_TESTS_IBM_Z_IMAGE: build_meko_tests_image, + MEKO_TESTS_IBM_POWER_IMAGE: build_meko_tests_image, OPERATOR_IMAGE: build_operator_image, OPERATOR_RACE_IMAGE: partial(build_operator_image, with_race_detection=True), MCO_TESTS_IMAGE: build_mco_tests_image, @@ -117,6 +122,7 @@ def image_build_config_from_args(args) -> ImageBuildConfiguration: # Resolve final values with overrides version = args.version dockerfile_path = image_build_info.dockerfile_path + builder = image_build_info.builder latest_tag = image_build_info.latest_tag olm_tag = image_build_info.olm_tag if args.registry: @@ -145,6 +151,7 @@ def image_build_config_from_args(args) -> ImageBuildConfiguration: olm_tag=olm_tag, registries=registries, dockerfile_path=dockerfile_path, + builder=builder, platforms=platforms, sign=sign, skip_if_exists=skip_if_exists, @@ -290,8 +297,10 @@ def main(): # Create buildx builder # It must be initialized here as opposed to in build_images.py so that parallel calls (such as agent builds) can access it - # and not face race conditions - ensure_buildx_builder(DEFAULT_BUILDER_NAME) + # and not face race conditions. For IBM Z and Power we use podman and cannot set docker buildx builder + if build_config.builder == BUILDER_DOCKER: + ensure_buildx_builder(DEFAULT_BUILDER_NAME) + build_image(args.image, build_config) diff --git a/scripts/release/tests/build_info_test.py b/scripts/release/tests/build_info_test.py index e6d44e9df..3942be55c 100644 --- a/scripts/release/tests/build_info_test.py +++ b/scripts/release/tests/build_info_test.py @@ -1,4 +1,5 @@ from scripts.release.build.build_info import ( + BUILDER_PODMAN, BinaryInfo, BuildInfo, HelmChartInfo, @@ -52,6 +53,20 @@ def test_load_build_info_development(): platforms=["linux/amd64"], dockerfile_path="docker/mongodb-kubernetes-tests/Dockerfile", ), + "meko-tests-ibm-power": ImageInfo( + repositories=["268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/mongodb-kubernetes-tests"], + platforms=["linux/ppc64le"], + dockerfile_path="docker/mongodb-kubernetes-tests/Dockerfile", + builder=BUILDER_PODMAN, + architecture_suffix=True, + ), + "meko-tests-ibm-z": ImageInfo( + repositories=["268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/mongodb-kubernetes-tests"], + platforms=["linux/s390x"], + dockerfile_path="docker/mongodb-kubernetes-tests/Dockerfile", + builder=BUILDER_PODMAN, + architecture_suffix=True, + ), "meko-tests-arm64": ImageInfo( repositories=["268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/mongodb-kubernetes-tests"], platforms=["linux/arm64"], @@ -152,6 +167,20 @@ def test_load_build_info_patch(): dockerfile_path="docker/mongodb-kubernetes-tests/Dockerfile", architecture_suffix=True, ), + "meko-tests-ibm-power": ImageInfo( + repositories=["268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/mongodb-kubernetes-tests"], + platforms=["linux/ppc64le"], + dockerfile_path="docker/mongodb-kubernetes-tests/Dockerfile", + builder=BUILDER_PODMAN, + architecture_suffix=True, + ), + "meko-tests-ibm-z": ImageInfo( + repositories=["268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/mongodb-kubernetes-tests"], + platforms=["linux/s390x"], + dockerfile_path="docker/mongodb-kubernetes-tests/Dockerfile", + builder=BUILDER_PODMAN, + architecture_suffix=True, + ), "readiness-probe": ImageInfo( repositories=["268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/mongodb-kubernetes-readinessprobe"], platforms=["linux/amd64"], @@ -259,6 +288,20 @@ def test_load_build_info_staging(): dockerfile_path="docker/mongodb-kubernetes-tests/Dockerfile", architecture_suffix=True, ), + "meko-tests-ibm-power": ImageInfo( + repositories=["268558157000.dkr.ecr.us-east-1.amazonaws.com/staging/mongodb-kubernetes-tests"], + platforms=["linux/ppc64le"], + dockerfile_path="docker/mongodb-kubernetes-tests/Dockerfile", + builder=BUILDER_PODMAN, + architecture_suffix=True, + ), + "meko-tests-ibm-z": ImageInfo( + repositories=["268558157000.dkr.ecr.us-east-1.amazonaws.com/staging/mongodb-kubernetes-tests"], + platforms=["linux/s390x"], + dockerfile_path="docker/mongodb-kubernetes-tests/Dockerfile", + builder=BUILDER_PODMAN, + architecture_suffix=True, + ), "readiness-probe": ImageInfo( repositories=["268558157000.dkr.ecr.us-east-1.amazonaws.com/staging/mongodb-kubernetes-readinessprobe"], platforms=["linux/arm64", "linux/amd64"], @@ -365,10 +408,29 @@ def test_load_build_info_release(): sign=True, ), "meko-tests": ImageInfo( - repositories=["quay.io/mongodb/mongodb-kubernetes-tests"], + repositories=["268558157000.dkr.ecr.us-east-1.amazonaws.com/staging/mongodb-kubernetes-tests"], platforms=["linux/amd64"], dockerfile_path="docker/mongodb-kubernetes-tests/Dockerfile", - skip_if_exists=True, + ), + "meko-tests-arm64": ImageInfo( + repositories=["268558157000.dkr.ecr.us-east-1.amazonaws.com/staging/mongodb-kubernetes-tests"], + platforms=["linux/arm64"], + dockerfile_path="docker/mongodb-kubernetes-tests/Dockerfile", + architecture_suffix=True, + ), + "meko-tests-ibm-power": ImageInfo( + repositories=["268558157000.dkr.ecr.us-east-1.amazonaws.com/staging/mongodb-kubernetes-tests"], + platforms=["linux/ppc64le"], + dockerfile_path="docker/mongodb-kubernetes-tests/Dockerfile", + builder=BUILDER_PODMAN, + architecture_suffix=True, + ), + "meko-tests-ibm-z": ImageInfo( + repositories=["268558157000.dkr.ecr.us-east-1.amazonaws.com/staging/mongodb-kubernetes-tests"], + platforms=["linux/s390x"], + dockerfile_path="docker/mongodb-kubernetes-tests/Dockerfile", + builder=BUILDER_PODMAN, + architecture_suffix=True, ), "readiness-probe": ImageInfo( repositories=["quay.io/mongodb/mongodb-kubernetes-readinessprobe"], From 8252467a7cecff5de1d97cc104191eaaa0291155 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Kara=C5=9B?= Date: Thu, 30 Oct 2025 12:03:07 +0100 Subject: [PATCH 2/5] Disable ibm z completely --- .evergreen-release.yml | 4 ++++ .evergreen.yml | 3 +++ 2 files changed, 7 insertions(+) diff --git a/.evergreen-release.yml b/.evergreen-release.yml index beeb22193..4b90c7a0b 100644 --- a/.evergreen-release.yml +++ b/.evergreen-release.yml @@ -313,6 +313,8 @@ buildvariants: - name: e2e_smoke_ibm_z_release display_name: e2e_smoke_ibm_z tags: [ "release", "e2e_smoke_release_test_suite" ] + # TODO: Re-enable when ibm_z series is stable + # https://jira.mongodb.org/browse/DEVPROD-23283 disable: true run_on: - rhel9-zseries-small @@ -329,6 +331,8 @@ buildvariants: - name: e2e_static_smoke_ibm_z_release display_name: e2e_static_smoke_ibm_z tags: [ "release", "e2e_smoke_release_test_suite", "static" ] + # TODO: Re-enable when ibm_z series is stable + # https://jira.mongodb.org/browse/DEVPROD-23283 disable: true run_on: - rhel9-zseries-small diff --git a/.evergreen.yml b/.evergreen.yml index 6ce411661..9b4aa721c 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -1754,6 +1754,9 @@ buildvariants: display_name: init_test_run_ibm_z max_hosts: -1 tags: [ "staging" ] + # TODO: Re-enable when ibm_z series is stable + # https://jira.mongodb.org/browse/DEVPROD-23283 + disable: true run_on: - rhel9-zseries-small - rhel9-zseries-large From 44307ae801c4974c1f40f63c06c9644b2ad7d851 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Kara=C5=9B?= Date: Thu, 30 Oct 2025 14:31:12 +0100 Subject: [PATCH 3/5] Fix release tasks --- .evergreen-functions.yml | 1 + .evergreen-release.yml | 9 +++++++++ .../release/kubectl_mongodb/promote_kubectl_plugin.py | 6 +++++- 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/.evergreen-functions.yml b/.evergreen-functions.yml index bd01d15d6..d67b7696e 100644 --- a/.evergreen-functions.yml +++ b/.evergreen-functions.yml @@ -844,6 +844,7 @@ functions: - workdir - triggered_by_git_tag - OPERATOR_VERSION + - STAGING_VERSION_OVERRIDE env: MACOS_NOTARY_KEY: ${macos_notary_keyid} MACOS_NOTARY_SECRET: ${macos_notary_secret} diff --git a/.evergreen-release.yml b/.evergreen-release.yml index 4b90c7a0b..1cdded9de 100644 --- a/.evergreen-release.yml +++ b/.evergreen-release.yml @@ -211,6 +211,9 @@ buildvariants: run_on: - release-ubuntu2404-small # This is required for CISA attestation https://jira.mongodb.org/browse/DEVPROD-17780 allowed_requesters: [ "patch", "github_tag" ] + depends_on: + - name: release_kubectl_mongodb_plugin + variant: release_kubectl_mongodb_plugin tasks: - name: build_test_image @@ -222,6 +225,9 @@ buildvariants: - release-rhel9-power-small # This is required for CISA attestation https://jira.mongodb.org/browse/DEVPROD-17780 - release-rhel9-power-large # This is required for CISA attestation https://jira.mongodb.org/browse/DEVPROD-17780 allowed_requesters: [ "patch", "github_tag" ] + depends_on: + - name: release_kubectl_mongodb_plugin + variant: release_kubectl_mongodb_plugin tasks: - name: build_test_image_ibm_power @@ -236,6 +242,9 @@ buildvariants: - release-rhel9-zseries-small # This is required for CISA attestation https://jira.mongodb.org/browse/DEVPROD-17780 - release-rhel9-zseries-large # This is required for CISA attestation https://jira.mongodb.org/browse/DEVPROD-17780 allowed_requesters: [ "patch", "github_tag" ] + depends_on: + - name: release_kubectl_mongodb_plugin + variant: release_kubectl_mongodb_plugin tasks: - name: build_test_image_ibm_z diff --git a/scripts/release/kubectl_mongodb/promote_kubectl_plugin.py b/scripts/release/kubectl_mongodb/promote_kubectl_plugin.py index bc7a02709..b305dd07b 100644 --- a/scripts/release/kubectl_mongodb/promote_kubectl_plugin.py +++ b/scripts/release/kubectl_mongodb/promote_kubectl_plugin.py @@ -39,7 +39,11 @@ def main(): kubectl_plugin_staging_info = load_build_info(BuildScenario.STAGING).binaries[KUBECTL_PLUGIN_BINARY] staging_scenario_bucket_name = kubectl_plugin_staging_info.s3_store - staging_version = get_commit_from_tag(release_version) + staging_version_override = os.environ.get("STAGING_VERSION_OVERRIDE") + if staging_version_override: + staging_version = staging_version_override + else: + staging_version = get_commit_from_tag(release_version) artifacts_dict = download_artifacts_from_s3( release_version, release_platforms, staging_version, staging_scenario_bucket_name From 97333b29c3595b062c0686e3f8ee5231caadd05c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Kara=C5=9B?= Date: Thu, 30 Oct 2025 14:54:29 +0100 Subject: [PATCH 4/5] Fix release tasks --- .evergreen-functions.yml | 1 + scripts/release/kubectl_mongodb/promote_kubectl_plugin.py | 5 +++-- scripts/release/publish_helm_chart.py | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.evergreen-functions.yml b/.evergreen-functions.yml index d67b7696e..0917c1de8 100644 --- a/.evergreen-functions.yml +++ b/.evergreen-functions.yml @@ -845,6 +845,7 @@ functions: - triggered_by_git_tag - OPERATOR_VERSION - STAGING_VERSION_OVERRIDE + - SKIP_GITHUB_RELEASE_UPLOAD env: MACOS_NOTARY_KEY: ${macos_notary_keyid} MACOS_NOTARY_SECRET: ${macos_notary_secret} diff --git a/scripts/release/kubectl_mongodb/promote_kubectl_plugin.py b/scripts/release/kubectl_mongodb/promote_kubectl_plugin.py index b305dd07b..47e9232ae 100644 --- a/scripts/release/kubectl_mongodb/promote_kubectl_plugin.py +++ b/scripts/release/kubectl_mongodb/promote_kubectl_plugin.py @@ -63,8 +63,9 @@ def main(): s3_artifacts = artifacts_dict | artifacts_tar_dict | checksum_file_dict promote_artifacts_to_s3(s3_artifacts, release_version, release_scenario_bucket_name) - github_artifacts = artifacts_tar + [checksum_file] - upload_assets_to_github_release(github_artifacts, release_version) + if os.environ.get("SKIP_GITHUB_RELEASE_UPLOAD", "false").lower() == "false": + github_artifacts = artifacts_tar + [checksum_file] + upload_assets_to_github_release(github_artifacts, release_version) # get_commit_from_tag gets the commit associated with a release tag, so that we can use that diff --git a/scripts/release/publish_helm_chart.py b/scripts/release/publish_helm_chart.py index cae99b19e..bfdc79483 100644 --- a/scripts/release/publish_helm_chart.py +++ b/scripts/release/publish_helm_chart.py @@ -79,7 +79,7 @@ def get_oci_registry(chart_info: HelmChartInfo) -> str: raise ValueError("Error: registry doesn't seem to be set in HelmChartInfo.") if not repo: - raise ValueError("Error: reposiotry doesn't seem to be set in HelmChartInfo.") + raise ValueError("Error: repository doesn't seem to be set in HelmChartInfo.") oci_registry = f"oci://{registry}/{repo}" logger.info(f"Determined OCI Registry: {oci_registry}") From 0cc987dcd7690837af9108a93ac1019065813f08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Kara=C5=9B?= Date: Fri, 31 Oct 2025 12:40:06 +0100 Subject: [PATCH 5/5] CLOUDP-295785 - copy kubectl-mongodb plugin to `./bin/kubectl-mongodb` for code snippets tests --- .evergreen-functions.yml | 2 + .evergreen-snippets.yml | 3 ++ .evergreen.yml | 8 ++++ .../download_kubectl_plugin.py | 38 +++++++++++-------- .../download_kubectl_plugin.sh | 2 +- 5 files changed, 37 insertions(+), 16 deletions(-) diff --git a/.evergreen-functions.yml b/.evergreen-functions.yml index 0917c1de8..0a24c6e1c 100644 --- a/.evergreen-functions.yml +++ b/.evergreen-functions.yml @@ -525,6 +525,8 @@ functions: params: working_dir: src/github.com/mongodb/mongodb-kubernetes binary: scripts/release/kubectl_mongodb/download_kubectl_plugin.sh + env: + PLATFORM: ${platform} pipeline: - command: subprocess.exec diff --git a/.evergreen-snippets.yml b/.evergreen-snippets.yml index 2446d08b8..b41f82d64 100644 --- a/.evergreen-snippets.yml +++ b/.evergreen-snippets.yml @@ -7,7 +7,10 @@ variables: - func: setup_mongosh - func: download_kube_tools - func: switch_context + - func: python_venv - func: download_multi_cluster_binary + vars: + platform: linux/amd64 teardown_task: - func: upload_e2e_logs - func: upload_code_snippets_logs diff --git a/.evergreen.yml b/.evergreen.yml index 9b4aa721c..cb431eb1f 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -358,6 +358,8 @@ tasks: - func: clone - func: setup_building_host - func: download_multi_cluster_binary + vars: + platform: linux/amd64 - func: pipeline vars: image_name: meko-tests @@ -367,6 +369,8 @@ tasks: - func: clone - func: setup_building_host - func: download_multi_cluster_binary + vars: + platform: linux/arm64 - func: pipeline vars: image_name: meko-tests-arm64 @@ -379,6 +383,8 @@ tasks: skip_minikube_setup: true skip_install_python_requirements: false - func: download_multi_cluster_binary + vars: + platform: linux/s390x - func: pipeline vars: image_name: meko-tests-ibm-z @@ -391,6 +397,8 @@ tasks: skip_minikube_setup: true skip_install_python_requirements: false - func: download_multi_cluster_binary + vars: + platform: linux/ppc64le - func: pipeline vars: image_name: meko-tests-ibm-power diff --git a/scripts/release/kubectl_mongodb/download_kubectl_plugin.py b/scripts/release/kubectl_mongodb/download_kubectl_plugin.py index 8d78f5d07..53e5853a3 100755 --- a/scripts/release/kubectl_mongodb/download_kubectl_plugin.py +++ b/scripts/release/kubectl_mongodb/download_kubectl_plugin.py @@ -1,15 +1,17 @@ import argparse import os +import shutil from botocore.exceptions import ClientError from lib.base_logger import logger -from scripts.release.argparse_utils import get_platforms_from_arg, get_scenario_from_arg +from scripts.release.argparse_utils import get_scenario_from_arg from scripts.release.build.build_info import ( KUBECTL_PLUGIN_BINARY, load_build_info, ) from scripts.release.build.build_scenario import SUPPORTED_SCENARIOS, BuildScenario +from scripts.release.build.image_build_configuration import SUPPORTED_PLATFORMS from scripts.release.kubectl_mongodb.build_kubectl_plugin import ( kubectl_plugin_name, parse_platform, @@ -17,12 +19,16 @@ ) from scripts.release.kubectl_mongodb.utils import create_s3_client +KUBECTL_MONGODB_PLUGIN_BIN_PATH = "bin/kubectl-mongodb" + def local_tests_plugin_path(arch_name: str) -> str: return f"docker/mongodb-kubernetes-tests/multi-cluster-kube-config-creator_{arch_name}" -def download_kubectl_plugin_from_s3(s3_bucket: str, s3_plugin_path: str, local_path: str): +def download_kubectl_plugin_from_s3( + s3_bucket: str, s3_plugin_path: str, local_path: str, copy_to_bin_path: bool = False +) -> None: """ Downloads the plugin for provided platform and puts it in the path expected by the tests image """ @@ -35,7 +41,12 @@ def download_kubectl_plugin_from_s3(s3_bucket: str, s3_plugin_path: str, local_p # change the file's permissions to make file executable os.chmod(local_path, 0o755) - logger.info(f"Successfully downloaded artifact to {local_path}") + if copy_to_bin_path: + shutil.copyfile(local_path, KUBECTL_MONGODB_PLUGIN_BIN_PATH) + + logger.info( + f"Successfully downloaded artifact to {local_path}{f" and {KUBECTL_MONGODB_PLUGIN_BIN_PATH}" if copy_to_bin_path else ""}" + ) except ClientError as e: if e.response["Error"]["Code"] == "404": raise Exception(f"Artifact not found at s3://{s3_bucket}/{s3_plugin_path}: {e}") @@ -74,28 +85,25 @@ def main(): "--platform", metavar="", action="store", + required=True, type=str, - help="Override the platforms instead of resolving from build scenario. Multi-arch builds are comma-separated. Example: linux/amd64,linux/arm64", + choices=SUPPORTED_PLATFORMS, + help=f"Specify kubectl-mongodb plugin platform to download. Options: {", ".join(SUPPORTED_PLATFORMS)}.", ) args = parser.parse_args() build_scenario = get_scenario_from_arg(args.build_scenario) build_info = load_build_info(build_scenario).binaries[KUBECTL_PLUGIN_BINARY] - platforms = get_platforms_from_arg(args.platform) or build_info.platforms + platform = args.platform version = args.version - for platform in platforms: - os_name, arch_name = parse_platform(platform) - if os_name != "linux": - logger.debug(f"Skipping non-linux platform {platform}, not used in e2e tests in Evergreen CI") - continue - - filename = kubectl_plugin_name(os_name, arch_name) - s3_plugin_path = s3_path(filename, version) - local_path = local_tests_plugin_path(arch_name) + os_name, arch_name = parse_platform(platform) + filename = kubectl_plugin_name(os_name, arch_name) + s3_plugin_path = s3_path(filename, version) + local_path = local_tests_plugin_path(arch_name) - download_kubectl_plugin_from_s3(build_info.s3_store, s3_plugin_path, local_path) + download_kubectl_plugin_from_s3(build_info.s3_store, s3_plugin_path, local_path, copy_to_bin_path=True) if __name__ == "__main__": diff --git a/scripts/release/kubectl_mongodb/download_kubectl_plugin.sh b/scripts/release/kubectl_mongodb/download_kubectl_plugin.sh index 2aaf5c5cb..546b2078c 100755 --- a/scripts/release/kubectl_mongodb/download_kubectl_plugin.sh +++ b/scripts/release/kubectl_mongodb/download_kubectl_plugin.sh @@ -4,4 +4,4 @@ set -Eeou pipefail source scripts/dev/set_env_context.sh -scripts/dev/run_python.sh scripts/release/kubectl_mongodb/download_kubectl_plugin.py --build-scenario "${BUILD_SCENARIO}" --version "${OPERATOR_VERSION}" +scripts/dev/run_python.sh scripts/release/kubectl_mongodb/download_kubectl_plugin.py --build-scenario "${BUILD_SCENARIO}" --version "${OPERATOR_VERSION}" --platform "${PLATFORM}"