diff --git a/.evergreen-release.yml b/.evergreen-release.yml index 1cdded9de..e86cdda13 100644 --- a/.evergreen-release.yml +++ b/.evergreen-release.yml @@ -214,6 +214,8 @@ buildvariants: depends_on: - name: release_kubectl_mongodb_plugin variant: release_kubectl_mongodb_plugin + - name: release_chart_to_oci_registry + variant: release_chart_to_oci_registry tasks: - name: build_test_image @@ -228,6 +230,8 @@ buildvariants: depends_on: - name: release_kubectl_mongodb_plugin variant: release_kubectl_mongodb_plugin + - name: release_chart_to_oci_registry + variant: release_chart_to_oci_registry tasks: - name: build_test_image_ibm_power @@ -245,6 +249,8 @@ buildvariants: depends_on: - name: release_kubectl_mongodb_plugin variant: release_kubectl_mongodb_plugin + - name: release_chart_to_oci_registry + variant: release_chart_to_oci_registry tasks: - name: build_test_image_ibm_z @@ -258,6 +264,8 @@ buildvariants: depends_on: - name: release_kubectl_mongodb_plugin variant: release_kubectl_mongodb_plugin + - name: release_chart_to_oci_registry + variant: release_chart_to_oci_registry tasks: - name: build_test_image_arm diff --git a/.evergreen.yml b/.evergreen.yml index d247f3b3b..bb218c013 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -37,6 +37,8 @@ variables: variant: init_test_run - name: build_init_om_images_ubi variant: init_test_run + - name: publish_helm_chart + variant: init_test_run - &base_no_om_image_dependency depends_on: @@ -54,6 +56,8 @@ variables: variant: init_test_run - name: build_init_appdb_images_ubi variant: init_test_run + - name: publish_helm_chart + variant: init_test_run - &community_dependency depends_on: @@ -67,6 +71,8 @@ variables: variant: init_test_run - name: build_mco_test_image variant: init_test_run + - name: publish_helm_chart + variant: init_test_run - &setup_group setup_group_can_fail_task: true @@ -138,6 +144,8 @@ variables: variant: init_test_run - name: build_init_om_images_ubi variant: init_test_run + - name: publish_helm_chart + variant: init_test_run - &base_om7_dependency_with_race depends_on: @@ -155,6 +163,8 @@ variables: variant: init_test_run - name: build_init_om_images_ubi variant: init_test_run + - name: publish_helm_chart + variant: init_test_run # Any change to base_om8_dependency should be reflected to its copy in .evergreen-snippets.yml - &base_om8_dependency @@ -173,6 +183,8 @@ variables: variant: init_test_run - name: build_init_om_images_ubi variant: init_test_run + - name: publish_helm_chart + variant: init_test_run parameters: - key: evergreen_retry @@ -1303,6 +1315,8 @@ buildvariants: variant: init_test_run - name: build_init_database_image_ubi variant: init_test_run + - name: publish_helm_chart + variant: init_test_run tasks: - name: e2e_custom_domain_task_group @@ -1440,6 +1454,8 @@ buildvariants: variant: init_test_run - name: build_test_image_ibm_power variant: init_test_run_ibm_power + - name: publish_helm_chart + variant: init_test_run tasks: - name: e2e_smoke_ibm_task_group @@ -1459,6 +1475,8 @@ buildvariants: variant: init_test_run - name: build_test_image_ibm_power variant: init_test_run_ibm_power + - name: publish_helm_chart + variant: init_test_run tasks: - name: e2e_smoke_ibm_task_group @@ -1481,6 +1499,8 @@ buildvariants: variant: init_test_run - name: build_test_image_ibm_z variant: init_test_run_ibm_z + - name: publish_helm_chart + variant: init_test_run tasks: - name: e2e_smoke_ibm_task_group @@ -1501,6 +1521,8 @@ buildvariants: variant: init_test_run - name: build_test_image_ibm_z variant: init_test_run_ibm_z + - name: publish_helm_chart + variant: init_test_run tasks: - name: e2e_smoke_ibm_task_group @@ -1519,6 +1541,8 @@ buildvariants: variant: init_test_run - name: build_test_image_arm variant: init_test_run_arm + - name: publish_helm_chart + variant: init_test_run tasks: - name: e2e_smoke_arm_task_group @@ -1535,6 +1559,8 @@ buildvariants: variant: init_test_run - name: build_test_image_arm variant: init_test_run_arm + - name: publish_helm_chart + variant: init_test_run tasks: - name: e2e_smoke_arm_task_group diff --git a/build_info.json b/build_info.json index 21efc0bd3..c1c28f6cb 100644 --- a/build_info.json +++ b/build_info.json @@ -453,12 +453,14 @@ "helm-charts": { "mongodb-kubernetes": { "patch": { + "version-prefix": "0.0.0+", "registry": "268558157000.dkr.ecr.us-east-1.amazonaws.com", "region": "us-east-1", "repository": "dev/mongodb/helm-charts" }, "staging": { "sign": true, + "version-prefix": "0.0.0+", "registry": "268558157000.dkr.ecr.us-east-1.amazonaws.com", "region": "us-east-1", "repository": "staging/mongodb/helm-charts" diff --git a/docker/mongodb-kubernetes-tests/Dockerfile b/docker/mongodb-kubernetes-tests/Dockerfile index 88be6b11d..b2ebf3cf9 100644 --- a/docker/mongodb-kubernetes-tests/Dockerfile +++ b/docker/mongodb-kubernetes-tests/Dockerfile @@ -16,7 +16,12 @@ RUN apt-get -qq update \ COPY requirements.txt requirements.txt -RUN python3 -m venv /venv && . /venv/bin/activate && pip install --upgrade pip && GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=1 pip install -r requirements.txt +# install aws, required to run helm registry login while running the tests +RUN python3 -m venv /venv \ + && . /venv/bin/activate \ + && pip install --upgrade pip \ + && GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=1 pip install -r requirements.txt \ + && pip install awscli FROM scratch AS tools_downloader @@ -75,8 +80,9 @@ WORKDIR /tests # copying the test files after python build, otherwise pip install will be called each time the tests change COPY . /tests -# copying the helm_chart directory as well to support installation of the Operator from the test application -COPY helm_chart /helm_chart +# copying the helm_chart/crds directory so that we can install CRDs before installing the operator helm chart to run a test. +# operator is installed via published OCI helm repo and not the local helm repo. +COPY helm_chart/crds /helm_chart/crds COPY release.json /release.json # we use the public directory to automatically test resources samples COPY public /mongodb-kubernetes/public diff --git a/docker/mongodb-kubernetes-tests/kubetester/consts.py b/docker/mongodb-kubernetes-tests/kubetester/consts.py new file mode 100644 index 000000000..20bed74c2 --- /dev/null +++ b/docker/mongodb-kubernetes-tests/kubetester/consts.py @@ -0,0 +1,7 @@ +OCI_HELM_VERSION = "OCI_HELM_VERSION" +OCI_HELM_REGISTRY_ENV_VAR_NAME = "OCI_HELM_REGISTRY" +OCI_HELM_REPOSITORY_ENV_VAR_NAME = "OCI_HELM_REPOSITORY" +OCI_HELM_REGION_ENV_VAR_NAME = "OCI_HELM_REGION" + +LEGACY_OPERATOR_CHART = "mongodb/enterprise-operator" +MCK_HELM_CHART = "mongodb/mongodb-kubernetes" diff --git a/docker/mongodb-kubernetes-tests/kubetester/helm.py b/docker/mongodb-kubernetes-tests/kubetester/helm.py index 9e0cca8fe..66523f012 100644 --- a/docker/mongodb-kubernetes-tests/kubetester/helm.py +++ b/docker/mongodb-kubernetes-tests/kubetester/helm.py @@ -1,15 +1,19 @@ import glob -import logging import os import re import subprocess import uuid from typing import Dict, List, Optional, Tuple +from kubetester.consts import * from tests import test_logger logger = test_logger.get_test_logger(__name__) +# LOCAL_CRDs_DIR is the dir where local helm chart's CRDs are copied in tests image +LOCAL_CRDs_DIR = "helm_chart/crds" +OCI_HELM_REGISTRY_ECR = "268558157000.dkr.ecr.us-east-1.amazonaws.com" + def helm_template( helm_args: Dict, @@ -25,7 +29,7 @@ def helm_template( command_args.append("--show-only") command_args.append(templates) - args = ("helm", "template", *(command_args), _helm_chart_dir(helm_chart_path)) + args = ("helm", "template", *command_args, _helm_chart_dir(helm_chart_path)) logger.info(" ".join(args)) yaml_file_name = "{}.yaml".format(str(uuid.uuid4())) @@ -145,6 +149,55 @@ def process_run_and_check(args, **kwargs): raise +def helm_registry_login_to_ecr(helm_registry: str, region: str): + logger.info(f"Attempting to log into ECR registry: {helm_registry}, using helm registry login.") + + aws_command = ["aws", "ecr", "get-login-password", "--region", region] + + # as we can see the password is being provided by stdin, that would mean we will have to + # pipe the aws_command (it figures out the password) into helm_command. + helm_command = ["helm", "registry", "login", "--username", "AWS", "--password-stdin", helm_registry] + + try: + logger.info("Starting AWS ECR credential retrieval.") + aws_proc = subprocess.Popen( + aws_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True # Treat input/output as text strings + ) + + logger.info("Starting Helm registry login.") + helm_proc = subprocess.Popen( + helm_command, stdin=aws_proc.stdout, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True + ) + + # Close the stdout stream of aws_proc in the parent process + # to prevent resource leakage (only needed if you plan to do more processing) + aws_proc.stdout.close() + + # Wait for the Helm command (helm_proc) to finish and capture its output + helm_stdout, helm_stderr = helm_proc.communicate() + + # Wait for the AWS process to finish as well + aws_proc.wait() + + if aws_proc.returncode != 0: + _, aws_stderr = aws_proc.communicate() + raise Exception(f"aws command to get password failed. Error: {aws_stderr}") + + if helm_proc.returncode == 0: + logger.info("Login to helm registry was successful.") + logger.info(helm_stdout.strip()) + else: + raise Exception( + f"Login to helm registry failed, Exit code: {helm_proc.returncode}, Error: {helm_stderr.strip()}" + ) + + except FileNotFoundError as e: + # This catches errors if 'aws' or 'helm' are not in the PATH + raise Exception(f"Command not found. Please ensure '{e.filename}' is installed and in your system's PATH.") + except Exception as e: + raise Exception(f"An unexpected error occurred: {e}.") + + def helm_upgrade( name: str, namespace: str, @@ -162,7 +215,7 @@ def helm_upgrade( chart_dir = helm_chart_path if helm_override_path else _helm_chart_dir(helm_chart_path) if apply_crds_first: - apply_crds_from_chart(chart_dir) + apply_crds_from_chart(LOCAL_CRDs_DIR) command_args = _create_helm_args(helm_args, helm_options) args = [ @@ -183,11 +236,24 @@ def helm_upgrade( process_run_and_check(command, check=True, capture_output=True, shell=True) -def apply_crds_from_chart(chart_dir: str): - crd_files = glob.glob(os.path.join(chart_dir, "crds", "*.yaml")) +# oci_chart_info returns the respective registry/repo and region information +# based on the build scenario (dev/staging) tests are being run in. These are +# read from build_info.json and then set to the tests image as env vars. +def oci_chart_info(): + registry = os.environ.get(OCI_HELM_REGISTRY_ENV_VAR_NAME) + repository = os.environ.get(OCI_HELM_REPOSITORY_ENV_VAR_NAME) + region = os.environ.get(OCI_HELM_REGION_ENV_VAR_NAME) + + logger.info(f"oci chart details in test image is registry {registry}, repo {repository}, region {region}") + + return registry, f"{repository}/mongodb-kubernetes", region + + +def apply_crds_from_chart(crds_dir: str): + crd_files = glob.glob(os.path.join(crds_dir, "*.yaml")) if not crd_files: - raise Exception(f"No CRD files found in chart directory: {chart_dir}") + raise Exception(f"No CRD files found in chart directory: {crds_dir}") logger.info(f"Found {len(crd_files)} CRD files to apply:") @@ -234,3 +300,35 @@ def _create_helm_args(helm_args: Dict[str, str], helm_options: Optional[List[str def _helm_chart_dir(default: Optional[str] = "helm_chart") -> str: return os.environ.get("HELM_CHART_DIR", default) + + +# helm_chart_path_and_version returns the chart path and version that we would like to install to run the E2E tests. +# for local tests it returns early with local helm chart dir and for other scenarios it figures out the chart and version +# based on the caller. In most of the cases we will install chart from OCI registry but for the tests where we would like +# to install MEKO's specific version or MCK's specific version, we would expect `helm_chart_path` to set already. +def helm_chart_path_and_version(helm_chart_path: str, operator_version: str) -> tuple[str, str]: + # these are imported here to resolve import cycle issue + from tests.conftest import LOCAL_HELM_CHART_DIR, local_operator + + if local_operator(): + return LOCAL_HELM_CHART_DIR, "" + + # helm_chart_path not being passed would mean we would like to install helm chart from OCI registry. + if not helm_chart_path: + # If operator_version is not passed, we want to install the current version. + if not operator_version: + operator_version = os.environ.get(OCI_HELM_VERSION) + + registry, repository, region = oci_chart_info() + # If ECR we need to login first to the OCI container registry + if registry == OCI_HELM_REGISTRY_ECR: + try: + helm_registry_login_to_ecr(registry, region) + except Exception as e: + raise Exception(f"Failed to login to ECR helm registry {registry}. Error: {e}") + + # figure out the registry URI, based on dev/staging scenario + chart_uri = f"oci://{registry}/{repository}" + helm_chart_path = chart_uri + + return helm_chart_path, operator_version diff --git a/docker/mongodb-kubernetes-tests/kubetester/operator.py b/docker/mongodb-kubernetes-tests/kubetester/operator.py index fa2b90344..27d0f4041 100644 --- a/docker/mongodb-kubernetes-tests/kubetester/operator.py +++ b/docker/mongodb-kubernetes-tests/kubetester/operator.py @@ -1,6 +1,5 @@ from __future__ import annotations -import logging import time from typing import Dict, List, Optional @@ -11,6 +10,7 @@ from kubetester import wait_for_webhook from kubetester.create_or_replace_from_yaml import create_or_replace_from_yaml from kubetester.helm import ( + helm_chart_path_and_version, helm_install, helm_repo_add, helm_template, @@ -25,6 +25,7 @@ "opsmanagers.mongodb.com", ) + logger = test_logger.get_test_logger(__name__) @@ -43,14 +44,16 @@ def __init__( namespace: str, helm_args: Optional[Dict] = None, helm_options: Optional[List[str]] = None, - helm_chart_path: Optional[str] = "helm_chart", + helm_chart_path: Optional[str] = None, name: Optional[str] = "mongodb-kubernetes-operator", api_client: Optional[client.api_client.ApiClient] = None, + operator_version: Optional[str] = None, ): - # The Operator will be installed from the following repo, so adding it first helm_repo_add("mongodb", "https://mongodb.github.io/helm-charts") + helm_chart_path, operator_version = helm_chart_path_and_version(helm_chart_path, operator_version) + if helm_args is None: helm_args = {} @@ -69,6 +72,7 @@ def __init__( self.helm_chart_path = helm_chart_path self.name = name self.api_client = api_client + self.operator_version = operator_version def install_from_template(self): """Uses helm to generate yaml specification and then uses python K8s client to apply them to the cluster @@ -82,6 +86,9 @@ def install_from_template(self): def install(self, custom_operator_version: Optional[str] = None) -> Operator: """Installs the Operator to Kubernetes cluster using 'helm install', waits until it's running""" + if not custom_operator_version: + custom_operator_version = self.operator_version + helm_install( self.name, self.namespace, @@ -99,6 +106,9 @@ def upgrade( self, multi_cluster: bool = False, custom_operator_version: Optional[str] = None, apply_crds_first: bool = False ) -> Operator: """Upgrades the Operator in Kubernetes cluster using 'helm upgrade', waits until it's running""" + if not custom_operator_version: + custom_operator_version = self.operator_version + helm_upgrade( self.name, self.namespace, diff --git a/docker/mongodb-kubernetes-tests/tests/conftest.py b/docker/mongodb-kubernetes-tests/tests/conftest.py index 15366fc79..818fce848 100644 --- a/docker/mongodb-kubernetes-tests/tests/conftest.py +++ b/docker/mongodb-kubernetes-tests/tests/conftest.py @@ -20,7 +20,12 @@ update_configmap, ) from kubetester.awss3client import AwsS3Client -from kubetester.helm import helm_install_from_chart, helm_repo_add +from kubetester.consts import * +from kubetester.helm import ( + helm_chart_path_and_version, + helm_install_from_chart, + helm_repo_add, +) from kubetester.kubetester import KubernetesTester from kubetester.kubetester import fixture as _fixture from kubetester.kubetester import running_locally @@ -55,9 +60,6 @@ LEGACY_CENTRAL_CLUSTER_NAME: str = "__default" LEGACY_DEPLOYMENT_STATE_VERSION: str = "1.27.0" -# Helm charts -LEGACY_OPERATOR_CHART = "mongodb/enterprise-operator" -MCK_HELM_CHART = "mongodb/mongodb-kubernetes" LOCAL_HELM_CHART_DIR = "helm_chart" OFFICIAL_OPERATOR_IMAGE_NAME = "mongodb-kubernetes" @@ -827,7 +829,7 @@ def _install_multi_cluster_operator( helm_opts: dict[str, str], central_cluster_name: str, operator_name: Optional[str] = MULTI_CLUSTER_OPERATOR_NAME, - helm_chart_path: Optional[str] = LOCAL_HELM_CHART_DIR, + helm_chart_path: Optional[str] = None, custom_operator_version: Optional[str] = None, apply_crds_first: bool = False, ) -> Operator: @@ -836,6 +838,8 @@ def _install_multi_cluster_operator( # The Operator will be installed from the following repo, so adding it first helm_repo_add("mongodb", "https://mongodb.github.io/helm-charts") + helm_chart_path, operator_version = helm_chart_path_and_version(helm_chart_path, custom_operator_version) + prepare_multi_cluster_namespaces( namespace, multi_cluster_operator_installation_config, @@ -851,6 +855,7 @@ def _install_multi_cluster_operator( helm_args=multi_cluster_operator_installation_config, api_client=central_cluster_client, helm_chart_path=helm_chart_path, + operator_version=custom_operator_version, ).upgrade(multi_cluster=True, custom_operator_version=custom_operator_version, apply_crds_first=apply_crds_first) # If we're running locally, then immediately after installing the deployment, we scale it to zero. diff --git a/scripts/evergreen/deployments/test-app/templates/mongodb-enterprise-tests.yaml b/scripts/evergreen/deployments/test-app/templates/mongodb-enterprise-tests.yaml index 2a3e37ff3..198374789 100644 --- a/scripts/evergreen/deployments/test-app/templates/mongodb-enterprise-tests.yaml +++ b/scripts/evergreen/deployments/test-app/templates/mongodb-enterprise-tests.yaml @@ -174,6 +174,22 @@ spec: - name: OM_DEBUG_HTTP value: "{{ .Values.omDebugHttp }}" {{ end }} + {{ if .Values.helm.oci.version }} + - name: OCI_HELM_VERSION + value: "{{ .Values.helm.oci.version }}" + {{ end }} + {{ if .Values.helm.oci.registry }} + - name: OCI_HELM_REGISTRY + value: "{{ .Values.helm.oci.registry }}" + {{ end }} + {{ if .Values.helm.oci.repository }} + - name: OCI_HELM_REPOSITORY + value: "{{ .Values.helm.oci.repository }}" + {{ end }} + {{ if .Values.helm.oci.region }} + - name: OCI_HELM_REGION + value: "{{ .Values.helm.oci.region }}" + {{ end }} - name: ops_manager_version value: "{{ .Values.opsManagerVersion }}" - name: cognito_user_pool_id diff --git a/scripts/evergreen/deployments/test-app/values.yaml b/scripts/evergreen/deployments/test-app/values.yaml index 13aba3df8..579bc426b 100644 --- a/scripts/evergreen/deployments/test-app/values.yaml +++ b/scripts/evergreen/deployments/test-app/values.yaml @@ -43,3 +43,12 @@ mdbImageType: "ubi8" # set to "true" to set OM_DEBUG_HTTP=true for the operator omDebugHttp: + +# sets the configuration using which we can figure out the helm OCI repo of the MCK operator while deploying it +# to run a test. +helm: + oci: + version: "" + registry: "" + repository: "" + region: "" diff --git a/scripts/evergreen/e2e/single_e2e.sh b/scripts/evergreen/e2e/single_e2e.sh index 766a1c146..3c881f19c 100755 --- a/scripts/evergreen/e2e/single_e2e.sh +++ b/scripts/evergreen/e2e/single_e2e.sh @@ -48,6 +48,16 @@ deploy_test_app() { BUILD_ID="${BUILD_ID:-default_build_id}" BUILD_VARIANT="${BUILD_VARIANT:-default_build_variant}" + if ! chart_info=$(scripts/dev/run_python.sh scripts/release/oci_chart_info.py --build-scenario "${BUILD_SCENARIO}" 2>&1); then + echo "${chart_info}" + exit 1 + fi + helm_oci_registry=$(echo "${chart_info}" | jq -r '.registry' ) + helm_oci_repository=$(echo "${chart_info}" | jq -r '.repository' ) + helm_oci_registry_region=$(echo "${chart_info}" | jq -r '.region' ) + helm_oci_version_prefix=$(echo "${chart_info}" | jq -r '.version_prefix // empty' ) + helm_oci_version="${helm_oci_version_prefix:-}${OPERATOR_VERSION}" + # note, that the 4 last parameters are used only for Mongodb resource testing - not for Ops Manager helm_params=( "--set" "taskId=${task_id:-'not-specified'}" @@ -76,6 +86,10 @@ deploy_test_app() { "--set" "cognito_user_password=${cognito_user_password}" "--set" "cognito_workload_url=${cognito_workload_url}" "--set" "cognito_workload_user_id=${cognito_workload_user_id}" + "--set" "helm.oci.version=${helm_oci_version}" + "--set" "helm.oci.registry=${helm_oci_registry}" + "--set" "helm.oci.repository=${helm_oci_repository}" + "--set" "helm.oci.region=${helm_oci_registry_region}" ) # shellcheck disable=SC2154 diff --git a/scripts/release/build/build_info.py b/scripts/release/build/build_info.py index 9b2b51091..142ad44a1 100644 --- a/scripts/release/build/build_info.py +++ b/scripts/release/build/build_info.py @@ -50,7 +50,8 @@ class BinaryInfo: class HelmChartInfo: repository: str registry: str - region: str + region: str = None + version_prefix: str = None sign: bool = False @@ -119,6 +120,7 @@ def load_build_info(scenario: BuildScenario) -> BuildInfo: helm_charts[name] = HelmChartInfo( repository=scenario_data.get("repository"), sign=scenario_data.get("sign", False), + version_prefix=scenario_data.get("version-prefix"), registry=scenario_data.get("registry"), region=scenario_data.get("region") ) diff --git a/scripts/release/helm_registry_login.py b/scripts/release/helm_registry_login.py index 3035d4197..dcdd14adc 100644 --- a/scripts/release/helm_registry_login.py +++ b/scripts/release/helm_registry_login.py @@ -88,7 +88,7 @@ def helm_registry_login_to_quay(registry): if not password: raise Exception(f"Env var {QUAY_PASSWORD_ENV_VAR} must be set with the quay password.") - # using quote will help us avoid command injectin issues, was reported by semggrep in PR + # using quote will help us avoid command injecting issues, was reported by semgrep in PR command = ["helm", "registry", "login", "--username", quote(username), "--password-stdin", quote(registry)] try: diff --git a/scripts/release/oci_chart_info.py b/scripts/release/oci_chart_info.py new file mode 100644 index 000000000..000cb5f75 --- /dev/null +++ b/scripts/release/oci_chart_info.py @@ -0,0 +1,40 @@ +import argparse +import json +from dataclasses import asdict + +from scripts.release.argparse_utils import get_scenario_from_arg +from scripts.release.build.build_info import load_build_info +from scripts.release.build.build_scenario import SUPPORTED_SCENARIOS, BuildScenario + + +def main(): + parser = argparse.ArgumentParser( + description="""Dump the Helm chart information for the 'mongodb-kubernetes' chart as JSON.""", + formatter_class=argparse.RawTextHelpFormatter, + ) + parser.add_argument( + "-b", + "--build-scenario", + metavar="", + action="store", + required=True, + type=str, + choices=SUPPORTED_SCENARIOS, + help=f"""Build scenario when reading configuration from 'build_info.json'. + Options: {", ".join(SUPPORTED_SCENARIOS)}. For '{BuildScenario.DEVELOPMENT}' the '{BuildScenario.PATCH}' scenario is used to read values from 'build_info.json'""", + ) + args = parser.parse_args() + + build_scenario = get_scenario_from_arg(args.build_scenario) + build_info = load_build_info(build_scenario) + chart_info = build_info.helm_charts["mongodb-kubernetes"] + + j = json.dumps(asdict(chart_info)) + print(j) + + +if __name__ == "__main__": + try: + main() + except Exception as e: + raise Exception(f"Failed while dumping the chart_info as json. Error: {e}") diff --git a/scripts/release/publish_helm_chart.py b/scripts/release/publish_helm_chart.py index bfdc79483..87fc27e08 100644 --- a/scripts/release/publish_helm_chart.py +++ b/scripts/release/publish_helm_chart.py @@ -1,14 +1,12 @@ import argparse -import os import subprocess -import sys - -import yaml from lib.base_logger import logger from scripts.release.build.build_info import * +from scripts.release.build.build_scenario import SUPPORTED_SCENARIOS CHART_DIR = "helm_chart" +MONGODB_KUBERNETES_CHART = "mongodb-kubernetes" def run_command(command: list[str]): @@ -26,51 +24,6 @@ def run_command(command: list[str]): ) -# update_chart_and_get_metadata updates the helm chart's Chart.yaml and sets the version -# to either evg patch id or commit which is set in OPERATOR_VERSION. -def update_chart_and_get_metadata(chart_dir: str, build_scenario) -> tuple[str, str]: - chart_path = os.path.join(chart_dir, "Chart.yaml") - version = os.environ.get("OPERATOR_VERSION") - if not version: - raise ValueError( - "Error: Environment variable 'OPERATOR_VERSION' must be set to determine the chart version to publish." - ) - - if not os.path.exists(chart_path): - raise FileNotFoundError( - f"Error: Chart.yaml not found in directory '{chart_dir}'. " - "Please ensure the directory exists and contains a valid Chart.yaml." - ) - - try: - with open(chart_path, "r") as f: - data = yaml.safe_load(f) - - chart_name = data.get("name") - if not chart_name: - raise ValueError("Chart.yaml is missing required 'name' field.") - except Exception as e: - raise Exception(f"Unable to load Chart.yaml from dir {chart_path}") - - # if build_scenario is release, the chart.yaml would already have correct chart version - if build_scenario == BuildScenario.RELEASE: - return chart_name, version - - new_version = f"0.0.0+{version}" - logger.info(f"New helm chart version will be: {new_version}") - - try: - data["version"] = new_version - - with open(chart_path, "w") as f: - yaml.safe_dump(data, f, sort_keys=False) - - logger.info(f"Successfully updated version for chart '{chart_name}' to '{new_version}'.") - return chart_name, new_version - except Exception as e: - raise RuntimeError(f"Failed to read or update Chart.yaml: {e}") - - def get_oci_registry(chart_info: HelmChartInfo) -> str: registry = chart_info.registry repo = chart_info.repository @@ -86,16 +39,21 @@ def get_oci_registry(chart_info: HelmChartInfo) -> str: return oci_registry -def publish_helm_chart(chart_info: HelmChartInfo, build_scenario): +def publish_helm_chart(chart_name: str, chart_info: HelmChartInfo, operator_version: str): try: - oci_registry = get_oci_registry(chart_info) - chart_name, chart_version = update_chart_and_get_metadata(CHART_DIR, build_scenario) + # If version_prefix is not specified, use the operator_version as is. + if chart_info.version_prefix is not None: + chart_version = f"{chart_info.version_prefix}{operator_version}" + else: + chart_version = operator_version + tgz_filename = f"{chart_name}-{chart_version}.tgz" logger.info(f"Packaging chart: {chart_name} with Version: {chart_version}") - package_command = ["helm", "package", CHART_DIR] + package_command = ["helm", "package", "--version", chart_version, CHART_DIR] run_command(package_command) + oci_registry = get_oci_registry(chart_info) logger.info(f"Pushing chart to registry: {oci_registry}") push_command = ["helm", "push", tgz_filename, oci_registry] run_command(push_command) @@ -107,20 +65,39 @@ def publish_helm_chart(chart_info: HelmChartInfo, build_scenario): def main(): parser = argparse.ArgumentParser( - description="Script to publish helm chart to the OCI container registry, based on the build scenario." + description="Script to publish helm chart to the OCI container registry, based on the build scenario.", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "-b", + "--build-scenario", + metavar="", + action="store", + required=True, + type=str, + choices=SUPPORTED_SCENARIOS, + help=f"""Build scenario when reading configuration from 'build_info.json'. +Options: {", ".join(SUPPORTED_SCENARIOS)}. For '{BuildScenario.DEVELOPMENT}' the '{BuildScenario.PATCH}' scenario is used to read values from 'build_info.json'""", + ) + parser.add_argument( + "-v", + "--version", + metavar="", + action="store", + required=True, + type=str, + help="Operator version to use when publishing helm chart", ) - parser.add_argument("--build_scenario", type=str, help="Build scenario (e.g., patch, staging etc).") args = parser.parse_args() build_scenario = args.build_scenario build_info = load_build_info(build_scenario) - return publish_helm_chart(build_info.helm_charts["mongodb-kubernetes"], build_scenario) + return publish_helm_chart(MONGODB_KUBERNETES_CHART, build_info.helm_charts[MONGODB_KUBERNETES_CHART], args.version) if __name__ == "__main__": try: main() - except Exception as e: - logger.error(f"Failure in the helm publishing process {e}") - sys.exit(1) + except Exception as main_err: + raise Exception(f"Failure in the helm publishing process {main_err}") diff --git a/scripts/release/publish_helm_chart.sh b/scripts/release/publish_helm_chart.sh index 5321bffd2..29f35d5f6 100755 --- a/scripts/release/publish_helm_chart.sh +++ b/scripts/release/publish_helm_chart.sh @@ -2,10 +2,10 @@ # Instead of calling the publish_helm_chart.py directly from .evergreen-functions.yaml # we are calling that via this .sh so that we can easily pass build_scenario from env var that -# is set via context files. Using the env vars, set via context files, in .evergreen configuraiton +# is set via context files. Using the env vars, set via context files, in .evergreen configuration # is not that straightforward. set -Eeou pipefail source scripts/dev/set_env_context.sh -scripts/dev/run_python.sh scripts/release/publish_helm_chart.py --build_scenario "${BUILD_SCENARIO}" +scripts/dev/run_python.sh scripts/release/publish_helm_chart.py --build-scenario "${BUILD_SCENARIO}" --version "${OPERATOR_VERSION}" diff --git a/scripts/release/tests/build_info_test.py b/scripts/release/tests/build_info_test.py index 3942be55c..e13e1c630 100644 --- a/scripts/release/tests/build_info_test.py +++ b/scripts/release/tests/build_info_test.py @@ -106,6 +106,7 @@ def test_load_build_info_development(): }, helm_charts={ "mongodb-kubernetes": HelmChartInfo( + version_prefix="0.0.0+", registry="268558157000.dkr.ecr.us-east-1.amazonaws.com", repository="dev/mongodb/helm-charts", region="us-east-1", @@ -214,6 +215,7 @@ def test_load_build_info_patch(): }, helm_charts={ "mongodb-kubernetes": HelmChartInfo( + version_prefix="0.0.0+", region="us-east-1", repository="dev/mongodb/helm-charts", registry="268558157000.dkr.ecr.us-east-1.amazonaws.com", @@ -351,10 +353,11 @@ def test_load_build_info_staging(): }, helm_charts={ "mongodb-kubernetes": HelmChartInfo( + sign=True, + version_prefix="0.0.0+", registry="268558157000.dkr.ecr.us-east-1.amazonaws.com", repository="staging/mongodb/helm-charts", region="us-east-1", - sign=True, ) }, ) @@ -481,10 +484,9 @@ def test_load_build_info_release(): }, helm_charts={ "mongodb-kubernetes": HelmChartInfo( + sign=True, registry="quay.io", repository="mongodb/helm-charts", - region=None, - sign=True, ) }, )