Showcase: yaml config management with secret env var placeholders

[ion-elgreco]

There isn't a discussion board, so I thought to share it in an issue since it's a pretty nice :). With the setup below, you can actually create yamls with this structure with env var placeholders using pydantic basesettings:

mcp_servers:
  - endpoint: "https://hello_world"
    api_key: ${EXAMPLE_KEY}

And the ${} are environment variables evaluated at runtime, when initializing the config.

You have to create a custom YamlConfigSettingSource, that does that:

import os
from functools import lru_cache
from pathlib import Path
from string import Template
from typing import Any

from pydantic import BaseModel, Field, SecretStr
from pydantic_settings import (
    BaseSettings,
    PydanticBaseSettingsSource,
    SettingsConfigDict,
    YamlConfigSettingsSource,
)

class EnvParsedYamlConfigSettingsSource(YamlConfigSettingsSource):
    """Yaml Settings Source that parses envVar references before loading.

    ENV VARS need to be configured like this: ${ENV_VAR}

    An example of a valid config.yaml would be:
        ```yaml
        config:
            api_key: ${OPENAI_KEY}
        ```
    """

    def _read_file(self, file_path: Path) -> dict[str, Any]:
        import json

        data = super()._read_file(file_path)
        data = json.dumps(data)
        data = Template(data).substitute(os.environ)
        return json.loads(data)

Now we can proceed with creating an example Config:

class MCPServer(BaseModel):
    """MCP Server connection"""

    endpoint: str
    api_key: SecretStr

class MCPServerConfiguration(BaseSettings):
    """MCP server configuration"""

    servers: list[MCPServer] | None = Field(alias="mcp_servers", default=None)
    model_config = SettingsConfigDict(
        extra="ignore",
        yaml_file=os.environ.get(
            "CONFIG_FILE_PATH",
            Path(__file__).parent.parent / "config.yaml",
        ),
    )

    @classmethod
    def settings_customise_sources(  # noqa: D102
        cls,
        settings_cls: type[BaseSettings],
        init_settings: PydanticBaseSettingsSource,  # noqa: ARG003
        env_settings: PydanticBaseSettingsSource,  # noqa: ARG003
        dotenv_settings: PydanticBaseSettingsSource,  # noqa: ARG003
        file_secret_settings: PydanticBaseSettingsSource,  # noqa: ARG003
    ) -> tuple[PydanticBaseSettingsSource, ...]:
        return (EnvParsedYamlConfigSettingsSource(settings_cls),)


class Config(BaseSettings):
    """Application settings"""

    mcp: MCPServerConfiguration = MCPServerConfiguration()  # type: ignore

    # API Configuration
    api_host: str = Field(default="0.0.0.0", description="API host")
    api_port: int = Field(default=8000, description="API port")

    class Config:
        """Config class for .env values"""

        env_file = ".env"
        env_file_encoding = "utf-8"
        case_sensitive = True
        extra = "ignore"


@lru_cache(1)
def get_config() -> Config:
    """Lazy static to get the config for the project"""
    config = Config()  # type: ignore[call-arg]`
    return config  # type: ignore

So let's bring this together and run:

print(config)
print(config.mcp.servers[0].api_key.get_secret_value())

export EXAMPLE_KEY='12345678' && python test.py, which will print out the following:

mcp=MCPServerConfiguration(servers=[MCPServer(endpoint='https://hello_world', api_key=SecretStr('**********'))]) api_host='0.0.0.0' api_port=8000
12345678