[DOP-39630] Hide secrets while logging settings value

data_rentgen/consumer/__init__.py

@@ -7,6 +7,7 @@
 from typing import Any
 
 import anyio
+from devtools import pformat
 from fast_depends import dependency_provider
 from faststream import ContextRepo, FastStream
 from faststream._internal._compat import ExceptionGroup
@@ -157,5 +158,5 @@ async def security_lifespan(context: ContextRepo):
 def get_application():
     settings = ConsumerApplicationSettings()
     setup_logging(settings.logging)
-    logger.info("Starting Data.Rentgen consumer with settings:\n%r", settings)
+    logger.info("Starting Data.Rentgen consumer with settings:\n%s", pformat(settings))
     return application_factory(settings=settings)

data_rentgen/db/factory.py

@@ -3,15 +3,15 @@
 
 from sqlalchemy.ext.asyncio import (
     AsyncSession,
-    async_engine_from_config,
     async_sessionmaker,
+    create_async_engine,
 )
 
 from data_rentgen.db.settings import DatabaseSettings
 
 
 def create_session_factory(settings: DatabaseSettings) -> async_sessionmaker[AsyncSession]:
-    engine = async_engine_from_config(settings.model_dump(), prefix="")
+    engine = create_async_engine(url=str(settings.url), **settings.model_dump(exclude={"url"}))
     return async_sessionmaker(
         bind=engine,
         class_=AsyncSession,

data_rentgen/db/migrations/env.py

@@ -19,7 +19,7 @@
 
 if not config.get_main_option("sqlalchemy.url"):
     # read application settings only if sqlalchemy.url is not being passed via cli arguments
-    config.set_main_option("sqlalchemy.url", DatabaseApplicationSettings().database.url)
+    config.set_main_option("sqlalchemy.url", str(DatabaseApplicationSettings().database.url))
 
 target_metadata = (Base.metadata,)
 

data_rentgen/db/settings.py

@@ -2,13 +2,36 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import textwrap
+from typing import Annotated
+from urllib.parse import urlsplit
 
-from pydantic import BaseModel, ConfigDict, Field
+from pydantic import AfterValidator, BaseModel, ConfigDict, Field, PostgresDsn, UrlConstraints
+from sqlalchemy import make_url
 
 from data_rentgen.logging import DEFAULT_LOGGING_SETTINGS, LoggingSettings
 from data_rentgen.settings import BaseSettings
 
 
+def validate_url(value: PostgresDsn):
+    if not value.path or len(value.path) <= 1:
+        msg = "Database URL must contain database name"
+        raise ValueError(msg)
+
+    split = urlsplit(str(value))
+    if not split.username or not split.password:
+        msg = "Database URL must contain username and password"
+        raise ValueError(msg)
+
+    return value
+
+
+PostgresURL = Annotated[
+    PostgresDsn,
+    UrlConstraints(allowed_schemes=["postgresql+asyncpg", "postgresql+psycopg"], host_required=True),
+    AfterValidator(validate_url),
+]
+
+
 class DatabaseSettings(BaseModel):
     """Data.Rentgen backend database settings.
 
@@ -29,7 +52,7 @@ class DatabaseSettings(BaseModel):
     ```
     """
 
-    url: str = Field(
+    url: PostgresURL = Field(
         description=textwrap.dedent(
             """
             Database connection URL.
@@ -38,13 +61,23 @@ class DatabaseSettings(BaseModel):
 
             !!! warning
 
-                Only async drivers are supported, e.g. `asyncpg`
+                Only async drivers are supported, e.g. `asyncpg` or `psycopg`
             """,
         ),
     )
 
     model_config = ConfigDict(extra="allow")
 
+    def __repr_args__(self):
+        safe_url = make_url(str(self.url)).render_as_string(
+            hide_password=True,
+        )
+        extra = super().__repr_args__()
+        return [
+            ("url", safe_url),
+            *[item for item in extra if item[0] != "url"],
+        ]
+
 
 class DatabaseApplicationSettings(BaseSettings):
     """Settings used by database migrations and maintenance scripts."""

data_rentgen/http2kafka/__init__.py

@@ -5,6 +5,7 @@
 from contextlib import asynccontextmanager
 
 import anyio
+from devtools import pformat
 from fastapi import FastAPI
 from faststream._internal._compat import ExceptionGroup
 from faststream.kafka import KafkaBroker
@@ -97,5 +98,5 @@ def application_factory(settings: Http2KafkaApplicationSettings) -> FastAPI:
 def get_application():
     settings = Http2KafkaApplicationSettings()
     setup_logging(settings.logging)
-    logger.info("Starting Data.Rentgen HTTP2Kafka with settings:\n%r", settings)
+    logger.info("Starting Data.Rentgen HTTP2Kafka with settings:\n%s", pformat(settings))
     return application_factory(settings=settings)

data_rentgen/server/__init__.py

@@ -4,6 +4,7 @@
 import logging
 from typing import TYPE_CHECKING
 
+from devtools import pformat
 from fastapi import FastAPI
 
 import data_rentgen
@@ -55,5 +56,5 @@ def application_factory(settings: ServerApplicationSettings) -> FastAPI:
 def get_application():
     settings = ServerApplicationSettings()
     setup_logging(settings.logging)
-    logger.info("Starting Data.Rentgen server with settings:\n%r", settings)
+    logger.info("Starting Data.Rentgen server with settings:\n%s", pformat(settings))
     return application_factory(settings=settings)

data_rentgen/server/providers/auth/dummy_provider.py

@@ -1,10 +1,10 @@
 # SPDX-FileCopyrightText: 2024-present MTS PJSC
 # SPDX-License-Identifier: Apache-2.0
 import logging
-from pprint import pformat
 from time import time
 from typing import Any
 
+from devtools import pformat
 from fastapi import FastAPI, Request
 
 from data_rentgen.db.models import User

data_rentgen/server/providers/auth/keycloak_provider.py

@@ -4,6 +4,7 @@
 import time
 from typing import Any, NoReturn
 
+from devtools import pformat
 from fastapi import FastAPI, Request
 from jwcrypto import jwk
 from jwcrypto.common import JWException
@@ -44,7 +45,7 @@ def setup(cls, app: FastAPI) -> FastAPI:
         settings = KeycloakAuthProviderSettings.model_validate(
             app.state.settings.auth.model_dump(exclude={"provider"}),
         )
-        logger.info("Using %s provider with settings:\n%s", cls.__name__, settings)
+        logger.info("Using %s provider with settings:\n%s", cls.__name__, pformat(settings))
 
         app.state.auth_provider = cls(settings)
         app.add_middleware(

data_rentgen/server/providers/auth/personal_token_provider.py

@@ -2,11 +2,11 @@
 # SPDX-License-Identifier: Apache-2.0
 import logging
 from datetime import UTC, datetime, timedelta
-from pprint import pformat
 from typing import Literal
 from uuid import UUID
 
 from cachetools import LRUCache, TTLCache
+from devtools import pformat
 from fastapi import FastAPI, Request
 
 from data_rentgen.db.models import PersonalToken, User

data_rentgen/server/settings/auth/__init__.py

@@ -41,6 +41,10 @@ class AuthSettings(BaseModel):
     @classmethod
     def _validate_provider(cls, value: type) -> type[AuthProvider]:
         if not issubclass(value, AuthProvider):
-            msg = f"Class {value.__qualname__} is not a subclass of {AuthProvider}"
+            msg = f"Class {value} is not a subclass of {AuthProvider}"
             raise TypeError(msg)
         return value
+
+    # prevent leaking provider secrets
+    def __repr_args__(self):
+        return [("provider", self.provider)]

mddocs/docs/changelog/next_release/516.improvement.md

@@ -0,0 +1 @@
+Hide auth provider secrets and database password from logs.

mddocs/docs/reference/http2kafka/configuration/debug.md

@@ -65,7 +65,7 @@ This is done by adding a specific filter to logging handler:
 logging:
     filters:
         correlation_id:
-            class: asgi_correlation_id.CorrelationIdFilter
+            (): asgi_correlation_id.CorrelationIdFilter
             uuid_length: 32
             default_value: '-'
 ```

mddocs/docs/reference/server/configuration/debug.md

@@ -64,7 +64,7 @@ This is done by adding a specific filter to logging handler:
 logging:
     filters:
         correlation_id:
-            class: asgi_correlation_id.CorrelationIdFilter
+            (): asgi_correlation_id.CorrelationIdFilter
             uuid_length: 32
             default_value: '-'
 ```

pyproject.toml

@@ -47,6 +47,7 @@ dependencies = [
   "python-dateutil~=2.9.0.post0",
   "packaging~=26.0",
   "cachetools~=7.1.1",
+  "devtools~=0.12.2",
 ]
 dynamic = ["version"]
 

tests/fixtures/async_engine.py

@@ -17,7 +17,7 @@
 @contextlib.asynccontextmanager
 async def get_async_engine(db_settings: DatabaseSettings) -> AsyncGenerator[AsyncEngine, None]:
     """Create test engine"""
-    connection_url = db_settings.url
+    connection_url = str(db_settings.url)
     engine = create_async_engine(connection_url)
     yield engine
     await engine.dispose()

tests/test_database/fixtures/alembic.py

@@ -30,7 +30,7 @@
 def empty_db_url(db_settings: DatabaseSettings) -> Generator[str, None, None]:
     """Create new test DB to run migrations"""
     new_db = secrets.token_hex(8)
-    original_url = urlparse(db_settings.url)
+    original_url = urlparse(str(db_settings.url))
 
     # updating original url with temp database name, and use it only for running migrations
     # sqlalchemy-utils does not support asyncio, so using sync action instead

tests/test_settings.py

@@ -27,7 +27,7 @@ def test_all_application_settings_are_loaded_from_default_yaml_file(
         dedent(
             """\
             database:
-              url: postgresql+asyncpg://yaml@localhost:5432/data_rentgen
+              url: postgresql+asyncpg://yaml:yaml@localhost:5432/data_rentgen
             kafka:
               bootstrap_servers: [yaml:9092]
             server:
@@ -45,10 +45,10 @@ def test_all_application_settings_are_loaded_from_default_yaml_file(
     http2kafka_settings = Http2KafkaApplicationSettings()
     database_settings = DatabaseApplicationSettings()
 
-    assert server_settings.database.url == "postgresql+asyncpg://yaml@localhost:5432/data_rentgen"
-    assert consumer_settings.database.url == "postgresql+asyncpg://yaml@localhost:5432/data_rentgen"
-    assert http2kafka_settings.database.url == "postgresql+asyncpg://yaml@localhost:5432/data_rentgen"
-    assert database_settings.database.url == "postgresql+asyncpg://yaml@localhost:5432/data_rentgen"
+    assert str(server_settings.database.url) == "postgresql+asyncpg://yaml:yaml@localhost:5432/data_rentgen"
+    assert str(consumer_settings.database.url) == "postgresql+asyncpg://yaml:yaml@localhost:5432/data_rentgen"
+    assert str(http2kafka_settings.database.url) == "postgresql+asyncpg://yaml:yaml@localhost:5432/data_rentgen"
+    assert str(database_settings.database.url) == "postgresql+asyncpg://yaml:yaml@localhost:5432/data_rentgen"
     assert consumer_settings.kafka.bootstrap_servers == ["yaml:9092"]
     assert http2kafka_settings.kafka.bootstrap_servers == ["yaml:9092"]
     assert server_settings.server.debug is True
@@ -66,20 +66,20 @@ def test_yaml_file_overrides_environment(
         dedent(
             """\
             database:
-              url: postgresql+asyncpg://yaml@localhost:5432/data_rentgen
+              url: postgresql+asyncpg://yaml:yaml@localhost:5432/data_rentgen
             """,
         ),
         encoding="utf-8",
     )
     monkeypatch.setenv("DATA_RENTGEN_CONFIG_FILE", str(config_path))
     monkeypatch.setenv(
         "DATA_RENTGEN__DATABASE__URL",
-        "postgresql+asyncpg://env@localhost:5432/data_rentgen",
+        "postgresql+asyncpg://env:env@localhost:5432/data_rentgen",
     )
 
     settings = DatabaseApplicationSettings()
 
-    assert settings.database.url == "postgresql+asyncpg://yaml@localhost:5432/data_rentgen"
+    assert str(settings.database.url) == "postgresql+asyncpg://yaml:yaml@localhost:5432/data_rentgen"
 
 
 def test_environment_fills_values_missing_from_yaml_file(
@@ -92,7 +92,7 @@ def test_environment_fills_values_missing_from_yaml_file(
         dedent(
             """\
             database:
-              url: postgresql+asyncpg://yaml@localhost:5432/data_rentgen
+              url: postgresql+asyncpg://yaml:yaml@localhost:5432/data_rentgen
             """,
         ),
         encoding="utf-8",
@@ -101,7 +101,7 @@ def test_environment_fills_values_missing_from_yaml_file(
 
     settings = DatabaseApplicationSettings()
 
-    assert settings.database.url == "postgresql+asyncpg://yaml@localhost:5432/data_rentgen"
+    assert str(settings.database.url) == "postgresql+asyncpg://yaml:yaml@localhost:5432/data_rentgen"
 
 
 def test_settings_can_be_loaded_from_environment_without_yaml_file(
@@ -112,9 +112,9 @@ def test_settings_can_be_loaded_from_environment_without_yaml_file(
     monkeypatch.setenv("DATA_RENTGEN_CONFIG_FILE", str(tmp_path / "missing.yml"))
     monkeypatch.setenv(
         "DATA_RENTGEN__DATABASE__URL",
-        "postgresql+asyncpg://env@localhost:5432/data_rentgen",
+        "postgresql+asyncpg://env:env@localhost:5432/data_rentgen",
     )
 
     settings = DatabaseApplicationSettings()
 
-    assert settings.database.url == "postgresql+asyncpg://env@localhost:5432/data_rentgen"
+    assert str(settings.database.url) == "postgresql+asyncpg://env:env@localhost:5432/data_rentgen"