mirror of
https://github.com/apache/superset.git
synced 2026-04-24 02:25:13 +00:00
feat: AWS Cross-Account IAM Authentication for Aurora (#37585)
This commit is contained in:
317
tests/unit_tests/db_engine_specs/test_aurora.py
Normal file
317
tests/unit_tests/db_engine_specs/test_aurora.py
Normal file
@@ -0,0 +1,317 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
# pylint: disable=import-outside-toplevel
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from superset.utils import json
|
||||
from tests.unit_tests.conftest import with_feature_flags
|
||||
|
||||
|
||||
def test_aurora_postgres_engine_spec_properties() -> None:
|
||||
from superset.db_engine_specs.aurora import AuroraPostgresEngineSpec
|
||||
|
||||
assert AuroraPostgresEngineSpec.engine == "postgresql"
|
||||
assert AuroraPostgresEngineSpec.engine_name == "Aurora PostgreSQL"
|
||||
assert AuroraPostgresEngineSpec.default_driver == "psycopg2"
|
||||
|
||||
|
||||
def test_update_params_from_encrypted_extra_without_iam() -> None:
|
||||
from superset.db_engine_specs.postgres import PostgresEngineSpec
|
||||
|
||||
database = MagicMock()
|
||||
database.encrypted_extra = json.dumps({})
|
||||
database.sqlalchemy_uri_decrypted = (
|
||||
"postgresql://user:password@mydb.us-east-1.rds.amazonaws.com:5432/mydb"
|
||||
)
|
||||
|
||||
params: dict[str, Any] = {}
|
||||
PostgresEngineSpec.update_params_from_encrypted_extra(database, params)
|
||||
|
||||
# No modifications should be made
|
||||
assert params == {}
|
||||
|
||||
|
||||
def test_update_params_from_encrypted_extra_iam_disabled() -> None:
|
||||
from superset.db_engine_specs.postgres import PostgresEngineSpec
|
||||
|
||||
database = MagicMock()
|
||||
database.encrypted_extra = json.dumps(
|
||||
{
|
||||
"aws_iam": {
|
||||
"enabled": False,
|
||||
"role_arn": "arn:aws:iam::123456789012:role/TestRole",
|
||||
"region": "us-east-1",
|
||||
"db_username": "superset_user",
|
||||
}
|
||||
}
|
||||
)
|
||||
database.sqlalchemy_uri_decrypted = (
|
||||
"postgresql://user:password@mydb.us-east-1.rds.amazonaws.com:5432/mydb"
|
||||
)
|
||||
|
||||
params: dict[str, Any] = {}
|
||||
PostgresEngineSpec.update_params_from_encrypted_extra(database, params)
|
||||
|
||||
# No modifications should be made when IAM is disabled
|
||||
assert params == {}
|
||||
|
||||
|
||||
@with_feature_flags(AWS_DATABASE_IAM_AUTH=True)
|
||||
def test_update_params_from_encrypted_extra_with_iam() -> None:
|
||||
from superset.db_engine_specs.aws_iam import AWSIAMAuthMixin
|
||||
from superset.db_engine_specs.postgres import PostgresEngineSpec
|
||||
|
||||
database = MagicMock()
|
||||
database.encrypted_extra = json.dumps(
|
||||
{
|
||||
"aws_iam": {
|
||||
"enabled": True,
|
||||
"role_arn": "arn:aws:iam::123456789012:role/TestRole",
|
||||
"region": "us-east-1",
|
||||
"db_username": "superset_iam_user",
|
||||
}
|
||||
}
|
||||
)
|
||||
database.sqlalchemy_uri_decrypted = (
|
||||
"postgresql://user@mydb.cluster-xyz.us-east-1.rds.amazonaws.com:5432/mydb"
|
||||
)
|
||||
|
||||
params: dict[str, Any] = {}
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
AWSIAMAuthMixin,
|
||||
"get_iam_credentials",
|
||||
return_value={
|
||||
"AccessKeyId": "ASIA...",
|
||||
"SecretAccessKey": "secret...",
|
||||
"SessionToken": "token...",
|
||||
},
|
||||
),
|
||||
patch.object(
|
||||
AWSIAMAuthMixin,
|
||||
"generate_rds_auth_token",
|
||||
return_value="iam-auth-token",
|
||||
),
|
||||
):
|
||||
PostgresEngineSpec.update_params_from_encrypted_extra(database, params)
|
||||
|
||||
assert "connect_args" in params
|
||||
assert params["connect_args"]["password"] == "iam-auth-token" # noqa: S105
|
||||
assert params["connect_args"]["user"] == "superset_iam_user"
|
||||
assert params["connect_args"]["sslmode"] == "require"
|
||||
|
||||
|
||||
def test_update_params_merges_remaining_encrypted_extra() -> None:
|
||||
from superset.db_engine_specs.postgres import PostgresEngineSpec
|
||||
|
||||
database = MagicMock()
|
||||
database.encrypted_extra = json.dumps(
|
||||
{
|
||||
"aws_iam": {"enabled": False},
|
||||
"pool_size": 10,
|
||||
}
|
||||
)
|
||||
database.sqlalchemy_uri_decrypted = (
|
||||
"postgresql://user:password@mydb.us-east-1.rds.amazonaws.com:5432/mydb"
|
||||
)
|
||||
|
||||
params: dict[str, Any] = {}
|
||||
PostgresEngineSpec.update_params_from_encrypted_extra(database, params)
|
||||
|
||||
# aws_iam should be consumed, pool_size should be merged
|
||||
assert "aws_iam" not in params
|
||||
assert params["pool_size"] == 10
|
||||
|
||||
|
||||
def test_update_params_from_encrypted_extra_no_encrypted_extra() -> None:
|
||||
from superset.db_engine_specs.postgres import PostgresEngineSpec
|
||||
|
||||
database = MagicMock()
|
||||
database.encrypted_extra = None
|
||||
|
||||
params: dict[str, Any] = {}
|
||||
PostgresEngineSpec.update_params_from_encrypted_extra(database, params)
|
||||
|
||||
# No modifications should be made
|
||||
assert params == {}
|
||||
|
||||
|
||||
def test_update_params_from_encrypted_extra_invalid_json() -> None:
|
||||
from superset.db_engine_specs.postgres import PostgresEngineSpec
|
||||
|
||||
database = MagicMock()
|
||||
database.encrypted_extra = "not-valid-json"
|
||||
|
||||
params: dict[str, Any] = {}
|
||||
|
||||
with pytest.raises(json.JSONDecodeError):
|
||||
PostgresEngineSpec.update_params_from_encrypted_extra(database, params)
|
||||
|
||||
|
||||
def test_encrypted_extra_sensitive_fields() -> None:
|
||||
from superset.db_engine_specs.postgres import PostgresEngineSpec
|
||||
|
||||
# Verify sensitive fields are properly defined
|
||||
assert (
|
||||
"$.aws_iam.external_id" in PostgresEngineSpec.encrypted_extra_sensitive_fields
|
||||
)
|
||||
assert "$.aws_iam.role_arn" in PostgresEngineSpec.encrypted_extra_sensitive_fields
|
||||
|
||||
|
||||
def test_mask_encrypted_extra() -> None:
|
||||
from superset.db_engine_specs.postgres import PostgresEngineSpec
|
||||
|
||||
encrypted_extra = json.dumps(
|
||||
{
|
||||
"aws_iam": {
|
||||
"enabled": True,
|
||||
"role_arn": "arn:aws:iam::123456789012:role/SecretRole",
|
||||
"external_id": "secret-external-id-12345",
|
||||
"region": "us-east-1",
|
||||
"db_username": "superset_user",
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
masked = PostgresEngineSpec.mask_encrypted_extra(encrypted_extra)
|
||||
assert masked is not None
|
||||
|
||||
masked_config = json.loads(masked)
|
||||
|
||||
# role_arn and external_id should be masked
|
||||
assert (
|
||||
masked_config["aws_iam"]["role_arn"]
|
||||
!= "arn:aws:iam::123456789012:role/SecretRole"
|
||||
)
|
||||
assert masked_config["aws_iam"]["external_id"] != "secret-external-id-12345"
|
||||
|
||||
# Non-sensitive fields should remain unchanged
|
||||
assert masked_config["aws_iam"]["enabled"] is True
|
||||
assert masked_config["aws_iam"]["region"] == "us-east-1"
|
||||
assert masked_config["aws_iam"]["db_username"] == "superset_user"
|
||||
|
||||
|
||||
def test_aurora_postgres_inherits_from_postgres() -> None:
|
||||
from superset.db_engine_specs.aurora import AuroraPostgresEngineSpec
|
||||
from superset.db_engine_specs.postgres import PostgresEngineSpec
|
||||
|
||||
# Verify inheritance
|
||||
assert issubclass(AuroraPostgresEngineSpec, PostgresEngineSpec)
|
||||
|
||||
# Verify it inherits PostgreSQL capabilities
|
||||
assert AuroraPostgresEngineSpec.supports_dynamic_schema is True
|
||||
assert AuroraPostgresEngineSpec.supports_catalog is True
|
||||
|
||||
|
||||
def test_aurora_mysql_engine_spec_properties() -> None:
|
||||
from superset.db_engine_specs.aurora import AuroraMySQLEngineSpec
|
||||
|
||||
assert AuroraMySQLEngineSpec.engine == "mysql"
|
||||
assert AuroraMySQLEngineSpec.engine_name == "Aurora MySQL"
|
||||
assert AuroraMySQLEngineSpec.default_driver == "mysqldb"
|
||||
|
||||
|
||||
def test_aurora_mysql_inherits_from_mysql() -> None:
|
||||
from superset.db_engine_specs.aurora import AuroraMySQLEngineSpec
|
||||
from superset.db_engine_specs.mysql import MySQLEngineSpec
|
||||
|
||||
assert issubclass(AuroraMySQLEngineSpec, MySQLEngineSpec)
|
||||
assert AuroraMySQLEngineSpec.supports_dynamic_schema is True
|
||||
|
||||
|
||||
def test_aurora_mysql_has_iam_support() -> None:
|
||||
from superset.db_engine_specs.aurora import AuroraMySQLEngineSpec
|
||||
|
||||
# Verify it inherits encrypted_extra_sensitive_fields
|
||||
assert (
|
||||
"$.aws_iam.external_id"
|
||||
in AuroraMySQLEngineSpec.encrypted_extra_sensitive_fields
|
||||
)
|
||||
assert (
|
||||
"$.aws_iam.role_arn" in AuroraMySQLEngineSpec.encrypted_extra_sensitive_fields
|
||||
)
|
||||
|
||||
|
||||
@with_feature_flags(AWS_DATABASE_IAM_AUTH=True)
|
||||
def test_aurora_mysql_update_params_from_encrypted_extra_with_iam() -> None:
|
||||
from superset.db_engine_specs.aurora import AuroraMySQLEngineSpec
|
||||
from superset.db_engine_specs.aws_iam import AWSIAMAuthMixin
|
||||
|
||||
database = MagicMock()
|
||||
database.encrypted_extra = json.dumps(
|
||||
{
|
||||
"aws_iam": {
|
||||
"enabled": True,
|
||||
"role_arn": "arn:aws:iam::123456789012:role/TestRole",
|
||||
"region": "us-east-1",
|
||||
"db_username": "superset_iam_user",
|
||||
}
|
||||
}
|
||||
)
|
||||
database.sqlalchemy_uri_decrypted = (
|
||||
"mysql://user@mydb.cluster-xyz.us-east-1.rds.amazonaws.com:3306/mydb"
|
||||
)
|
||||
|
||||
params: dict[str, Any] = {}
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
AWSIAMAuthMixin,
|
||||
"get_iam_credentials",
|
||||
return_value={
|
||||
"AccessKeyId": "ASIA...",
|
||||
"SecretAccessKey": "secret...",
|
||||
"SessionToken": "token...",
|
||||
},
|
||||
),
|
||||
patch.object(
|
||||
AWSIAMAuthMixin,
|
||||
"generate_rds_auth_token",
|
||||
return_value="iam-auth-token",
|
||||
),
|
||||
):
|
||||
AuroraMySQLEngineSpec.update_params_from_encrypted_extra(database, params)
|
||||
|
||||
assert "connect_args" in params
|
||||
assert params["connect_args"]["password"] == "iam-auth-token" # noqa: S105
|
||||
assert params["connect_args"]["user"] == "superset_iam_user"
|
||||
# Note: ssl_mode is not set because MySQL drivers don't support it.
|
||||
# SSL should be configured via the database's extra settings.
|
||||
|
||||
|
||||
def test_aurora_data_api_classes_unchanged() -> None:
|
||||
from superset.db_engine_specs.aurora import (
|
||||
AuroraMySQLDataAPI,
|
||||
AuroraPostgresDataAPI,
|
||||
)
|
||||
|
||||
# Verify Data API classes are still available and unchanged
|
||||
assert AuroraMySQLDataAPI.engine == "mysql"
|
||||
assert AuroraMySQLDataAPI.default_driver == "auroradataapi"
|
||||
assert AuroraMySQLDataAPI.engine_name == "Aurora MySQL (Data API)"
|
||||
|
||||
assert AuroraPostgresDataAPI.engine == "postgresql"
|
||||
assert AuroraPostgresDataAPI.default_driver == "auroradataapi"
|
||||
assert AuroraPostgresDataAPI.engine_name == "Aurora PostgreSQL (Data API)"
|
||||
1045
tests/unit_tests/db_engine_specs/test_aws_iam.py
Normal file
1045
tests/unit_tests/db_engine_specs/test_aws_iam.py
Normal file
File diff suppressed because it is too large
Load Diff
236
tests/unit_tests/db_engine_specs/test_mysql_iam.py
Normal file
236
tests/unit_tests/db_engine_specs/test_mysql_iam.py
Normal file
@@ -0,0 +1,236 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
# pylint: disable=import-outside-toplevel
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from superset.utils import json
|
||||
from tests.unit_tests.conftest import with_feature_flags
|
||||
|
||||
|
||||
def test_mysql_encrypted_extra_sensitive_fields() -> None:
|
||||
from superset.db_engine_specs.mysql import MySQLEngineSpec
|
||||
|
||||
assert "$.aws_iam.external_id" in MySQLEngineSpec.encrypted_extra_sensitive_fields
|
||||
assert "$.aws_iam.role_arn" in MySQLEngineSpec.encrypted_extra_sensitive_fields
|
||||
|
||||
|
||||
def test_mysql_update_params_no_encrypted_extra() -> None:
|
||||
from superset.db_engine_specs.mysql import MySQLEngineSpec
|
||||
|
||||
database = MagicMock()
|
||||
database.encrypted_extra = None
|
||||
|
||||
params: dict[str, Any] = {}
|
||||
MySQLEngineSpec.update_params_from_encrypted_extra(database, params)
|
||||
|
||||
assert params == {}
|
||||
|
||||
|
||||
def test_mysql_update_params_empty_encrypted_extra() -> None:
|
||||
from superset.db_engine_specs.mysql import MySQLEngineSpec
|
||||
|
||||
database = MagicMock()
|
||||
database.encrypted_extra = json.dumps({})
|
||||
|
||||
params: dict[str, Any] = {}
|
||||
MySQLEngineSpec.update_params_from_encrypted_extra(database, params)
|
||||
|
||||
assert params == {}
|
||||
|
||||
|
||||
def test_mysql_update_params_iam_disabled() -> None:
|
||||
from superset.db_engine_specs.mysql import MySQLEngineSpec
|
||||
|
||||
database = MagicMock()
|
||||
database.encrypted_extra = json.dumps(
|
||||
{
|
||||
"aws_iam": {
|
||||
"enabled": False,
|
||||
"role_arn": "arn:aws:iam::123456789012:role/TestRole",
|
||||
"region": "us-east-1",
|
||||
"db_username": "superset_user",
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
params: dict[str, Any] = {}
|
||||
MySQLEngineSpec.update_params_from_encrypted_extra(database, params)
|
||||
|
||||
assert params == {}
|
||||
|
||||
|
||||
@with_feature_flags(AWS_DATABASE_IAM_AUTH=True)
|
||||
def test_mysql_update_params_with_iam() -> None:
|
||||
from superset.db_engine_specs.aws_iam import AWSIAMAuthMixin
|
||||
from superset.db_engine_specs.mysql import MySQLEngineSpec
|
||||
|
||||
database = MagicMock()
|
||||
database.encrypted_extra = json.dumps(
|
||||
{
|
||||
"aws_iam": {
|
||||
"enabled": True,
|
||||
"role_arn": "arn:aws:iam::123456789012:role/TestRole",
|
||||
"region": "us-east-1",
|
||||
"db_username": "superset_iam_user",
|
||||
}
|
||||
}
|
||||
)
|
||||
database.sqlalchemy_uri_decrypted = (
|
||||
"mysql://user@mydb.cluster-xyz.us-east-1.rds.amazonaws.com:3306/mydb"
|
||||
)
|
||||
|
||||
params: dict[str, Any] = {}
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
AWSIAMAuthMixin,
|
||||
"get_iam_credentials",
|
||||
return_value={
|
||||
"AccessKeyId": "ASIA...",
|
||||
"SecretAccessKey": "secret...",
|
||||
"SessionToken": "token...",
|
||||
},
|
||||
),
|
||||
patch.object(
|
||||
AWSIAMAuthMixin,
|
||||
"generate_rds_auth_token",
|
||||
return_value="iam-auth-token",
|
||||
),
|
||||
):
|
||||
MySQLEngineSpec.update_params_from_encrypted_extra(database, params)
|
||||
|
||||
assert "connect_args" in params
|
||||
assert params["connect_args"]["password"] == "iam-auth-token" # noqa: S105
|
||||
assert params["connect_args"]["user"] == "superset_iam_user"
|
||||
# Note: ssl_mode is not set because MySQL drivers don't support it.
|
||||
# SSL should be configured via the database's extra settings.
|
||||
|
||||
|
||||
@with_feature_flags(AWS_DATABASE_IAM_AUTH=True)
|
||||
def test_mysql_update_params_iam_uses_mysql_port() -> None:
|
||||
from superset.db_engine_specs.aws_iam import AWSIAMAuthMixin
|
||||
from superset.db_engine_specs.mysql import MySQLEngineSpec
|
||||
|
||||
database = MagicMock()
|
||||
database.encrypted_extra = json.dumps(
|
||||
{
|
||||
"aws_iam": {
|
||||
"enabled": True,
|
||||
"role_arn": "arn:aws:iam::123456789012:role/TestRole",
|
||||
"region": "us-east-1",
|
||||
"db_username": "superset_iam_user",
|
||||
}
|
||||
}
|
||||
)
|
||||
# URI without explicit port
|
||||
database.sqlalchemy_uri_decrypted = (
|
||||
"mysql://user@mydb.cluster-xyz.us-east-1.rds.amazonaws.com/mydb"
|
||||
)
|
||||
|
||||
params: dict[str, Any] = {}
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
AWSIAMAuthMixin,
|
||||
"get_iam_credentials",
|
||||
return_value={
|
||||
"AccessKeyId": "ASIA...",
|
||||
"SecretAccessKey": "secret...",
|
||||
"SessionToken": "token...",
|
||||
},
|
||||
),
|
||||
patch.object(
|
||||
AWSIAMAuthMixin,
|
||||
"generate_rds_auth_token",
|
||||
return_value="iam-auth-token",
|
||||
) as mock_gen_token,
|
||||
):
|
||||
MySQLEngineSpec.update_params_from_encrypted_extra(database, params)
|
||||
|
||||
# Should use default MySQL port 3306
|
||||
token_call_kwargs = mock_gen_token.call_args[1]
|
||||
assert token_call_kwargs["port"] == 3306
|
||||
|
||||
|
||||
def test_mysql_update_params_merges_remaining_encrypted_extra() -> None:
|
||||
from superset.db_engine_specs.mysql import MySQLEngineSpec
|
||||
|
||||
database = MagicMock()
|
||||
database.encrypted_extra = json.dumps(
|
||||
{
|
||||
"aws_iam": {"enabled": False},
|
||||
"pool_size": 10,
|
||||
}
|
||||
)
|
||||
|
||||
params: dict[str, Any] = {}
|
||||
MySQLEngineSpec.update_params_from_encrypted_extra(database, params)
|
||||
|
||||
assert "aws_iam" not in params
|
||||
assert params["pool_size"] == 10
|
||||
|
||||
|
||||
def test_mysql_update_params_invalid_json() -> None:
|
||||
from superset.db_engine_specs.mysql import MySQLEngineSpec
|
||||
|
||||
database = MagicMock()
|
||||
database.encrypted_extra = "not-valid-json"
|
||||
|
||||
params: dict[str, Any] = {}
|
||||
|
||||
with pytest.raises(json.JSONDecodeError):
|
||||
MySQLEngineSpec.update_params_from_encrypted_extra(database, params)
|
||||
|
||||
|
||||
def test_mysql_mask_encrypted_extra() -> None:
|
||||
from superset.db_engine_specs.mysql import MySQLEngineSpec
|
||||
|
||||
encrypted_extra = json.dumps(
|
||||
{
|
||||
"aws_iam": {
|
||||
"enabled": True,
|
||||
"role_arn": "arn:aws:iam::123456789012:role/SecretRole",
|
||||
"external_id": "secret-external-id-12345",
|
||||
"region": "us-east-1",
|
||||
"db_username": "superset_user",
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
masked = MySQLEngineSpec.mask_encrypted_extra(encrypted_extra)
|
||||
assert masked is not None
|
||||
|
||||
masked_config = json.loads(masked)
|
||||
|
||||
# role_arn and external_id should be masked
|
||||
assert (
|
||||
masked_config["aws_iam"]["role_arn"]
|
||||
!= "arn:aws:iam::123456789012:role/SecretRole"
|
||||
)
|
||||
assert masked_config["aws_iam"]["external_id"] != "secret-external-id-12345"
|
||||
|
||||
# Non-sensitive fields should remain unchanged
|
||||
assert masked_config["aws_iam"]["enabled"] is True
|
||||
assert masked_config["aws_iam"]["region"] == "us-east-1"
|
||||
assert masked_config["aws_iam"]["db_username"] == "superset_user"
|
||||
387
tests/unit_tests/db_engine_specs/test_redshift_iam.py
Normal file
387
tests/unit_tests/db_engine_specs/test_redshift_iam.py
Normal file
@@ -0,0 +1,387 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
# pylint: disable=import-outside-toplevel
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from superset.utils import json
|
||||
from tests.unit_tests.conftest import with_feature_flags
|
||||
|
||||
|
||||
def test_redshift_encrypted_extra_sensitive_fields() -> None:
|
||||
from superset.db_engine_specs.redshift import RedshiftEngineSpec
|
||||
|
||||
assert (
|
||||
"$.aws_iam.external_id" in RedshiftEngineSpec.encrypted_extra_sensitive_fields
|
||||
)
|
||||
assert "$.aws_iam.role_arn" in RedshiftEngineSpec.encrypted_extra_sensitive_fields
|
||||
|
||||
|
||||
def test_redshift_update_params_no_encrypted_extra() -> None:
|
||||
from superset.db_engine_specs.redshift import RedshiftEngineSpec
|
||||
|
||||
database = MagicMock()
|
||||
database.encrypted_extra = None
|
||||
|
||||
params: dict[str, Any] = {}
|
||||
RedshiftEngineSpec.update_params_from_encrypted_extra(database, params)
|
||||
|
||||
assert params == {}
|
||||
|
||||
|
||||
def test_redshift_update_params_empty_encrypted_extra() -> None:
|
||||
from superset.db_engine_specs.redshift import RedshiftEngineSpec
|
||||
|
||||
database = MagicMock()
|
||||
database.encrypted_extra = json.dumps({})
|
||||
|
||||
params: dict[str, Any] = {}
|
||||
RedshiftEngineSpec.update_params_from_encrypted_extra(database, params)
|
||||
|
||||
assert params == {}
|
||||
|
||||
|
||||
def test_redshift_update_params_iam_disabled() -> None:
|
||||
from superset.db_engine_specs.redshift import RedshiftEngineSpec
|
||||
|
||||
database = MagicMock()
|
||||
database.encrypted_extra = json.dumps(
|
||||
{
|
||||
"aws_iam": {
|
||||
"enabled": False,
|
||||
"role_arn": "arn:aws:iam::123456789012:role/TestRole",
|
||||
"region": "us-east-1",
|
||||
"workgroup_name": "my-workgroup",
|
||||
"db_name": "dev",
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
params: dict[str, Any] = {}
|
||||
RedshiftEngineSpec.update_params_from_encrypted_extra(database, params)
|
||||
|
||||
assert params == {}
|
||||
|
||||
|
||||
@with_feature_flags(AWS_DATABASE_IAM_AUTH=True)
|
||||
def test_redshift_update_params_with_iam() -> None:
|
||||
from superset.db_engine_specs.aws_iam import AWSIAMAuthMixin
|
||||
from superset.db_engine_specs.redshift import RedshiftEngineSpec
|
||||
|
||||
database = MagicMock()
|
||||
database.encrypted_extra = json.dumps(
|
||||
{
|
||||
"aws_iam": {
|
||||
"enabled": True,
|
||||
"role_arn": "arn:aws:iam::123456789012:role/RedshiftRole",
|
||||
"region": "us-east-1",
|
||||
"workgroup_name": "my-workgroup",
|
||||
"db_name": "dev",
|
||||
}
|
||||
}
|
||||
)
|
||||
database.sqlalchemy_uri_decrypted = (
|
||||
"redshift+psycopg2://user@my-workgroup.123456789012.us-east-1"
|
||||
".redshift-serverless.amazonaws.com:5439/dev"
|
||||
)
|
||||
|
||||
params: dict[str, Any] = {}
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
AWSIAMAuthMixin,
|
||||
"get_iam_credentials",
|
||||
return_value={
|
||||
"AccessKeyId": "ASIA...",
|
||||
"SecretAccessKey": "secret...",
|
||||
"SessionToken": "token...",
|
||||
},
|
||||
),
|
||||
patch.object(
|
||||
AWSIAMAuthMixin,
|
||||
"generate_redshift_credentials",
|
||||
return_value=("IAM:admin", "redshift-temp-password"),
|
||||
),
|
||||
):
|
||||
RedshiftEngineSpec.update_params_from_encrypted_extra(database, params)
|
||||
|
||||
assert "connect_args" in params
|
||||
assert params["connect_args"]["password"] == "redshift-temp-password" # noqa: S105
|
||||
assert params["connect_args"]["user"] == "IAM:admin"
|
||||
assert params["connect_args"]["sslmode"] == "verify-ca"
|
||||
|
||||
|
||||
@with_feature_flags(AWS_DATABASE_IAM_AUTH=True)
|
||||
def test_redshift_update_params_with_external_id() -> None:
|
||||
from superset.db_engine_specs.aws_iam import AWSIAMAuthMixin
|
||||
from superset.db_engine_specs.redshift import RedshiftEngineSpec
|
||||
|
||||
database = MagicMock()
|
||||
database.encrypted_extra = json.dumps(
|
||||
{
|
||||
"aws_iam": {
|
||||
"enabled": True,
|
||||
"role_arn": "arn:aws:iam::222222222222:role/CrossAccountRedshift",
|
||||
"external_id": "superset-prod-12345",
|
||||
"region": "us-west-2",
|
||||
"workgroup_name": "prod-workgroup",
|
||||
"db_name": "analytics",
|
||||
"session_duration": 1800,
|
||||
}
|
||||
}
|
||||
)
|
||||
database.sqlalchemy_uri_decrypted = (
|
||||
"redshift+psycopg2://user@prod-workgroup.222222222222.us-west-2"
|
||||
".redshift-serverless.amazonaws.com:5439/analytics"
|
||||
)
|
||||
|
||||
params: dict[str, Any] = {}
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
AWSIAMAuthMixin,
|
||||
"get_iam_credentials",
|
||||
return_value={
|
||||
"AccessKeyId": "ASIA...",
|
||||
"SecretAccessKey": "secret...",
|
||||
"SessionToken": "token...",
|
||||
},
|
||||
) as mock_get_creds,
|
||||
patch.object(
|
||||
AWSIAMAuthMixin,
|
||||
"generate_redshift_credentials",
|
||||
return_value=("IAM:admin", "redshift-temp-password"),
|
||||
),
|
||||
):
|
||||
RedshiftEngineSpec.update_params_from_encrypted_extra(database, params)
|
||||
|
||||
mock_get_creds.assert_called_once_with(
|
||||
role_arn="arn:aws:iam::222222222222:role/CrossAccountRedshift",
|
||||
region="us-west-2",
|
||||
external_id="superset-prod-12345",
|
||||
session_duration=1800,
|
||||
)
|
||||
|
||||
|
||||
def test_redshift_update_params_merges_remaining_encrypted_extra() -> None:
|
||||
from superset.db_engine_specs.redshift import RedshiftEngineSpec
|
||||
|
||||
database = MagicMock()
|
||||
database.encrypted_extra = json.dumps(
|
||||
{
|
||||
"aws_iam": {"enabled": False},
|
||||
"pool_size": 5,
|
||||
}
|
||||
)
|
||||
|
||||
params: dict[str, Any] = {}
|
||||
RedshiftEngineSpec.update_params_from_encrypted_extra(database, params)
|
||||
|
||||
assert "aws_iam" not in params
|
||||
assert params["pool_size"] == 5
|
||||
|
||||
|
||||
def test_redshift_update_params_invalid_json() -> None:
|
||||
from superset.db_engine_specs.redshift import RedshiftEngineSpec
|
||||
|
||||
database = MagicMock()
|
||||
database.encrypted_extra = "not-valid-json"
|
||||
|
||||
params: dict[str, Any] = {}
|
||||
|
||||
with pytest.raises(json.JSONDecodeError):
|
||||
RedshiftEngineSpec.update_params_from_encrypted_extra(database, params)
|
||||
|
||||
|
||||
def test_redshift_mask_encrypted_extra() -> None:
|
||||
from superset.db_engine_specs.redshift import RedshiftEngineSpec
|
||||
|
||||
encrypted_extra = json.dumps(
|
||||
{
|
||||
"aws_iam": {
|
||||
"enabled": True,
|
||||
"role_arn": "arn:aws:iam::123456789012:role/SecretRole",
|
||||
"external_id": "secret-external-id-12345",
|
||||
"region": "us-east-1",
|
||||
"workgroup_name": "my-workgroup",
|
||||
"db_name": "dev",
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
masked = RedshiftEngineSpec.mask_encrypted_extra(encrypted_extra)
|
||||
assert masked is not None
|
||||
|
||||
masked_config = json.loads(masked)
|
||||
|
||||
# role_arn and external_id should be masked
|
||||
assert (
|
||||
masked_config["aws_iam"]["role_arn"]
|
||||
!= "arn:aws:iam::123456789012:role/SecretRole"
|
||||
)
|
||||
assert masked_config["aws_iam"]["external_id"] != "secret-external-id-12345"
|
||||
|
||||
# Non-sensitive fields should remain unchanged
|
||||
assert masked_config["aws_iam"]["enabled"] is True
|
||||
assert masked_config["aws_iam"]["region"] == "us-east-1"
|
||||
assert masked_config["aws_iam"]["workgroup_name"] == "my-workgroup"
|
||||
assert masked_config["aws_iam"]["db_name"] == "dev"
|
||||
|
||||
|
||||
@with_feature_flags(AWS_DATABASE_IAM_AUTH=True)
|
||||
def test_redshift_update_params_with_iam_provisioned_cluster() -> None:
|
||||
from superset.db_engine_specs.aws_iam import AWSIAMAuthMixin
|
||||
from superset.db_engine_specs.redshift import RedshiftEngineSpec
|
||||
|
||||
database = MagicMock()
|
||||
database.encrypted_extra = json.dumps(
|
||||
{
|
||||
"aws_iam": {
|
||||
"enabled": True,
|
||||
"role_arn": "arn:aws:iam::123456789012:role/RedshiftRole",
|
||||
"region": "us-east-1",
|
||||
"cluster_identifier": "my-redshift-cluster",
|
||||
"db_username": "superset_user",
|
||||
"db_name": "analytics",
|
||||
}
|
||||
}
|
||||
)
|
||||
database.sqlalchemy_uri_decrypted = (
|
||||
"redshift+psycopg2://user@my-redshift-cluster.abc123.us-east-1"
|
||||
".redshift.amazonaws.com:5439/analytics"
|
||||
)
|
||||
|
||||
params: dict[str, Any] = {}
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
AWSIAMAuthMixin,
|
||||
"get_iam_credentials",
|
||||
return_value={
|
||||
"AccessKeyId": "ASIA...",
|
||||
"SecretAccessKey": "secret...",
|
||||
"SessionToken": "token...",
|
||||
},
|
||||
),
|
||||
patch.object(
|
||||
AWSIAMAuthMixin,
|
||||
"generate_redshift_cluster_credentials",
|
||||
return_value=("IAM:superset_user", "cluster-temp-password"),
|
||||
),
|
||||
):
|
||||
RedshiftEngineSpec.update_params_from_encrypted_extra(database, params)
|
||||
|
||||
assert "connect_args" in params
|
||||
assert params["connect_args"]["password"] == "cluster-temp-password" # noqa: S105
|
||||
assert params["connect_args"]["user"] == "IAM:superset_user"
|
||||
assert params["connect_args"]["sslmode"] == "verify-ca"
|
||||
|
||||
|
||||
@with_feature_flags(AWS_DATABASE_IAM_AUTH=True)
|
||||
def test_redshift_update_params_provisioned_cluster_with_external_id() -> None:
|
||||
from superset.db_engine_specs.aws_iam import AWSIAMAuthMixin
|
||||
from superset.db_engine_specs.redshift import RedshiftEngineSpec
|
||||
|
||||
database = MagicMock()
|
||||
database.encrypted_extra = json.dumps(
|
||||
{
|
||||
"aws_iam": {
|
||||
"enabled": True,
|
||||
"role_arn": "arn:aws:iam::222222222222:role/CrossAccountRedshift",
|
||||
"external_id": "superset-prod-12345",
|
||||
"region": "us-west-2",
|
||||
"cluster_identifier": "prod-cluster",
|
||||
"db_username": "analytics_user",
|
||||
"db_name": "prod_db",
|
||||
"session_duration": 1800,
|
||||
}
|
||||
}
|
||||
)
|
||||
database.sqlalchemy_uri_decrypted = (
|
||||
"redshift+psycopg2://user@prod-cluster.xyz789.us-west-2"
|
||||
".redshift.amazonaws.com:5439/prod_db"
|
||||
)
|
||||
|
||||
params: dict[str, Any] = {}
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
AWSIAMAuthMixin,
|
||||
"get_iam_credentials",
|
||||
return_value={
|
||||
"AccessKeyId": "ASIA...",
|
||||
"SecretAccessKey": "secret...",
|
||||
"SessionToken": "token...",
|
||||
},
|
||||
) as mock_get_creds,
|
||||
patch.object(
|
||||
AWSIAMAuthMixin,
|
||||
"generate_redshift_cluster_credentials",
|
||||
return_value=("IAM:analytics_user", "cluster-temp-password"),
|
||||
),
|
||||
):
|
||||
RedshiftEngineSpec.update_params_from_encrypted_extra(database, params)
|
||||
|
||||
mock_get_creds.assert_called_once_with(
|
||||
role_arn="arn:aws:iam::222222222222:role/CrossAccountRedshift",
|
||||
region="us-west-2",
|
||||
external_id="superset-prod-12345",
|
||||
session_duration=1800,
|
||||
)
|
||||
|
||||
|
||||
def test_redshift_mask_encrypted_extra_provisioned_cluster() -> None:
|
||||
from superset.db_engine_specs.redshift import RedshiftEngineSpec
|
||||
|
||||
encrypted_extra = json.dumps(
|
||||
{
|
||||
"aws_iam": {
|
||||
"enabled": True,
|
||||
"role_arn": "arn:aws:iam::123456789012:role/SecretRole",
|
||||
"external_id": "secret-external-id-12345",
|
||||
"region": "us-east-1",
|
||||
"cluster_identifier": "my-cluster",
|
||||
"db_username": "superset_user",
|
||||
"db_name": "analytics",
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
masked = RedshiftEngineSpec.mask_encrypted_extra(encrypted_extra)
|
||||
assert masked is not None
|
||||
|
||||
masked_config = json.loads(masked)
|
||||
|
||||
# role_arn and external_id should be masked
|
||||
assert (
|
||||
masked_config["aws_iam"]["role_arn"]
|
||||
!= "arn:aws:iam::123456789012:role/SecretRole"
|
||||
)
|
||||
assert masked_config["aws_iam"]["external_id"] != "secret-external-id-12345"
|
||||
|
||||
# Non-sensitive fields should remain unchanged
|
||||
assert masked_config["aws_iam"]["enabled"] is True
|
||||
assert masked_config["aws_iam"]["region"] == "us-east-1"
|
||||
assert masked_config["aws_iam"]["cluster_identifier"] == "my-cluster"
|
||||
assert masked_config["aws_iam"]["db_username"] == "superset_user"
|
||||
assert masked_config["aws_iam"]["db_name"] == "analytics"
|
||||
Reference in New Issue
Block a user