From ce6e8b919e79ab5b8d93cdc88e0a91ee9be57231 Mon Sep 17 00:00:00 2001 From: Prabhjyot Singh Date: Fri, 14 Nov 2025 14:28:11 -0500 Subject: [PATCH 1/9] AMBARI-26556: Add a way to encrypted configurations.json file in the ambari-agent's cache --- ambari-agent/conf/unix/ambari-agent.ini | 2 + .../main/python/ambari_agent/AmbariConfig.py | 16 +++++ .../ClusterAlertDefinitionsCache.py | 4 +- .../main/python/ambari_agent/ClusterCache.py | 64 +++++++++++++++++-- .../ambari_agent/ClusterConfigurationCache.py | 4 +- .../ClusterHostLevelParamsCache.py | 4 +- .../ambari_agent/ClusterMetadataCache.py | 2 +- .../ambari_agent/ClusterTopologyCache.py | 2 +- .../python/ambari_agent/InitializerModule.py | 14 +++- 9 files changed, 94 insertions(+), 18 deletions(-) diff --git a/ambari-agent/conf/unix/ambari-agent.ini b/ambari-agent/conf/unix/ambari-agent.ini index 2e403a00995..1779417e429 100644 --- a/ambari-agent/conf/unix/ambari-agent.ini +++ b/ambari-agent/conf/unix/ambari-agent.ini @@ -57,6 +57,8 @@ ssl_verify_cert=0 credential_lib_dir=/var/lib/ambari-agent/cred/lib credential_conf_dir=/var/lib/ambari-agent/cred/conf credential_shell_cmd=org.apache.hadoop.security.alias.CredentialShell +agent_secret=default-secret-change-me +agent_salt=ambari-agent-cache-salt [network] ; this option apply only for Agent communication diff --git a/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py b/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py index a61be081411..436e1c0cffb 100644 --- a/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py +++ b/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py @@ -463,6 +463,22 @@ def get_ca_cert_file_path(self): """ return self.get("security", "ca_cert_path", default="") + def get_agent_secret(self): + """ + Get agent secret used to authenticate with the server. + + :return: agent secret string + """ + return self.get('security', 'agent_secret', default="") + + def get_agent_salt(self): + """ + Get agent salt used for hashing/encryption. + + :return: agent salt string + """ + return self.get('security', 'agent_salt', default="") + @property def send_alert_changes_only(self): return bool(self.get("agent", "send_alert_changes_only", "0")) diff --git a/ambari-agent/src/main/python/ambari_agent/ClusterAlertDefinitionsCache.py b/ambari-agent/src/main/python/ambari_agent/ClusterAlertDefinitionsCache.py index bbee0b28d2d..3013386e4c4 100644 --- a/ambari-agent/src/main/python/ambari_agent/ClusterAlertDefinitionsCache.py +++ b/ambari-agent/src/main/python/ambari_agent/ClusterAlertDefinitionsCache.py @@ -34,13 +34,13 @@ class ClusterAlertDefinitionsCache(ClusterCache): differently for every host. """ - def __init__(self, cluster_cache_dir): + def __init__(self, cluster_cache_dir, secret=None, salt=None): """ Initializes the host level params cache. :param cluster_cache_dir: :return: """ - super(ClusterAlertDefinitionsCache, self).__init__(cluster_cache_dir) + super(ClusterAlertDefinitionsCache, self).__init__(cluster_cache_dir, secret, salt) def get_alert_definition_index_by_id(self, alert_dict, cluster_id, alert_id): definitions = alert_dict[cluster_id]["alertDefinitions"] diff --git a/ambari-agent/src/main/python/ambari_agent/ClusterCache.py b/ambari-agent/src/main/python/ambari_agent/ClusterCache.py index 2c924b1ed12..2b38a85d671 100644 --- a/ambari-agent/src/main/python/ambari_agent/ClusterCache.py +++ b/ambari-agent/src/main/python/ambari_agent/ClusterCache.py @@ -22,6 +22,11 @@ import os import threading from collections import defaultdict +from cryptography.fernet import Fernet +import base64 +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC + from ambari_agent.Utils import Utils @@ -38,7 +43,7 @@ class ClusterCache(dict): file_locks = defaultdict(threading.RLock) - def __init__(self, cluster_cache_dir): + def __init__(self, cluster_cache_dir, secret=None, salt=None): """ Initializes the cache. :param cluster_cache_dir: @@ -46,6 +51,8 @@ def __init__(self, cluster_cache_dir): """ self.cluster_cache_dir = cluster_cache_dir + self.secret = secret + self.salt = salt self.__current_cache_json_file = os.path.join( self.cluster_cache_dir, self.get_cache_name() + ".json" @@ -63,8 +70,10 @@ def __init__(self, cluster_cache_dir): try: with self.__file_lock: if os.path.isfile(self.__current_cache_json_file): - with open(self.__current_cache_json_file, "r") as fp: - cache_dict = json.load(fp) + with open(self.__current_cache_json_file, "rb") as fp: # Note: 'rb' for binary + encrypted_data = fp.read() + decrypted_json = self._decrypt_data(encrypted_data) + cache_dict = json.loads(decrypted_json) if os.path.isfile(self.__current_cache_hash_file): with open(self.__current_cache_hash_file, "r") as fp: @@ -83,6 +92,39 @@ def __init__(self, cluster_cache_dir): logger.exception(f"Loading saved cache for {self.__class__.__name__} failed") self.rewrite_cache({}, None) + def _get_encryption_key(self): + """ + Generate encryption key from a secret and PBKDF2-HMAC. + """ + kdf = PBKDF2HMAC( + algorithm=hashes.SHA256(), + length=32, + salt=self.salt.encode('utf-8'), + iterations=100000, + ) + + key = base64.urlsafe_b64encode(kdf.derive(self.secret.encode())) + return Fernet(key) + + def _is_encryption_enabled(self): + return not self.secret or not self.salt + + def _encrypt_data(self, data): + """Encrypt string data""" + if self._is_encryption_enabled(): + return data + else: + fernet = self._get_encryption_key() + return fernet.encrypt(data.encode()) + + def _decrypt_data(self, encrypted_data): + """Decrypt encrypted bytes to string""" + if self._is_encryption_enabled(): + return encrypted_data + else: + fernet = self._get_encryption_key() + return fernet.decrypt(encrypted_data).decode() + def get_cluster_indepedent_data(self): return self[ClusterCache.COMMON_DATA_CLUSTER] @@ -141,8 +183,16 @@ def persist_cache(self, cache_hash): os.makedirs(self.cluster_cache_dir) with self.__file_lock: - with open(self.__current_cache_json_file, "w") as f: - json.dump(self, f, indent=2) + # Encrypt JSON data + json_str = json.dumps(self, indent=2) + encrypted_json = self._encrypt_data(json_str) + + if self._is_encryption_enabled(): + with open(self.__current_cache_json_file, "w") as f: + f.write(encrypted_json) + else: + with open(self.__current_cache_json_file, "wb") as f: # Note: 'wb' for binary + f.write(encrypted_json) if self.hash is not None: with open(self.__current_cache_hash_file, "w") as fp: @@ -173,7 +223,7 @@ def get_cache_name(self): raise NotImplemented() def __deepcopy__(self, memo): - return self.__class__(self.cluster_cache_dir) + return self.__class__(self.cluster_cache_dir, self.secret, self.salt) def __copy__(self): - return self.__class__(self.cluster_cache_dir) + return self.__class__(self.cluster_cache_dir, self.secret, self.salt) diff --git a/ambari-agent/src/main/python/ambari_agent/ClusterConfigurationCache.py b/ambari-agent/src/main/python/ambari_agent/ClusterConfigurationCache.py index ec5d9044139..f8f1069fca6 100644 --- a/ambari-agent/src/main/python/ambari_agent/ClusterConfigurationCache.py +++ b/ambari-agent/src/main/python/ambari_agent/ClusterConfigurationCache.py @@ -31,13 +31,13 @@ class ClusterConfigurationCache(ClusterCache): configuration properties. """ - def __init__(self, cluster_cache_dir): + def __init__(self, cluster_cache_dir, secret=None, salt=None): """ Initializes the configuration cache. :param cluster_cache_dir: directory the changed json are saved :return: """ - super(ClusterConfigurationCache, self).__init__(cluster_cache_dir) + super(ClusterConfigurationCache, self).__init__(cluster_cache_dir, secret, salt) def get_cache_name(self): return "configurations" diff --git a/ambari-agent/src/main/python/ambari_agent/ClusterHostLevelParamsCache.py b/ambari-agent/src/main/python/ambari_agent/ClusterHostLevelParamsCache.py index e31717649fa..0ca5792bfb1 100644 --- a/ambari-agent/src/main/python/ambari_agent/ClusterHostLevelParamsCache.py +++ b/ambari-agent/src/main/python/ambari_agent/ClusterHostLevelParamsCache.py @@ -34,13 +34,13 @@ class ClusterHostLevelParamsCache(ClusterCache): differently for every host. """ - def __init__(self, cluster_cache_dir): + def __init__(self, cluster_cache_dir, secret=None, salt=None): """ Initializes the host level params cache. :param cluster_cache_dir: :return: """ - super(ClusterHostLevelParamsCache, self).__init__(cluster_cache_dir) + super(ClusterHostLevelParamsCache, self).__init__(cluster_cache_dir, secret, salt) def get_cache_name(self): return "host_level_params" diff --git a/ambari-agent/src/main/python/ambari_agent/ClusterMetadataCache.py b/ambari-agent/src/main/python/ambari_agent/ClusterMetadataCache.py index db5d70fcdbb..006154502fb 100644 --- a/ambari-agent/src/main/python/ambari_agent/ClusterMetadataCache.py +++ b/ambari-agent/src/main/python/ambari_agent/ClusterMetadataCache.py @@ -38,7 +38,7 @@ def __init__(self, cluster_cache_dir, config): :return: """ self.config = config - super(ClusterMetadataCache, self).__init__(cluster_cache_dir) + super(ClusterMetadataCache, self).__init__(cluster_cache_dir, config.get_agent_secret(), config.get_agent_salt()) def on_cache_update(self): try: diff --git a/ambari-agent/src/main/python/ambari_agent/ClusterTopologyCache.py b/ambari-agent/src/main/python/ambari_agent/ClusterTopologyCache.py index 22abd1c789c..14cdd1ecbf1 100644 --- a/ambari-agent/src/main/python/ambari_agent/ClusterTopologyCache.py +++ b/ambari-agent/src/main/python/ambari_agent/ClusterTopologyCache.py @@ -50,7 +50,7 @@ def __init__(self, cluster_cache_dir, config): self.cluster_local_components = {} self.cluster_host_info = None self.component_version_map = {} - super(ClusterTopologyCache, self).__init__(cluster_cache_dir) + super(ClusterTopologyCache, self).__init__(cluster_cache_dir, config.get_agent_secret(), config.get_agent_salt()) def get_cache_name(self): return "topology" diff --git a/ambari-agent/src/main/python/ambari_agent/InitializerModule.py b/ambari-agent/src/main/python/ambari_agent/InitializerModule.py index 72c8b620f01..8e86ddf26fe 100644 --- a/ambari-agent/src/main/python/ambari_agent/InitializerModule.py +++ b/ambari-agent/src/main/python/ambari_agent/InitializerModule.py @@ -90,11 +90,19 @@ def init(self): self.config.cluster_cache_dir, self.config ) self.host_level_params_cache = ClusterHostLevelParamsCache( - self.config.cluster_cache_dir + self.config.cluster_cache_dir, + self.config.get_agent_secret(), + self.config.get_agent_salt() + ) + self.configurations_cache = ClusterConfigurationCache( + self.config.cluster_cache_dir, + self.config.get_agent_secret(), + self.config.get_agent_salt() ) - self.configurations_cache = ClusterConfigurationCache(self.config.cluster_cache_dir) self.alert_definitions_cache = ClusterAlertDefinitionsCache( - self.config.cluster_cache_dir + self.config.cluster_cache_dir, + self.config.get_agent_secret(), + self.config.get_agent_salt() ) self.configuration_builder = ConfigurationBuilder(self) self.stale_alerts_monitor = StaleAlertsMonitor(self) From d7c33c745d7b196dfa867fe6eca28e255fcb199a Mon Sep 17 00:00:00 2001 From: Prabhjyot Singh Date: Fri, 14 Nov 2025 21:48:54 -0500 Subject: [PATCH 2/9] add unit test --- .../python/ambari_agent/TestClusterCache.py | 64 +++++++++++++++++++ setup.py | 2 +- 2 files changed, 65 insertions(+), 1 deletion(-) create mode 100644 ambari-agent/src/test/python/ambari_agent/TestClusterCache.py diff --git a/ambari-agent/src/test/python/ambari_agent/TestClusterCache.py b/ambari-agent/src/test/python/ambari_agent/TestClusterCache.py new file mode 100644 index 00000000000..8973b0d7478 --- /dev/null +++ b/ambari-agent/src/test/python/ambari_agent/TestClusterCache.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 + +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from ambari_agent import main + +main.MEMORY_LEAK_DEBUG_FILEPATH = "/tmp/memory_leak_debug.out" +import os +import tempfile +import shutil +from unittest import TestCase + +from ambari_agent.ClusterCache import ClusterCache +from mock.mock import patch, MagicMock +from ambari_commons import OSCheck +from only_for_platform import os_distro_value + + +class TestCertGeneration(TestCase): + @patch.object(OSCheck, "os_distribution", new=MagicMock(return_value=os_distro_value)) + def setUp(self): + self.tmpdir = tempfile.mkdtemp() + + + @patch.object(os, "chmod") + def test_enc(self, chmod_mock): + cluster_cache_dir = self.tmpdir + "/cluster_cache_dir" + clusterCacheEncrypted = DummyClusterCache(cluster_cache_dir, "super_secret", "super_secret_salt") + string_json = '{"a": 1, "b": 2}' + encrypted = clusterCacheEncrypted._encrypt_data(string_json) + self.assertNotEqual(string_json, encrypted) + decrypted = clusterCacheEncrypted._decrypt_data(encrypted) + self.assertEqual(string_json, decrypted) + + clusterCacheUnEncrypted = DummyClusterCache(cluster_cache_dir) + string_json = '{"a": 1, "b": 2}' + encrypted = clusterCacheUnEncrypted._encrypt_data(string_json) + self.assertEqual(string_json, encrypted) + decrypted = clusterCacheUnEncrypted._decrypt_data(encrypted) + self.assertEqual(string_json, decrypted) + + def tearDown(self): + shutil.rmtree(self.tmpdir) + +class DummyClusterCache(ClusterCache): + def get_cache_name(self): + # Dummy implementation just for tests + return "configuration" diff --git a/setup.py b/setup.py index 452e098cd3c..05589d546bf 100755 --- a/setup.py +++ b/setup.py @@ -131,7 +131,7 @@ def get_version(): + get_ambari_server_stack_package() + get_extra_common_packages(), package_dir=create_package_dir_map(), - install_requires=["coilmq==1.0.1"], + install_requires=["coilmq==1.0.1", "cryptography==46.0.3"], include_package_data=True, long_description="The Apache Ambari project is aimed at making Hadoop management simpler by developing software for provisioning, managing, and monitoring Apache Hadoop clusters. " "Ambari provides an intuitive, easy-to-use Hadoop management web UI backed by its RESTful APIs.", From 7204f3286ef6f5a04e61fd77dc1bbd4f26352da5 Mon Sep 17 00:00:00 2001 From: Prabhjyot Singh Date: Fri, 14 Nov 2025 22:00:10 -0500 Subject: [PATCH 3/9] fix unit test and add some comments --- .../python/ambari_agent/TestClusterCache.py | 58 ++++++++++++++++--- 1 file changed, 50 insertions(+), 8 deletions(-) diff --git a/ambari-agent/src/test/python/ambari_agent/TestClusterCache.py b/ambari-agent/src/test/python/ambari_agent/TestClusterCache.py index 8973b0d7478..bbe8fec074a 100644 --- a/ambari-agent/src/test/python/ambari_agent/TestClusterCache.py +++ b/ambari-agent/src/test/python/ambari_agent/TestClusterCache.py @@ -33,32 +33,74 @@ class TestCertGeneration(TestCase): + """ + Test suite for verifying encryption behavior of ClusterCache. + + It covers: + - encryption/decryption round-trip when secret/salt are provided + - behavior when encryption is effectively disabled (no secret/salt) + """ + + # so that ClusterCache initialization is OS-agnostic in this test. @patch.object(OSCheck, "os_distribution", new=MagicMock(return_value=os_distro_value)) def setUp(self): + # Create a temporary directory that will be cleaned up after each test. self.tmpdir = tempfile.mkdtemp() + cluster_cache_dir = self.tmpdir + "/cluster_cache_dir" + # Instance with encryption enabled (secret and salt provided). + self.cluster_cache_encrypted = DummyClusterCache( + cluster_cache_dir, + "super_secret", + "super_secret_salt" + ) + + # Instance with encryption disabled (no secret or salt). + self.cluster_cache_unencrypted = DummyClusterCache(cluster_cache_dir) @patch.object(os, "chmod") def test_enc(self, chmod_mock): - cluster_cache_dir = self.tmpdir + "/cluster_cache_dir" - clusterCacheEncrypted = DummyClusterCache(cluster_cache_dir, "super_secret", "super_secret_salt") + """ + Verify that: + - encrypted instance changes the data and can restore it back + - unencrypted instance is a no-op for encrypt/decrypt + """ string_json = '{"a": 1, "b": 2}' - encrypted = clusterCacheEncrypted._encrypt_data(string_json) + + # Encrypted cache should not store raw JSON. + encrypted = self.cluster_cache_encrypted._encrypt_data(string_json) self.assertNotEqual(string_json, encrypted) - decrypted = clusterCacheEncrypted._decrypt_data(encrypted) + # Round-trip must produce original JSON. + decrypted = self.cluster_cache_encrypted._decrypt_data(encrypted) self.assertEqual(string_json, decrypted) - clusterCacheUnEncrypted = DummyClusterCache(cluster_cache_dir) + # For unencrypted cache, encrypt/decrypt should behave as pass-through. string_json = '{"a": 1, "b": 2}' - encrypted = clusterCacheUnEncrypted._encrypt_data(string_json) + encrypted = self.cluster_cache_unencrypted._encrypt_data(string_json) self.assertEqual(string_json, encrypted) - decrypted = clusterCacheUnEncrypted._decrypt_data(encrypted) + decrypted = self.cluster_cache_unencrypted._decrypt_data(encrypted) self.assertEqual(string_json, decrypted) + @patch.object(os, "chmod") + def test_encryption_enable(self, chmod_mock): + """ + Verify that _is_encryption_enabled reflects whether secret/salt were provided. + """ + # When secret/salt are given, encryption flag should reflect enabled status. + self.assertFalse(self.cluster_cache_encrypted._is_encryption_enabled()) + + # When no secret/salt, encryption should be reported as disabled. + self.assertTrue(self.cluster_cache_unencrypted._is_encryption_enabled()) + def tearDown(self): shutil.rmtree(self.tmpdir) class DummyClusterCache(ClusterCache): + """ + Minimal ClusterCache subclass used only for unit testing. + + It overrides get_cache_name to avoid depending on real production cache names. + """ def get_cache_name(self): - # Dummy implementation just for tests + # Dummy implementation just for tests. return "configuration" From 75ca12e32eaa11315bac2148c5ef4874976b89de Mon Sep 17 00:00:00 2001 From: Prabhjyot Singh Date: Fri, 14 Nov 2025 22:13:41 -0500 Subject: [PATCH 4/9] add comment for the new value --- ambari-agent/conf/unix/ambari-agent.ini | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/ambari-agent/conf/unix/ambari-agent.ini b/ambari-agent/conf/unix/ambari-agent.ini index 1779417e429..884f5febc45 100644 --- a/ambari-agent/conf/unix/ambari-agent.ini +++ b/ambari-agent/conf/unix/ambari-agent.ini @@ -57,8 +57,11 @@ ssl_verify_cert=0 credential_lib_dir=/var/lib/ambari-agent/cred/lib credential_conf_dir=/var/lib/ambari-agent/cred/conf credential_shell_cmd=org.apache.hadoop.security.alias.CredentialShell -agent_secret=default-secret-change-me -agent_salt=ambari-agent-cache-salt +; Both agent_secret and agent_salt are required if the user wants to enable encryption, and they +; should be changed from the default values for production clusters. After enabling this for the +; first time, the user will also need to clear the cache directory so it can be rebuilt in an encrypted form. +; agent_secret=default-secret-change-me +; agent_salt=ambari-agent-cache-salt [network] ; this option apply only for Agent communication @@ -80,4 +83,4 @@ idle_interval_max=10 [logging] -syslog_enabled=0 \ No newline at end of file +syslog_enabled=0 From 3cc00bb846d809c918e46ebb828cb783fe24f834 Mon Sep 17 00:00:00 2001 From: Prabhjyot Singh Date: Tue, 2 Dec 2025 16:14:06 -0500 Subject: [PATCH 5/9] fix cryptography installation script --- ambari-agent/conf/unix/install-helper.sh | 2 ++ setup.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/ambari-agent/conf/unix/install-helper.sh b/ambari-agent/conf/unix/install-helper.sh index cc73ad059d1..788e906e068 100644 --- a/ambari-agent/conf/unix/install-helper.sh +++ b/ambari-agent/conf/unix/install-helper.sh @@ -168,6 +168,8 @@ do_install(){ else ln -s "${ambari_python}" "${PYTHON_WRAPER_TARGET}" + "$ambari_python" -m pip install "cryptography==46.0.3" + if [ -f ${bak} ]; then if [ -f "${upgrade_agent_configs_script}" ]; then ${upgrade_agent_configs_script} diff --git a/setup.py b/setup.py index 05589d546bf..452e098cd3c 100755 --- a/setup.py +++ b/setup.py @@ -131,7 +131,7 @@ def get_version(): + get_ambari_server_stack_package() + get_extra_common_packages(), package_dir=create_package_dir_map(), - install_requires=["coilmq==1.0.1", "cryptography==46.0.3"], + install_requires=["coilmq==1.0.1"], include_package_data=True, long_description="The Apache Ambari project is aimed at making Hadoop management simpler by developing software for provisioning, managing, and monitoring Apache Hadoop clusters. " "Ambari provides an intuitive, easy-to-use Hadoop management web UI backed by its RESTful APIs.", From 6873e63a200835e74adbf1eb39a23b91054baa6a Mon Sep 17 00:00:00 2001 From: Prabhjyot Singh Date: Thu, 4 Dec 2025 09:32:21 -0500 Subject: [PATCH 6/9] use ambari_pyaes instead of downloading a new one --- ambari-agent/conf/unix/install-helper.sh | 4 +- .../main/python/ambari_agent/ClusterCache.py | 90 +++++++++++++------ 2 files changed, 66 insertions(+), 28 deletions(-) diff --git a/ambari-agent/conf/unix/install-helper.sh b/ambari-agent/conf/unix/install-helper.sh index 788e906e068..929a6eab89e 100644 --- a/ambari-agent/conf/unix/install-helper.sh +++ b/ambari-agent/conf/unix/install-helper.sh @@ -137,7 +137,7 @@ do_install(){ mv /etc/ambari-agent/conf.save /etc/ambari-agent/conf_$(date '+%d_%m_%y_%H_%M').save fi - # these symlinks (or directories) where created in ambari releases prior to ambari-2.6.2. Do clean up. + # these symlinks (or directories) where created in ambari releases prior to ambari-2.6.2. Do clean up. rm -rf "${OLD_COMMON_DIR}" "${OLD_RESOURCE_MANAGEMENT_DIR}" "${OLD_JINJA_DIR}" "${OLD_SIMPLEJSON_DIR}" "${OLD_OLD_COMMON_DIR}" "${OLD_AMBARI_AGENT_DIR}" # setting up /usr/sbin/ambari-agent symlink @@ -168,8 +168,6 @@ do_install(){ else ln -s "${ambari_python}" "${PYTHON_WRAPER_TARGET}" - "$ambari_python" -m pip install "cryptography==46.0.3" - if [ -f ${bak} ]; then if [ -f "${upgrade_agent_configs_script}" ]; then ${upgrade_agent_configs_script} diff --git a/ambari-agent/src/main/python/ambari_agent/ClusterCache.py b/ambari-agent/src/main/python/ambari_agent/ClusterCache.py index 2b38a85d671..6fb5c74bdd7 100644 --- a/ambari-agent/src/main/python/ambari_agent/ClusterCache.py +++ b/ambari-agent/src/main/python/ambari_agent/ClusterCache.py @@ -22,10 +22,8 @@ import os import threading from collections import defaultdict -from cryptography.fernet import Fernet -import base64 -from cryptography.hazmat.primitives import hashes -from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC +import ambari_pyaes +from ambari_pbkdf2.pbkdf2 import PBKDF2 from ambari_agent.Utils import Utils @@ -92,19 +90,67 @@ def __init__(self, cluster_cache_dir, secret=None, salt=None): logger.exception(f"Loading saved cache for {self.__class__.__name__} failed") self.rewrite_cache({}, None) - def _get_encryption_key(self): - """ - Generate encryption key from a secret and PBKDF2-HMAC. - """ - kdf = PBKDF2HMAC( - algorithm=hashes.SHA256(), - length=32, - salt=self.salt.encode('utf-8'), - iterations=100000, + def encrypt(self, plaintext, encryption_key): + salt = os.urandom(16) + iv = os.urandom(16) + + key = PBKDF2(encryption_key, salt, iterations=65536).read(16) + aes = ambari_pyaes.AESModeOfOperationCBC(key, iv=iv) + + # ensure bytes + if not isinstance(plaintext, bytes): + plaintext = plaintext.encode() + + # PKCS7 pad + padded = ambari_pyaes.util.append_PKCS7_padding(plaintext) + + # CBC encrypt block-by-block + ciphertext = b"" + for i in range(0, len(padded), 16): + block = padded[i:i + 16] + encrypted_block = aes.encrypt(block) # must be exactly 16 bytes + ciphertext += encrypted_block + + inner = "::".join([ + salt.hex(), + iv.hex(), + ciphertext.hex() + ]).encode() + + return f"${{enc=aes128_hex, value={inner.hex()}}}" + + def decrypt(self, encrypted_value, encryption_key): + if isinstance(encrypted_value, bytes): + try: + ev_str = encrypted_value.decode() + except Exception: + ev_str = None + else: + ev_str = encrypted_value + + if not ev_str or "value=" not in ev_str: + return encrypted_value + + enc_text = encrypted_value.split("value=")[1][:-1] + # salt::iv::ciphertext(hex) + salt_hex, iv_hex, data_hex = ( + bytes.fromhex(part) + for part in bytes.fromhex(enc_text).decode().split("::") ) - key = base64.urlsafe_b64encode(kdf.derive(self.secret.encode())) - return Fernet(key) + key = PBKDF2(encryption_key, salt_hex, iterations=65536).read(16) + aes = ambari_pyaes.AESModeOfOperationCBC(key, iv=iv_hex) + + data = data_hex + + # Decrypt block-by-block (required) + plaintext = b"" + for i in range(0, len(data), 16): + block = data[i:i + 16] + plaintext += aes.decrypt(block) + + # Remove padding + return ambari_pyaes.util.strip_PKCS7_padding(plaintext) def _is_encryption_enabled(self): return not self.secret or not self.salt @@ -114,16 +160,14 @@ def _encrypt_data(self, data): if self._is_encryption_enabled(): return data else: - fernet = self._get_encryption_key() - return fernet.encrypt(data.encode()) + return self.encrypt(data.encode(), self.secret) def _decrypt_data(self, encrypted_data): """Decrypt encrypted bytes to string""" if self._is_encryption_enabled(): return encrypted_data else: - fernet = self._get_encryption_key() - return fernet.decrypt(encrypted_data).decode() + return self.decrypt(encrypted_data, self.secret).decode() def get_cluster_indepedent_data(self): return self[ClusterCache.COMMON_DATA_CLUSTER] @@ -187,12 +231,8 @@ def persist_cache(self, cache_hash): json_str = json.dumps(self, indent=2) encrypted_json = self._encrypt_data(json_str) - if self._is_encryption_enabled(): - with open(self.__current_cache_json_file, "w") as f: - f.write(encrypted_json) - else: - with open(self.__current_cache_json_file, "wb") as f: # Note: 'wb' for binary - f.write(encrypted_json) + with open(self.__current_cache_json_file, "w") as f: + f.write(encrypted_json) if self.hash is not None: with open(self.__current_cache_hash_file, "w") as fp: From 50b907f54d3dff9cf99c4ec0dc09820a0518c3e7 Mon Sep 17 00:00:00 2001 From: Prabhjyot Singh Date: Thu, 4 Dec 2025 09:36:16 -0500 Subject: [PATCH 7/9] revert install-helper.sh --- ambari-agent/conf/unix/install-helper.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ambari-agent/conf/unix/install-helper.sh b/ambari-agent/conf/unix/install-helper.sh index 929a6eab89e..cc73ad059d1 100644 --- a/ambari-agent/conf/unix/install-helper.sh +++ b/ambari-agent/conf/unix/install-helper.sh @@ -137,7 +137,7 @@ do_install(){ mv /etc/ambari-agent/conf.save /etc/ambari-agent/conf_$(date '+%d_%m_%y_%H_%M').save fi - # these symlinks (or directories) where created in ambari releases prior to ambari-2.6.2. Do clean up. + # these symlinks (or directories) where created in ambari releases prior to ambari-2.6.2. Do clean up. rm -rf "${OLD_COMMON_DIR}" "${OLD_RESOURCE_MANAGEMENT_DIR}" "${OLD_JINJA_DIR}" "${OLD_SIMPLEJSON_DIR}" "${OLD_OLD_COMMON_DIR}" "${OLD_AMBARI_AGENT_DIR}" # setting up /usr/sbin/ambari-agent symlink From 9f9d9bfc437171a9bd48af0c822ae4621a4b4256 Mon Sep 17 00:00:00 2001 From: Prabhjyot Singh Date: Thu, 4 Dec 2025 10:05:07 -0500 Subject: [PATCH 8/9] fix typo --- ambari-agent/src/main/python/ambari_agent/ClusterCache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ambari-agent/src/main/python/ambari_agent/ClusterCache.py b/ambari-agent/src/main/python/ambari_agent/ClusterCache.py index 6fb5c74bdd7..990ba300e04 100644 --- a/ambari-agent/src/main/python/ambari_agent/ClusterCache.py +++ b/ambari-agent/src/main/python/ambari_agent/ClusterCache.py @@ -131,7 +131,7 @@ def decrypt(self, encrypted_value, encryption_key): if not ev_str or "value=" not in ev_str: return encrypted_value - enc_text = encrypted_value.split("value=")[1][:-1] + enc_text = ev_str.split("value=")[1][:-1] # salt::iv::ciphertext(hex) salt_hex, iv_hex, data_hex = ( bytes.fromhex(part) From 19f722af919cf51e6ad25b3b145ca793ff5fab53 Mon Sep 17 00:00:00 2001 From: Prabhjyot Singh Date: Fri, 5 Dec 2025 12:14:32 -0500 Subject: [PATCH 9/9] AMBARI-26556: remove salt from config and make it random --- ambari-agent/conf/unix/ambari-agent.ini | 3 +-- .../main/python/ambari_agent/AmbariConfig.py | 8 -------- .../ClusterAlertDefinitionsCache.py | 4 ++-- .../main/python/ambari_agent/ClusterCache.py | 9 ++++----- .../ambari_agent/ClusterConfigurationCache.py | 4 ++-- .../ClusterHostLevelParamsCache.py | 4 ++-- .../ambari_agent/ClusterMetadataCache.py | 2 +- .../ambari_agent/ClusterTopologyCache.py | 2 +- .../python/ambari_agent/InitializerModule.py | 9 +++------ .../python/ambari_agent/TestClusterCache.py | 19 +++++++++---------- 10 files changed, 25 insertions(+), 39 deletions(-) diff --git a/ambari-agent/conf/unix/ambari-agent.ini b/ambari-agent/conf/unix/ambari-agent.ini index 884f5febc45..d792a288743 100644 --- a/ambari-agent/conf/unix/ambari-agent.ini +++ b/ambari-agent/conf/unix/ambari-agent.ini @@ -57,11 +57,10 @@ ssl_verify_cert=0 credential_lib_dir=/var/lib/ambari-agent/cred/lib credential_conf_dir=/var/lib/ambari-agent/cred/conf credential_shell_cmd=org.apache.hadoop.security.alias.CredentialShell -; Both agent_secret and agent_salt are required if the user wants to enable encryption, and they +; agent_secret required if the user wants to enable encryption, and they ; should be changed from the default values for production clusters. After enabling this for the ; first time, the user will also need to clear the cache directory so it can be rebuilt in an encrypted form. ; agent_secret=default-secret-change-me -; agent_salt=ambari-agent-cache-salt [network] ; this option apply only for Agent communication diff --git a/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py b/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py index f47c67d3386..c37ac84611b 100644 --- a/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py +++ b/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py @@ -477,14 +477,6 @@ def get_agent_secret(self): """ return self.get('security', 'agent_secret', default="") - def get_agent_salt(self): - """ - Get agent salt used for hashing/encryption. - - :return: agent salt string - """ - return self.get('security', 'agent_salt', default="") - @property def send_alert_changes_only(self): return bool(self.get("agent", "send_alert_changes_only", "0")) diff --git a/ambari-agent/src/main/python/ambari_agent/ClusterAlertDefinitionsCache.py b/ambari-agent/src/main/python/ambari_agent/ClusterAlertDefinitionsCache.py index 3013386e4c4..1100ffc0b62 100644 --- a/ambari-agent/src/main/python/ambari_agent/ClusterAlertDefinitionsCache.py +++ b/ambari-agent/src/main/python/ambari_agent/ClusterAlertDefinitionsCache.py @@ -34,13 +34,13 @@ class ClusterAlertDefinitionsCache(ClusterCache): differently for every host. """ - def __init__(self, cluster_cache_dir, secret=None, salt=None): + def __init__(self, cluster_cache_dir, secret=None): """ Initializes the host level params cache. :param cluster_cache_dir: :return: """ - super(ClusterAlertDefinitionsCache, self).__init__(cluster_cache_dir, secret, salt) + super(ClusterAlertDefinitionsCache, self).__init__(cluster_cache_dir, secret) def get_alert_definition_index_by_id(self, alert_dict, cluster_id, alert_id): definitions = alert_dict[cluster_id]["alertDefinitions"] diff --git a/ambari-agent/src/main/python/ambari_agent/ClusterCache.py b/ambari-agent/src/main/python/ambari_agent/ClusterCache.py index 990ba300e04..8ea93c09423 100644 --- a/ambari-agent/src/main/python/ambari_agent/ClusterCache.py +++ b/ambari-agent/src/main/python/ambari_agent/ClusterCache.py @@ -41,7 +41,7 @@ class ClusterCache(dict): file_locks = defaultdict(threading.RLock) - def __init__(self, cluster_cache_dir, secret=None, salt=None): + def __init__(self, cluster_cache_dir, secret=None): """ Initializes the cache. :param cluster_cache_dir: @@ -50,7 +50,6 @@ def __init__(self, cluster_cache_dir, secret=None, salt=None): self.cluster_cache_dir = cluster_cache_dir self.secret = secret - self.salt = salt self.__current_cache_json_file = os.path.join( self.cluster_cache_dir, self.get_cache_name() + ".json" @@ -153,7 +152,7 @@ def decrypt(self, encrypted_value, encryption_key): return ambari_pyaes.util.strip_PKCS7_padding(plaintext) def _is_encryption_enabled(self): - return not self.secret or not self.salt + return not self.secret def _encrypt_data(self, data): """Encrypt string data""" @@ -263,7 +262,7 @@ def get_cache_name(self): raise NotImplemented() def __deepcopy__(self, memo): - return self.__class__(self.cluster_cache_dir, self.secret, self.salt) + return self.__class__(self.cluster_cache_dir, self.secret) def __copy__(self): - return self.__class__(self.cluster_cache_dir, self.secret, self.salt) + return self.__class__(self.cluster_cache_dir, self.secret) diff --git a/ambari-agent/src/main/python/ambari_agent/ClusterConfigurationCache.py b/ambari-agent/src/main/python/ambari_agent/ClusterConfigurationCache.py index f8f1069fca6..ec8a796da11 100644 --- a/ambari-agent/src/main/python/ambari_agent/ClusterConfigurationCache.py +++ b/ambari-agent/src/main/python/ambari_agent/ClusterConfigurationCache.py @@ -31,13 +31,13 @@ class ClusterConfigurationCache(ClusterCache): configuration properties. """ - def __init__(self, cluster_cache_dir, secret=None, salt=None): + def __init__(self, cluster_cache_dir, secret=None): """ Initializes the configuration cache. :param cluster_cache_dir: directory the changed json are saved :return: """ - super(ClusterConfigurationCache, self).__init__(cluster_cache_dir, secret, salt) + super(ClusterConfigurationCache, self).__init__(cluster_cache_dir, secret) def get_cache_name(self): return "configurations" diff --git a/ambari-agent/src/main/python/ambari_agent/ClusterHostLevelParamsCache.py b/ambari-agent/src/main/python/ambari_agent/ClusterHostLevelParamsCache.py index 0ca5792bfb1..21ce4a2e259 100644 --- a/ambari-agent/src/main/python/ambari_agent/ClusterHostLevelParamsCache.py +++ b/ambari-agent/src/main/python/ambari_agent/ClusterHostLevelParamsCache.py @@ -34,13 +34,13 @@ class ClusterHostLevelParamsCache(ClusterCache): differently for every host. """ - def __init__(self, cluster_cache_dir, secret=None, salt=None): + def __init__(self, cluster_cache_dir, secret=None): """ Initializes the host level params cache. :param cluster_cache_dir: :return: """ - super(ClusterHostLevelParamsCache, self).__init__(cluster_cache_dir, secret, salt) + super(ClusterHostLevelParamsCache, self).__init__(cluster_cache_dir, secret) def get_cache_name(self): return "host_level_params" diff --git a/ambari-agent/src/main/python/ambari_agent/ClusterMetadataCache.py b/ambari-agent/src/main/python/ambari_agent/ClusterMetadataCache.py index 006154502fb..840da4c8503 100644 --- a/ambari-agent/src/main/python/ambari_agent/ClusterMetadataCache.py +++ b/ambari-agent/src/main/python/ambari_agent/ClusterMetadataCache.py @@ -38,7 +38,7 @@ def __init__(self, cluster_cache_dir, config): :return: """ self.config = config - super(ClusterMetadataCache, self).__init__(cluster_cache_dir, config.get_agent_secret(), config.get_agent_salt()) + super(ClusterMetadataCache, self).__init__(cluster_cache_dir, config.get_agent_secret()) def on_cache_update(self): try: diff --git a/ambari-agent/src/main/python/ambari_agent/ClusterTopologyCache.py b/ambari-agent/src/main/python/ambari_agent/ClusterTopologyCache.py index 14cdd1ecbf1..7aa8ef5e07c 100644 --- a/ambari-agent/src/main/python/ambari_agent/ClusterTopologyCache.py +++ b/ambari-agent/src/main/python/ambari_agent/ClusterTopologyCache.py @@ -50,7 +50,7 @@ def __init__(self, cluster_cache_dir, config): self.cluster_local_components = {} self.cluster_host_info = None self.component_version_map = {} - super(ClusterTopologyCache, self).__init__(cluster_cache_dir, config.get_agent_secret(), config.get_agent_salt()) + super(ClusterTopologyCache, self).__init__(cluster_cache_dir, config.get_agent_secret()) def get_cache_name(self): return "topology" diff --git a/ambari-agent/src/main/python/ambari_agent/InitializerModule.py b/ambari-agent/src/main/python/ambari_agent/InitializerModule.py index 8e86ddf26fe..f14ca0f13c2 100644 --- a/ambari-agent/src/main/python/ambari_agent/InitializerModule.py +++ b/ambari-agent/src/main/python/ambari_agent/InitializerModule.py @@ -91,18 +91,15 @@ def init(self): ) self.host_level_params_cache = ClusterHostLevelParamsCache( self.config.cluster_cache_dir, - self.config.get_agent_secret(), - self.config.get_agent_salt() + self.config.get_agent_secret() ) self.configurations_cache = ClusterConfigurationCache( self.config.cluster_cache_dir, - self.config.get_agent_secret(), - self.config.get_agent_salt() + self.config.get_agent_secret() ) self.alert_definitions_cache = ClusterAlertDefinitionsCache( self.config.cluster_cache_dir, - self.config.get_agent_secret(), - self.config.get_agent_salt() + self.config.get_agent_secret() ) self.configuration_builder = ConfigurationBuilder(self) self.stale_alerts_monitor = StaleAlertsMonitor(self) diff --git a/ambari-agent/src/test/python/ambari_agent/TestClusterCache.py b/ambari-agent/src/test/python/ambari_agent/TestClusterCache.py index bbe8fec074a..2deb212e74d 100644 --- a/ambari-agent/src/test/python/ambari_agent/TestClusterCache.py +++ b/ambari-agent/src/test/python/ambari_agent/TestClusterCache.py @@ -32,13 +32,13 @@ from only_for_platform import os_distro_value -class TestCertGeneration(TestCase): +class TestClusterCache(TestCase): """ Test suite for verifying encryption behavior of ClusterCache. It covers: - - encryption/decryption round-trip when secret/salt are provided - - behavior when encryption is effectively disabled (no secret/salt) + - encryption/decryption round-trip when secret are provided + - behavior when encryption is effectively disabled (no secret) """ # so that ClusterCache initialization is OS-agnostic in this test. @@ -48,14 +48,13 @@ def setUp(self): self.tmpdir = tempfile.mkdtemp() cluster_cache_dir = self.tmpdir + "/cluster_cache_dir" - # Instance with encryption enabled (secret and salt provided). + # Instance with encryption enabled (secret provided). self.cluster_cache_encrypted = DummyClusterCache( cluster_cache_dir, - "super_secret", - "super_secret_salt" + "super_secret" ) - # Instance with encryption disabled (no secret or salt). + # Instance with encryption disabled (no secret). self.cluster_cache_unencrypted = DummyClusterCache(cluster_cache_dir) @patch.object(os, "chmod") @@ -84,12 +83,12 @@ def test_enc(self, chmod_mock): @patch.object(os, "chmod") def test_encryption_enable(self, chmod_mock): """ - Verify that _is_encryption_enabled reflects whether secret/salt were provided. + Verify that _is_encryption_enabled reflects whether secret were provided. """ - # When secret/salt are given, encryption flag should reflect enabled status. + # When secret are given, encryption flag should reflect enabled status. self.assertFalse(self.cluster_cache_encrypted._is_encryption_enabled()) - # When no secret/salt, encryption should be reported as disabled. + # When no secret, encryption should be reported as disabled. self.assertTrue(self.cluster_cache_unencrypted._is_encryption_enabled()) def tearDown(self):