From fa7b74ea6424aa0fef6b51a2539a77922813195d Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 17 Oct 2025 15:49:02 +0000 Subject: [PATCH 1/6] Merge pull request #88341 from ClickHouse/disable_catalogs_in_system_tables Disable catalogs in system tables --- src/Backups/BackupEntriesCollector.cpp | 2 +- src/Core/Settings.cpp | 2 +- src/Core/SettingsChangesHistory.cpp | 59 +++++++++++++++++ src/Databases/DataLake/DatabaseDataLake.cpp | 6 +- src/Databases/DataLake/DatabaseDataLake.h | 4 +- src/Databases/IDatabase.h | 4 +- src/Interpreters/DatabaseCatalog.cpp | 45 ++++++++++--- src/Interpreters/DatabaseCatalog.h | 16 ++++- src/Interpreters/InterpreterCheckQuery.cpp | 2 +- src/Interpreters/InterpreterCreateQuery.cpp | 2 +- .../InterpreterShowTablesQuery.cpp | 15 ++++- src/Interpreters/InterpreterSystemQuery.cpp | 14 ++-- .../ServerAsynchronousMetrics.cpp | 4 +- src/Interpreters/loadMetadata.cpp | 2 +- src/Server/ReplicasStatusHandler.cpp | 2 +- src/Storages/RocksDB/StorageSystemRocksDB.cpp | 2 +- src/Storages/StorageMerge.cpp | 2 +- src/Storages/System/StorageSystemClusters.cpp | 2 +- src/Storages/System/StorageSystemColumns.cpp | 5 +- .../System/StorageSystemCompletions.cpp | 4 +- .../StorageSystemDataSkippingIndices.cpp | 2 +- .../System/StorageSystemDatabases.cpp | 11 +++- .../System/StorageSystemDistributionQueue.cpp | 2 +- src/Storages/System/StorageSystemGraphite.cpp | 2 +- .../System/StorageSystemIcebergHistory.cpp | 2 +- .../System/StorageSystemKafkaConsumers.cpp | 2 +- .../System/StorageSystemMutations.cpp | 2 +- ...torageSystemObjectStorageQueueSettings.cpp | 2 +- .../StorageSystemPartMovesBetweenShards.cpp | 2 +- .../System/StorageSystemPartsBase.cpp | 2 +- .../System/StorageSystemProjections.cpp | 2 +- src/Storages/System/StorageSystemReplicas.cpp | 2 +- .../System/StorageSystemReplicationQueue.cpp | 2 +- src/Storages/System/StorageSystemTables.cpp | 9 ++- tests/integration/test_database_glue/test.py | 66 +++++++++++++++++-- .../integration/test_database_iceberg/test.py | 6 +- .../test.py | 2 +- 37 files changed, 247 insertions(+), 65 deletions(-) diff --git a/src/Backups/BackupEntriesCollector.cpp b/src/Backups/BackupEntriesCollector.cpp index 4ce7a5f99edc..cf73f97e80ed 100644 --- a/src/Backups/BackupEntriesCollector.cpp +++ b/src/Backups/BackupEntriesCollector.cpp @@ -397,7 +397,7 @@ void BackupEntriesCollector::gatherDatabasesMetadata() case ASTBackupQuery::ElementType::ALL: { - for (const auto & [database_name, database] : DatabaseCatalog::instance().getDatabases()) + for (const auto & [database_name, database] : DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = true})) { if (!element.except_databases.contains(database_name)) { diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index e37c189d3c2b..4a5fc0821e8a 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -6472,7 +6472,7 @@ Query Iceberg table using the snapshot that was current at a specific timestamp. DECLARE(Int64, iceberg_snapshot_id, 0, R"( Query Iceberg table using the specific snapshot id. )", 0) \ - DECLARE(Bool, show_data_lake_catalogs_in_system_tables, true, R"( + DECLARE(Bool, show_data_lake_catalogs_in_system_tables, false, R"( Enables showing data lake catalogs in system tables. )", 0) \ DECLARE(Bool, delta_lake_enable_expression_visitor_logging, false, R"( diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index eaa78f0a3661..ac7db7994280 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -39,6 +39,65 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() /// controls new feature and it's 'true' by default, use 'false' as previous_value). /// It's used to implement `compatibility` setting (see https://github.com/ClickHouse/ClickHouse/issues/35972) /// Note: please check if the key already exists to prevent duplicate entries. + addSettingsChanges(settings_changes_history, "25.10", + { + {"show_data_lake_catalogs_in_system_tables", true, false, "Disable catalogs in system tables by default"}, + {"optimize_rewrite_like_perfect_affix", false, true, "New setting"}, + {"allow_dynamic_type_in_join_keys", true, false, "Disallow using Dynamic type in JOIN keys by default"}, + {"use_skip_indexes_on_data_read", false, true, "Enabled skip index usage in read phase by default"}, + {"s3queue_keeper_fault_injection_probablility", 0, 0, "New setting."}, + {"enable_join_runtime_filters", false, false, "New setting"}, + {"join_runtime_filter_exact_values_limit", 10000, 10000, "New setting"}, + {"join_runtime_bloom_filter_bytes", 512_KiB, 512_KiB, "New setting"}, + {"join_runtime_bloom_filter_hash_functions", 3, 3, "New setting"}, + {"use_join_disjunctions_push_down", false, false, "New setting."}, + {"joined_block_split_single_row", false, false, "New setting"}, + {"rewrite_in_to_join", false, false, "New experimental setting"}, + {"iceberg_insert_max_rows_in_data_file", 100000, 1000000, "New setting."}, + {"iceberg_insert_max_bytes_in_data_file", 100000000, 100000000, "New setting."}, + {"delta_lake_insert_max_rows_in_data_file", 100000, 1000000, "New setting."}, + {"delta_lake_log_metadata", false, false, "New setting."}, + {"distributed_cache_prefer_bigger_buffer_size", false, false, "New setting."}, + {"allow_experimental_qbit_type", false, false, "New experimental setting"}, + {"optimize_qbit_distance_function_reads", true, true, "New setting"}, + {"read_from_distributed_cache_if_exists_otherwise_bypass_cache", false, false, "New setting"}, + {"s3_slow_all_threads_after_retryable_error", false, false, "Disable the setting by default"}, + {"backup_slow_all_threads_after_retryable_s3_error", false, false, "Disable the setting by default"}, + {"enable_http_compression", false, true, "It should be beneficial in general"}, + {"inject_random_order_for_select_without_order_by", false, false, "New setting"}, + {"exclude_materialize_skip_indexes_on_insert", "", "", "New setting."}, + {"optimize_empty_string_comparisons", false, true, "A new setting."}, + {"query_plan_use_logical_join_step", true, true, "Added alias"}, + {"schema_inference_make_columns_nullable", 1, 3, "Take nullability information from Parquet/ORC/Arrow metadata by default, instead of making everything nullable."}, + {"materialized_views_squash_parallel_inserts", false, true, "Added setting to preserve old behavior if needed."}, + }); + addSettingsChanges(settings_changes_history, "25.9", + { + {"input_format_protobuf_oneof_presence", false, false, "New setting"}, + {"iceberg_delete_data_on_drop", false, false, "New setting"}, + {"use_skip_indexes_on_data_read", false, false, "New setting"}, + {"s3_slow_all_threads_after_retryable_error", false, false, "Added an alias for setting `backup_slow_all_threads_after_retryable_s3_error`"}, + {"iceberg_metadata_log_level", "none", "none", "New setting."}, + {"iceberg_insert_max_rows_in_data_file", 100000, 100000, "New setting."}, + {"iceberg_insert_max_bytes_in_data_file", 100000000, 100000000, "New setting."}, + {"query_plan_optimize_join_order_limit", 0, 1, "New setting"}, + {"query_plan_display_internal_aliases", false, false, "New setting"}, + {"query_plan_max_step_description_length", 1000000000, 500, "New setting"}, + {"allow_experimental_delta_lake_writes", false, false, "New setting."}, + {"query_plan_convert_any_join_to_semi_or_anti_join", true, true, "New setting."}, + {"text_index_use_bloom_filter", true, true, "New setting."}, + {"query_plan_direct_read_from_text_index", true, true, "New setting."}, + {"enable_producing_buckets_out_of_order_in_aggregation", false, true, "New setting"}, + {"jemalloc_enable_profiler", false, false, "New setting"}, + {"jemalloc_collect_profile_samples_in_trace_log", false, false, "New setting"}, + {"delta_lake_insert_max_bytes_in_data_file", 1_GiB, 1_GiB, "New setting."}, + {"delta_lake_insert_max_rows_in_data_file", 100000, 100000, "New setting."}, + {"promql_evaluation_time", Field{"auto"}, Field{"auto"}, "The setting was renamed. The previous name is `evaluation_time`."}, + {"evaluation_time", 0, 0, "Old setting which popped up here being renamed."}, + {"os_threads_nice_value_query", 0, 0, "New setting."}, + {"os_threads_nice_value_materialized_view", 0, 0, "New setting."}, + {"os_thread_priority", 0, 0, "Alias for os_threads_nice_value_query."}, + }); addSettingsChanges(settings_changes_history, "25.8", { {"output_format_json_quote_64bit_integers", true, false, "Disable quoting of the 64 bit integers in JSON by default"}, diff --git a/src/Databases/DataLake/DatabaseDataLake.cpp b/src/Databases/DataLake/DatabaseDataLake.cpp index 15ca4b9dd3e7..2679ae0de56c 100644 --- a/src/Databases/DataLake/DatabaseDataLake.cpp +++ b/src/Databases/DataLake/DatabaseDataLake.cpp @@ -532,14 +532,10 @@ DatabaseTablesIteratorPtr DatabaseDataLake::getTablesIterator( DatabaseTablesIteratorPtr DatabaseDataLake::getLightweightTablesIterator( ContextPtr context_, const FilterByNameFunction & filter_by_table_name, - bool skip_not_loaded, - bool skip_data_lake_catalog) const + bool skip_not_loaded) const { Tables tables; - if (skip_data_lake_catalog) - return std::make_unique(tables, getDatabaseName()); - auto catalog = getCatalog(); DB::Names iceberg_tables; diff --git a/src/Databases/DataLake/DatabaseDataLake.h b/src/Databases/DataLake/DatabaseDataLake.h index c46b0eb3dcd2..b13cbf866009 100644 --- a/src/Databases/DataLake/DatabaseDataLake.h +++ b/src/Databases/DataLake/DatabaseDataLake.h @@ -31,6 +31,7 @@ class DatabaseDataLake final : public IDatabase, WithContext bool canContainDistributedTables() const override { return false; } bool canContainRocksDBTables() const override { return false; } bool shouldBeEmptyOnDetach() const override { return false; } + bool isDatalakeCatalog() const override { return true; } bool empty() const override; @@ -47,8 +48,7 @@ class DatabaseDataLake final : public IDatabase, WithContext DatabaseTablesIteratorPtr getLightweightTablesIterator( ContextPtr context, const FilterByNameFunction & filter_by_table_name, - bool skip_not_loaded, - bool skip_data_lake_catalog) const override; + bool skip_not_loaded) const override; void shutdown() override {} diff --git a/src/Databases/IDatabase.h b/src/Databases/IDatabase.h index eb7d5f2c75ce..9f4fa18cc71a 100644 --- a/src/Databases/IDatabase.h +++ b/src/Databases/IDatabase.h @@ -179,6 +179,8 @@ class IDatabase : public std::enable_shared_from_this virtual bool canContainRocksDBTables() const { return true; } + virtual bool isDatalakeCatalog() const { return false; } + /// Load a set of existing tables. /// You can call only once, right after the object is created. virtual void loadStoredObjects( /// NOLINT @@ -267,7 +269,7 @@ class IDatabase : public std::enable_shared_from_this /// Same as above, but may return non-fully initialized StoragePtr objects which are not suitable for reading. /// Useful for queries like "SHOW TABLES" - virtual DatabaseTablesIteratorPtr getLightweightTablesIterator(ContextPtr context, const FilterByNameFunction & filter_by_table_name = {}, bool skip_not_loaded = false, [[maybe_unused]] bool skip_data_lake_catalog = false) const /// NOLINT + virtual DatabaseTablesIteratorPtr getLightweightTablesIterator(ContextPtr context, const FilterByNameFunction & filter_by_table_name = {}, bool skip_not_loaded = false) const /// NOLINT { return getTablesIterator(context, filter_by_table_name, skip_not_loaded); } diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index b6c3ef92c90f..727a05a22f4a 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -115,7 +115,7 @@ class DatabaseNameHints : public IHints<> const bool need_to_check_access_for_databases = !access->isGranted(AccessType::SHOW_DATABASES); Names result; - auto databases_list = database_catalog.getDatabases(); + auto databases_list = database_catalog.getDatabases(GetDatabasesOptions{.with_datalake_catalogs = true}); for (const auto & database_name : databases_list | boost::adaptors::map_keys) { if (need_to_check_access_for_databases && !access->isGranted(AccessType::SHOW_DATABASES, database_name)) @@ -341,7 +341,10 @@ void DatabaseCatalog::shutdownImpl(std::function shutdown_system_logs) auto it = std::find_if(elem.map.begin(), elem.map.end(), not_empty_mapping); return it != elem.map.end(); }) == uuid_map.end()); + databases.clear(); + databases_without_datalake_catalogs.clear(); + referential_dependencies.clear(); loading_dependencies.clear(); view_dependencies.clear(); @@ -576,6 +579,18 @@ void DatabaseCatalog::assertDatabaseExists(const String & database_name) const } } +bool DatabaseCatalog::hasDatalakeCatalogs() const +{ + std::lock_guard lock{databases_mutex}; + return databases.size() != databases_without_datalake_catalogs.size(); +} + +bool DatabaseCatalog::isDatalakeCatalog(const String & database_name) const +{ + std::lock_guard lock{databases_mutex}; + return databases.contains(database_name) && !databases_without_datalake_catalogs.contains(database_name); +} + void DatabaseCatalog::assertDatabaseDoesntExist(const String & database_name) const { std::lock_guard lock{databases_mutex}; @@ -594,6 +609,9 @@ void DatabaseCatalog::attachDatabase(const String & database_name, const Databas std::lock_guard lock{databases_mutex}; assertDatabaseDoesntExistUnlocked(database_name); databases.emplace(database_name, database); + if (!database->isDatalakeCatalog()) + databases_without_datalake_catalogs.emplace(database_name, database); + NOEXCEPT_SCOPE({ UUID db_uuid = database->getUUID(); if (db_uuid != UUIDHelpers::Nil) @@ -618,8 +636,9 @@ DatabasePtr DatabaseCatalog::detachDatabase(ContextPtr local_context, const Stri if (db_uuid != UUIDHelpers::Nil) removeUUIDMapping(db_uuid); databases.erase(database_name); - } + if (auto it = databases_without_datalake_catalogs.find(database_name); it != databases_without_datalake_catalogs.end()) + databases_without_datalake_catalogs.erase(it); } if (!db) { @@ -685,7 +704,13 @@ void DatabaseCatalog::updateDatabaseName(const String & old_name, const String & databases.erase(it); databases.emplace(new_name, db); - /// Update dependencies. + auto no_catalogs_it = databases_without_datalake_catalogs.find(old_name); + if (no_catalogs_it != databases_without_datalake_catalogs.end()) + { + databases_without_datalake_catalogs.erase(no_catalogs_it); + databases_without_datalake_catalogs.emplace(new_name, db); + } + for (const auto & table_name : tables_in_database) { auto removed_ref_deps = referential_dependencies.removeDependencies(StorageID{old_name, table_name}, /* remove_isolated_tables= */ true); @@ -806,10 +831,13 @@ bool DatabaseCatalog::isDatabaseExist(const String & database_name) const return databases.end() != databases.find(database_name); } -Databases DatabaseCatalog::getDatabases() const +Databases DatabaseCatalog::getDatabases(GetDatabasesOptions options) const { std::lock_guard lock{databases_mutex}; - return databases; + if (options.with_datalake_catalogs) + return databases; + + return databases_without_datalake_catalogs; } bool DatabaseCatalog::isTableExist(const DB::StorageID & table_id, ContextPtr context_) const @@ -1075,7 +1103,7 @@ void DatabaseCatalog::loadMarkedAsDroppedTables() std::map> dropped_metadata; String path = fs::path("metadata_dropped") / ""; - auto db_map = getDatabases(); + auto db_map = getDatabases(GetDatabasesOptions{.with_datalake_catalogs = true}); std::set metadata_disk_list; for (const auto & [_, db] : db_map) { @@ -1965,7 +1993,7 @@ void DatabaseCatalog::reloadDisksTask() disks.swap(disks_to_reload); } - for (auto & database : getDatabases()) + for (auto & database : getDatabases(GetDatabasesOptions{.with_datalake_catalogs = false})) { // WARNING: In case of `async_load_databases = true` getTablesIterator() call wait for all table in the database to be loaded. // WARNING: It means that no database will be able to update configuration until all databases are fully loaded. @@ -2120,7 +2148,8 @@ std::pair TableNameHints::getExtendedHintForTable(const String & { /// load all available databases from the DatabaseCatalog instance auto & database_catalog = DatabaseCatalog::instance(); - auto all_databases = database_catalog.getDatabases(); + /// NOTE Skip datalake catalogs to avoid unnecessary access to remote catalogs (can be expensive) + auto all_databases = database_catalog.getDatabases(GetDatabasesOptions{.with_datalake_catalogs = false}); for (const auto & [db_name, db] : all_databases) { diff --git a/src/Interpreters/DatabaseCatalog.h b/src/Interpreters/DatabaseCatalog.h index e695f7c22521..c95aaf4654d9 100644 --- a/src/Interpreters/DatabaseCatalog.h +++ b/src/Interpreters/DatabaseCatalog.h @@ -121,6 +121,11 @@ using TemporaryTablesMapping = std::map; class BackgroundSchedulePoolTaskHolder; +struct GetDatabasesOptions +{ + bool with_datalake_catalogs{false}; +}; + /// For some reason Context is required to get Storage from Database object class DatabaseCatalog : boost::noncopyable, WithMutableContext { @@ -171,7 +176,13 @@ class DatabaseCatalog : boost::noncopyable, WithMutableContext DatabasePtr getDatabase(const UUID & uuid) const; DatabasePtr tryGetDatabase(const UUID & uuid) const; bool isDatabaseExist(const String & database_name) const; - Databases getDatabases() const; + /// Datalake catalogs are implement at IDatabase level in ClickHouse. + /// In general case Datalake catalog is a some remote service which contains iceberg/delta tables. + /// Sometimes this service charges money for requests. With this flag we explicitly protect ourself + /// to not accidentally query external non-free service for some trivial things like + /// autocompletion hints or system.tables query. We have a setting which allow to show + /// these databases everywhere, but user must explicitly specify it. + Databases getDatabases(GetDatabasesOptions options) const; /// Same as getDatabase(const String & database_name), but if database_name is empty, current database of local_context is used DatabasePtr getDatabase(const String & database_name, ContextPtr local_context) const; @@ -272,6 +283,8 @@ class DatabaseCatalog : boost::noncopyable, WithMutableContext bool canPerformReplicatedDDLQueries() const; void updateMetadataFile(const DatabasePtr & database); + bool hasDatalakeCatalogs() const; + bool isDatalakeCatalog(const String & database_name) const; private: // The global instance of database catalog. unique_ptr is to allow @@ -319,6 +332,7 @@ class DatabaseCatalog : boost::noncopyable, WithMutableContext mutable std::mutex databases_mutex; Databases databases TSA_GUARDED_BY(databases_mutex); + Databases databases_without_datalake_catalogs TSA_GUARDED_BY(databases_mutex); UUIDToStorageMap uuid_map; /// Referential dependencies between tables: table "A" depends on table "B" diff --git a/src/Interpreters/InterpreterCheckQuery.cpp b/src/Interpreters/InterpreterCheckQuery.cpp index 2a9682905a8c..6391608f4902 100644 --- a/src/Interpreters/InterpreterCheckQuery.cpp +++ b/src/Interpreters/InterpreterCheckQuery.cpp @@ -395,7 +395,7 @@ InterpreterCheckQuery::InterpreterCheckQuery(const ASTPtr & query_ptr_, ContextP static Strings getAllDatabases(const ContextPtr & context) { Strings res; - const auto & databases = DatabaseCatalog::instance().getDatabases(); + const auto & databases = DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = false}); res.reserve(databases.size()); for (const auto & [database_name, _] : databases) { diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 0e859d775db3..d9ce22442b9d 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -206,7 +206,7 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create) auto db_num_limit = getContext()->getGlobalContext()->getServerSettings()[ServerSetting::max_database_num_to_throw].value; if (db_num_limit > 0 && !internal) { - size_t db_count = DatabaseCatalog::instance().getDatabases().size(); + size_t db_count = DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = true}).size(); std::initializer_list system_databases = { DatabaseCatalog::TEMPORARY_DATABASE, diff --git a/src/Interpreters/InterpreterShowTablesQuery.cpp b/src/Interpreters/InterpreterShowTablesQuery.cpp index 8a159ffdcbb3..d8e01ecd722a 100644 --- a/src/Interpreters/InterpreterShowTablesQuery.cpp +++ b/src/Interpreters/InterpreterShowTablesQuery.cpp @@ -14,6 +14,7 @@ #include #include #include +#include namespace DB @@ -229,8 +230,20 @@ BlockIO InterpreterShowTablesQuery::execute() return res; } + auto rewritten_query = getRewrittenQuery(); + String database = getContext()->resolveDatabase(query.getFrom()); + if (DatabaseCatalog::instance().isDatalakeCatalog(database)) + { + auto context_copy = Context::createCopy(getContext()); + /// HACK To always show them in explicit "SHOW TABLES" queries + context_copy->setSetting("show_data_lake_catalogs_in_system_tables", true); + return executeQuery(rewritten_query, context_copy, QueryFlags{ .internal = true }).second; + } + else + { + return executeQuery(rewritten_query, getContext(), QueryFlags{ .internal = true }).second; + } - return executeQuery(getRewrittenQuery(), getContext(), QueryFlags{ .internal = true }).second; } /// (*) Sorting is strictly speaking not necessary but 1. it is convenient for users, 2. SQL currently does not allow to diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp index f02a6e8ddd6c..58d228d5f189 100644 --- a/src/Interpreters/InterpreterSystemQuery.cpp +++ b/src/Interpreters/InterpreterSystemQuery.cpp @@ -242,7 +242,7 @@ void InterpreterSystemQuery::startStopAction(StorageActionBlockType action_type, } else { - for (auto & elem : DatabaseCatalog::instance().getDatabases()) + for (auto & elem : DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = false})) { startStopActionInDatabase(action_type, start, elem.first, elem.second, getContext(), log); } @@ -1075,7 +1075,7 @@ void InterpreterSystemQuery::restartReplicas(ContextMutablePtr system_context) bool access_is_granted_globally = access->isGranted(AccessType::SYSTEM_RESTART_REPLICA); bool show_tables_is_granted_globally = access->isGranted(AccessType::SHOW_TABLES); - for (auto & elem : catalog.getDatabases()) + for (auto & elem : catalog.getDatabases(GetDatabasesOptions{.with_datalake_catalogs = false})) { if (!elem.second->canContainMergeTreeTables()) continue; @@ -1137,7 +1137,7 @@ void InterpreterSystemQuery::dropReplica(ASTSystemQuery & query) } else if (query.is_drop_whole_replica) { - auto databases = DatabaseCatalog::instance().getDatabases(); + auto databases = DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = false}); auto access = getContext()->getAccess(); bool access_is_granted_globally = access->isGranted(AccessType::SYSTEM_DROP_REPLICA); @@ -1178,7 +1178,7 @@ void InterpreterSystemQuery::dropReplica(ASTSystemQuery & query) String remote_replica_path = fs::path(query.replica_zk_path) / "replicas" / query.replica; /// This check is actually redundant, but it may prevent from some user mistakes - for (auto & elem : DatabaseCatalog::instance().getDatabases()) + for (auto & elem : DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = false})) { DatabasePtr & database = elem.second; for (auto iterator = database->getTablesIterator(getContext()); iterator->isValid(); iterator->next()) @@ -1272,7 +1272,7 @@ void InterpreterSystemQuery::dropDatabaseReplica(ASTSystemQuery & query) } else if (query.is_drop_whole_replica) { - auto databases = DatabaseCatalog::instance().getDatabases(); + auto databases = DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = false}); auto access = getContext()->getAccess(); bool access_is_granted_globally = access->isGranted(AccessType::SYSTEM_DROP_REPLICA); @@ -1298,7 +1298,7 @@ void InterpreterSystemQuery::dropDatabaseReplica(ASTSystemQuery & query) getContext()->checkAccess(AccessType::SYSTEM_DROP_REPLICA); /// This check is actually redundant, but it may prevent from some user mistakes - for (auto & elem : DatabaseCatalog::instance().getDatabases()) + for (auto & elem : DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = false})) if (auto * replicated = dynamic_cast(elem.second.get())) check_not_local_replica(replicated, query); @@ -1416,7 +1416,7 @@ void InterpreterSystemQuery::loadOrUnloadPrimaryKeysImpl(bool load) getContext()->checkAccess(load ? AccessType::SYSTEM_LOAD_PRIMARY_KEY : AccessType::SYSTEM_UNLOAD_PRIMARY_KEY); LOG_TRACE(log, "{} primary keys for all tables", load ? "Loading" : "Unloading"); - for (auto & database : DatabaseCatalog::instance().getDatabases()) + for (auto & database : DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = false})) { for (auto it = database.second->getTablesIterator(getContext()); it->isValid(); it->next()) { diff --git a/src/Interpreters/ServerAsynchronousMetrics.cpp b/src/Interpreters/ServerAsynchronousMetrics.cpp index 80af95332a12..0e3713d66c94 100644 --- a/src/Interpreters/ServerAsynchronousMetrics.cpp +++ b/src/Interpreters/ServerAsynchronousMetrics.cpp @@ -196,7 +196,7 @@ void ServerAsynchronousMetrics::updateImpl(TimePoint update_time, TimePoint curr } { - auto databases = DatabaseCatalog::instance().getDatabases(); + auto databases = DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = false}); size_t max_queue_size = 0; size_t max_inserts_in_queue = 0; @@ -371,7 +371,7 @@ void ServerAsynchronousMetrics::updateMutationAndDetachedPartsStats() DetachedPartsStats current_values{}; MutationStats current_mutation_stats{}; - for (const auto & db : DatabaseCatalog::instance().getDatabases()) + for (const auto & db : DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = false})) { if (!db.second->canContainMergeTreeTables()) continue; diff --git a/src/Interpreters/loadMetadata.cpp b/src/Interpreters/loadMetadata.cpp index e2c448c0bc93..f05508975a5b 100644 --- a/src/Interpreters/loadMetadata.cpp +++ b/src/Interpreters/loadMetadata.cpp @@ -538,7 +538,7 @@ void convertDatabasesEnginesIfNeed(const LoadTaskPtrs & load_metadata, ContextMu // Wait for all table to be loaded and started waitLoad(TablesLoaderForegroundPoolId, load_metadata); - for (const auto & [name, _] : DatabaseCatalog::instance().getDatabases()) + for (const auto & [name, _] : DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = false})) if (name != DatabaseCatalog::SYSTEM_DATABASE) maybeConvertOrdinaryDatabaseToAtomic(context, name); diff --git a/src/Server/ReplicasStatusHandler.cpp b/src/Server/ReplicasStatusHandler.cpp index a44a02f26ae6..4129144deea5 100644 --- a/src/Server/ReplicasStatusHandler.cpp +++ b/src/Server/ReplicasStatusHandler.cpp @@ -55,7 +55,7 @@ void ReplicasStatusHandler::handleRequest(HTTPServerRequest & request, HTTPServe bool ok = true; WriteBufferFromOwnString message; - auto databases = DatabaseCatalog::instance().getDatabases(); + auto databases = DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = false}); /// Iterate through all the replicated tables. for (const auto & db : databases) diff --git a/src/Storages/RocksDB/StorageSystemRocksDB.cpp b/src/Storages/RocksDB/StorageSystemRocksDB.cpp index 9099671592a4..27606bdd0c33 100644 --- a/src/Storages/RocksDB/StorageSystemRocksDB.cpp +++ b/src/Storages/RocksDB/StorageSystemRocksDB.cpp @@ -54,7 +54,7 @@ void StorageSystemRocksDB::fillData(MutableColumns & res_columns, ContextPtr con using RocksDBStoragePtr = std::shared_ptr; std::map> tables; - for (const auto & db : DatabaseCatalog::instance().getDatabases()) + for (const auto & db : DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = false})) { if (!db.second->canContainRocksDBTables()) continue; diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index c858186e8a27..c6097b4388c7 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -1436,7 +1436,7 @@ StorageMerge::DatabaseTablesIterators StorageMerge::DatabaseNameOrRegexp::getDat else { /// database_name argument is a regexp - auto databases = DatabaseCatalog::instance().getDatabases(); + auto databases = DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = true}); for (const auto & db : databases) { diff --git a/src/Storages/System/StorageSystemClusters.cpp b/src/Storages/System/StorageSystemClusters.cpp index 3acd3beb4be4..828918952e41 100644 --- a/src/Storages/System/StorageSystemClusters.cpp +++ b/src/Storages/System/StorageSystemClusters.cpp @@ -52,7 +52,7 @@ void StorageSystemClusters::fillData(MutableColumns & res_columns, ContextPtr co for (const auto & name_and_cluster : context->getClusters()) writeCluster(res_columns, columns_mask, name_and_cluster, /* replicated= */ nullptr); - const auto databases = DatabaseCatalog::instance().getDatabases(); + const auto databases = DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = false}); for (const auto & name_and_database : databases) { if (const auto * replicated = typeid_cast(name_and_database.second.get())) diff --git a/src/Storages/System/StorageSystemColumns.cpp b/src/Storages/System/StorageSystemColumns.cpp index 2425358c9447..8ae1fbb16bfc 100644 --- a/src/Storages/System/StorageSystemColumns.cpp +++ b/src/Storages/System/StorageSystemColumns.cpp @@ -27,6 +27,7 @@ namespace DB namespace Setting { extern const SettingsSeconds lock_acquire_timeout; + extern const SettingsBool show_data_lake_catalogs_in_system_tables; } namespace ErrorCodes { @@ -441,7 +442,9 @@ void ReadFromSystemColumns::initializePipeline(QueryPipelineBuilder & pipeline, /// Add `database` column. MutableColumnPtr database_column_mut = ColumnString::create(); - const auto databases = DatabaseCatalog::instance().getDatabases(); + const auto & context = getContext(); + const auto & settings = context->getSettingsRef(); + const auto databases = DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = settings[Setting::show_data_lake_catalogs_in_system_tables]}); for (const auto & [database_name, database] : databases) { if (database_name == DatabaseCatalog::TEMPORARY_DATABASE) diff --git a/src/Storages/System/StorageSystemCompletions.cpp b/src/Storages/System/StorageSystemCompletions.cpp index 16bdee3ff29d..3664e841c099 100644 --- a/src/Storages/System/StorageSystemCompletions.cpp +++ b/src/Storages/System/StorageSystemCompletions.cpp @@ -32,6 +32,7 @@ namespace Setting { extern const SettingsUInt64 readonly; extern const SettingsSeconds lock_acquire_timeout; + extern const SettingsBool show_data_lake_catalogs_in_system_tables; } static constexpr const char * DATABASE_CONTEXT = "database"; @@ -96,7 +97,8 @@ void fillDataWithTableColumns(const String & database_name, const String & table void fillDataWithDatabasesTablesColumns(MutableColumns & res_columns, const ContextPtr & context) { const auto & access = context->getAccess(); - const auto & databases = DatabaseCatalog::instance().getDatabases(); + const auto & settings = context->getSettingsRef(); + const auto & databases = DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = settings[Setting::show_data_lake_catalogs_in_system_tables]}); for (const auto & [database_name, database_ptr] : databases) { if (!access->isGranted(AccessType::SHOW_DATABASES) || !access->isGranted(AccessType::SHOW_DATABASES, database_name)) diff --git a/src/Storages/System/StorageSystemDataSkippingIndices.cpp b/src/Storages/System/StorageSystemDataSkippingIndices.cpp index fa70a42cce0d..808caa7dab78 100644 --- a/src/Storages/System/StorageSystemDataSkippingIndices.cpp +++ b/src/Storages/System/StorageSystemDataSkippingIndices.cpp @@ -261,7 +261,7 @@ void ReadFromSystemDataSkippingIndices::initializePipeline(QueryPipelineBuilder { MutableColumnPtr column = ColumnString::create(); - const auto databases = DatabaseCatalog::instance().getDatabases(); + const auto databases = DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = false}); for (const auto & [database_name, database] : databases) { if (database_name == DatabaseCatalog::TEMPORARY_DATABASE) diff --git a/src/Storages/System/StorageSystemDatabases.cpp b/src/Storages/System/StorageSystemDatabases.cpp index 12b3b266c301..dceb291d5d50 100644 --- a/src/Storages/System/StorageSystemDatabases.cpp +++ b/src/Storages/System/StorageSystemDatabases.cpp @@ -11,11 +11,18 @@ #include #include #include +#include namespace DB { +namespace Setting +{ + extern const SettingsBool show_data_lake_catalogs_in_system_tables; +} + + ColumnsDescription StorageSystemDatabases::getColumnsDescription() { auto description = ColumnsDescription @@ -111,8 +118,8 @@ void StorageSystemDatabases::fillData(MutableColumns & res_columns, ContextPtr c { const auto access = context->getAccess(); const bool need_to_check_access_for_databases = !access->isGranted(AccessType::SHOW_DATABASES); - - const auto databases = DatabaseCatalog::instance().getDatabases(); + const auto & settings = context->getSettingsRef(); + const auto databases = DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = settings[Setting::show_data_lake_catalogs_in_system_tables]}); ColumnPtr filtered_databases_column = getFilteredDatabases(databases, predicate, context); for (size_t i = 0; i < filtered_databases_column->size(); ++i) diff --git a/src/Storages/System/StorageSystemDistributionQueue.cpp b/src/Storages/System/StorageSystemDistributionQueue.cpp index ff9cb6aa50c4..5c364622bbff 100644 --- a/src/Storages/System/StorageSystemDistributionQueue.cpp +++ b/src/Storages/System/StorageSystemDistributionQueue.cpp @@ -121,7 +121,7 @@ void StorageSystemDistributionQueue::fillData(MutableColumns & res_columns, Cont const bool check_access_for_databases = !access->isGranted(AccessType::SHOW_TABLES); std::map> tables; - for (const auto & db : DatabaseCatalog::instance().getDatabases()) + for (const auto & db : DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = false})) { /// Check if database can contain distributed tables if (!db.second->canContainDistributedTables()) diff --git a/src/Storages/System/StorageSystemGraphite.cpp b/src/Storages/System/StorageSystemGraphite.cpp index ef13c3c24da0..8e0dd1e20260 100644 --- a/src/Storages/System/StorageSystemGraphite.cpp +++ b/src/Storages/System/StorageSystemGraphite.cpp @@ -35,7 +35,7 @@ ColumnsDescription StorageSystemGraphite::getColumnsDescription() */ static StorageSystemGraphite::Configs getConfigs(ContextPtr context) { - const Databases databases = DatabaseCatalog::instance().getDatabases(); + const Databases databases = DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = false}); StorageSystemGraphite::Configs graphite_configs; for (const auto & db : databases) diff --git a/src/Storages/System/StorageSystemIcebergHistory.cpp b/src/Storages/System/StorageSystemIcebergHistory.cpp index f2148a753bb5..410dfcbc40a6 100644 --- a/src/Storages/System/StorageSystemIcebergHistory.cpp +++ b/src/Storages/System/StorageSystemIcebergHistory.cpp @@ -93,7 +93,7 @@ void StorageSystemIcebergHistory::fillData([[maybe_unused]] MutableColumns & res if (show_tables_granted) { - auto databases = DatabaseCatalog::instance().getDatabases(); + auto databases = DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = true}); for (const auto & db: databases) { /// with last flag we are filtering out all non iceberg table diff --git a/src/Storages/System/StorageSystemKafkaConsumers.cpp b/src/Storages/System/StorageSystemKafkaConsumers.cpp index 8a0e1dc3c7b8..a8ecd6034f42 100644 --- a/src/Storages/System/StorageSystemKafkaConsumers.cpp +++ b/src/Storages/System/StorageSystemKafkaConsumers.cpp @@ -163,7 +163,7 @@ void StorageSystemKafkaConsumers::fillData(MutableColumns & res_columns, Context return; add_rows(it, kafka_table); }; - auto databases = DatabaseCatalog::instance().getDatabases(); + auto databases = DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = false}); for (const auto & db : databases) { for (auto it = db.second->getTablesIterator(context); it->isValid(); it->next()) diff --git a/src/Storages/System/StorageSystemMutations.cpp b/src/Storages/System/StorageSystemMutations.cpp index f2b54b361806..738ca1c38c92 100644 --- a/src/Storages/System/StorageSystemMutations.cpp +++ b/src/Storages/System/StorageSystemMutations.cpp @@ -68,7 +68,7 @@ void StorageSystemMutations::fillData(MutableColumns & res_columns, ContextPtr c /// Collect a set of *MergeTree tables. std::map> merge_tree_tables; - for (const auto & db : DatabaseCatalog::instance().getDatabases()) + for (const auto & db : DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = false})) { /// Check if database can contain MergeTree tables if (!db.second->canContainMergeTreeTables()) diff --git a/src/Storages/System/StorageSystemObjectStorageQueueSettings.cpp b/src/Storages/System/StorageSystemObjectStorageQueueSettings.cpp index a6cf0ab255cd..d9b08ae474cb 100644 --- a/src/Storages/System/StorageSystemObjectStorageQueueSettings.cpp +++ b/src/Storages/System/StorageSystemObjectStorageQueueSettings.cpp @@ -58,7 +58,7 @@ void StorageSystemObjectStorageQueueSettings::fillData( const bool show_tables_granted = access->isGranted(AccessType::SHOW_TABLES); if (show_tables_granted) { - auto databases = DatabaseCatalog::instance().getDatabases(); + auto databases = DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = false}); for (const auto & db : databases) { for (auto iterator = db.second->getTablesIterator(context); iterator->isValid(); iterator->next()) diff --git a/src/Storages/System/StorageSystemPartMovesBetweenShards.cpp b/src/Storages/System/StorageSystemPartMovesBetweenShards.cpp index 58c61ca42aa0..65db49a8c887 100644 --- a/src/Storages/System/StorageSystemPartMovesBetweenShards.cpp +++ b/src/Storages/System/StorageSystemPartMovesBetweenShards.cpp @@ -59,7 +59,7 @@ void StorageSystemPartMovesBetweenShards::fillData(MutableColumns & res_columns, const bool check_access_for_databases = !access->isGranted(AccessType::SHOW_TABLES); std::map> replicated_tables; - for (const auto & db : DatabaseCatalog::instance().getDatabases()) + for (const auto & db : DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = false})) { /// Check if database can contain replicated tables if (!db.second->canContainMergeTreeTables()) diff --git a/src/Storages/System/StorageSystemPartsBase.cpp b/src/Storages/System/StorageSystemPartsBase.cpp index 51ba81ef0169..3fd6a8f37b83 100644 --- a/src/Storages/System/StorageSystemPartsBase.cpp +++ b/src/Storages/System/StorageSystemPartsBase.cpp @@ -118,7 +118,7 @@ StoragesInfoStream::StoragesInfoStream(std::optional filter_by_datab const bool check_access_for_tables = !access->isGranted(AccessType::SHOW_TABLES); { - Databases databases = DatabaseCatalog::instance().getDatabases(); + Databases databases = DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = false}); /// Add column 'database'. MutableColumnPtr database_column_mut = ColumnString::create(); diff --git a/src/Storages/System/StorageSystemProjections.cpp b/src/Storages/System/StorageSystemProjections.cpp index e88763f2bf3f..44b8ce12a3a7 100644 --- a/src/Storages/System/StorageSystemProjections.cpp +++ b/src/Storages/System/StorageSystemProjections.cpp @@ -248,7 +248,7 @@ void ReadFromSystemProjections::initializePipeline(QueryPipelineBuilder & pipeli { MutableColumnPtr column = ColumnString::create(); - const auto databases = DatabaseCatalog::instance().getDatabases(); + const auto databases = DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = false}); for (const auto & [database_name, database] : databases) { if (database_name == DatabaseCatalog::TEMPORARY_DATABASE) diff --git a/src/Storages/System/StorageSystemReplicas.cpp b/src/Storages/System/StorageSystemReplicas.cpp index aac54c244d23..03479fa7c507 100644 --- a/src/Storages/System/StorageSystemReplicas.cpp +++ b/src/Storages/System/StorageSystemReplicas.cpp @@ -332,7 +332,7 @@ void StorageSystemReplicas::read( /// We collect a set of replicated tables. std::map> replicated_tables; - for (const auto & db : DatabaseCatalog::instance().getDatabases()) + for (const auto & db : DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = false})) { /// Check if database can contain replicated tables if (!db.second->canContainMergeTreeTables()) diff --git a/src/Storages/System/StorageSystemReplicationQueue.cpp b/src/Storages/System/StorageSystemReplicationQueue.cpp index 4faaee1ebb73..645a0e387b91 100644 --- a/src/Storages/System/StorageSystemReplicationQueue.cpp +++ b/src/Storages/System/StorageSystemReplicationQueue.cpp @@ -78,7 +78,7 @@ void StorageSystemReplicationQueue::fillData(MutableColumns & res_columns, Conte const bool check_access_for_databases = !access->isGranted(AccessType::SHOW_TABLES); std::map> replicated_tables; - for (const auto & db : DatabaseCatalog::instance().getDatabases()) + for (const auto & db : DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = false})) { /// Check if database can contain replicated tables if (!db.second->canContainMergeTreeTables()) diff --git a/src/Storages/System/StorageSystemTables.cpp b/src/Storages/System/StorageSystemTables.cpp index ff1d570a8024..76785716bfc7 100644 --- a/src/Storages/System/StorageSystemTables.cpp +++ b/src/Storages/System/StorageSystemTables.cpp @@ -70,7 +70,8 @@ ColumnPtr getFilteredDatabases(const ActionsDAG::Node * predicate, ContextPtr co { MutableColumnPtr column = ColumnString::create(); - const auto databases = DatabaseCatalog::instance().getDatabases(); + const auto & settings = context->getSettingsRef(); + const auto databases = DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = settings[Setting::show_data_lake_catalogs_in_system_tables]}); for (const auto & database_name : databases | boost::adaptors::map_keys) { if (database_name == DatabaseCatalog::TEMPORARY_DATABASE) @@ -125,8 +126,7 @@ ColumnPtr getFilteredTables( { auto table_it = database->getLightweightTablesIterator(context, /* filter_by_table_name */ {}, - /* skip_not_loaded */ false, - !context->getSettingsRef()[DB::Setting::show_data_lake_catalogs_in_system_tables]); + /* skip_not_loaded */ false); for (; table_it->isValid(); table_it->next()) { database_column->insert(table_it->name()); @@ -444,8 +444,7 @@ class TablesBlockSource : public ISource if (!tables_it || !tables_it->isValid()) tables_it = database->getLightweightTablesIterator(context, /* filter_by_table_name */ {}, - /* skip_not_loaded */ false, - !context->getSettingsRef()[DB::Setting::show_data_lake_catalogs_in_system_tables]); + /* skip_not_loaded */ false); const bool need_table = needTable(database, getPort().getHeader()); diff --git a/tests/integration/test_database_glue/test.py b/tests/integration/test_database_glue/test.py index 70db874cd3c8..7ba239484640 100644 --- a/tests/integration/test_database_glue/test.py +++ b/tests/integration/test_database_glue/test.py @@ -295,14 +295,14 @@ def test_list_tables(started_cluster): assert ( tables_list == node.query( - f"SELECT name FROM system.tables WHERE database = '{CATALOG_NAME}' and name ILIKE '{root_namespace}%' ORDER BY name" + f"SELECT name FROM system.tables WHERE database = '{CATALOG_NAME}' and name ILIKE '{root_namespace}%' ORDER BY name SETTINGS show_data_lake_catalogs_in_system_tables = true" ).strip() ) node.restart_clickhouse() assert ( tables_list == node.query( - f"SELECT name FROM system.tables WHERE database = '{CATALOG_NAME}' and name ILIKE '{root_namespace}%' ORDER BY name" + f"SELECT name FROM system.tables WHERE database = '{CATALOG_NAME}' and name ILIKE '{root_namespace}%' ORDER BY name SETTINGS show_data_lake_catalogs_in_system_tables = true" ).strip() ) @@ -354,7 +354,7 @@ def test_select(started_cluster): ) assert int(node.query(f"SELECT count() FROM system.iceberg_history WHERE database = '{CATALOG_NAME}' and table ilike '%{root_namespace}%'").strip()) == 4 - assert int(node.query(f"SELECT count() FROM system.tables WHERE database = '{CATALOG_NAME}' and table ilike '%{root_namespace}%'").strip()) == 4 + assert int(node.query(f"SELECT count() FROM system.tables WHERE database = '{CATALOG_NAME}' and table ilike '%{root_namespace}%' SETTINGS show_data_lake_catalogs_in_system_tables = true").strip()) == 4 assert int(node.query(f"SELECT count() FROM system.tables WHERE database = '{CATALOG_NAME}' and table ilike '%{root_namespace}%' SETTINGS show_data_lake_catalogs_in_system_tables = false").strip()) == 0 @@ -384,7 +384,6 @@ def test_hide_sensitive_info(started_cluster): assert "SECRET_2" not in node.query(f"SHOW CREATE DATABASE {CATALOG_NAME}") - def test_select_after_rename(started_cluster): node = started_cluster.instances["node1"] @@ -578,3 +577,62 @@ def test_drop_table(started_cluster): drop_clickhouse_glue_table(node, root_namespace, table_name) assert len(catalog.list_tables(root_namespace)) == 0 + + +def test_system_tables(started_cluster): + node = started_cluster.instances["node1"] + + test_ref = f"test_system_tables_{uuid.uuid4()}" + table_name = f"{test_ref}_table" + root_namespace = f"{test_ref}_namespace" + + namespaces_to_create = [ + root_namespace, + f"{root_namespace}_A", + f"{root_namespace}_B", + f"{root_namespace}_C", + ] + + catalog = load_catalog_impl(started_cluster) + + for namespace in namespaces_to_create: + catalog.create_namespace(namespace) + assert len(catalog.list_tables(namespace)) == 0 + + for namespace in namespaces_to_create: + table = create_table(catalog, namespace, table_name) + + num_rows = 10 + df = generate_arrow_data(num_rows) + table.append(df) + + create_clickhouse_glue_database(started_cluster, node, CATALOG_NAME) + + expected = DEFAULT_CREATE_TABLE.format(CATALOG_NAME, namespace, table_name) + assert expected == node.query( + f"SHOW CREATE TABLE {CATALOG_NAME}.`{namespace}.{table_name}`" + ) + + assert num_rows == int( + node.query(f"SELECT count() FROM {CATALOG_NAME}.`{namespace}.{table_name}`") + ) + + # system.tables + assert int(node.query(f"SELECT count() FROM system.tables WHERE database = '{CATALOG_NAME}' and table ilike '%{root_namespace}%' SETTINGS show_data_lake_catalogs_in_system_tables = true").strip()) == 4 + assert int(node.query(f"SELECT count() FROM system.tables WHERE database = '{CATALOG_NAME}' and table ilike '%{root_namespace}%'").strip()) == 0 + + # system.iceberg_history - always displays irrespective of show_data_lake_catalogs_in_system_tables + assert int(node.query(f"SELECT count() FROM system.iceberg_history WHERE database = '{CATALOG_NAME}' and table ilike '%{root_namespace}%' SETTINGS show_data_lake_catalogs_in_system_tables = true").strip()) == 4 + assert int(node.query(f"SELECT count() FROM system.iceberg_history WHERE database = '{CATALOG_NAME}' and table ilike '%{root_namespace}%' SETTINGS show_data_lake_catalogs_in_system_tables = false").strip()) == 4 + + # system.databases + assert int(node.query(f"SELECT count() FROM system.databases WHERE name = '{CATALOG_NAME}' SETTINGS show_data_lake_catalogs_in_system_tables = true").strip()) == 1 + assert int(node.query(f"SELECT count() FROM system.databases WHERE name = '{CATALOG_NAME}'").strip()) == 0 + + # system.columns + assert int(node.query(f"SELECT count() FROM system.columns WHERE database = '{CATALOG_NAME}' and table ilike '%{root_namespace}%' SETTINGS show_data_lake_catalogs_in_system_tables = true").strip()) == 24 + assert int(node.query(f"SELECT count() FROM system.columns WHERE database = '{CATALOG_NAME}' and table ilike '%{root_namespace}%'").strip()) == 0 + + # system.completions + assert int(node.query(f"SELECT count() FROM system.completions WHERE startsWith(word, '{test_ref}') SETTINGS show_data_lake_catalogs_in_system_tables = true").strip()) != 0 + assert int(node.query(f"SELECT count() FROM system.completions WHERE startsWith(word, '{test_ref}')").strip()) == 0 diff --git a/tests/integration/test_database_iceberg/test.py b/tests/integration/test_database_iceberg/test.py index 662b51e0f687..f142ab8c7752 100644 --- a/tests/integration/test_database_iceberg/test.py +++ b/tests/integration/test_database_iceberg/test.py @@ -251,14 +251,14 @@ def test_list_tables(started_cluster): assert ( tables_list == node.query( - f"SELECT name FROM system.tables WHERE database = '{CATALOG_NAME}' and name ILIKE '{root_namespace}%' ORDER BY name" + f"SELECT name FROM system.tables WHERE database = '{CATALOG_NAME}' and name ILIKE '{root_namespace}%' ORDER BY name SETTINGS show_data_lake_catalogs_in_system_tables = true" ).strip() ) node.restart_clickhouse() assert ( tables_list == node.query( - f"SELECT name FROM system.tables WHERE database = '{CATALOG_NAME}' and name ILIKE '{root_namespace}%' ORDER BY name" + f"SELECT name FROM system.tables WHERE database = '{CATALOG_NAME}' and name ILIKE '{root_namespace}%' ORDER BY name SETTINGS show_data_lake_catalogs_in_system_tables = true" ).strip() ) @@ -296,7 +296,7 @@ def test_many_namespaces(started_cluster): table_name = f"{namespace}.{table}" assert int( node.query( - f"SELECT count() FROM system.tables WHERE database = '{CATALOG_NAME}' and name = '{table_name}'" + f"SELECT count() FROM system.tables WHERE database = '{CATALOG_NAME}' and name = '{table_name}' SETTINGS show_data_lake_catalogs_in_system_tables = true" ) ) diff --git a/tests/integration/test_database_iceberg_lakekeeper_catalog/test.py b/tests/integration/test_database_iceberg_lakekeeper_catalog/test.py index 9488f39473f3..d670b4997cde 100644 --- a/tests/integration/test_database_iceberg_lakekeeper_catalog/test.py +++ b/tests/integration/test_database_iceberg_lakekeeper_catalog/test.py @@ -168,7 +168,7 @@ def test_list_tables(started_cluster): assert ( tables_list == node.query( - f"SELECT name FROM system.tables WHERE database = '{CATALOG_NAME}' and name ILIKE '{namespace_prefix}%' ORDER BY name" + f"SELECT name FROM system.tables WHERE database = '{CATALOG_NAME}' and name ILIKE '{namespace_prefix}%' ORDER BY name SETTINGS show_data_lake_catalogs_in_system_tables = true" ).strip() ) From be5e096494f0221ac089e982bed9c02289483a4a Mon Sep 17 00:00:00 2001 From: Andrey Zvonov Date: Fri, 19 Dec 2025 11:57:44 +0100 Subject: [PATCH 2/6] fix SettingsChangesHistory --- src/Core/SettingsChangesHistory.cpp | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index ac7db7994280..3b7895b04334 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -97,6 +97,24 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() {"os_threads_nice_value_query", 0, 0, "New setting."}, {"os_threads_nice_value_materialized_view", 0, 0, "New setting."}, {"os_thread_priority", 0, 0, "Alias for os_threads_nice_value_query."}, + {"allow_experimental_iceberg_read_optimization", true, true, "New setting."}, + {"object_storage_cluster_join_mode", "allow", "allow", "New setting"}, + {"lock_object_storage_task_distribution_ms", 500, 500, "Raised the value to 500 to avoid hoping tasks between executors."}, + {"allow_retries_in_cluster_requests", false, false, "New setting"}, + {"object_storage_remote_initiator", false, false, "New setting."}, + {"allow_experimental_export_merge_tree_part", false, false, "New setting."}, + {"export_merge_tree_part_overwrite_file_if_exists", false, false, "New setting."}, + {"allow_experimental_export_merge_tree_part", false, true, "Turned ON by default for Antalya."}, + {"export_merge_tree_partition_force_export", false, false, "New setting."}, + {"export_merge_tree_partition_max_retries", 3, 3, "New setting."}, + {"export_merge_tree_partition_manifest_ttl", 180, 180, "New setting."}, + {"export_merge_tree_part_file_already_exists_policy", "skip", "skip", "New setting."}, + {"iceberg_timezone_for_timestamptz", "UTC", "UTC", "New setting."}, + {"hybrid_table_auto_cast_columns", true, true, "New setting to automatically cast Hybrid table columns when segments disagree on types. Default enabled."}, + {"allow_experimental_hybrid_table", false, false, "Added new setting to allow the Hybrid table engine."}, + {"export_merge_tree_part_max_bytes_per_file", 0, 0, "New setting."}, + {"export_merge_tree_part_max_rows_per_file", 0, 0, "New setting."}, + {"show_data_lake_catalogs_in_system_tables", true, false, "Disable catalogs in system tables by default"}, }); addSettingsChanges(settings_changes_history, "25.8", { From 906a335c686cfda21a9aba7f528ef01784ee7977 Mon Sep 17 00:00:00 2001 From: Andrey Zvonov Date: Fri, 26 Dec 2025 14:05:26 +0100 Subject: [PATCH 3/6] fix version and history --- src/Core/SettingsChangesHistory.cpp | 80 ++--------------------------- 1 file changed, 4 insertions(+), 76 deletions(-) diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index 3b7895b04334..3dbf458bb695 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -39,82 +39,10 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() /// controls new feature and it's 'true' by default, use 'false' as previous_value). /// It's used to implement `compatibility` setting (see https://github.com/ClickHouse/ClickHouse/issues/35972) /// Note: please check if the key already exists to prevent duplicate entries. - addSettingsChanges(settings_changes_history, "25.10", - { - {"show_data_lake_catalogs_in_system_tables", true, false, "Disable catalogs in system tables by default"}, - {"optimize_rewrite_like_perfect_affix", false, true, "New setting"}, - {"allow_dynamic_type_in_join_keys", true, false, "Disallow using Dynamic type in JOIN keys by default"}, - {"use_skip_indexes_on_data_read", false, true, "Enabled skip index usage in read phase by default"}, - {"s3queue_keeper_fault_injection_probablility", 0, 0, "New setting."}, - {"enable_join_runtime_filters", false, false, "New setting"}, - {"join_runtime_filter_exact_values_limit", 10000, 10000, "New setting"}, - {"join_runtime_bloom_filter_bytes", 512_KiB, 512_KiB, "New setting"}, - {"join_runtime_bloom_filter_hash_functions", 3, 3, "New setting"}, - {"use_join_disjunctions_push_down", false, false, "New setting."}, - {"joined_block_split_single_row", false, false, "New setting"}, - {"rewrite_in_to_join", false, false, "New experimental setting"}, - {"iceberg_insert_max_rows_in_data_file", 100000, 1000000, "New setting."}, - {"iceberg_insert_max_bytes_in_data_file", 100000000, 100000000, "New setting."}, - {"delta_lake_insert_max_rows_in_data_file", 100000, 1000000, "New setting."}, - {"delta_lake_log_metadata", false, false, "New setting."}, - {"distributed_cache_prefer_bigger_buffer_size", false, false, "New setting."}, - {"allow_experimental_qbit_type", false, false, "New experimental setting"}, - {"optimize_qbit_distance_function_reads", true, true, "New setting"}, - {"read_from_distributed_cache_if_exists_otherwise_bypass_cache", false, false, "New setting"}, - {"s3_slow_all_threads_after_retryable_error", false, false, "Disable the setting by default"}, - {"backup_slow_all_threads_after_retryable_s3_error", false, false, "Disable the setting by default"}, - {"enable_http_compression", false, true, "It should be beneficial in general"}, - {"inject_random_order_for_select_without_order_by", false, false, "New setting"}, - {"exclude_materialize_skip_indexes_on_insert", "", "", "New setting."}, - {"optimize_empty_string_comparisons", false, true, "A new setting."}, - {"query_plan_use_logical_join_step", true, true, "Added alias"}, - {"schema_inference_make_columns_nullable", 1, 3, "Take nullability information from Parquet/ORC/Arrow metadata by default, instead of making everything nullable."}, - {"materialized_views_squash_parallel_inserts", false, true, "Added setting to preserve old behavior if needed."}, - }); - addSettingsChanges(settings_changes_history, "25.9", - { - {"input_format_protobuf_oneof_presence", false, false, "New setting"}, - {"iceberg_delete_data_on_drop", false, false, "New setting"}, - {"use_skip_indexes_on_data_read", false, false, "New setting"}, - {"s3_slow_all_threads_after_retryable_error", false, false, "Added an alias for setting `backup_slow_all_threads_after_retryable_s3_error`"}, - {"iceberg_metadata_log_level", "none", "none", "New setting."}, - {"iceberg_insert_max_rows_in_data_file", 100000, 100000, "New setting."}, - {"iceberg_insert_max_bytes_in_data_file", 100000000, 100000000, "New setting."}, - {"query_plan_optimize_join_order_limit", 0, 1, "New setting"}, - {"query_plan_display_internal_aliases", false, false, "New setting"}, - {"query_plan_max_step_description_length", 1000000000, 500, "New setting"}, - {"allow_experimental_delta_lake_writes", false, false, "New setting."}, - {"query_plan_convert_any_join_to_semi_or_anti_join", true, true, "New setting."}, - {"text_index_use_bloom_filter", true, true, "New setting."}, - {"query_plan_direct_read_from_text_index", true, true, "New setting."}, - {"enable_producing_buckets_out_of_order_in_aggregation", false, true, "New setting"}, - {"jemalloc_enable_profiler", false, false, "New setting"}, - {"jemalloc_collect_profile_samples_in_trace_log", false, false, "New setting"}, - {"delta_lake_insert_max_bytes_in_data_file", 1_GiB, 1_GiB, "New setting."}, - {"delta_lake_insert_max_rows_in_data_file", 100000, 100000, "New setting."}, - {"promql_evaluation_time", Field{"auto"}, Field{"auto"}, "The setting was renamed. The previous name is `evaluation_time`."}, - {"evaluation_time", 0, 0, "Old setting which popped up here being renamed."}, - {"os_threads_nice_value_query", 0, 0, "New setting."}, - {"os_threads_nice_value_materialized_view", 0, 0, "New setting."}, - {"os_thread_priority", 0, 0, "Alias for os_threads_nice_value_query."}, - {"allow_experimental_iceberg_read_optimization", true, true, "New setting."}, - {"object_storage_cluster_join_mode", "allow", "allow", "New setting"}, - {"lock_object_storage_task_distribution_ms", 500, 500, "Raised the value to 500 to avoid hoping tasks between executors."}, - {"allow_retries_in_cluster_requests", false, false, "New setting"}, - {"object_storage_remote_initiator", false, false, "New setting."}, - {"allow_experimental_export_merge_tree_part", false, false, "New setting."}, - {"export_merge_tree_part_overwrite_file_if_exists", false, false, "New setting."}, - {"allow_experimental_export_merge_tree_part", false, true, "Turned ON by default for Antalya."}, - {"export_merge_tree_partition_force_export", false, false, "New setting."}, - {"export_merge_tree_partition_max_retries", 3, 3, "New setting."}, - {"export_merge_tree_partition_manifest_ttl", 180, 180, "New setting."}, - {"export_merge_tree_part_file_already_exists_policy", "skip", "skip", "New setting."}, - {"iceberg_timezone_for_timestamptz", "UTC", "UTC", "New setting."}, - {"hybrid_table_auto_cast_columns", true, true, "New setting to automatically cast Hybrid table columns when segments disagree on types. Default enabled."}, - {"allow_experimental_hybrid_table", false, false, "Added new setting to allow the Hybrid table engine."}, - {"export_merge_tree_part_max_bytes_per_file", 0, 0, "New setting."}, - {"export_merge_tree_part_max_rows_per_file", 0, 0, "New setting."}, - {"show_data_lake_catalogs_in_system_tables", true, false, "Disable catalogs in system tables by default"}, + addSettingsChanges(settings_changes_history, "25.8.12.1000", + { + {"show_data_lake_catalogs_in_system_tables", false, true, "Disable catalogs in system tables by default"}, + }); addSettingsChanges(settings_changes_history, "25.8", { From f38f666d8ee6b9f321bf0df9c9b085168108a3ab Mon Sep 17 00:00:00 2001 From: Andrey Zvonov Date: Fri, 26 Dec 2025 17:37:41 +0100 Subject: [PATCH 4/6] bump version --- src/Core/SettingsChangesHistory.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index 3dbf458bb695..2d1ec60f4d0d 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -39,7 +39,7 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() /// controls new feature and it's 'true' by default, use 'false' as previous_value). /// It's used to implement `compatibility` setting (see https://github.com/ClickHouse/ClickHouse/issues/35972) /// Note: please check if the key already exists to prevent duplicate entries. - addSettingsChanges(settings_changes_history, "25.8.12.1000", + addSettingsChanges(settings_changes_history, "25.8.12.10000", { {"show_data_lake_catalogs_in_system_tables", false, true, "Disable catalogs in system tables by default"}, From 2229800397c2626e4cfd040b68503a44b1ff4157 Mon Sep 17 00:00:00 2001 From: Andrey Zvonov Date: Fri, 26 Dec 2025 23:14:42 +0100 Subject: [PATCH 5/6] fix test accordingly --- tests/integration/test_database_hms/test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_database_hms/test.py b/tests/integration/test_database_hms/test.py index d64bb63699d7..65b07a352d00 100644 --- a/tests/integration/test_database_hms/test.py +++ b/tests/integration/test_database_hms/test.py @@ -199,14 +199,14 @@ def test_list_tables(started_cluster): assert ( tables_list == node.query( - f"SELECT name FROM system.tables WHERE database = '{CATALOG_NAME}' and name ILIKE '{root_namespace}%' ORDER BY name" + f"SELECT name FROM system.tables WHERE database = '{CATALOG_NAME}' and name ILIKE '{root_namespace}%' ORDER BY name SETTINGS show_data_lake_catalogs_in_system_tables=1" ).strip() ) node.restart_clickhouse() assert ( tables_list == node.query( - f"SELECT name FROM system.tables WHERE database = '{CATALOG_NAME}' and name ILIKE '{root_namespace}%' ORDER BY name" + f"SELECT name FROM system.tables WHERE database = '{CATALOG_NAME}' and name ILIKE '{root_namespace}%' ORDER BY name SETTINGS show_data_lake_catalogs_in_system_tables=1" ).strip() ) @@ -235,7 +235,7 @@ def test_many_namespaces(started_cluster): table_name = f"{namespace}.{table}" assert int( node.query( - f"SELECT count() FROM system.tables WHERE database = '{CATALOG_NAME}' and name = '{table_name}'" + f"SELECT count() FROM system.tables WHERE database = '{CATALOG_NAME}' and name = '{table_name}' SETTINGS show_data_lake_catalogs_in_system_tables=1" ) ) From e089108bc4d9ccb4bd8c5f08acb2fd36ccbb019b Mon Sep 17 00:00:00 2001 From: Andrey Zvonov <32552679+zvonand@users.noreply.github.com> Date: Sat, 27 Dec 2025 13:22:16 +0100 Subject: [PATCH 6/6] Update settings change history for version 25.8.13.10000 --- src/Core/SettingsChangesHistory.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index 2d1ec60f4d0d..9dc03ed64207 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -39,7 +39,7 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() /// controls new feature and it's 'true' by default, use 'false' as previous_value). /// It's used to implement `compatibility` setting (see https://github.com/ClickHouse/ClickHouse/issues/35972) /// Note: please check if the key already exists to prevent duplicate entries. - addSettingsChanges(settings_changes_history, "25.8.12.10000", + addSettingsChanges(settings_changes_history, "25.8.13.10000", { {"show_data_lake_catalogs_in_system_tables", false, true, "Disable catalogs in system tables by default"},