diff --git a/src/Backups/BackupEntriesCollector.cpp b/src/Backups/BackupEntriesCollector.cpp index 4ce7a5f99edc..cf73f97e80ed 100644 --- a/src/Backups/BackupEntriesCollector.cpp +++ b/src/Backups/BackupEntriesCollector.cpp @@ -397,7 +397,7 @@ void BackupEntriesCollector::gatherDatabasesMetadata() case ASTBackupQuery::ElementType::ALL: { - for (const auto & [database_name, database] : DatabaseCatalog::instance().getDatabases()) + for (const auto & [database_name, database] : DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = true})) { if (!element.except_databases.contains(database_name)) { diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index e37c189d3c2b..4a5fc0821e8a 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -6472,7 +6472,7 @@ Query Iceberg table using the snapshot that was current at a specific timestamp. DECLARE(Int64, iceberg_snapshot_id, 0, R"( Query Iceberg table using the specific snapshot id. )", 0) \ - DECLARE(Bool, show_data_lake_catalogs_in_system_tables, true, R"( + DECLARE(Bool, show_data_lake_catalogs_in_system_tables, false, R"( Enables showing data lake catalogs in system tables. )", 0) \ DECLARE(Bool, delta_lake_enable_expression_visitor_logging, false, R"( diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index eaa78f0a3661..9dc03ed64207 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -39,6 +39,11 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() /// controls new feature and it's 'true' by default, use 'false' as previous_value). /// It's used to implement `compatibility` setting (see https://github.com/ClickHouse/ClickHouse/issues/35972) /// Note: please check if the key already exists to prevent duplicate entries. + addSettingsChanges(settings_changes_history, "25.8.13.10000", + { + {"show_data_lake_catalogs_in_system_tables", false, true, "Disable catalogs in system tables by default"}, + + }); addSettingsChanges(settings_changes_history, "25.8", { {"output_format_json_quote_64bit_integers", true, false, "Disable quoting of the 64 bit integers in JSON by default"}, diff --git a/src/Databases/DataLake/DatabaseDataLake.cpp b/src/Databases/DataLake/DatabaseDataLake.cpp index 15ca4b9dd3e7..2679ae0de56c 100644 --- a/src/Databases/DataLake/DatabaseDataLake.cpp +++ b/src/Databases/DataLake/DatabaseDataLake.cpp @@ -532,14 +532,10 @@ DatabaseTablesIteratorPtr DatabaseDataLake::getTablesIterator( DatabaseTablesIteratorPtr DatabaseDataLake::getLightweightTablesIterator( ContextPtr context_, const FilterByNameFunction & filter_by_table_name, - bool skip_not_loaded, - bool skip_data_lake_catalog) const + bool skip_not_loaded) const { Tables tables; - if (skip_data_lake_catalog) - return std::make_unique(tables, getDatabaseName()); - auto catalog = getCatalog(); DB::Names iceberg_tables; diff --git a/src/Databases/DataLake/DatabaseDataLake.h b/src/Databases/DataLake/DatabaseDataLake.h index c46b0eb3dcd2..b13cbf866009 100644 --- a/src/Databases/DataLake/DatabaseDataLake.h +++ b/src/Databases/DataLake/DatabaseDataLake.h @@ -31,6 +31,7 @@ class DatabaseDataLake final : public IDatabase, WithContext bool canContainDistributedTables() const override { return false; } bool canContainRocksDBTables() const override { return false; } bool shouldBeEmptyOnDetach() const override { return false; } + bool isDatalakeCatalog() const override { return true; } bool empty() const override; @@ -47,8 +48,7 @@ class DatabaseDataLake final : public IDatabase, WithContext DatabaseTablesIteratorPtr getLightweightTablesIterator( ContextPtr context, const FilterByNameFunction & filter_by_table_name, - bool skip_not_loaded, - bool skip_data_lake_catalog) const override; + bool skip_not_loaded) const override; void shutdown() override {} diff --git a/src/Databases/IDatabase.h b/src/Databases/IDatabase.h index eb7d5f2c75ce..9f4fa18cc71a 100644 --- a/src/Databases/IDatabase.h +++ b/src/Databases/IDatabase.h @@ -179,6 +179,8 @@ class IDatabase : public std::enable_shared_from_this virtual bool canContainRocksDBTables() const { return true; } + virtual bool isDatalakeCatalog() const { return false; } + /// Load a set of existing tables. /// You can call only once, right after the object is created. virtual void loadStoredObjects( /// NOLINT @@ -267,7 +269,7 @@ class IDatabase : public std::enable_shared_from_this /// Same as above, but may return non-fully initialized StoragePtr objects which are not suitable for reading. /// Useful for queries like "SHOW TABLES" - virtual DatabaseTablesIteratorPtr getLightweightTablesIterator(ContextPtr context, const FilterByNameFunction & filter_by_table_name = {}, bool skip_not_loaded = false, [[maybe_unused]] bool skip_data_lake_catalog = false) const /// NOLINT + virtual DatabaseTablesIteratorPtr getLightweightTablesIterator(ContextPtr context, const FilterByNameFunction & filter_by_table_name = {}, bool skip_not_loaded = false) const /// NOLINT { return getTablesIterator(context, filter_by_table_name, skip_not_loaded); } diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index b6c3ef92c90f..727a05a22f4a 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -115,7 +115,7 @@ class DatabaseNameHints : public IHints<> const bool need_to_check_access_for_databases = !access->isGranted(AccessType::SHOW_DATABASES); Names result; - auto databases_list = database_catalog.getDatabases(); + auto databases_list = database_catalog.getDatabases(GetDatabasesOptions{.with_datalake_catalogs = true}); for (const auto & database_name : databases_list | boost::adaptors::map_keys) { if (need_to_check_access_for_databases && !access->isGranted(AccessType::SHOW_DATABASES, database_name)) @@ -341,7 +341,10 @@ void DatabaseCatalog::shutdownImpl(std::function shutdown_system_logs) auto it = std::find_if(elem.map.begin(), elem.map.end(), not_empty_mapping); return it != elem.map.end(); }) == uuid_map.end()); + databases.clear(); + databases_without_datalake_catalogs.clear(); + referential_dependencies.clear(); loading_dependencies.clear(); view_dependencies.clear(); @@ -576,6 +579,18 @@ void DatabaseCatalog::assertDatabaseExists(const String & database_name) const } } +bool DatabaseCatalog::hasDatalakeCatalogs() const +{ + std::lock_guard lock{databases_mutex}; + return databases.size() != databases_without_datalake_catalogs.size(); +} + +bool DatabaseCatalog::isDatalakeCatalog(const String & database_name) const +{ + std::lock_guard lock{databases_mutex}; + return databases.contains(database_name) && !databases_without_datalake_catalogs.contains(database_name); +} + void DatabaseCatalog::assertDatabaseDoesntExist(const String & database_name) const { std::lock_guard lock{databases_mutex}; @@ -594,6 +609,9 @@ void DatabaseCatalog::attachDatabase(const String & database_name, const Databas std::lock_guard lock{databases_mutex}; assertDatabaseDoesntExistUnlocked(database_name); databases.emplace(database_name, database); + if (!database->isDatalakeCatalog()) + databases_without_datalake_catalogs.emplace(database_name, database); + NOEXCEPT_SCOPE({ UUID db_uuid = database->getUUID(); if (db_uuid != UUIDHelpers::Nil) @@ -618,8 +636,9 @@ DatabasePtr DatabaseCatalog::detachDatabase(ContextPtr local_context, const Stri if (db_uuid != UUIDHelpers::Nil) removeUUIDMapping(db_uuid); databases.erase(database_name); - } + if (auto it = databases_without_datalake_catalogs.find(database_name); it != databases_without_datalake_catalogs.end()) + databases_without_datalake_catalogs.erase(it); } if (!db) { @@ -685,7 +704,13 @@ void DatabaseCatalog::updateDatabaseName(const String & old_name, const String & databases.erase(it); databases.emplace(new_name, db); - /// Update dependencies. + auto no_catalogs_it = databases_without_datalake_catalogs.find(old_name); + if (no_catalogs_it != databases_without_datalake_catalogs.end()) + { + databases_without_datalake_catalogs.erase(no_catalogs_it); + databases_without_datalake_catalogs.emplace(new_name, db); + } + for (const auto & table_name : tables_in_database) { auto removed_ref_deps = referential_dependencies.removeDependencies(StorageID{old_name, table_name}, /* remove_isolated_tables= */ true); @@ -806,10 +831,13 @@ bool DatabaseCatalog::isDatabaseExist(const String & database_name) const return databases.end() != databases.find(database_name); } -Databases DatabaseCatalog::getDatabases() const +Databases DatabaseCatalog::getDatabases(GetDatabasesOptions options) const { std::lock_guard lock{databases_mutex}; - return databases; + if (options.with_datalake_catalogs) + return databases; + + return databases_without_datalake_catalogs; } bool DatabaseCatalog::isTableExist(const DB::StorageID & table_id, ContextPtr context_) const @@ -1075,7 +1103,7 @@ void DatabaseCatalog::loadMarkedAsDroppedTables() std::map> dropped_metadata; String path = fs::path("metadata_dropped") / ""; - auto db_map = getDatabases(); + auto db_map = getDatabases(GetDatabasesOptions{.with_datalake_catalogs = true}); std::set metadata_disk_list; for (const auto & [_, db] : db_map) { @@ -1965,7 +1993,7 @@ void DatabaseCatalog::reloadDisksTask() disks.swap(disks_to_reload); } - for (auto & database : getDatabases()) + for (auto & database : getDatabases(GetDatabasesOptions{.with_datalake_catalogs = false})) { // WARNING: In case of `async_load_databases = true` getTablesIterator() call wait for all table in the database to be loaded. // WARNING: It means that no database will be able to update configuration until all databases are fully loaded. @@ -2120,7 +2148,8 @@ std::pair TableNameHints::getExtendedHintForTable(const String & { /// load all available databases from the DatabaseCatalog instance auto & database_catalog = DatabaseCatalog::instance(); - auto all_databases = database_catalog.getDatabases(); + /// NOTE Skip datalake catalogs to avoid unnecessary access to remote catalogs (can be expensive) + auto all_databases = database_catalog.getDatabases(GetDatabasesOptions{.with_datalake_catalogs = false}); for (const auto & [db_name, db] : all_databases) { diff --git a/src/Interpreters/DatabaseCatalog.h b/src/Interpreters/DatabaseCatalog.h index e695f7c22521..c95aaf4654d9 100644 --- a/src/Interpreters/DatabaseCatalog.h +++ b/src/Interpreters/DatabaseCatalog.h @@ -121,6 +121,11 @@ using TemporaryTablesMapping = std::map; class BackgroundSchedulePoolTaskHolder; +struct GetDatabasesOptions +{ + bool with_datalake_catalogs{false}; +}; + /// For some reason Context is required to get Storage from Database object class DatabaseCatalog : boost::noncopyable, WithMutableContext { @@ -171,7 +176,13 @@ class DatabaseCatalog : boost::noncopyable, WithMutableContext DatabasePtr getDatabase(const UUID & uuid) const; DatabasePtr tryGetDatabase(const UUID & uuid) const; bool isDatabaseExist(const String & database_name) const; - Databases getDatabases() const; + /// Datalake catalogs are implement at IDatabase level in ClickHouse. + /// In general case Datalake catalog is a some remote service which contains iceberg/delta tables. + /// Sometimes this service charges money for requests. With this flag we explicitly protect ourself + /// to not accidentally query external non-free service for some trivial things like + /// autocompletion hints or system.tables query. We have a setting which allow to show + /// these databases everywhere, but user must explicitly specify it. + Databases getDatabases(GetDatabasesOptions options) const; /// Same as getDatabase(const String & database_name), but if database_name is empty, current database of local_context is used DatabasePtr getDatabase(const String & database_name, ContextPtr local_context) const; @@ -272,6 +283,8 @@ class DatabaseCatalog : boost::noncopyable, WithMutableContext bool canPerformReplicatedDDLQueries() const; void updateMetadataFile(const DatabasePtr & database); + bool hasDatalakeCatalogs() const; + bool isDatalakeCatalog(const String & database_name) const; private: // The global instance of database catalog. unique_ptr is to allow @@ -319,6 +332,7 @@ class DatabaseCatalog : boost::noncopyable, WithMutableContext mutable std::mutex databases_mutex; Databases databases TSA_GUARDED_BY(databases_mutex); + Databases databases_without_datalake_catalogs TSA_GUARDED_BY(databases_mutex); UUIDToStorageMap uuid_map; /// Referential dependencies between tables: table "A" depends on table "B" diff --git a/src/Interpreters/InterpreterCheckQuery.cpp b/src/Interpreters/InterpreterCheckQuery.cpp index 2a9682905a8c..6391608f4902 100644 --- a/src/Interpreters/InterpreterCheckQuery.cpp +++ b/src/Interpreters/InterpreterCheckQuery.cpp @@ -395,7 +395,7 @@ InterpreterCheckQuery::InterpreterCheckQuery(const ASTPtr & query_ptr_, ContextP static Strings getAllDatabases(const ContextPtr & context) { Strings res; - const auto & databases = DatabaseCatalog::instance().getDatabases(); + const auto & databases = DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = false}); res.reserve(databases.size()); for (const auto & [database_name, _] : databases) { diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 0e859d775db3..d9ce22442b9d 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -206,7 +206,7 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create) auto db_num_limit = getContext()->getGlobalContext()->getServerSettings()[ServerSetting::max_database_num_to_throw].value; if (db_num_limit > 0 && !internal) { - size_t db_count = DatabaseCatalog::instance().getDatabases().size(); + size_t db_count = DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = true}).size(); std::initializer_list system_databases = { DatabaseCatalog::TEMPORARY_DATABASE, diff --git a/src/Interpreters/InterpreterShowTablesQuery.cpp b/src/Interpreters/InterpreterShowTablesQuery.cpp index 8a159ffdcbb3..d8e01ecd722a 100644 --- a/src/Interpreters/InterpreterShowTablesQuery.cpp +++ b/src/Interpreters/InterpreterShowTablesQuery.cpp @@ -14,6 +14,7 @@ #include #include #include +#include namespace DB @@ -229,8 +230,20 @@ BlockIO InterpreterShowTablesQuery::execute() return res; } + auto rewritten_query = getRewrittenQuery(); + String database = getContext()->resolveDatabase(query.getFrom()); + if (DatabaseCatalog::instance().isDatalakeCatalog(database)) + { + auto context_copy = Context::createCopy(getContext()); + /// HACK To always show them in explicit "SHOW TABLES" queries + context_copy->setSetting("show_data_lake_catalogs_in_system_tables", true); + return executeQuery(rewritten_query, context_copy, QueryFlags{ .internal = true }).second; + } + else + { + return executeQuery(rewritten_query, getContext(), QueryFlags{ .internal = true }).second; + } - return executeQuery(getRewrittenQuery(), getContext(), QueryFlags{ .internal = true }).second; } /// (*) Sorting is strictly speaking not necessary but 1. it is convenient for users, 2. SQL currently does not allow to diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp index f02a6e8ddd6c..58d228d5f189 100644 --- a/src/Interpreters/InterpreterSystemQuery.cpp +++ b/src/Interpreters/InterpreterSystemQuery.cpp @@ -242,7 +242,7 @@ void InterpreterSystemQuery::startStopAction(StorageActionBlockType action_type, } else { - for (auto & elem : DatabaseCatalog::instance().getDatabases()) + for (auto & elem : DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = false})) { startStopActionInDatabase(action_type, start, elem.first, elem.second, getContext(), log); } @@ -1075,7 +1075,7 @@ void InterpreterSystemQuery::restartReplicas(ContextMutablePtr system_context) bool access_is_granted_globally = access->isGranted(AccessType::SYSTEM_RESTART_REPLICA); bool show_tables_is_granted_globally = access->isGranted(AccessType::SHOW_TABLES); - for (auto & elem : catalog.getDatabases()) + for (auto & elem : catalog.getDatabases(GetDatabasesOptions{.with_datalake_catalogs = false})) { if (!elem.second->canContainMergeTreeTables()) continue; @@ -1137,7 +1137,7 @@ void InterpreterSystemQuery::dropReplica(ASTSystemQuery & query) } else if (query.is_drop_whole_replica) { - auto databases = DatabaseCatalog::instance().getDatabases(); + auto databases = DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = false}); auto access = getContext()->getAccess(); bool access_is_granted_globally = access->isGranted(AccessType::SYSTEM_DROP_REPLICA); @@ -1178,7 +1178,7 @@ void InterpreterSystemQuery::dropReplica(ASTSystemQuery & query) String remote_replica_path = fs::path(query.replica_zk_path) / "replicas" / query.replica; /// This check is actually redundant, but it may prevent from some user mistakes - for (auto & elem : DatabaseCatalog::instance().getDatabases()) + for (auto & elem : DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = false})) { DatabasePtr & database = elem.second; for (auto iterator = database->getTablesIterator(getContext()); iterator->isValid(); iterator->next()) @@ -1272,7 +1272,7 @@ void InterpreterSystemQuery::dropDatabaseReplica(ASTSystemQuery & query) } else if (query.is_drop_whole_replica) { - auto databases = DatabaseCatalog::instance().getDatabases(); + auto databases = DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = false}); auto access = getContext()->getAccess(); bool access_is_granted_globally = access->isGranted(AccessType::SYSTEM_DROP_REPLICA); @@ -1298,7 +1298,7 @@ void InterpreterSystemQuery::dropDatabaseReplica(ASTSystemQuery & query) getContext()->checkAccess(AccessType::SYSTEM_DROP_REPLICA); /// This check is actually redundant, but it may prevent from some user mistakes - for (auto & elem : DatabaseCatalog::instance().getDatabases()) + for (auto & elem : DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = false})) if (auto * replicated = dynamic_cast(elem.second.get())) check_not_local_replica(replicated, query); @@ -1416,7 +1416,7 @@ void InterpreterSystemQuery::loadOrUnloadPrimaryKeysImpl(bool load) getContext()->checkAccess(load ? AccessType::SYSTEM_LOAD_PRIMARY_KEY : AccessType::SYSTEM_UNLOAD_PRIMARY_KEY); LOG_TRACE(log, "{} primary keys for all tables", load ? "Loading" : "Unloading"); - for (auto & database : DatabaseCatalog::instance().getDatabases()) + for (auto & database : DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = false})) { for (auto it = database.second->getTablesIterator(getContext()); it->isValid(); it->next()) { diff --git a/src/Interpreters/ServerAsynchronousMetrics.cpp b/src/Interpreters/ServerAsynchronousMetrics.cpp index 80af95332a12..0e3713d66c94 100644 --- a/src/Interpreters/ServerAsynchronousMetrics.cpp +++ b/src/Interpreters/ServerAsynchronousMetrics.cpp @@ -196,7 +196,7 @@ void ServerAsynchronousMetrics::updateImpl(TimePoint update_time, TimePoint curr } { - auto databases = DatabaseCatalog::instance().getDatabases(); + auto databases = DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = false}); size_t max_queue_size = 0; size_t max_inserts_in_queue = 0; @@ -371,7 +371,7 @@ void ServerAsynchronousMetrics::updateMutationAndDetachedPartsStats() DetachedPartsStats current_values{}; MutationStats current_mutation_stats{}; - for (const auto & db : DatabaseCatalog::instance().getDatabases()) + for (const auto & db : DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = false})) { if (!db.second->canContainMergeTreeTables()) continue; diff --git a/src/Interpreters/loadMetadata.cpp b/src/Interpreters/loadMetadata.cpp index e2c448c0bc93..f05508975a5b 100644 --- a/src/Interpreters/loadMetadata.cpp +++ b/src/Interpreters/loadMetadata.cpp @@ -538,7 +538,7 @@ void convertDatabasesEnginesIfNeed(const LoadTaskPtrs & load_metadata, ContextMu // Wait for all table to be loaded and started waitLoad(TablesLoaderForegroundPoolId, load_metadata); - for (const auto & [name, _] : DatabaseCatalog::instance().getDatabases()) + for (const auto & [name, _] : DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = false})) if (name != DatabaseCatalog::SYSTEM_DATABASE) maybeConvertOrdinaryDatabaseToAtomic(context, name); diff --git a/src/Server/ReplicasStatusHandler.cpp b/src/Server/ReplicasStatusHandler.cpp index a44a02f26ae6..4129144deea5 100644 --- a/src/Server/ReplicasStatusHandler.cpp +++ b/src/Server/ReplicasStatusHandler.cpp @@ -55,7 +55,7 @@ void ReplicasStatusHandler::handleRequest(HTTPServerRequest & request, HTTPServe bool ok = true; WriteBufferFromOwnString message; - auto databases = DatabaseCatalog::instance().getDatabases(); + auto databases = DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = false}); /// Iterate through all the replicated tables. for (const auto & db : databases) diff --git a/src/Storages/RocksDB/StorageSystemRocksDB.cpp b/src/Storages/RocksDB/StorageSystemRocksDB.cpp index 9099671592a4..27606bdd0c33 100644 --- a/src/Storages/RocksDB/StorageSystemRocksDB.cpp +++ b/src/Storages/RocksDB/StorageSystemRocksDB.cpp @@ -54,7 +54,7 @@ void StorageSystemRocksDB::fillData(MutableColumns & res_columns, ContextPtr con using RocksDBStoragePtr = std::shared_ptr; std::map> tables; - for (const auto & db : DatabaseCatalog::instance().getDatabases()) + for (const auto & db : DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = false})) { if (!db.second->canContainRocksDBTables()) continue; diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index c858186e8a27..c6097b4388c7 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -1436,7 +1436,7 @@ StorageMerge::DatabaseTablesIterators StorageMerge::DatabaseNameOrRegexp::getDat else { /// database_name argument is a regexp - auto databases = DatabaseCatalog::instance().getDatabases(); + auto databases = DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = true}); for (const auto & db : databases) { diff --git a/src/Storages/System/StorageSystemClusters.cpp b/src/Storages/System/StorageSystemClusters.cpp index 3acd3beb4be4..828918952e41 100644 --- a/src/Storages/System/StorageSystemClusters.cpp +++ b/src/Storages/System/StorageSystemClusters.cpp @@ -52,7 +52,7 @@ void StorageSystemClusters::fillData(MutableColumns & res_columns, ContextPtr co for (const auto & name_and_cluster : context->getClusters()) writeCluster(res_columns, columns_mask, name_and_cluster, /* replicated= */ nullptr); - const auto databases = DatabaseCatalog::instance().getDatabases(); + const auto databases = DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = false}); for (const auto & name_and_database : databases) { if (const auto * replicated = typeid_cast(name_and_database.second.get())) diff --git a/src/Storages/System/StorageSystemColumns.cpp b/src/Storages/System/StorageSystemColumns.cpp index 2425358c9447..8ae1fbb16bfc 100644 --- a/src/Storages/System/StorageSystemColumns.cpp +++ b/src/Storages/System/StorageSystemColumns.cpp @@ -27,6 +27,7 @@ namespace DB namespace Setting { extern const SettingsSeconds lock_acquire_timeout; + extern const SettingsBool show_data_lake_catalogs_in_system_tables; } namespace ErrorCodes { @@ -441,7 +442,9 @@ void ReadFromSystemColumns::initializePipeline(QueryPipelineBuilder & pipeline, /// Add `database` column. MutableColumnPtr database_column_mut = ColumnString::create(); - const auto databases = DatabaseCatalog::instance().getDatabases(); + const auto & context = getContext(); + const auto & settings = context->getSettingsRef(); + const auto databases = DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = settings[Setting::show_data_lake_catalogs_in_system_tables]}); for (const auto & [database_name, database] : databases) { if (database_name == DatabaseCatalog::TEMPORARY_DATABASE) diff --git a/src/Storages/System/StorageSystemCompletions.cpp b/src/Storages/System/StorageSystemCompletions.cpp index 16bdee3ff29d..3664e841c099 100644 --- a/src/Storages/System/StorageSystemCompletions.cpp +++ b/src/Storages/System/StorageSystemCompletions.cpp @@ -32,6 +32,7 @@ namespace Setting { extern const SettingsUInt64 readonly; extern const SettingsSeconds lock_acquire_timeout; + extern const SettingsBool show_data_lake_catalogs_in_system_tables; } static constexpr const char * DATABASE_CONTEXT = "database"; @@ -96,7 +97,8 @@ void fillDataWithTableColumns(const String & database_name, const String & table void fillDataWithDatabasesTablesColumns(MutableColumns & res_columns, const ContextPtr & context) { const auto & access = context->getAccess(); - const auto & databases = DatabaseCatalog::instance().getDatabases(); + const auto & settings = context->getSettingsRef(); + const auto & databases = DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = settings[Setting::show_data_lake_catalogs_in_system_tables]}); for (const auto & [database_name, database_ptr] : databases) { if (!access->isGranted(AccessType::SHOW_DATABASES) || !access->isGranted(AccessType::SHOW_DATABASES, database_name)) diff --git a/src/Storages/System/StorageSystemDataSkippingIndices.cpp b/src/Storages/System/StorageSystemDataSkippingIndices.cpp index fa70a42cce0d..808caa7dab78 100644 --- a/src/Storages/System/StorageSystemDataSkippingIndices.cpp +++ b/src/Storages/System/StorageSystemDataSkippingIndices.cpp @@ -261,7 +261,7 @@ void ReadFromSystemDataSkippingIndices::initializePipeline(QueryPipelineBuilder { MutableColumnPtr column = ColumnString::create(); - const auto databases = DatabaseCatalog::instance().getDatabases(); + const auto databases = DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = false}); for (const auto & [database_name, database] : databases) { if (database_name == DatabaseCatalog::TEMPORARY_DATABASE) diff --git a/src/Storages/System/StorageSystemDatabases.cpp b/src/Storages/System/StorageSystemDatabases.cpp index 12b3b266c301..dceb291d5d50 100644 --- a/src/Storages/System/StorageSystemDatabases.cpp +++ b/src/Storages/System/StorageSystemDatabases.cpp @@ -11,11 +11,18 @@ #include #include #include +#include namespace DB { +namespace Setting +{ + extern const SettingsBool show_data_lake_catalogs_in_system_tables; +} + + ColumnsDescription StorageSystemDatabases::getColumnsDescription() { auto description = ColumnsDescription @@ -111,8 +118,8 @@ void StorageSystemDatabases::fillData(MutableColumns & res_columns, ContextPtr c { const auto access = context->getAccess(); const bool need_to_check_access_for_databases = !access->isGranted(AccessType::SHOW_DATABASES); - - const auto databases = DatabaseCatalog::instance().getDatabases(); + const auto & settings = context->getSettingsRef(); + const auto databases = DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = settings[Setting::show_data_lake_catalogs_in_system_tables]}); ColumnPtr filtered_databases_column = getFilteredDatabases(databases, predicate, context); for (size_t i = 0; i < filtered_databases_column->size(); ++i) diff --git a/src/Storages/System/StorageSystemDistributionQueue.cpp b/src/Storages/System/StorageSystemDistributionQueue.cpp index ff9cb6aa50c4..5c364622bbff 100644 --- a/src/Storages/System/StorageSystemDistributionQueue.cpp +++ b/src/Storages/System/StorageSystemDistributionQueue.cpp @@ -121,7 +121,7 @@ void StorageSystemDistributionQueue::fillData(MutableColumns & res_columns, Cont const bool check_access_for_databases = !access->isGranted(AccessType::SHOW_TABLES); std::map> tables; - for (const auto & db : DatabaseCatalog::instance().getDatabases()) + for (const auto & db : DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = false})) { /// Check if database can contain distributed tables if (!db.second->canContainDistributedTables()) diff --git a/src/Storages/System/StorageSystemGraphite.cpp b/src/Storages/System/StorageSystemGraphite.cpp index ef13c3c24da0..8e0dd1e20260 100644 --- a/src/Storages/System/StorageSystemGraphite.cpp +++ b/src/Storages/System/StorageSystemGraphite.cpp @@ -35,7 +35,7 @@ ColumnsDescription StorageSystemGraphite::getColumnsDescription() */ static StorageSystemGraphite::Configs getConfigs(ContextPtr context) { - const Databases databases = DatabaseCatalog::instance().getDatabases(); + const Databases databases = DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = false}); StorageSystemGraphite::Configs graphite_configs; for (const auto & db : databases) diff --git a/src/Storages/System/StorageSystemIcebergHistory.cpp b/src/Storages/System/StorageSystemIcebergHistory.cpp index f2148a753bb5..410dfcbc40a6 100644 --- a/src/Storages/System/StorageSystemIcebergHistory.cpp +++ b/src/Storages/System/StorageSystemIcebergHistory.cpp @@ -93,7 +93,7 @@ void StorageSystemIcebergHistory::fillData([[maybe_unused]] MutableColumns & res if (show_tables_granted) { - auto databases = DatabaseCatalog::instance().getDatabases(); + auto databases = DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = true}); for (const auto & db: databases) { /// with last flag we are filtering out all non iceberg table diff --git a/src/Storages/System/StorageSystemKafkaConsumers.cpp b/src/Storages/System/StorageSystemKafkaConsumers.cpp index 8a0e1dc3c7b8..a8ecd6034f42 100644 --- a/src/Storages/System/StorageSystemKafkaConsumers.cpp +++ b/src/Storages/System/StorageSystemKafkaConsumers.cpp @@ -163,7 +163,7 @@ void StorageSystemKafkaConsumers::fillData(MutableColumns & res_columns, Context return; add_rows(it, kafka_table); }; - auto databases = DatabaseCatalog::instance().getDatabases(); + auto databases = DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = false}); for (const auto & db : databases) { for (auto it = db.second->getTablesIterator(context); it->isValid(); it->next()) diff --git a/src/Storages/System/StorageSystemMutations.cpp b/src/Storages/System/StorageSystemMutations.cpp index f2b54b361806..738ca1c38c92 100644 --- a/src/Storages/System/StorageSystemMutations.cpp +++ b/src/Storages/System/StorageSystemMutations.cpp @@ -68,7 +68,7 @@ void StorageSystemMutations::fillData(MutableColumns & res_columns, ContextPtr c /// Collect a set of *MergeTree tables. std::map> merge_tree_tables; - for (const auto & db : DatabaseCatalog::instance().getDatabases()) + for (const auto & db : DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = false})) { /// Check if database can contain MergeTree tables if (!db.second->canContainMergeTreeTables()) diff --git a/src/Storages/System/StorageSystemObjectStorageQueueSettings.cpp b/src/Storages/System/StorageSystemObjectStorageQueueSettings.cpp index a6cf0ab255cd..d9b08ae474cb 100644 --- a/src/Storages/System/StorageSystemObjectStorageQueueSettings.cpp +++ b/src/Storages/System/StorageSystemObjectStorageQueueSettings.cpp @@ -58,7 +58,7 @@ void StorageSystemObjectStorageQueueSettings::fillData( const bool show_tables_granted = access->isGranted(AccessType::SHOW_TABLES); if (show_tables_granted) { - auto databases = DatabaseCatalog::instance().getDatabases(); + auto databases = DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = false}); for (const auto & db : databases) { for (auto iterator = db.second->getTablesIterator(context); iterator->isValid(); iterator->next()) diff --git a/src/Storages/System/StorageSystemPartMovesBetweenShards.cpp b/src/Storages/System/StorageSystemPartMovesBetweenShards.cpp index 58c61ca42aa0..65db49a8c887 100644 --- a/src/Storages/System/StorageSystemPartMovesBetweenShards.cpp +++ b/src/Storages/System/StorageSystemPartMovesBetweenShards.cpp @@ -59,7 +59,7 @@ void StorageSystemPartMovesBetweenShards::fillData(MutableColumns & res_columns, const bool check_access_for_databases = !access->isGranted(AccessType::SHOW_TABLES); std::map> replicated_tables; - for (const auto & db : DatabaseCatalog::instance().getDatabases()) + for (const auto & db : DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = false})) { /// Check if database can contain replicated tables if (!db.second->canContainMergeTreeTables()) diff --git a/src/Storages/System/StorageSystemPartsBase.cpp b/src/Storages/System/StorageSystemPartsBase.cpp index 51ba81ef0169..3fd6a8f37b83 100644 --- a/src/Storages/System/StorageSystemPartsBase.cpp +++ b/src/Storages/System/StorageSystemPartsBase.cpp @@ -118,7 +118,7 @@ StoragesInfoStream::StoragesInfoStream(std::optional filter_by_datab const bool check_access_for_tables = !access->isGranted(AccessType::SHOW_TABLES); { - Databases databases = DatabaseCatalog::instance().getDatabases(); + Databases databases = DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = false}); /// Add column 'database'. MutableColumnPtr database_column_mut = ColumnString::create(); diff --git a/src/Storages/System/StorageSystemProjections.cpp b/src/Storages/System/StorageSystemProjections.cpp index e88763f2bf3f..44b8ce12a3a7 100644 --- a/src/Storages/System/StorageSystemProjections.cpp +++ b/src/Storages/System/StorageSystemProjections.cpp @@ -248,7 +248,7 @@ void ReadFromSystemProjections::initializePipeline(QueryPipelineBuilder & pipeli { MutableColumnPtr column = ColumnString::create(); - const auto databases = DatabaseCatalog::instance().getDatabases(); + const auto databases = DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = false}); for (const auto & [database_name, database] : databases) { if (database_name == DatabaseCatalog::TEMPORARY_DATABASE) diff --git a/src/Storages/System/StorageSystemReplicas.cpp b/src/Storages/System/StorageSystemReplicas.cpp index aac54c244d23..03479fa7c507 100644 --- a/src/Storages/System/StorageSystemReplicas.cpp +++ b/src/Storages/System/StorageSystemReplicas.cpp @@ -332,7 +332,7 @@ void StorageSystemReplicas::read( /// We collect a set of replicated tables. std::map> replicated_tables; - for (const auto & db : DatabaseCatalog::instance().getDatabases()) + for (const auto & db : DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = false})) { /// Check if database can contain replicated tables if (!db.second->canContainMergeTreeTables()) diff --git a/src/Storages/System/StorageSystemReplicationQueue.cpp b/src/Storages/System/StorageSystemReplicationQueue.cpp index 4faaee1ebb73..645a0e387b91 100644 --- a/src/Storages/System/StorageSystemReplicationQueue.cpp +++ b/src/Storages/System/StorageSystemReplicationQueue.cpp @@ -78,7 +78,7 @@ void StorageSystemReplicationQueue::fillData(MutableColumns & res_columns, Conte const bool check_access_for_databases = !access->isGranted(AccessType::SHOW_TABLES); std::map> replicated_tables; - for (const auto & db : DatabaseCatalog::instance().getDatabases()) + for (const auto & db : DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = false})) { /// Check if database can contain replicated tables if (!db.second->canContainMergeTreeTables()) diff --git a/src/Storages/System/StorageSystemTables.cpp b/src/Storages/System/StorageSystemTables.cpp index ff1d570a8024..76785716bfc7 100644 --- a/src/Storages/System/StorageSystemTables.cpp +++ b/src/Storages/System/StorageSystemTables.cpp @@ -70,7 +70,8 @@ ColumnPtr getFilteredDatabases(const ActionsDAG::Node * predicate, ContextPtr co { MutableColumnPtr column = ColumnString::create(); - const auto databases = DatabaseCatalog::instance().getDatabases(); + const auto & settings = context->getSettingsRef(); + const auto databases = DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = settings[Setting::show_data_lake_catalogs_in_system_tables]}); for (const auto & database_name : databases | boost::adaptors::map_keys) { if (database_name == DatabaseCatalog::TEMPORARY_DATABASE) @@ -125,8 +126,7 @@ ColumnPtr getFilteredTables( { auto table_it = database->getLightweightTablesIterator(context, /* filter_by_table_name */ {}, - /* skip_not_loaded */ false, - !context->getSettingsRef()[DB::Setting::show_data_lake_catalogs_in_system_tables]); + /* skip_not_loaded */ false); for (; table_it->isValid(); table_it->next()) { database_column->insert(table_it->name()); @@ -444,8 +444,7 @@ class TablesBlockSource : public ISource if (!tables_it || !tables_it->isValid()) tables_it = database->getLightweightTablesIterator(context, /* filter_by_table_name */ {}, - /* skip_not_loaded */ false, - !context->getSettingsRef()[DB::Setting::show_data_lake_catalogs_in_system_tables]); + /* skip_not_loaded */ false); const bool need_table = needTable(database, getPort().getHeader()); diff --git a/tests/integration/test_database_glue/test.py b/tests/integration/test_database_glue/test.py index 70db874cd3c8..7ba239484640 100644 --- a/tests/integration/test_database_glue/test.py +++ b/tests/integration/test_database_glue/test.py @@ -295,14 +295,14 @@ def test_list_tables(started_cluster): assert ( tables_list == node.query( - f"SELECT name FROM system.tables WHERE database = '{CATALOG_NAME}' and name ILIKE '{root_namespace}%' ORDER BY name" + f"SELECT name FROM system.tables WHERE database = '{CATALOG_NAME}' and name ILIKE '{root_namespace}%' ORDER BY name SETTINGS show_data_lake_catalogs_in_system_tables = true" ).strip() ) node.restart_clickhouse() assert ( tables_list == node.query( - f"SELECT name FROM system.tables WHERE database = '{CATALOG_NAME}' and name ILIKE '{root_namespace}%' ORDER BY name" + f"SELECT name FROM system.tables WHERE database = '{CATALOG_NAME}' and name ILIKE '{root_namespace}%' ORDER BY name SETTINGS show_data_lake_catalogs_in_system_tables = true" ).strip() ) @@ -354,7 +354,7 @@ def test_select(started_cluster): ) assert int(node.query(f"SELECT count() FROM system.iceberg_history WHERE database = '{CATALOG_NAME}' and table ilike '%{root_namespace}%'").strip()) == 4 - assert int(node.query(f"SELECT count() FROM system.tables WHERE database = '{CATALOG_NAME}' and table ilike '%{root_namespace}%'").strip()) == 4 + assert int(node.query(f"SELECT count() FROM system.tables WHERE database = '{CATALOG_NAME}' and table ilike '%{root_namespace}%' SETTINGS show_data_lake_catalogs_in_system_tables = true").strip()) == 4 assert int(node.query(f"SELECT count() FROM system.tables WHERE database = '{CATALOG_NAME}' and table ilike '%{root_namespace}%' SETTINGS show_data_lake_catalogs_in_system_tables = false").strip()) == 0 @@ -384,7 +384,6 @@ def test_hide_sensitive_info(started_cluster): assert "SECRET_2" not in node.query(f"SHOW CREATE DATABASE {CATALOG_NAME}") - def test_select_after_rename(started_cluster): node = started_cluster.instances["node1"] @@ -578,3 +577,62 @@ def test_drop_table(started_cluster): drop_clickhouse_glue_table(node, root_namespace, table_name) assert len(catalog.list_tables(root_namespace)) == 0 + + +def test_system_tables(started_cluster): + node = started_cluster.instances["node1"] + + test_ref = f"test_system_tables_{uuid.uuid4()}" + table_name = f"{test_ref}_table" + root_namespace = f"{test_ref}_namespace" + + namespaces_to_create = [ + root_namespace, + f"{root_namespace}_A", + f"{root_namespace}_B", + f"{root_namespace}_C", + ] + + catalog = load_catalog_impl(started_cluster) + + for namespace in namespaces_to_create: + catalog.create_namespace(namespace) + assert len(catalog.list_tables(namespace)) == 0 + + for namespace in namespaces_to_create: + table = create_table(catalog, namespace, table_name) + + num_rows = 10 + df = generate_arrow_data(num_rows) + table.append(df) + + create_clickhouse_glue_database(started_cluster, node, CATALOG_NAME) + + expected = DEFAULT_CREATE_TABLE.format(CATALOG_NAME, namespace, table_name) + assert expected == node.query( + f"SHOW CREATE TABLE {CATALOG_NAME}.`{namespace}.{table_name}`" + ) + + assert num_rows == int( + node.query(f"SELECT count() FROM {CATALOG_NAME}.`{namespace}.{table_name}`") + ) + + # system.tables + assert int(node.query(f"SELECT count() FROM system.tables WHERE database = '{CATALOG_NAME}' and table ilike '%{root_namespace}%' SETTINGS show_data_lake_catalogs_in_system_tables = true").strip()) == 4 + assert int(node.query(f"SELECT count() FROM system.tables WHERE database = '{CATALOG_NAME}' and table ilike '%{root_namespace}%'").strip()) == 0 + + # system.iceberg_history - always displays irrespective of show_data_lake_catalogs_in_system_tables + assert int(node.query(f"SELECT count() FROM system.iceberg_history WHERE database = '{CATALOG_NAME}' and table ilike '%{root_namespace}%' SETTINGS show_data_lake_catalogs_in_system_tables = true").strip()) == 4 + assert int(node.query(f"SELECT count() FROM system.iceberg_history WHERE database = '{CATALOG_NAME}' and table ilike '%{root_namespace}%' SETTINGS show_data_lake_catalogs_in_system_tables = false").strip()) == 4 + + # system.databases + assert int(node.query(f"SELECT count() FROM system.databases WHERE name = '{CATALOG_NAME}' SETTINGS show_data_lake_catalogs_in_system_tables = true").strip()) == 1 + assert int(node.query(f"SELECT count() FROM system.databases WHERE name = '{CATALOG_NAME}'").strip()) == 0 + + # system.columns + assert int(node.query(f"SELECT count() FROM system.columns WHERE database = '{CATALOG_NAME}' and table ilike '%{root_namespace}%' SETTINGS show_data_lake_catalogs_in_system_tables = true").strip()) == 24 + assert int(node.query(f"SELECT count() FROM system.columns WHERE database = '{CATALOG_NAME}' and table ilike '%{root_namespace}%'").strip()) == 0 + + # system.completions + assert int(node.query(f"SELECT count() FROM system.completions WHERE startsWith(word, '{test_ref}') SETTINGS show_data_lake_catalogs_in_system_tables = true").strip()) != 0 + assert int(node.query(f"SELECT count() FROM system.completions WHERE startsWith(word, '{test_ref}')").strip()) == 0 diff --git a/tests/integration/test_database_hms/test.py b/tests/integration/test_database_hms/test.py index d64bb63699d7..65b07a352d00 100644 --- a/tests/integration/test_database_hms/test.py +++ b/tests/integration/test_database_hms/test.py @@ -199,14 +199,14 @@ def test_list_tables(started_cluster): assert ( tables_list == node.query( - f"SELECT name FROM system.tables WHERE database = '{CATALOG_NAME}' and name ILIKE '{root_namespace}%' ORDER BY name" + f"SELECT name FROM system.tables WHERE database = '{CATALOG_NAME}' and name ILIKE '{root_namespace}%' ORDER BY name SETTINGS show_data_lake_catalogs_in_system_tables=1" ).strip() ) node.restart_clickhouse() assert ( tables_list == node.query( - f"SELECT name FROM system.tables WHERE database = '{CATALOG_NAME}' and name ILIKE '{root_namespace}%' ORDER BY name" + f"SELECT name FROM system.tables WHERE database = '{CATALOG_NAME}' and name ILIKE '{root_namespace}%' ORDER BY name SETTINGS show_data_lake_catalogs_in_system_tables=1" ).strip() ) @@ -235,7 +235,7 @@ def test_many_namespaces(started_cluster): table_name = f"{namespace}.{table}" assert int( node.query( - f"SELECT count() FROM system.tables WHERE database = '{CATALOG_NAME}' and name = '{table_name}'" + f"SELECT count() FROM system.tables WHERE database = '{CATALOG_NAME}' and name = '{table_name}' SETTINGS show_data_lake_catalogs_in_system_tables=1" ) ) diff --git a/tests/integration/test_database_iceberg/test.py b/tests/integration/test_database_iceberg/test.py index 662b51e0f687..f142ab8c7752 100644 --- a/tests/integration/test_database_iceberg/test.py +++ b/tests/integration/test_database_iceberg/test.py @@ -251,14 +251,14 @@ def test_list_tables(started_cluster): assert ( tables_list == node.query( - f"SELECT name FROM system.tables WHERE database = '{CATALOG_NAME}' and name ILIKE '{root_namespace}%' ORDER BY name" + f"SELECT name FROM system.tables WHERE database = '{CATALOG_NAME}' and name ILIKE '{root_namespace}%' ORDER BY name SETTINGS show_data_lake_catalogs_in_system_tables = true" ).strip() ) node.restart_clickhouse() assert ( tables_list == node.query( - f"SELECT name FROM system.tables WHERE database = '{CATALOG_NAME}' and name ILIKE '{root_namespace}%' ORDER BY name" + f"SELECT name FROM system.tables WHERE database = '{CATALOG_NAME}' and name ILIKE '{root_namespace}%' ORDER BY name SETTINGS show_data_lake_catalogs_in_system_tables = true" ).strip() ) @@ -296,7 +296,7 @@ def test_many_namespaces(started_cluster): table_name = f"{namespace}.{table}" assert int( node.query( - f"SELECT count() FROM system.tables WHERE database = '{CATALOG_NAME}' and name = '{table_name}'" + f"SELECT count() FROM system.tables WHERE database = '{CATALOG_NAME}' and name = '{table_name}' SETTINGS show_data_lake_catalogs_in_system_tables = true" ) ) diff --git a/tests/integration/test_database_iceberg_lakekeeper_catalog/test.py b/tests/integration/test_database_iceberg_lakekeeper_catalog/test.py index 9488f39473f3..d670b4997cde 100644 --- a/tests/integration/test_database_iceberg_lakekeeper_catalog/test.py +++ b/tests/integration/test_database_iceberg_lakekeeper_catalog/test.py @@ -168,7 +168,7 @@ def test_list_tables(started_cluster): assert ( tables_list == node.query( - f"SELECT name FROM system.tables WHERE database = '{CATALOG_NAME}' and name ILIKE '{namespace_prefix}%' ORDER BY name" + f"SELECT name FROM system.tables WHERE database = '{CATALOG_NAME}' and name ILIKE '{namespace_prefix}%' ORDER BY name SETTINGS show_data_lake_catalogs_in_system_tables = true" ).strip() )