From d75a03e91354c42cf133b2098076826b9e5d6d33 Mon Sep 17 00:00:00 2001 From: Krish Date: Wed, 9 Apr 2025 18:06:28 +0530 Subject: [PATCH 01/16] Added TRANSACTION_ISOLATION_LEVEL in AbstractDBSpecificConnectorConfig --- .../io/cdap/plugin/db/ConnectionConfig.java | 2 + .../AbstractDBSpecificConnectorConfig.java | 26 +++++++++ .../widgets/Postgres-batchsink.json | 56 +++++++++++++++++++ .../widgets/Postgres-batchsource.json | 56 +++++++++++++++++++ 4 files changed, 140 insertions(+) diff --git a/database-commons/src/main/java/io/cdap/plugin/db/ConnectionConfig.java b/database-commons/src/main/java/io/cdap/plugin/db/ConnectionConfig.java index 588ed78b8..61256d24e 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/ConnectionConfig.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/ConnectionConfig.java @@ -45,6 +45,8 @@ public abstract class ConnectionConfig extends PluginConfig implements DatabaseC public static final String CONNECTION_ARGUMENTS = "connectionArguments"; public static final String JDBC_PLUGIN_NAME = "jdbcPluginName"; public static final String JDBC_PLUGIN_TYPE = "jdbc"; + public static final String TRANSACTION_ISOLATION_LEVEL = "transactionIsolationLevel"; + public static final String ROLE = "role"; @Name(JDBC_PLUGIN_NAME) @Description("Name of the JDBC driver to use. This is the value of the 'jdbcPluginName' key defined in the JSON " + diff --git a/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnectorConfig.java b/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnectorConfig.java index 5c6b08031..e39a7497c 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnectorConfig.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnectorConfig.java @@ -20,6 +20,7 @@ import io.cdap.cdap.api.annotation.Macro; import io.cdap.cdap.api.annotation.Name; import io.cdap.plugin.db.ConnectionConfig; +import io.cdap.plugin.db.TransactionIsolationLevel; import java.util.Collections; import java.util.Map; @@ -30,6 +31,8 @@ */ public abstract class AbstractDBSpecificConnectorConfig extends AbstractDBConnectorConfig { + private static final String ROLE_NORMAL = "normal"; + @Name(ConnectionConfig.HOST) @Description("Database host") @Macro @@ -42,6 +45,17 @@ public abstract class AbstractDBSpecificConnectorConfig extends AbstractDBConnec @Nullable protected Integer port; + @Name(ConnectionConfig.ROLE) + @Description("Login role of the user when connecting to the database.") + @Nullable + protected String role; + + @Name(ConnectionConfig.TRANSACTION_ISOLATION_LEVEL) + @Description("The transaction isolation level for the database session.") + @Macro + @Nullable + protected String transactionIsolationLevel; + public String getHost() { return host; } @@ -55,4 +69,16 @@ public int getPort() { public boolean canConnect() { return super.canConnect() && !containsMacro(ConnectionConfig.HOST) && !containsMacro(ConnectionConfig.PORT); } + + public String getRole() { + return role == null ? "normal" : role; + } + + public String getTransactionIsolationLevel() { + if (transactionIsolationLevel == null) { + transactionIsolationLevel = TransactionIsolationLevel.Level.TRANSACTION_SERIALIZABLE.name(); + } + return (!getRole().equals(ROLE_NORMAL)) ? TransactionIsolationLevel.Level.TRANSACTION_READ_COMMITTED.name() : + TransactionIsolationLevel.Level.valueOf(transactionIsolationLevel).name(); + } } diff --git a/postgresql-plugin/widgets/Postgres-batchsink.json b/postgresql-plugin/widgets/Postgres-batchsink.json index 6aa2dad8a..a986c6030 100644 --- a/postgresql-plugin/widgets/Postgres-batchsink.json +++ b/postgresql-plugin/widgets/Postgres-batchsink.json @@ -65,6 +65,42 @@ "label": "Password", "name": "password" }, + { + "label": "Role", + "name": "role", + "widget-type": "radio-group", + "widget-attributes": { + "layout": "inline", + "default": "normal", + "options": [ + { + "id": "normal", + "label": "Normal" + }, + { + "id": "sysdba", + "label": "SYSDBA" + }, + { + "id": "sysoper", + "label": "SYSOPER" + } + ] + } + }, + { + "widget-type": "select", + "label": "Transaction Isolation Level", + "name": "transactionIsolationLevel", + "widget-attributes": { + "values": [ + "TRANSACTION_READ_COMMITTED", + "TRANSACTION_REPEATABLE_READ", + "TRANSACTION_SERIALIZABLE" + ], + "default": "TRANSACTION_SERIALIZABLE" + } + }, { "widget-type": "keyvalue", "label": "Connection Arguments", @@ -160,6 +196,18 @@ ], "outputs": [], "filters": [ + { + "name": "showIsolationLevels", + "condition": { + "expression": "role == 'normal'" + }, + "show": [ + { + "type": "property", + "name": "transactionIsolationLevel" + } + ] + }, { "name": "showConnectionProperties ", "condition": { @@ -186,6 +234,14 @@ "type": "property", "name": "port" }, + { + "type": "property", + "name": "role" + }, + { + "type": "property", + "name": "transactionIsolationLevel" + }, { "type": "property", "name": "database" diff --git a/postgresql-plugin/widgets/Postgres-batchsource.json b/postgresql-plugin/widgets/Postgres-batchsource.json index 0e4ba28c1..d76dacb16 100644 --- a/postgresql-plugin/widgets/Postgres-batchsource.json +++ b/postgresql-plugin/widgets/Postgres-batchsource.json @@ -65,6 +65,42 @@ "label": "Password", "name": "password" }, + { + "label": "Role", + "name": "role", + "widget-type": "radio-group", + "widget-attributes": { + "layout": "inline", + "default": "normal", + "options": [ + { + "id": "normal", + "label": "Normal" + }, + { + "id": "sysdba", + "label": "SYSDBA" + }, + { + "id": "sysoper", + "label": "SYSOPER" + } + ] + } + }, + { + "widget-type": "select", + "label": "Transaction Isolation Level", + "name": "transactionIsolationLevel", + "widget-attributes": { + "values": [ + "TRANSACTION_READ_COMMITTED", + "TRANSACTION_REPEATABLE_READ", + "TRANSACTION_SERIALIZABLE" + ], + "default": "TRANSACTION_SERIALIZABLE" + } + }, { "widget-type": "keyvalue", "label": "Connection Arguments", @@ -180,6 +216,18 @@ } ], "filters": [ + { + "name": "showIsolationLevels", + "condition": { + "expression": "role == 'normal'" + }, + "show": [ + { + "type": "property", + "name": "transactionIsolationLevel" + } + ] + }, { "name": "showConnectionProperties ", "condition": { @@ -206,6 +254,14 @@ "type": "property", "name": "port" }, + { + "type": "property", + "name": "role" + }, + { + "type": "property", + "name": "transactionIsolationLevel" + }, { "type": "property", "name": "database" From 5da84dabf3c290b5c972371e54d0788e21969e67 Mon Sep 17 00:00:00 2001 From: Krish Date: Thu, 10 Apr 2025 14:46:50 +0530 Subject: [PATCH 02/16] Added Transactio Isolation levels to MySQL --- mysql-plugin/widgets/MySQL-connector.json | 37 +++++++++++++++++++++ mysql-plugin/widgets/Mysql-batchsink.json | 37 +++++++++++++++++++++ mysql-plugin/widgets/Mysql-batchsource.json | 37 +++++++++++++++++++++ 3 files changed, 111 insertions(+) diff --git a/mysql-plugin/widgets/MySQL-connector.json b/mysql-plugin/widgets/MySQL-connector.json index 9064d1bf6..06d973095 100644 --- a/mysql-plugin/widgets/MySQL-connector.json +++ b/mysql-plugin/widgets/MySQL-connector.json @@ -30,6 +30,43 @@ "widget-attributes": { "default": "3306" } + }, + { + "label": "Role", + "name": "role", + "widget-type": "radio-group", + "widget-attributes": { + "layout": "inline", + "default": "normal", + "options": [ + { + "id": "normal", + "label": "Normal" + }, + { + "id": "sysdba", + "label": "SYSDBA" + }, + { + "id": "sysoper", + "label": "SYSOPER" + } + ] + } + }, + { + "widget-type": "select", + "label": "Transaction Isolation Level", + "name": "transactionIsolationLevel", + "widget-attributes": { + "values": [ + "TRANSACTION_READ_UNCOMMITTED", + "TRANSACTION_READ_COMMITTED", + "TRANSACTION_REPEATABLE_READ", + "TRANSACTION_SERIALIZABLE" + ], + "default": "TRANSACTION_SERIALIZABLE" + } } ] }, diff --git a/mysql-plugin/widgets/Mysql-batchsink.json b/mysql-plugin/widgets/Mysql-batchsink.json index c525ead40..295c442f2 100644 --- a/mysql-plugin/widgets/Mysql-batchsink.json +++ b/mysql-plugin/widgets/Mysql-batchsink.json @@ -65,6 +65,43 @@ "label": "Password", "name": "password" }, + { + "label": "Role", + "name": "role", + "widget-type": "radio-group", + "widget-attributes": { + "layout": "inline", + "default": "normal", + "options": [ + { + "id": "normal", + "label": "Normal" + }, + { + "id": "sysdba", + "label": "SYSDBA" + }, + { + "id": "sysoper", + "label": "SYSOPER" + } + ] + } + }, + { + "widget-type": "select", + "label": "Transaction Isolation Level", + "name": "transactionIsolationLevel", + "widget-attributes": { + "values": [ + "TRANSACTION_READ_UNCOMMITTED", + "TRANSACTION_READ_COMMITTED", + "TRANSACTION_REPEATABLE_READ", + "TRANSACTION_SERIALIZABLE" + ], + "default": "TRANSACTION_SERIALIZABLE" + } + }, { "widget-type": "keyvalue", "label": "Connection Arguments", diff --git a/mysql-plugin/widgets/Mysql-batchsource.json b/mysql-plugin/widgets/Mysql-batchsource.json index 9175bd5ed..768c5faf9 100644 --- a/mysql-plugin/widgets/Mysql-batchsource.json +++ b/mysql-plugin/widgets/Mysql-batchsource.json @@ -65,6 +65,43 @@ "label": "Password", "name": "password" }, + { + "label": "Role", + "name": "role", + "widget-type": "radio-group", + "widget-attributes": { + "layout": "inline", + "default": "normal", + "options": [ + { + "id": "normal", + "label": "Normal" + }, + { + "id": "sysdba", + "label": "SYSDBA" + }, + { + "id": "sysoper", + "label": "SYSOPER" + } + ] + } + }, + { + "widget-type": "select", + "label": "Transaction Isolation Level", + "name": "transactionIsolationLevel", + "widget-attributes": { + "values": [ + "TRANSACTION_READ_UNCOMMITTED", + "TRANSACTION_READ_COMMITTED", + "TRANSACTION_REPEATABLE_READ", + "TRANSACTION_SERIALIZABLE" + ], + "default": "TRANSACTION_SERIALIZABLE" + } + }, { "widget-type": "keyvalue", "label": "Connection Arguments", From 144de23822f3957cde548d92859e56fcdbb44acb Mon Sep 17 00:00:00 2001 From: Krish Date: Thu, 10 Apr 2025 14:53:17 +0530 Subject: [PATCH 03/16] Added widgets to MySQL --- mysql-plugin/widgets/Mysql-batchsink.json | 16 ++++++++++++++++ mysql-plugin/widgets/Mysql-batchsource.json | 16 ++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/mysql-plugin/widgets/Mysql-batchsink.json b/mysql-plugin/widgets/Mysql-batchsink.json index 295c442f2..101935ce0 100644 --- a/mysql-plugin/widgets/Mysql-batchsink.json +++ b/mysql-plugin/widgets/Mysql-batchsink.json @@ -244,6 +244,18 @@ ], "outputs": [], "filters": [ + { + "name": "showIsolationLevels", + "condition": { + "expression": "role == 'normal'" + }, + "show": [ + { + "type": "property", + "name": "transactionIsolationLevel" + } + ] + }, { "name": "showConnectionProperties ", "condition": { @@ -262,6 +274,10 @@ "type": "property", "name": "password" }, + { + "type": "property", + "name": "transactionIsolationLevel" + }, { "type": "property", "name": "host" diff --git a/mysql-plugin/widgets/Mysql-batchsource.json b/mysql-plugin/widgets/Mysql-batchsource.json index 768c5faf9..55cf03576 100644 --- a/mysql-plugin/widgets/Mysql-batchsource.json +++ b/mysql-plugin/widgets/Mysql-batchsource.json @@ -296,6 +296,18 @@ } ], "filters": [ + { + "name": "showIsolationLevels", + "condition": { + "expression": "role == 'normal'" + }, + "show": [ + { + "type": "property", + "name": "transactionIsolationLevel" + } + ] + }, { "name": "showConnectionProperties ", "condition": { @@ -314,6 +326,10 @@ "type": "property", "name": "password" }, + { + "type": "property", + "name": "transactionIsolationLevel" + }, { "type": "property", "name": "host" From 34169854f0a7072e83883b222f410ace546f3088 Mon Sep 17 00:00:00 2001 From: Krish Date: Thu, 10 Apr 2025 15:04:03 +0530 Subject: [PATCH 04/16] Added transaction isolation level to postgresql connector --- .../widgets/PostgreSQL-connector.json | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/postgresql-plugin/widgets/PostgreSQL-connector.json b/postgresql-plugin/widgets/PostgreSQL-connector.json index 091afc972..e8d02f0a4 100644 --- a/postgresql-plugin/widgets/PostgreSQL-connector.json +++ b/postgresql-plugin/widgets/PostgreSQL-connector.json @@ -31,6 +31,42 @@ "default": "5432" } }, + { + "label": "Role", + "name": "role", + "widget-type": "radio-group", + "widget-attributes": { + "layout": "inline", + "default": "normal", + "options": [ + { + "id": "normal", + "label": "Normal" + }, + { + "id": "sysdba", + "label": "SYSDBA" + }, + { + "id": "sysoper", + "label": "SYSOPER" + } + ] + } + }, + { + "widget-type": "select", + "label": "Transaction Isolation Level", + "name": "transactionIsolationLevel", + "widget-attributes": { + "values": [ + "TRANSACTION_READ_COMMITTED", + "TRANSACTION_REPEATABLE_READ", + "TRANSACTION_SERIALIZABLE" + ], + "default": "TRANSACTION_SERIALIZABLE" + } + }, { "widget-type": "textbox", "label": "Database", From 8946d5e0a42f546c90962be8b593e805a31c6d84 Mon Sep 17 00:00:00 2001 From: Krish Date: Thu, 10 Apr 2025 15:46:30 +0530 Subject: [PATCH 05/16] Removed transaction isolation from oracle --- .../AbstractDBSpecificConnectorConfig.java | 11 +---------- .../plugin/oracle/OracleConnectorConfig.java | 18 ------------------ 2 files changed, 1 insertion(+), 28 deletions(-) diff --git a/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnectorConfig.java b/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnectorConfig.java index e39a7497c..6f8df105d 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnectorConfig.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnectorConfig.java @@ -45,10 +45,6 @@ public abstract class AbstractDBSpecificConnectorConfig extends AbstractDBConnec @Nullable protected Integer port; - @Name(ConnectionConfig.ROLE) - @Description("Login role of the user when connecting to the database.") - @Nullable - protected String role; @Name(ConnectionConfig.TRANSACTION_ISOLATION_LEVEL) @Description("The transaction isolation level for the database session.") @@ -70,15 +66,10 @@ public boolean canConnect() { return super.canConnect() && !containsMacro(ConnectionConfig.HOST) && !containsMacro(ConnectionConfig.PORT); } - public String getRole() { - return role == null ? "normal" : role; - } - public String getTransactionIsolationLevel() { if (transactionIsolationLevel == null) { transactionIsolationLevel = TransactionIsolationLevel.Level.TRANSACTION_SERIALIZABLE.name(); } - return (!getRole().equals(ROLE_NORMAL)) ? TransactionIsolationLevel.Level.TRANSACTION_READ_COMMITTED.name() : - TransactionIsolationLevel.Level.valueOf(transactionIsolationLevel).name(); + return TransactionIsolationLevel.Level.valueOf(transactionIsolationLevel).name(); } } diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java index 10022364a..9a297ee89 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java @@ -81,12 +81,6 @@ public String getConnectionString() { @Macro private String database; - @Name(OracleConstants.TRANSACTION_ISOLATION_LEVEL) - @Description("The transaction isolation level for the database session.") - @Macro - @Nullable - private String transactionIsolationLevel; - @Name(OracleConstants.USE_SSL) @Description("Turns on SSL encryption. Connection will fail if SSL is not available") @Nullable @@ -124,18 +118,6 @@ public Properties getConnectionArgumentsProperties() { return prop; } - public String getTransactionIsolationLevel() { - //if null default to the highest isolation level possible - if (transactionIsolationLevel == null) { - transactionIsolationLevel = TransactionIsolationLevel.Level.TRANSACTION_SERIALIZABLE.name(); - } - //To solve the problem of ORA-08178: illegal SERIALIZABLE clause specified for user INTERNAL - //This ensures that the role is mapped to the right serialization level, even w/ incorrect user input - //if role is SYSDBA or SYSOP it will map to read_committed. else serialized - return (!getRole().equals(ROLE_NORMAL)) ? TransactionIsolationLevel.Level.TRANSACTION_READ_COMMITTED.name() : - TransactionIsolationLevel.Level.valueOf(transactionIsolationLevel).name(); - } - @Override public Map getAdditionalArguments() { Map additonalArguments = new HashMap<>(); From 8b01c447a066f934e3eb7551a8ff1f3b1f351e43 Mon Sep 17 00:00:00 2001 From: Krish Date: Thu, 10 Apr 2025 15:57:37 +0530 Subject: [PATCH 06/16] Removed role from MySQL and PostgreSQL --- .../AbstractDBSpecificConnectorConfig.java | 2 - mysql-plugin/widgets/MySQL-connector.json | 23 ----------- mysql-plugin/widgets/Mysql-batchsink.json | 35 ----------------- mysql-plugin/widgets/Mysql-batchsource.json | 35 ----------------- .../widgets/PostgreSQL-connector.json | 23 ----------- .../widgets/Postgres-batchsink.json | 39 ------------------- .../widgets/Postgres-batchsource.json | 39 ------------------- 7 files changed, 196 deletions(-) diff --git a/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnectorConfig.java b/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnectorConfig.java index 6f8df105d..2220f02d8 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnectorConfig.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnectorConfig.java @@ -31,8 +31,6 @@ */ public abstract class AbstractDBSpecificConnectorConfig extends AbstractDBConnectorConfig { - private static final String ROLE_NORMAL = "normal"; - @Name(ConnectionConfig.HOST) @Description("Database host") @Macro diff --git a/mysql-plugin/widgets/MySQL-connector.json b/mysql-plugin/widgets/MySQL-connector.json index 06d973095..f60f5526f 100644 --- a/mysql-plugin/widgets/MySQL-connector.json +++ b/mysql-plugin/widgets/MySQL-connector.json @@ -31,29 +31,6 @@ "default": "3306" } }, - { - "label": "Role", - "name": "role", - "widget-type": "radio-group", - "widget-attributes": { - "layout": "inline", - "default": "normal", - "options": [ - { - "id": "normal", - "label": "Normal" - }, - { - "id": "sysdba", - "label": "SYSDBA" - }, - { - "id": "sysoper", - "label": "SYSOPER" - } - ] - } - }, { "widget-type": "select", "label": "Transaction Isolation Level", diff --git a/mysql-plugin/widgets/Mysql-batchsink.json b/mysql-plugin/widgets/Mysql-batchsink.json index 101935ce0..58596aae2 100644 --- a/mysql-plugin/widgets/Mysql-batchsink.json +++ b/mysql-plugin/widgets/Mysql-batchsink.json @@ -65,29 +65,6 @@ "label": "Password", "name": "password" }, - { - "label": "Role", - "name": "role", - "widget-type": "radio-group", - "widget-attributes": { - "layout": "inline", - "default": "normal", - "options": [ - { - "id": "normal", - "label": "Normal" - }, - { - "id": "sysdba", - "label": "SYSDBA" - }, - { - "id": "sysoper", - "label": "SYSOPER" - } - ] - } - }, { "widget-type": "select", "label": "Transaction Isolation Level", @@ -244,18 +221,6 @@ ], "outputs": [], "filters": [ - { - "name": "showIsolationLevels", - "condition": { - "expression": "role == 'normal'" - }, - "show": [ - { - "type": "property", - "name": "transactionIsolationLevel" - } - ] - }, { "name": "showConnectionProperties ", "condition": { diff --git a/mysql-plugin/widgets/Mysql-batchsource.json b/mysql-plugin/widgets/Mysql-batchsource.json index 55cf03576..506e837f7 100644 --- a/mysql-plugin/widgets/Mysql-batchsource.json +++ b/mysql-plugin/widgets/Mysql-batchsource.json @@ -65,29 +65,6 @@ "label": "Password", "name": "password" }, - { - "label": "Role", - "name": "role", - "widget-type": "radio-group", - "widget-attributes": { - "layout": "inline", - "default": "normal", - "options": [ - { - "id": "normal", - "label": "Normal" - }, - { - "id": "sysdba", - "label": "SYSDBA" - }, - { - "id": "sysoper", - "label": "SYSOPER" - } - ] - } - }, { "widget-type": "select", "label": "Transaction Isolation Level", @@ -296,18 +273,6 @@ } ], "filters": [ - { - "name": "showIsolationLevels", - "condition": { - "expression": "role == 'normal'" - }, - "show": [ - { - "type": "property", - "name": "transactionIsolationLevel" - } - ] - }, { "name": "showConnectionProperties ", "condition": { diff --git a/postgresql-plugin/widgets/PostgreSQL-connector.json b/postgresql-plugin/widgets/PostgreSQL-connector.json index e8d02f0a4..9a7a02e14 100644 --- a/postgresql-plugin/widgets/PostgreSQL-connector.json +++ b/postgresql-plugin/widgets/PostgreSQL-connector.json @@ -31,29 +31,6 @@ "default": "5432" } }, - { - "label": "Role", - "name": "role", - "widget-type": "radio-group", - "widget-attributes": { - "layout": "inline", - "default": "normal", - "options": [ - { - "id": "normal", - "label": "Normal" - }, - { - "id": "sysdba", - "label": "SYSDBA" - }, - { - "id": "sysoper", - "label": "SYSOPER" - } - ] - } - }, { "widget-type": "select", "label": "Transaction Isolation Level", diff --git a/postgresql-plugin/widgets/Postgres-batchsink.json b/postgresql-plugin/widgets/Postgres-batchsink.json index a986c6030..14e6f8154 100644 --- a/postgresql-plugin/widgets/Postgres-batchsink.json +++ b/postgresql-plugin/widgets/Postgres-batchsink.json @@ -65,29 +65,6 @@ "label": "Password", "name": "password" }, - { - "label": "Role", - "name": "role", - "widget-type": "radio-group", - "widget-attributes": { - "layout": "inline", - "default": "normal", - "options": [ - { - "id": "normal", - "label": "Normal" - }, - { - "id": "sysdba", - "label": "SYSDBA" - }, - { - "id": "sysoper", - "label": "SYSOPER" - } - ] - } - }, { "widget-type": "select", "label": "Transaction Isolation Level", @@ -196,18 +173,6 @@ ], "outputs": [], "filters": [ - { - "name": "showIsolationLevels", - "condition": { - "expression": "role == 'normal'" - }, - "show": [ - { - "type": "property", - "name": "transactionIsolationLevel" - } - ] - }, { "name": "showConnectionProperties ", "condition": { @@ -234,10 +199,6 @@ "type": "property", "name": "port" }, - { - "type": "property", - "name": "role" - }, { "type": "property", "name": "transactionIsolationLevel" diff --git a/postgresql-plugin/widgets/Postgres-batchsource.json b/postgresql-plugin/widgets/Postgres-batchsource.json index d76dacb16..60de4725f 100644 --- a/postgresql-plugin/widgets/Postgres-batchsource.json +++ b/postgresql-plugin/widgets/Postgres-batchsource.json @@ -65,29 +65,6 @@ "label": "Password", "name": "password" }, - { - "label": "Role", - "name": "role", - "widget-type": "radio-group", - "widget-attributes": { - "layout": "inline", - "default": "normal", - "options": [ - { - "id": "normal", - "label": "Normal" - }, - { - "id": "sysdba", - "label": "SYSDBA" - }, - { - "id": "sysoper", - "label": "SYSOPER" - } - ] - } - }, { "widget-type": "select", "label": "Transaction Isolation Level", @@ -216,18 +193,6 @@ } ], "filters": [ - { - "name": "showIsolationLevels", - "condition": { - "expression": "role == 'normal'" - }, - "show": [ - { - "type": "property", - "name": "transactionIsolationLevel" - } - ] - }, { "name": "showConnectionProperties ", "condition": { @@ -254,10 +219,6 @@ "type": "property", "name": "port" }, - { - "type": "property", - "name": "role" - }, { "type": "property", "name": "transactionIsolationLevel" From 085e047eac5975697cec9ecef63dd687e25286db Mon Sep 17 00:00:00 2001 From: Krish Date: Thu, 10 Apr 2025 16:39:44 +0530 Subject: [PATCH 07/16] Added md doc for MySQL and PostgreSQL --- mysql-plugin/docs/MySQL-connector.md | 7 +++++++ mysql-plugin/docs/Mysql-batchsink.md | 7 +++++++ mysql-plugin/docs/Mysql-batchsource.md | 7 +++++++ postgresql-plugin/docs/PostgreSQL-connector.md | 5 +++++ postgresql-plugin/docs/Postgres-batchsink.md | 5 +++++ postgresql-plugin/docs/Postgres-batchsource.md | 5 +++++ 6 files changed, 36 insertions(+) diff --git a/mysql-plugin/docs/MySQL-connector.md b/mysql-plugin/docs/MySQL-connector.md index fb5c1fbb8..22af6e04a 100644 --- a/mysql-plugin/docs/MySQL-connector.md +++ b/mysql-plugin/docs/MySQL-connector.md @@ -22,6 +22,13 @@ authentication. Optional for databases that do not require authentication. **Password:** Password to use to connect to the specified database. +**Transaction Isolation Level** The transaction isolation level of the databse connection +- TRANSACTION_READ_COMMITTED: No dirty reads. Non-repeatable reads and phantom reads are possible. +- TRANSACTION_SERIALIZABLE (default): No dirty reads. Non-repeatable and phantom reads are prevented. +- TRANSACTION_REPEATABLE_READ: No dirty reads. Prevents non-repeatable reads, but phantom reads are still possible. +- TRANSACTION_READ_UNCOMMITTED: Allows dirty reads (reading uncommitted changes from other transactions). Non-repeatable + reads and phantom reads are possible. + **Connection Arguments:** A list of arbitrary string tag/value pairs as connection arguments. These arguments will be passed to the JDBC driver, as connection arguments, for JDBC drivers that may need additional configurations. This is a semicolon-separated list of key-value pairs, where each pair is separated by a equals '=' and specifies diff --git a/mysql-plugin/docs/Mysql-batchsink.md b/mysql-plugin/docs/Mysql-batchsink.md index b28a28618..3340268aa 100644 --- a/mysql-plugin/docs/Mysql-batchsink.md +++ b/mysql-plugin/docs/Mysql-batchsink.md @@ -39,6 +39,13 @@ You also can use the macro function ${conn(connection-name)}. **Password:** Password to use to connect to the specified database. +**Transaction Isolation Level** The transaction isolation level of the databse connection +- TRANSACTION_READ_COMMITTED: No dirty reads. Non-repeatable reads and phantom reads are possible. +- TRANSACTION_SERIALIZABLE (default): No dirty reads. Non-repeatable and phantom reads are prevented. +- TRANSACTION_REPEATABLE_READ: No dirty reads. Prevents non-repeatable reads, but phantom reads are still possible. +- TRANSACTION_READ_UNCOMMITTED: Allows dirty reads (reading uncommitted changes from other transactions). Non-repeatable +reads and phantom reads are possible. + **Connection Arguments:** A list of arbitrary string key/value pairs as connection arguments. These arguments will be passed to the JDBC driver as connection arguments for JDBC drivers that may need additional configurations. diff --git a/mysql-plugin/docs/Mysql-batchsource.md b/mysql-plugin/docs/Mysql-batchsource.md index 010e08216..08e533ea2 100644 --- a/mysql-plugin/docs/Mysql-batchsource.md +++ b/mysql-plugin/docs/Mysql-batchsource.md @@ -49,6 +49,13 @@ For example, 'SELECT MIN(id),MAX(id) FROM table'. Not required if numSplits is s **Password:** Password to use to connect to the specified database. +**Transaction Isolation Level** The transaction isolation level of the databse connection +- TRANSACTION_READ_COMMITTED: No dirty reads. Non-repeatable reads and phantom reads are possible. +- TRANSACTION_SERIALIZABLE (default): No dirty reads. Non-repeatable and phantom reads are prevented. +- TRANSACTION_REPEATABLE_READ: No dirty reads. Prevents non-repeatable reads, but phantom reads are still possible. +- TRANSACTION_READ_UNCOMMITTED: Allows dirty reads (reading uncommitted changes from other transactions). Non-repeatable + reads and phantom reads are possible. + **Connection Arguments:** A list of arbitrary string key/value pairs as connection arguments. These arguments will be passed to the JDBC driver as connection arguments for JDBC drivers that may need additional configurations. diff --git a/postgresql-plugin/docs/PostgreSQL-connector.md b/postgresql-plugin/docs/PostgreSQL-connector.md index 739c678e3..96c3ac5bb 100644 --- a/postgresql-plugin/docs/PostgreSQL-connector.md +++ b/postgresql-plugin/docs/PostgreSQL-connector.md @@ -22,6 +22,11 @@ authentication. Optional for databases that do not require authentication. **Password:** Password to use to connect to the specified database. +**Transaction Isolation Level** The transaction isolation level of the databse connection +- TRANSACTION_READ_COMMITTED: No dirty reads. Non-repeatable reads and phantom reads are possible. +- TRANSACTION_SERIALIZABLE (default): No dirty reads. Non-repeatable and phantom reads are prevented. +- TRANSACTION_REPEATABLE_READ: No dirty reads. Prevents non-repeatable reads, but phantom reads are still possible. + **Database:** The name of the database to connect to. **Connection Arguments:** A list of arbitrary string tag/value pairs as connection arguments. These arguments diff --git a/postgresql-plugin/docs/Postgres-batchsink.md b/postgresql-plugin/docs/Postgres-batchsink.md index b8a996463..a0c2b144d 100644 --- a/postgresql-plugin/docs/Postgres-batchsink.md +++ b/postgresql-plugin/docs/Postgres-batchsink.md @@ -39,6 +39,11 @@ You also can use the macro function ${conn(connection-name)}. **Password:** Password to use to connect to the specified database. +**Transaction Isolation Level** The transaction isolation level of the databse connection +- TRANSACTION_READ_COMMITTED: No dirty reads. Non-repeatable reads and phantom reads are possible. +- TRANSACTION_SERIALIZABLE (default): No dirty reads. Non-repeatable and phantom reads are prevented. +- TRANSACTION_REPEATABLE_READ: No dirty reads. Prevents non-repeatable reads, but phantom reads are still possible. + **Connection Arguments:** A list of arbitrary string key/value pairs as connection arguments. These arguments will be passed to the JDBC driver as connection arguments for JDBC drivers that may need additional configurations. diff --git a/postgresql-plugin/docs/Postgres-batchsource.md b/postgresql-plugin/docs/Postgres-batchsource.md index af359022d..46a5e56ac 100644 --- a/postgresql-plugin/docs/Postgres-batchsource.md +++ b/postgresql-plugin/docs/Postgres-batchsource.md @@ -49,6 +49,11 @@ For example, 'SELECT MIN(id),MAX(id) FROM table'. Not required if numSplits is s **Password:** Password to use to connect to the specified database. +**Transaction Isolation Level** The transaction isolation level of the databse connection +- TRANSACTION_READ_COMMITTED: No dirty reads. Non-repeatable reads and phantom reads are possible. +- TRANSACTION_SERIALIZABLE (default): No dirty reads. Non-repeatable and phantom reads are prevented. +- TRANSACTION_REPEATABLE_READ: No dirty reads. Prevents non-repeatable reads, but phantom reads are still possible. + **Connection Arguments:** A list of arbitrary string key/value pairs as connection arguments. These arguments will be passed to the JDBC driver as connection arguments for JDBC drivers that may need additional configurations. From 4f77a05fc800ff7522186a4aadc6b7601582f248 Mon Sep 17 00:00:00 2001 From: Krish Date: Fri, 11 Apr 2025 11:02:42 +0530 Subject: [PATCH 08/16] Reverted getTransactionIsolationLevel for oracel --- .../io/cdap/plugin/oracle/OracleConnectorConfig.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java index 9a297ee89..a7e22e2e6 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java @@ -118,6 +118,18 @@ public Properties getConnectionArgumentsProperties() { return prop; } + public String getTransactionIsolationLevel() { + //if null default to the highest isolation level possible + if (transactionIsolationLevel == null) { + transactionIsolationLevel = TransactionIsolationLevel.Level.TRANSACTION_SERIALIZABLE.name(); + } + //To solve the problem of ORA-08178: illegal SERIALIZABLE clause specified for user INTERNAL + //This ensures that the role is mapped to the right serialization level, even w/ incorrect user input + //if role is SYSDBA or SYSOP it will map to read_committed. else serialized + return (!getRole().equals(ROLE_NORMAL)) ? TransactionIsolationLevel.Level.TRANSACTION_READ_COMMITTED.name() : + TransactionIsolationLevel.Level.valueOf(transactionIsolationLevel).name(); + } + @Override public Map getAdditionalArguments() { Map additonalArguments = new HashMap<>(); From 178bfb59c7065183bbc01f165a86700b13db08ae Mon Sep 17 00:00:00 2001 From: Krish Date: Fri, 11 Apr 2025 11:26:58 +0530 Subject: [PATCH 09/16] Changed Transaction Isolation Level to READ_COMMITTED for normal roles --- .../plugin/db/connector/AbstractDBSpecificConnectorConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnectorConfig.java b/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnectorConfig.java index 2220f02d8..b3df3935b 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnectorConfig.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnectorConfig.java @@ -66,7 +66,7 @@ public boolean canConnect() { public String getTransactionIsolationLevel() { if (transactionIsolationLevel == null) { - transactionIsolationLevel = TransactionIsolationLevel.Level.TRANSACTION_SERIALIZABLE.name(); + transactionIsolationLevel = TransactionIsolationLevel.Level.TRANSACTION_READ_COMMITTED.name(); } return TransactionIsolationLevel.Level.valueOf(transactionIsolationLevel).name(); } From 4af4f5e2172946845f5f3056ac8236f88ad711f0 Mon Sep 17 00:00:00 2001 From: Krish Date: Fri, 11 Apr 2025 12:02:10 +0530 Subject: [PATCH 10/16] Updated md doc --- mysql-plugin/docs/MySQL-connector.md | 3 +-- mysql-plugin/docs/Mysql-batchsink.md | 3 +-- mysql-plugin/docs/Mysql-batchsource.md | 3 +-- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/mysql-plugin/docs/MySQL-connector.md b/mysql-plugin/docs/MySQL-connector.md index 22af6e04a..8ece1a5bc 100644 --- a/mysql-plugin/docs/MySQL-connector.md +++ b/mysql-plugin/docs/MySQL-connector.md @@ -26,8 +26,7 @@ authentication. Optional for databases that do not require authentication. - TRANSACTION_READ_COMMITTED: No dirty reads. Non-repeatable reads and phantom reads are possible. - TRANSACTION_SERIALIZABLE (default): No dirty reads. Non-repeatable and phantom reads are prevented. - TRANSACTION_REPEATABLE_READ: No dirty reads. Prevents non-repeatable reads, but phantom reads are still possible. -- TRANSACTION_READ_UNCOMMITTED: Allows dirty reads (reading uncommitted changes from other transactions). Non-repeatable - reads and phantom reads are possible. +- TRANSACTION_READ_UNCOMMITTED: Allows dirty reads (reading uncommitted changes from other transactions). Non-repeatable reads and phantom reads are possible. **Connection Arguments:** A list of arbitrary string tag/value pairs as connection arguments. These arguments will be passed to the JDBC driver, as connection arguments, for JDBC drivers that may need additional configurations. diff --git a/mysql-plugin/docs/Mysql-batchsink.md b/mysql-plugin/docs/Mysql-batchsink.md index 3340268aa..aaf6764e7 100644 --- a/mysql-plugin/docs/Mysql-batchsink.md +++ b/mysql-plugin/docs/Mysql-batchsink.md @@ -43,8 +43,7 @@ You also can use the macro function ${conn(connection-name)}. - TRANSACTION_READ_COMMITTED: No dirty reads. Non-repeatable reads and phantom reads are possible. - TRANSACTION_SERIALIZABLE (default): No dirty reads. Non-repeatable and phantom reads are prevented. - TRANSACTION_REPEATABLE_READ: No dirty reads. Prevents non-repeatable reads, but phantom reads are still possible. -- TRANSACTION_READ_UNCOMMITTED: Allows dirty reads (reading uncommitted changes from other transactions). Non-repeatable -reads and phantom reads are possible. +- TRANSACTION_READ_UNCOMMITTED: Allows dirty reads (reading uncommitted changes from other transactions). Non-repeatable reads and phantom reads are possible. **Connection Arguments:** A list of arbitrary string key/value pairs as connection arguments. These arguments will be passed to the JDBC driver as connection arguments for JDBC drivers that may need additional configurations. diff --git a/mysql-plugin/docs/Mysql-batchsource.md b/mysql-plugin/docs/Mysql-batchsource.md index 08e533ea2..69723eef1 100644 --- a/mysql-plugin/docs/Mysql-batchsource.md +++ b/mysql-plugin/docs/Mysql-batchsource.md @@ -53,8 +53,7 @@ For example, 'SELECT MIN(id),MAX(id) FROM table'. Not required if numSplits is s - TRANSACTION_READ_COMMITTED: No dirty reads. Non-repeatable reads and phantom reads are possible. - TRANSACTION_SERIALIZABLE (default): No dirty reads. Non-repeatable and phantom reads are prevented. - TRANSACTION_REPEATABLE_READ: No dirty reads. Prevents non-repeatable reads, but phantom reads are still possible. -- TRANSACTION_READ_UNCOMMITTED: Allows dirty reads (reading uncommitted changes from other transactions). Non-repeatable - reads and phantom reads are possible. +- TRANSACTION_READ_UNCOMMITTED: Allows dirty reads (reading uncommitted changes from other transactions). Non-repeatable reads and phantom reads are possible. **Connection Arguments:** A list of arbitrary string key/value pairs as connection arguments. These arguments will be passed to the JDBC driver as connection arguments for JDBC drivers that may need additional configurations. From d6628b5a2f734a9c852d9f1276ca52795c18f5d5 Mon Sep 17 00:00:00 2001 From: Krish Date: Mon, 14 Apr 2025 13:42:59 +0530 Subject: [PATCH 11/16] Made default isolation level as null --- .../plugin/db/connector/AbstractDBSpecificConnectorConfig.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnectorConfig.java b/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnectorConfig.java index b3df3935b..977315033 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnectorConfig.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnectorConfig.java @@ -43,7 +43,6 @@ public abstract class AbstractDBSpecificConnectorConfig extends AbstractDBConnec @Nullable protected Integer port; - @Name(ConnectionConfig.TRANSACTION_ISOLATION_LEVEL) @Description("The transaction isolation level for the database session.") @Macro @@ -66,7 +65,7 @@ public boolean canConnect() { public String getTransactionIsolationLevel() { if (transactionIsolationLevel == null) { - transactionIsolationLevel = TransactionIsolationLevel.Level.TRANSACTION_READ_COMMITTED.name(); + return null; } return TransactionIsolationLevel.Level.valueOf(transactionIsolationLevel).name(); } From d7a8032fe22d91d7da02044745bde809398257e1 Mon Sep 17 00:00:00 2001 From: Krish Date: Mon, 14 Apr 2025 15:47:18 +0530 Subject: [PATCH 12/16] Added getAdditionalArgument in AbstractDBSpecificConnectorConfig --- .../connector/AbstractDBSpecificConnectorConfig.java | 10 ++++++++++ mysql-plugin/docs/MySQL-connector.md | 2 +- mysql-plugin/docs/Mysql-batchsink.md | 2 +- mysql-plugin/docs/Mysql-batchsource.md | 2 +- .../io/cdap/plugin/oracle/OracleConnectorConfig.java | 10 +--------- postgresql-plugin/docs/PostgreSQL-connector.md | 2 +- postgresql-plugin/docs/Postgres-batchsink.md | 2 +- postgresql-plugin/docs/Postgres-batchsource.md | 2 +- 8 files changed, 17 insertions(+), 15 deletions(-) diff --git a/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnectorConfig.java b/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnectorConfig.java index 977315033..a4337f542 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnectorConfig.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnectorConfig.java @@ -23,6 +23,7 @@ import io.cdap.plugin.db.TransactionIsolationLevel; import java.util.Collections; +import java.util.HashMap; import java.util.Map; import javax.annotation.Nullable; @@ -63,6 +64,15 @@ public boolean canConnect() { return super.canConnect() && !containsMacro(ConnectionConfig.HOST) && !containsMacro(ConnectionConfig.PORT); } + @Override + public Map getAdditionalArguments() { + Map additonalArguments = new HashMap<>(); + if (getTransactionIsolationLevel() != null) { + additonalArguments.put(TransactionIsolationLevel.CONF_KEY, getTransactionIsolationLevel()); + } + return additonalArguments; + } + public String getTransactionIsolationLevel() { if (transactionIsolationLevel == null) { return null; diff --git a/mysql-plugin/docs/MySQL-connector.md b/mysql-plugin/docs/MySQL-connector.md index 8ece1a5bc..3363b6e31 100644 --- a/mysql-plugin/docs/MySQL-connector.md +++ b/mysql-plugin/docs/MySQL-connector.md @@ -24,7 +24,7 @@ authentication. Optional for databases that do not require authentication. **Transaction Isolation Level** The transaction isolation level of the databse connection - TRANSACTION_READ_COMMITTED: No dirty reads. Non-repeatable reads and phantom reads are possible. -- TRANSACTION_SERIALIZABLE (default): No dirty reads. Non-repeatable and phantom reads are prevented. +- TRANSACTION_SERIALIZABLE: No dirty reads. Non-repeatable and phantom reads are prevented. - TRANSACTION_REPEATABLE_READ: No dirty reads. Prevents non-repeatable reads, but phantom reads are still possible. - TRANSACTION_READ_UNCOMMITTED: Allows dirty reads (reading uncommitted changes from other transactions). Non-repeatable reads and phantom reads are possible. diff --git a/mysql-plugin/docs/Mysql-batchsink.md b/mysql-plugin/docs/Mysql-batchsink.md index aaf6764e7..c75fb5a95 100644 --- a/mysql-plugin/docs/Mysql-batchsink.md +++ b/mysql-plugin/docs/Mysql-batchsink.md @@ -41,7 +41,7 @@ You also can use the macro function ${conn(connection-name)}. **Transaction Isolation Level** The transaction isolation level of the databse connection - TRANSACTION_READ_COMMITTED: No dirty reads. Non-repeatable reads and phantom reads are possible. -- TRANSACTION_SERIALIZABLE (default): No dirty reads. Non-repeatable and phantom reads are prevented. +- TRANSACTION_SERIALIZABLE: No dirty reads. Non-repeatable and phantom reads are prevented. - TRANSACTION_REPEATABLE_READ: No dirty reads. Prevents non-repeatable reads, but phantom reads are still possible. - TRANSACTION_READ_UNCOMMITTED: Allows dirty reads (reading uncommitted changes from other transactions). Non-repeatable reads and phantom reads are possible. diff --git a/mysql-plugin/docs/Mysql-batchsource.md b/mysql-plugin/docs/Mysql-batchsource.md index 69723eef1..3fdb892aa 100644 --- a/mysql-plugin/docs/Mysql-batchsource.md +++ b/mysql-plugin/docs/Mysql-batchsource.md @@ -51,7 +51,7 @@ For example, 'SELECT MIN(id),MAX(id) FROM table'. Not required if numSplits is s **Transaction Isolation Level** The transaction isolation level of the databse connection - TRANSACTION_READ_COMMITTED: No dirty reads. Non-repeatable reads and phantom reads are possible. -- TRANSACTION_SERIALIZABLE (default): No dirty reads. Non-repeatable and phantom reads are prevented. +- TRANSACTION_SERIALIZABLE: No dirty reads. Non-repeatable and phantom reads are prevented. - TRANSACTION_REPEATABLE_READ: No dirty reads. Prevents non-repeatable reads, but phantom reads are still possible. - TRANSACTION_READ_UNCOMMITTED: Allows dirty reads (reading uncommitted changes from other transactions). Non-repeatable reads and phantom reads are possible. diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java index a7e22e2e6..c3ce051e5 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java @@ -118,6 +118,7 @@ public Properties getConnectionArgumentsProperties() { return prop; } + @Override public String getTransactionIsolationLevel() { //if null default to the highest isolation level possible if (transactionIsolationLevel == null) { @@ -130,15 +131,6 @@ public String getTransactionIsolationLevel() { TransactionIsolationLevel.Level.valueOf(transactionIsolationLevel).name(); } - @Override - public Map getAdditionalArguments() { - Map additonalArguments = new HashMap<>(); - if (getTransactionIsolationLevel() != null) { - additonalArguments.put(TransactionIsolationLevel.CONF_KEY, getTransactionIsolationLevel()); - } - return additonalArguments; - } - @Override public boolean canConnect() { return super.canConnect() && !containsMacro(OracleConstants.NAME_DATABASE); diff --git a/postgresql-plugin/docs/PostgreSQL-connector.md b/postgresql-plugin/docs/PostgreSQL-connector.md index 96c3ac5bb..033957aeb 100644 --- a/postgresql-plugin/docs/PostgreSQL-connector.md +++ b/postgresql-plugin/docs/PostgreSQL-connector.md @@ -24,7 +24,7 @@ authentication. Optional for databases that do not require authentication. **Transaction Isolation Level** The transaction isolation level of the databse connection - TRANSACTION_READ_COMMITTED: No dirty reads. Non-repeatable reads and phantom reads are possible. -- TRANSACTION_SERIALIZABLE (default): No dirty reads. Non-repeatable and phantom reads are prevented. +- TRANSACTION_SERIALIZABLE: No dirty reads. Non-repeatable and phantom reads are prevented. - TRANSACTION_REPEATABLE_READ: No dirty reads. Prevents non-repeatable reads, but phantom reads are still possible. **Database:** The name of the database to connect to. diff --git a/postgresql-plugin/docs/Postgres-batchsink.md b/postgresql-plugin/docs/Postgres-batchsink.md index a0c2b144d..cbbd1bb61 100644 --- a/postgresql-plugin/docs/Postgres-batchsink.md +++ b/postgresql-plugin/docs/Postgres-batchsink.md @@ -41,7 +41,7 @@ You also can use the macro function ${conn(connection-name)}. **Transaction Isolation Level** The transaction isolation level of the databse connection - TRANSACTION_READ_COMMITTED: No dirty reads. Non-repeatable reads and phantom reads are possible. -- TRANSACTION_SERIALIZABLE (default): No dirty reads. Non-repeatable and phantom reads are prevented. +- TRANSACTION_SERIALIZABLE: No dirty reads. Non-repeatable and phantom reads are prevented. - TRANSACTION_REPEATABLE_READ: No dirty reads. Prevents non-repeatable reads, but phantom reads are still possible. **Connection Arguments:** A list of arbitrary string key/value pairs as connection arguments. These arguments diff --git a/postgresql-plugin/docs/Postgres-batchsource.md b/postgresql-plugin/docs/Postgres-batchsource.md index 46a5e56ac..5b2eb8428 100644 --- a/postgresql-plugin/docs/Postgres-batchsource.md +++ b/postgresql-plugin/docs/Postgres-batchsource.md @@ -51,7 +51,7 @@ For example, 'SELECT MIN(id),MAX(id) FROM table'. Not required if numSplits is s **Transaction Isolation Level** The transaction isolation level of the databse connection - TRANSACTION_READ_COMMITTED: No dirty reads. Non-repeatable reads and phantom reads are possible. -- TRANSACTION_SERIALIZABLE (default): No dirty reads. Non-repeatable and phantom reads are prevented. +- TRANSACTION_SERIALIZABLE: No dirty reads. Non-repeatable and phantom reads are prevented. - TRANSACTION_REPEATABLE_READ: No dirty reads. Prevents non-repeatable reads, but phantom reads are still possible. **Connection Arguments:** A list of arbitrary string key/value pairs as connection arguments. These arguments From 1cf173d8adc1a74a8c38a4ee401825f2e2ab838d Mon Sep 17 00:00:00 2001 From: Krish Date: Tue, 15 Apr 2025 15:02:45 +0530 Subject: [PATCH 13/16] Fixed log error for transaction isolation level --- .../db/config/AbstractDBSpecificSourceConfig.java | 4 ++++ .../db/connector/AbstractDBConnectorConfig.java | 5 +++++ .../db/connector/AbstractDBSpecificConnector.java | 4 ++++ .../connector/AbstractDBSpecificConnectorConfig.java | 7 +++++++ .../main/java/io/cdap/plugin/mysql/MysqlSource.java | 5 +++++ .../cdap/plugin/postgres/PostgresConnectorConfig.java | 1 + .../java/io/cdap/plugin/postgres/PostgresSource.java | 11 +++++++++++ 7 files changed, 37 insertions(+) diff --git a/database-commons/src/main/java/io/cdap/plugin/db/config/AbstractDBSpecificSourceConfig.java b/database-commons/src/main/java/io/cdap/plugin/db/config/AbstractDBSpecificSourceConfig.java index 41c577397..7cadced76 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/config/AbstractDBSpecificSourceConfig.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/config/AbstractDBSpecificSourceConfig.java @@ -28,6 +28,8 @@ import io.cdap.plugin.db.TransactionIsolationLevel; import io.cdap.plugin.db.connector.AbstractDBConnectorConfig; import io.cdap.plugin.db.source.AbstractDBSource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.Collections; @@ -49,6 +51,7 @@ public abstract class AbstractDBSpecificSourceConfig extends PluginConfig implem public static final String DATABASE = "database"; public static final String FETCH_SIZE = "fetchSize"; public static final String DEFAULT_FETCH_SIZE = "1000"; + public static final Logger LOG = LoggerFactory.getLogger(AbstractDBSpecificSourceConfig.class); @Name(Constants.Reference.REFERENCE_NAME) @Description(Constants.Reference.REFERENCE_NAME_DESCRIPTION) @@ -205,6 +208,7 @@ public Schema getSchema() { } public String getTransactionIsolationLevel() { + LOG.debug("Hi Krish Inside AbstractDBSpecificSourceConfig getTransactionIsolationLevel "); return null; } diff --git a/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBConnectorConfig.java b/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBConnectorConfig.java index 4bee056f8..fa4257d13 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBConnectorConfig.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBConnectorConfig.java @@ -25,6 +25,8 @@ import io.cdap.plugin.common.KeyValueListParser; import io.cdap.plugin.common.db.DBConnectorProperties; import io.cdap.plugin.db.ConnectionConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.Collections; import java.util.HashMap; @@ -37,6 +39,8 @@ */ public abstract class AbstractDBConnectorConfig extends PluginConfig implements DBConnectorProperties { + private static final Logger LOG = LoggerFactory.getLogger(AbstractDBConnectorConfig.class); + @Name(ConnectionConfig.JDBC_PLUGIN_NAME) @Description("Name of the JDBC driver to use. This is the value of the 'jdbcPluginName' key defined in the JSON " + "file for the JDBC plugin.") @@ -113,6 +117,7 @@ protected static Properties getConnectionArgumentsProperties(@Nullable String co } public Map getAdditionalArguments() { + LOG.debug("inside get Additional argument of AbstractDBConnectorConfig"); return Collections.emptyMap(); } diff --git a/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnector.java b/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnector.java index 8a9b7b6e4..334635653 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnector.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnector.java @@ -37,6 +37,8 @@ import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.lib.db.DBConfiguration; import org.apache.hadoop.mapreduce.lib.db.DBWritable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; import java.sql.Connection; @@ -56,6 +58,7 @@ public abstract class AbstractDBSpecificConnector extends implements BatchConnector { private final AbstractDBConnectorConfig config; + private static final Logger LOG = LoggerFactory.getLogger(AbstractDBSpecificConnector.class); protected AbstractDBSpecificConnector(AbstractDBConnectorConfig config) { super(config); @@ -99,6 +102,7 @@ public InputFormatProvider getInputFormatProvider(ConnectorContext context, Samp tableQuery, null, false); connectionConfigAccessor.setConnectionArguments(Maps.fromProperties(config.getConnectionArgumentsProperties())); connectionConfigAccessor.getConfiguration().setInt(MRJobConfig.NUM_MAPS, 1); + LOG.debug("Moving inside AbstractDBConnectorConfig"); Map additionalArguments = config.getAdditionalArguments(); for (Map.Entry argument : additionalArguments.entrySet()) { connectionConfigAccessor.getConfiguration().set(argument.getKey(), argument.getValue()); diff --git a/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnectorConfig.java b/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnectorConfig.java index a4337f542..ea9edcda0 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnectorConfig.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnectorConfig.java @@ -21,6 +21,8 @@ import io.cdap.cdap.api.annotation.Name; import io.cdap.plugin.db.ConnectionConfig; import io.cdap.plugin.db.TransactionIsolationLevel; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.Collections; import java.util.HashMap; @@ -32,9 +34,11 @@ */ public abstract class AbstractDBSpecificConnectorConfig extends AbstractDBConnectorConfig { + private static final Logger LOG = LoggerFactory.getLogger(AbstractDBSpecificConnectorConfig.class); @Name(ConnectionConfig.HOST) @Description("Database host") @Macro + @Nullable protected String host; @@ -67,6 +71,7 @@ public boolean canConnect() { @Override public Map getAdditionalArguments() { Map additonalArguments = new HashMap<>(); + LOG.debug("inside get AdditionalArguemnts of AbstractDBSpecificConnectorConfig"); if (getTransactionIsolationLevel() != null) { additonalArguments.put(TransactionIsolationLevel.CONF_KEY, getTransactionIsolationLevel()); } @@ -74,9 +79,11 @@ public Map getAdditionalArguments() { } public String getTransactionIsolationLevel() { + LOG.debug("Hi Krish, INSIDE AbstractDBSpecificConnectorCOnfig getTransactionIsolationLevel"); if (transactionIsolationLevel == null) { return null; } return TransactionIsolationLevel.Level.valueOf(transactionIsolationLevel).name(); } } + diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java index 971b76809..38642468c 100644 --- a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java @@ -197,6 +197,11 @@ public MysqlConnectorConfig getConnection() { return connection; } + @Override + public String getTransactionIsolationLevel() { + return connection.getTransactionIsolationLevel(); + } + @Override public void validate(FailureCollector collector) { ConfigUtil.validateConnection(this, useConnection, connection, collector); diff --git a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresConnectorConfig.java b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresConnectorConfig.java index 2688d14ed..33ac0f08a 100644 --- a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresConnectorConfig.java +++ b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresConnectorConfig.java @@ -19,6 +19,7 @@ import io.cdap.cdap.api.annotation.Description; import io.cdap.cdap.api.annotation.Macro; import io.cdap.cdap.api.annotation.Name; +import io.cdap.plugin.db.TransactionIsolationLevel; import io.cdap.plugin.db.connector.AbstractDBSpecificConnectorConfig; /** diff --git a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSource.java b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSource.java index 8e3c091f9..de5e4d3a6 100644 --- a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSource.java +++ b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSource.java @@ -36,6 +36,8 @@ import io.cdap.plugin.db.source.AbstractDBSource; import io.cdap.plugin.util.DBUtils; import org.apache.hadoop.mapreduce.lib.db.DBWritable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.Map; import javax.annotation.Nullable; @@ -50,6 +52,8 @@ @Metadata(properties = {@MetadataProperty(key = Connector.PLUGIN_TYPE, value = PostgresConnector.NAME)}) public class PostgresSource extends AbstractDBSource { + private static final Logger LOG = LoggerFactory.getLogger(PostgresSource.class); + private final PostgresSourceConfig postgresSourceConfig; public PostgresSource(PostgresSourceConfig postgresSourceConfig) { @@ -143,9 +147,16 @@ protected PostgresConnectorConfig getConnection() { return connection; } + @Override + public String getTransactionIsolationLevel() { + return connection.getTransactionIsolationLevel(); + } + @Override public void validate(FailureCollector collector) { ConfigUtil.validateConnection(this, useConnection, connection, collector); + LOG.debug("Hi Krish, transaction level is: {}", connection.getTransactionIsolationLevel()); + connection.getAdditionalArguments(); super.validate(collector); } From 9874ebfe0611c522725db5bc647fac165a38d100 Mon Sep 17 00:00:00 2001 From: Krish Date: Thu, 17 Apr 2025 11:40:35 +0530 Subject: [PATCH 14/16] Removed test logs --- .../io/cdap/plugin/db/config/AbstractDBSpecificSourceConfig.java | 1 - .../plugin/db/connector/AbstractDBSpecificConnectorConfig.java | 1 - mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java | 1 + .../src/main/java/io/cdap/plugin/postgres/PostgresSource.java | 1 - 4 files changed, 1 insertion(+), 3 deletions(-) diff --git a/database-commons/src/main/java/io/cdap/plugin/db/config/AbstractDBSpecificSourceConfig.java b/database-commons/src/main/java/io/cdap/plugin/db/config/AbstractDBSpecificSourceConfig.java index 7cadced76..2757b3ac5 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/config/AbstractDBSpecificSourceConfig.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/config/AbstractDBSpecificSourceConfig.java @@ -208,7 +208,6 @@ public Schema getSchema() { } public String getTransactionIsolationLevel() { - LOG.debug("Hi Krish Inside AbstractDBSpecificSourceConfig getTransactionIsolationLevel "); return null; } diff --git a/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnectorConfig.java b/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnectorConfig.java index ea9edcda0..077fc6098 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnectorConfig.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnectorConfig.java @@ -79,7 +79,6 @@ public Map getAdditionalArguments() { } public String getTransactionIsolationLevel() { - LOG.debug("Hi Krish, INSIDE AbstractDBSpecificConnectorCOnfig getTransactionIsolationLevel"); if (transactionIsolationLevel == null) { return null; } diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java index 38642468c..8aa7fa27d 100644 --- a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java @@ -205,6 +205,7 @@ public String getTransactionIsolationLevel() { @Override public void validate(FailureCollector collector) { ConfigUtil.validateConnection(this, useConnection, connection, collector); + connection.getAdditionalArguments(); super.validate(collector); } diff --git a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSource.java b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSource.java index de5e4d3a6..f6804eccd 100644 --- a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSource.java +++ b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSource.java @@ -155,7 +155,6 @@ public String getTransactionIsolationLevel() { @Override public void validate(FailureCollector collector) { ConfigUtil.validateConnection(this, useConnection, connection, collector); - LOG.debug("Hi Krish, transaction level is: {}", connection.getTransactionIsolationLevel()); connection.getAdditionalArguments(); super.validate(collector); } From ed2c392d8211c7fda54ba1be6e63433e15cc329b Mon Sep 17 00:00:00 2001 From: Krish Date: Thu, 17 Apr 2025 13:38:22 +0530 Subject: [PATCH 15/16] Removed role from ConnectionConfig --- .../src/main/java/io/cdap/plugin/db/ConnectionConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database-commons/src/main/java/io/cdap/plugin/db/ConnectionConfig.java b/database-commons/src/main/java/io/cdap/plugin/db/ConnectionConfig.java index 61256d24e..beeaaaf6f 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/ConnectionConfig.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/ConnectionConfig.java @@ -46,7 +46,7 @@ public abstract class ConnectionConfig extends PluginConfig implements DatabaseC public static final String JDBC_PLUGIN_NAME = "jdbcPluginName"; public static final String JDBC_PLUGIN_TYPE = "jdbc"; public static final String TRANSACTION_ISOLATION_LEVEL = "transactionIsolationLevel"; - public static final String ROLE = "role"; + @Name(JDBC_PLUGIN_NAME) @Description("Name of the JDBC driver to use. This is the value of the 'jdbcPluginName' key defined in the JSON " + From 4b27a1bfef271fc32ee258d540ad6eef0d92db3c Mon Sep 17 00:00:00 2001 From: Krish Date: Thu, 17 Apr 2025 13:39:37 +0530 Subject: [PATCH 16/16] Removed test logs --- .../io/cdap/plugin/db/connector/AbstractDBConnectorConfig.java | 1 - 1 file changed, 1 deletion(-) diff --git a/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBConnectorConfig.java b/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBConnectorConfig.java index fa4257d13..2be1786d2 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBConnectorConfig.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBConnectorConfig.java @@ -117,7 +117,6 @@ protected static Properties getConnectionArgumentsProperties(@Nullable String co } public Map getAdditionalArguments() { - LOG.debug("inside get Additional argument of AbstractDBConnectorConfig"); return Collections.emptyMap(); }