From bc307ea4e34f292cca86939a07ff78d3d43f52bf Mon Sep 17 00:00:00 2001 From: samik Date: Tue, 9 Jan 2024 21:53:03 +0530 Subject: [PATCH 01/52] Remove SNAPSHOT for 6.10 release --- aurora-mysql-plugin/pom.xml | 2 +- aurora-postgresql-plugin/pom.xml | 2 +- cloudsql-mysql-plugin/pom.xml | 4 ++-- cloudsql-postgresql-plugin/pom.xml | 2 +- database-commons/pom.xml | 2 +- db2-plugin/pom.xml | 2 +- generic-database-plugin/pom.xml | 2 +- generic-db-argument-setter/pom.xml | 2 +- mariadb-plugin/pom.xml | 2 +- memsql-plugin/pom.xml | 2 +- mssql-plugin/pom.xml | 2 +- mysql-plugin/pom.xml | 2 +- netezza-plugin/pom.xml | 2 +- oracle-plugin/pom.xml | 2 +- pom.xml | 4 ++-- postgresql-plugin/pom.xml | 2 +- saphana-plugin/pom.xml | 2 +- teradata-plugin/pom.xml | 2 +- 18 files changed, 20 insertions(+), 20 deletions(-) diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index 51ff6fb06..7550eccb4 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.0-SNAPSHOT + 1.11.0 Aurora DB MySQL plugin diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index 28de0db21..f3ee25ff0 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.0-SNAPSHOT + 1.11.0 Aurora DB PostgreSQL plugin diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index d05e592f5..3d1eea97e 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.0-SNAPSHOT + 1.11.0 CloudSQL MySQL plugin @@ -45,7 +45,7 @@ io.cdap.plugin mysql-plugin - 1.11.0-SNAPSHOT + 1.11.0 diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index 2f974e854..52eac90f8 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.0-SNAPSHOT + 1.11.0 CloudSQL PostgreSQL plugin diff --git a/database-commons/pom.xml b/database-commons/pom.xml index 0ecbfb445..ec0bec2eb 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.0-SNAPSHOT + 1.11.0 Database Commons diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index a43bcb92e..714a008cf 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.0-SNAPSHOT + 1.11.0 IBM DB2 plugin diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index dbcd46d47..abfcbb7c2 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.0-SNAPSHOT + 1.11.0 Generic database plugin diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index 8a8dcd1c4..838e0d4e3 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.0-SNAPSHOT + 1.11.0 Generic database argument setter plugin diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index 0e9a09e02..2376f4fba 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.0-SNAPSHOT + 1.11.0 Maria DB plugin diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index 5c50a857e..dacec62d0 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.0-SNAPSHOT + 1.11.0 Memsql plugin diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index 45e2b9c03..72e3e100b 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.0-SNAPSHOT + 1.11.0 Microsoft SQL Server plugin diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index f691a15f2..a72a88545 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.0-SNAPSHOT + 1.11.0 Mysql plugin diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index 900e430fe..e6ea65fc8 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.0-SNAPSHOT + 1.11.0 Netezza plugin diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index e0ed7ff50..2b0f9a447 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.0-SNAPSHOT + 1.11.0 Oracle plugin diff --git a/pom.xml b/pom.xml index a6b40960c..5221c25ba 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin database-plugins-parent - 1.11.0-SNAPSHOT + 1.11.0 pom Database Plugins Collection of database plugins @@ -723,7 +723,7 @@ io.cdap.tests.e2e cdap-e2e-framework - 0.3.0-SNAPSHOT + 0.3.0 test diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index 7f3e6f14c..b25312406 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.0-SNAPSHOT + 1.11.0 PostgreSQL plugin diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index 86b40a38e..856e241c4 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.0-SNAPSHOT + 1.11.0 SAP HANA plugin diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index fa770a19a..88f58b7bb 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -21,7 +21,7 @@ database-plugins-parent io.cdap.plugin - 1.11.0-SNAPSHOT + 1.11.0 teradata-plugin From 3289df464027d4186170aa2c0f4077e84cca6428 Mon Sep 17 00:00:00 2001 From: vikasrathee-cs Date: Thu, 8 Feb 2024 13:43:22 +0530 Subject: [PATCH 02/52] bumpup from 1.11.0 to 1.11.1-SNAPSHOT. --- aurora-mysql-plugin/pom.xml | 2 +- aurora-postgresql-plugin/pom.xml | 16 +-- cloudsql-mysql-plugin/pom.xml | 22 ++-- cloudsql-postgresql-plugin/pom.xml | 16 +-- database-commons/pom.xml | 2 +- db2-plugin/pom.xml | 14 +- generic-database-plugin/pom.xml | 8 +- generic-db-argument-setter/pom.xml | 14 +- mariadb-plugin/pom.xml | 202 ++++++++++++++--------------- memsql-plugin/pom.xml | 14 +- mssql-plugin/pom.xml | 2 +- mysql-plugin/pom.xml | 4 +- netezza-plugin/pom.xml | 14 +- oracle-plugin/pom.xml | 14 +- pom.xml | 6 +- postgresql-plugin/pom.xml | 8 +- saphana-plugin/pom.xml | 16 +-- teradata-plugin/pom.xml | 18 +-- 18 files changed, 197 insertions(+), 195 deletions(-) diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index 7550eccb4..4399d5276 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.0 + 1.11.1-SNAPSHOT Aurora DB MySQL plugin diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index f3ee25ff0..81494de40 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.0 + 1.11.1-SNAPSHOT Aurora DB PostgreSQL plugin @@ -103,13 +103,13 @@ <_exportcontents> - io.cdap.plugin.auroradb.postgres.*; - io.cdap.plugin.postgres.*; - io.cdap.plugin.db.source.*; - io.cdap.plugin.db.sink.*; - org.apache.commons.lang; - org.apache.commons.logging.*; - org.codehaus.jackson.* + io.cdap.plugin.auroradb.postgres.*; + io.cdap.plugin.postgres.*; + io.cdap.plugin.db.source.*; + io.cdap.plugin.db.sink.*; + org.apache.commons.lang; + org.apache.commons.logging.*; + org.codehaus.jackson.* *;inline=false;scope=compile true diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index 3d1eea97e..10c662ed2 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -20,13 +20,13 @@ database-plugins-parent io.cdap.plugin - 1.11.0 + 1.11.1-SNAPSHOT CloudSQL MySQL plugin cloudsql-mysql-plugin 4.0.0 - + io.cdap.cdap @@ -45,7 +45,7 @@ io.cdap.plugin mysql-plugin - 1.11.0 + 1.11.1-SNAPSHOT @@ -99,7 +99,7 @@ test - + @@ -128,13 +128,13 @@ <_exportcontents> - io.cdap.plugin.cloudsql.mysql.*; - io.cdap.plugin.db.source.*; - io.cdap.plugin.db.sink.*; - io.cdap.plugin.mysql.*; - org.apache.commons.lang.*; - org.apache.commons.logging.*; - org.codehaus.jackson.* + io.cdap.plugin.cloudsql.mysql.*; + io.cdap.plugin.db.source.*; + io.cdap.plugin.db.sink.*; + io.cdap.plugin.mysql.*; + org.apache.commons.lang.*; + org.apache.commons.logging.*; + org.codehaus.jackson.* *;inline=false;scope=compile true diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index 52eac90f8..614b55a6e 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.0 + 1.11.1-SNAPSHOT CloudSQL PostgreSQL plugin @@ -133,13 +133,13 @@ <_exportcontents> - io.cdap.plugin.cloudsql.postgres.*; - io.cdap.plugin.postgres.*; - io.cdap.plugin.db.source.*; - io.cdap.plugin.db.sink.*; - org.apache.commons.lang.*; - org.apache.commons.logging.*; - org.codehaus.jackson.* + io.cdap.plugin.cloudsql.postgres.*; + io.cdap.plugin.postgres.*; + io.cdap.plugin.db.source.*; + io.cdap.plugin.db.sink.*; + org.apache.commons.lang.*; + org.apache.commons.logging.*; + org.codehaus.jackson.* *;inline=false;scope=compile true diff --git a/database-commons/pom.xml b/database-commons/pom.xml index ec0bec2eb..262e0b695 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.0 + 1.11.1-SNAPSHOT Database Commons diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index 714a008cf..77f64ce0c 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.0 + 1.11.1-SNAPSHOT IBM DB2 plugin @@ -98,12 +98,12 @@ <_exportcontents> - io.cdap.plugin.db2.*; - io.cdap.plugin.db.source.*; - io.cdap.plugin.db.sink.*; - org.apache.commons.lang; - org.apache.commons.logging.*; - org.codehaus.jackson.* + io.cdap.plugin.db2.*; + io.cdap.plugin.db.source.*; + io.cdap.plugin.db.sink.*; + org.apache.commons.lang; + org.apache.commons.logging.*; + org.codehaus.jackson.* *;inline=false;scope=compile true diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index abfcbb7c2..218a1e71b 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.0 + 1.11.1-SNAPSHOT Generic database plugin @@ -97,9 +97,9 @@ <_exportcontents> - io.cdap.plugin.jdbc.*; - io.cdap.plugin.db.source.*; - io.cdap.plugin.db.sink.*; + io.cdap.plugin.jdbc.*; + io.cdap.plugin.db.source.*; + io.cdap.plugin.db.sink.*; *;inline=false;scope=compile true diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index 838e0d4e3..1fd9d97ef 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.0 + 1.11.1-SNAPSHOT Generic database argument setter plugin @@ -97,12 +97,12 @@ <_exportcontents> - io.cdap.plugin.jdbc.*; - io.cdap.plugin.db.source.*; - io.cdap.plugin.db.sink.*; - org.apache.commons.lang; - org.apache.commons.logging.*; - org.codehaus.jackson.* + io.cdap.plugin.jdbc.*; + io.cdap.plugin.db.source.*; + io.cdap.plugin.db.sink.*; + org.apache.commons.lang; + org.apache.commons.logging.*; + org.codehaus.jackson.* *;inline=false;scope=compile true diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index 2376f4fba..5ba0cad4e 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -17,108 +17,108 @@ - - database-plugins-parent - io.cdap.plugin - 1.11.0 - + + database-plugins-parent + io.cdap.plugin + 1.11.1-SNAPSHOT + - Maria DB plugin - mariadb-plugin - 4.0.0 + Maria DB plugin + mariadb-plugin + 4.0.0 - - - io.cdap.cdap - cdap-etl-api - - - io.cdap.plugin - database-commons - ${project.version} - - - io.cdap.plugin - hydrator-common - - - com.google.guava - guava - + + + io.cdap.cdap + cdap-etl-api + + + io.cdap.plugin + database-commons + ${project.version} + + + io.cdap.plugin + hydrator-common + + + com.google.guava + guava + - - - io.cdap.plugin - database-commons - ${project.version} - test-jar - test - - - io.cdap.cdap - hydrator-test - - - io.cdap.cdap - cdap-data-pipeline3_2.12 - - - junit - junit - - - io.cdap.cdap - cdap-api - provided - - - org.mariadb.jdbc - mariadb-java-client - 2.7.3 - test - - - org.jetbrains - annotations - RELEASE - compile - - - - - - io.cdap - cdap-maven-plugin - - - org.apache.felix - maven-bundle-plugin - 5.1.2 - true - - - <_exportcontents> - io.cdap.plugin.mariadb.*; - io.cdap.plugin.db.source.*; - io.cdap.plugin.db.sink.*; - org.apache.commons.lang; - org.apache.commons.logging.*; - org.codehaus.jackson.* - - *;inline=false;scope=compile - true - lib - - - - - package - - bundle - - - - - - + + + io.cdap.plugin + database-commons + ${project.version} + test-jar + test + + + io.cdap.cdap + hydrator-test + + + io.cdap.cdap + cdap-data-pipeline3_2.12 + + + junit + junit + + + io.cdap.cdap + cdap-api + provided + + + org.mariadb.jdbc + mariadb-java-client + 2.7.3 + test + + + org.jetbrains + annotations + RELEASE + compile + + + + + + io.cdap + cdap-maven-plugin + + + org.apache.felix + maven-bundle-plugin + 5.1.2 + true + + + <_exportcontents> + io.cdap.plugin.mariadb.*; + io.cdap.plugin.db.source.*; + io.cdap.plugin.db.sink.*; + org.apache.commons.lang; + org.apache.commons.logging.*; + org.codehaus.jackson.* + + *;inline=false;scope=compile + true + lib + + + + + package + + bundle + + + + + + diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index dacec62d0..88e491000 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.0 + 1.11.1-SNAPSHOT Memsql plugin @@ -95,12 +95,12 @@ <_exportcontents> - io.cdap.plugin.memsql.*; - io.cdap.plugin.db.source.*; - io.cdap.plugin.db.sink.*; - org.apache.commons.lang; - org.apache.commons.logging.*; - org.codehaus.jackson.* + io.cdap.plugin.memsql.*; + io.cdap.plugin.db.source.*; + io.cdap.plugin.db.sink.*; + org.apache.commons.lang; + org.apache.commons.logging.*; + org.codehaus.jackson.* *;inline=false;scope=compile true diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index 72e3e100b..26575edf8 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.0 + 1.11.1-SNAPSHOT Microsoft SQL Server plugin diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index a72a88545..efb2cce40 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -20,13 +20,13 @@ database-plugins-parent io.cdap.plugin - 1.11.0 + 1.11.1-SNAPSHOT Mysql plugin mysql-plugin 4.0.0 - + io.cdap.cdap diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index e6ea65fc8..74100cc01 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.0 + 1.11.1-SNAPSHOT Netezza plugin @@ -92,12 +92,12 @@ <_exportcontents> - io.cdap.plugin.netezza.*; - io.cdap.plugin.db.source.*; - io.cdap.plugin.db.sink.*; - org.apache.commons.lang; - org.apache.commons.logging.*; - org.codehaus.jackson.* + io.cdap.plugin.netezza.*; + io.cdap.plugin.db.source.*; + io.cdap.plugin.db.sink.*; + org.apache.commons.lang; + org.apache.commons.logging.*; + org.codehaus.jackson.* *;inline=false;scope=compile true diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index 2b0f9a447..e701b6a78 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.0 + 1.11.1-SNAPSHOT Oracle plugin @@ -113,12 +113,12 @@ <_exportcontents> - io.cdap.plugin.oracle.*; - io.cdap.plugin.db.source.*; - io.cdap.plugin.db.sink.*; - org.apache.commons.lang; - org.apache.commons.logging.*; - org.codehaus.jackson.* + io.cdap.plugin.oracle.*; + io.cdap.plugin.db.source.*; + io.cdap.plugin.db.sink.*; + org.apache.commons.lang; + org.apache.commons.logging.*; + org.codehaus.jackson.* *;inline=false;scope=compile true diff --git a/pom.xml b/pom.xml index 5221c25ba..08928d5c2 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin database-plugins-parent - 1.11.0 + 1.11.1-SNAPSHOT pom Database Plugins Collection of database plugins @@ -399,7 +399,9 @@ maven-surefire-plugin 2.22.0 - -Xmx3g -Djava.awt.headless=true -XX:MaxPermSize=256m -XX:+UseConcMarkSweepGC -Djava.net.preferIPv4Stack=true + -Xmx3g -Djava.awt.headless=true -XX:MaxPermSize=256m -XX:+UseConcMarkSweepGC + -Djava.net.preferIPv4Stack=true + ${surefire.redirectTestOutputToFile} false plain diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index b25312406..c4bec7485 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.0 + 1.11.1-SNAPSHOT PostgreSQL plugin @@ -100,9 +100,9 @@ <_exportcontents> - io.cdap.plugin.postgres.*; - io.cdap.plugin.db.source.*; - io.cdap.plugin.db.sink.*; + io.cdap.plugin.postgres.*; + io.cdap.plugin.db.source.*; + io.cdap.plugin.db.sink.*; *;inline=false;scope=compile true diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index 856e241c4..10902c00d 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.0 + 1.11.1-SNAPSHOT SAP HANA plugin @@ -85,13 +85,13 @@ <_exportcontents> - io.cdap.plugin.saphana.*; - io.cdap.plugin.db.source.*; - io.cdap.plugin.db.sink.*; - io.cdap.plugin.saphana.*; - org.apache.commons.lang; - org.apache.commons.logging.*; - org.codehaus.jackson.* + io.cdap.plugin.saphana.*; + io.cdap.plugin.db.source.*; + io.cdap.plugin.db.sink.*; + io.cdap.plugin.saphana.*; + org.apache.commons.lang; + org.apache.commons.logging.*; + org.codehaus.jackson.* *;inline=false;scope=compile true diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index 88f58b7bb..38741b6a3 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -21,7 +21,7 @@ database-plugins-parent io.cdap.plugin - 1.11.0 + 1.11.1-SNAPSHOT teradata-plugin @@ -90,14 +90,14 @@ <_exportcontents> - io.cdap.plugin.teradata.*; - io.cdap.plugin.util.*; - io.cdap.plugin.db.source.*; - io.cdap.plugin.db.sink.*; - io.cdap.plugin.saphana.*; - org.apache.commons.lang; - org.apache.commons.logging.*; - org.codehaus.jackson.* + io.cdap.plugin.teradata.*; + io.cdap.plugin.util.*; + io.cdap.plugin.db.source.*; + io.cdap.plugin.db.sink.*; + io.cdap.plugin.saphana.*; + org.apache.commons.lang; + org.apache.commons.logging.*; + org.codehaus.jackson.* *;inline=false;scope=compile true From ab14b0f404307f1b172159cc2a904b16a2a24735 Mon Sep 17 00:00:00 2001 From: vikasrathee-cs Date: Tue, 5 Dec 2023 17:00:10 +0530 Subject: [PATCH 03/52] Fix for connection arguments not getting updated. --- .../mysql/CloudSQLMySQLConnectorConfig.java | 6 ++--- .../plugin/mysql/MysqlConnectorConfig.java | 6 ++--- .../mysql/MysqlFailedConnectionTest.java | 26 +++++++++++++++---- 3 files changed, 27 insertions(+), 11 deletions(-) diff --git a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnectorConfig.java b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnectorConfig.java index 1e89d5a95..e763f6235 100644 --- a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnectorConfig.java +++ b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnectorConfig.java @@ -105,14 +105,14 @@ public String getConnectionString() { @Override public Properties getConnectionArgumentsProperties() { Properties properties = super.getConnectionArgumentsProperties(); - properties.put(JDBC_PROPERTY_CONNECT_TIMEOUT_MILLIS, "20000"); - properties.put(JDBC_PROPERTY_SOCKET_TIMEOUT_MILLIS, "20000"); + properties.putIfAbsent(JDBC_PROPERTY_CONNECT_TIMEOUT_MILLIS, "20000"); + properties.putIfAbsent(JDBC_PROPERTY_SOCKET_TIMEOUT_MILLIS, "20000"); return properties; } @Override public boolean canConnect() { return super.canConnect() && !containsMacro(CloudSQLUtil.CONNECTION_NAME) && - !containsMacro(ConnectionConfig.PORT) && !containsMacro(ConnectionConfig.DATABASE); + !containsMacro(ConnectionConfig.PORT) && !containsMacro(ConnectionConfig.DATABASE); } } diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConnectorConfig.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConnectorConfig.java index 9b481e4fe..8c20798d3 100644 --- a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConnectorConfig.java +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConnectorConfig.java @@ -57,9 +57,9 @@ public int getDefaultPort() { public Properties getConnectionArgumentsProperties() { Properties prop = super.getConnectionArgumentsProperties(); // the unit below is milli-second - prop.put(JDBC_PROPERTY_CONNECT_TIMEOUT, "20000"); - prop.put(JDBC_PROPERTY_SOCKET_TIMEOUT, "20000"); - prop.put(JDBC_REWRITE_BATCHED_STATEMENTS, "true"); + prop.putIfAbsent(JDBC_PROPERTY_CONNECT_TIMEOUT, "20000"); + prop.putIfAbsent(JDBC_PROPERTY_SOCKET_TIMEOUT, "20000"); + prop.putIfAbsent(JDBC_REWRITE_BATCHED_STATEMENTS, "true"); // MySQL property to ensure that TINYINT(1) type data is not converted to MySQL Bit/Boolean type in the ResultSet. prop.putIfAbsent(MYSQL_TINYINT1_IS_BIT, "false"); return prop; diff --git a/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlFailedConnectionTest.java b/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlFailedConnectionTest.java index a1be6a754..5c4f35828 100644 --- a/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlFailedConnectionTest.java +++ b/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlFailedConnectionTest.java @@ -31,10 +31,26 @@ public void test() throws ClassNotFoundException, IOException { new MysqlConnectorConfig("localhost", 3306, "username", "password", "jdbc", "")); super.test(JDBC_DRIVER_CLASS_NAME, connector, "Failed to create connection to database via connection string: " + - "jdbc:mysql://localhost:3306 and arguments: {user=username, " + - "rewriteBatchedStatements=true, " + - "connectTimeout=20000, tinyInt1isBit=false, " + - "socketTimeout=20000}. Error: " + - "ConnectException: Connection refused (Connection refused)."); + "jdbc:mysql://localhost:3306 and arguments: {user=username, " + + "rewriteBatchedStatements=true, " + + "connectTimeout=20000, tinyInt1isBit=false, " + + "socketTimeout=20000}. Error: " + + "ConnectException: Connection refused (Connection refused)."); } + + @Test + public void testWithUpdatedConnectionArguments() throws ClassNotFoundException, IOException { + + MysqlConnector connector = new MysqlConnector( + new MysqlConnectorConfig("localhost", 3306, "username", "password", "jdbc", + "connectTimeout=30000;socketTimeout=30000")); + + super.test(JDBC_DRIVER_CLASS_NAME, connector, "Failed to create connection to database via connection string: " + + "jdbc:mysql://localhost:3306 and arguments: {user=username, " + + "rewriteBatchedStatements=true, " + + "connectTimeout=30000, tinyInt1isBit=false, " + + "socketTimeout=30000}. Error: " + + "ConnectException: Connection refused (Connection refused)."); + } + } From 4e3b51febce76889fcb3e6c273538f4f5daa6d6b Mon Sep 17 00:00:00 2001 From: vikasrathee-cs Date: Wed, 20 Sep 2023 17:09:15 +0530 Subject: [PATCH 04/52] Redshift source and connector plugin added. --- .../docs/Redshift-batchsource.md | 102 ++++++ .../docs/Redshift-connector.md | 26 ++ .../icons/Redshift-batchsource.png | Bin 0 -> 6145 bytes amazon-redshift-plugin/pom.xml | 139 ++++++++ .../amazon/redshift/RedshiftConnector.java | 117 ++++++ .../redshift/RedshiftConnectorConfig.java | 87 +++++ .../amazon/redshift/RedshiftConstants.java | 27 ++ .../amazon/redshift/RedshiftDBRecord.java | 129 +++++++ .../amazon/redshift/RedshiftSchemaReader.java | 117 ++++++ .../amazon/redshift/RedshiftSource.java | 128 +++++++ .../redshift/RedshiftConnectorTest.java | 39 ++ .../redshift/RedshiftConnectorUnitTest.java | 55 +++ .../redshift/RedshiftDBRecordUnitTest.java | 155 ++++++++ .../RedshiftFailedConnectionTest.java | 38 ++ .../redshift/RedshiftPluginTestBase.java | 218 ++++++++++++ .../redshift/RedshiftPluginTestSuite.java | 31 ++ .../redshift/RedshiftSourceTestRun.java | 332 ++++++++++++++++++ .../widgets/Redshift-batchsource.json | 240 +++++++++++++ .../widgets/Redshift-connector.json | 75 ++++ pom.xml | 1 + 20 files changed, 2056 insertions(+) create mode 100644 amazon-redshift-plugin/docs/Redshift-batchsource.md create mode 100644 amazon-redshift-plugin/docs/Redshift-connector.md create mode 100644 amazon-redshift-plugin/icons/Redshift-batchsource.png create mode 100644 amazon-redshift-plugin/pom.xml create mode 100644 amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftConnector.java create mode 100644 amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftConnectorConfig.java create mode 100644 amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftConstants.java create mode 100644 amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftDBRecord.java create mode 100644 amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftSchemaReader.java create mode 100644 amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftSource.java create mode 100644 amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftConnectorTest.java create mode 100644 amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftConnectorUnitTest.java create mode 100644 amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftDBRecordUnitTest.java create mode 100644 amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftFailedConnectionTest.java create mode 100644 amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftPluginTestBase.java create mode 100644 amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftPluginTestSuite.java create mode 100644 amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftSourceTestRun.java create mode 100644 amazon-redshift-plugin/widgets/Redshift-batchsource.json create mode 100644 amazon-redshift-plugin/widgets/Redshift-connector.json diff --git a/amazon-redshift-plugin/docs/Redshift-batchsource.md b/amazon-redshift-plugin/docs/Redshift-batchsource.md new file mode 100644 index 000000000..38873b15a --- /dev/null +++ b/amazon-redshift-plugin/docs/Redshift-batchsource.md @@ -0,0 +1,102 @@ +# Amazon Redshift Batch Source + +Description +----------- +Reads from an Amazon Redshift database using a configurable SQL query. +Outputs one record for each row returned by the query. + + +Use Case +-------- +The source is used whenever you need to read from an Amazon Redshift database. For example, you may want +to create daily snapshots of a database table by using this source and writing to +a TimePartitionedFileSet. + + +Properties +---------- +**Reference Name:** Name used to uniquely identify this source for lineage, annotating metadata, etc. + +**JDBC Driver name:** Name of the JDBC driver to use. + +**Host:** Host URL of the current master instance of Redshift cluster. + +**Port:** Port that Redshift master instance is listening to. + +**Database:** Redshift database name. + +**Import Query:** The SELECT query to use to import data from the specified table. +You can specify an arbitrary number of columns to import, or import all columns using \*. The Query should +contain the '$CONDITIONS' string. For example, 'SELECT * FROM table WHERE $CONDITIONS'. +The '$CONDITIONS' string will be replaced by 'splitBy' field limits specified by the bounding query. +The '$CONDITIONS' string is not required if numSplits is set to one. + +**Bounding Query:** Bounding Query should return the min and max of the values of the 'splitBy' field. +For example, 'SELECT MIN(id),MAX(id) FROM table'. Not required if numSplits is set to one. + +**Split-By Field Name:** Field Name which will be used to generate splits. Not required if numSplits is set to one. + +**Number of Splits to Generate:** Number of splits to generate. + +**Username:** User identity for connecting to the specified database. + +**Password:** Password to use to connect to the specified database. + +**Connection Arguments:** A list of arbitrary string key/value pairs as connection arguments. These arguments +will be passed to the JDBC driver as connection arguments for JDBC drivers that may need additional configurations. + +**Schema:** The schema of records output by the source. This will be used in place of whatever schema comes +back from the query. However, it must match the schema that comes back from the query, +except it can mark fields as nullable and can contain a subset of the fields. + +**Fetch Size:** The number of rows to fetch at a time per split. Larger fetch size can result in faster import, +with the tradeoff of higher memory usage. + +Example +------ +Suppose you want to read data from an Amazon Redshift database named "prod" that is running on +"redshift.xyz.eu-central-1.redshift.amazonaws.com", port 5439, as "sa" user with "Test11" password. +Ensure that the driver for Redshift is installed (you can also provide driver name for some specific driver, +otherwise "redshift" will be used), then configure the plugin with:then configure plugin with: + +``` +Reference Name: "src1" +Driver Name: "redshift" +Host: "redshift.xyz.eu-central-1.redshift.amazonaws.com" +Port: 5439 +Database: "prod" +Import Query: "select id, name, email, phone from users;" +Number of Splits to Generate: 1 +Username: "sa" +Password: "Test11" +``` + +Data Types Mapping +------------------ + +Mapping of Redshift types to CDAP schema: + +| Redshift Data Type | CDAP Schema Data Type | Comment | +|-----------------------------------------------------|-----------------------|----------------------------------| +| bigint | long | | +| boolean | boolean | | +| character | string | | +| character varying | string | | +| double precision | double | | +| integer | int | | +| numeric(precision, scale)/decimal(precision, scale) | decimal | | +| numeric(with 0 precision) | string | | +| real | float | | +| smallint | int | | +| smallserial | int | | +| text | string | | +| date | date | | +| time [ (p) ] [ without time zone ] | time | | +| time [ (p) ] with time zone | string | | +| timestamp [ (p) ] [ without time zone ] | timestamp | | +| timestamp [ (p) ] with time zone | timestamp | stored in UTC format in database | +| xml | string | | +| json | string | | +| super | string | | +| geometry | bytes | | +| hllsketch | string | | diff --git a/amazon-redshift-plugin/docs/Redshift-connector.md b/amazon-redshift-plugin/docs/Redshift-connector.md new file mode 100644 index 000000000..368d9e09f --- /dev/null +++ b/amazon-redshift-plugin/docs/Redshift-connector.md @@ -0,0 +1,26 @@ +# Amazon Redshift Connection + +Description +----------- +Use this connection to access data in an Amazon Redshift database using JDBC. + +Properties +---------- +**Name:** Name of the connection. Connection names must be unique in a namespace. + +**Description:** Description of the connection. + +**JDBC Driver name:** Name of the JDBC driver to use. + +**Host:** Host of the current master instance of Redshift cluster. + +**Port:** Port that Redshift master instance is listening to. + +**Database:** Redshift database name. + +**Username:** User identity for connecting to the specified database. + +**Password:** Password to use to connect to the specified database. + +**Connection Arguments:** A list of arbitrary string key/value pairs as connection arguments. These arguments +will be passed to the JDBC driver as connection arguments for JDBC drivers that may need additional configurations. diff --git a/amazon-redshift-plugin/icons/Redshift-batchsource.png b/amazon-redshift-plugin/icons/Redshift-batchsource.png new file mode 100644 index 0000000000000000000000000000000000000000..11c334799daa890a320e5d4aa934d44a71bc988f GIT binary patch literal 6145 zcmZu#1yqzx+a^||Q;_adIwY4aC6*4Qdx2$PS&1bT0YOSB5m-P{K$=Aw=@M2-atS3C zqy(hl$M=1|@B8aHXP%k4=epy%=FEA{GcgAGT4cnG#5g!OWDm8~jc)poKaG&!=A9Vs z$#c`;`Wk7e;#7<2-`bB9h`*& zAl^4<92|uJxtk%x8E(%W0P*zll?woJ{)Ldc8UHbhaI*gefqMWsE%Xf7)u1qEb}3;= zVNp&+Vs>_R1(=hIoRPZbKkhd(Ag3D~?ky)G;_vS->@OhDAdiwoT# zgnR?N;PwGRUcOv^2l-bVb!T5k7}y&QhI+C8iEHlweF_J1a{fv5@8_R7A>RK^_$QiVSudEWxWws zk@zOlzcxydSU~s-5C?~L^r5W!WrW127Uq3WOr7C^6(52)IPjce(E0uiBta9&8DqCFWCwAOt zVy|_7-r0B-*v%1JTRD@l({?(tjYRg193K8W96r3aP&m^#90cvn@-(`yf90YLk9&&-Wm|%K8QXz1(z{; z>fEL99bj9DIec3ulx8C}OpWjHwr+s8)3J#3tZ5n7udB!qJ@wY|ZEGjDk_*vgq{KP& zS4WOq%C}#=*-Ob9vj=J>pnzCmVZw6)7ZV4mmT5~ z3Y|0z;lfl4UFs~uZ$%59szrESZZHl5rfCE?>Hzm?*D92sX+=6C`$LXVL*3GmmkL89 z;n1|zm$UwoDx=7Wl=raXq;FLpgA%-YjV{^WpQ&+XFlfEJHLY<(s>risp&+3!NS!h1 z6+i*{GC2O4znqEDEl%-g8K>Y;?d?@Z@xx>1% z$H1z?VjN4kp+h6NI+GSmZ=^^#%E~8_=JFM9d~li2vk>><5k+EE!4@{J*I(DJ5xMhx zhd;Y(o*xXLcr2bzl-G6OD#Ae-3~Cs@TMATiC$a@7L$h)%-~A@M(_|j$6MpXtUZ+g$ z>$h78Q)O3vOmJ@4R@Chwq>WVdTncqw76~wwN|&&H&6hVJ(@Oqv>zWqN@++JTS{9=R z`b3Ob>l`XXjPLmsGC50cRSEo%UTc`kmHu@|Jhw_Gm`qJv0UnTCe@@*N$Q@ksVEO*> z&Ub%Dp5F-^?`-c?MFx~8s&uK97nRFSJ1aJmA?|}j4wi$B*Yvk!(#{`s-rloSMuiO@ zT+E6AFC)22KU69A?DW8fVPE+hC6x!rd=BK|Q*4 zky(#(W#M3#nm4o7J|FTU8@eJ}6U?I|6%=|z0XH}K5L4-fmCAJT!KiE6bV7dpoWd;l zn$`n3I{Mzc=wa#4EIggz#E5z+uUylABTUkvq8~ zpwZ{Cp(;lKhIb2<)rm%k$gV&9cDKKJCxG|}6sAtOahZD60&}7=lhT#hWjW;fm41QJ z$FNr*gGN}bz;{!i5>SIl72-2Zw=u#E*ZyokTVUO*gTEdJ9d2g7pL?(rEx3QVm_b>l zgnS+7H&ynAs&ao)6~F(mXh)&oi)Z_@oaQFFCSwi0zL;Ml(=`}V9w_4_(Zz#O#AjtK zU}pT!q}Zy@)2PUz-dDXC$upv~`s zi76W(Uq!`&b8|G=V4_*Czj=O>anN-A?@|MHU$0feFOmL1-+IrHt%~{k+jO;Fj|Jbg zr`p`HJ?Uz{!rtd8+nZ+?sWb>H-MrlFB&5J=Q`$(SF8=lNV-PtsO?A#AJvN92``HKB z8jH;^Z4iU(&zV&lpmk=?{nH4oXut`A>OtQ zRs)gi&-Fa7jjzrspwt|n9CHr}+MQf6tuKs3w7Uoo>*dIK59OEw?HrF4>kVnn^kgo4 z!cTZ)I?YnHsi!~RCgxYLH@>%Gh1}Q=x zxgDFMcv+{9=f={SInK9n)=3POT`G#R;_oqEzc*GvHNNaC?bIIO%e9bEd^8lx_Q|k1 z7xl4zqC@8H3dkqUW0#HjYQV?qU>&41pyXLpD(7==s(SUxwH;sc3U3O9RQ^o5DS~=4 za2PNkIfEL7I_5J;zbTGLXEYil#}RHZU*LPC{+S_4=QYlJ2jQN!)#=HYh7WcnRc`>$ z8Y$^m+jE(^gk5_5n=XQK9fT*d>HIhO><7>VV*rL#9rbF>AnPPA@AWq0K4<6UChR>jFpIrzeC zndogOJnp9@@_rs$DtDrvG+zL%Q+XLxtFQ#1!I}Zh&a`(;CI9OF;vNyyC7s z2&ozVdZyS<%mmA*HxN|-GoPS<`)o<~S1`L#QW-v3fH9!Re2G`0xq(F_k8R@SAZpNr zXJVzenD*$7g_>@Y_LsVEl#FcJU$PCjIDJsg-+R5u($c1bUd&ZiSMJJWA^{W#(t6K- z)Ry*$7vWz{ouU}jnnpq7=-*7d?sy|z_C-jG8U!4oj&q|z(q+XG4ol{b-GNU>8R+y= zk9ujAemw7HCnA&B&>cg$#4EdMmyu=}m7}|u0;O1H3V2Ld=plmMIlyw458z?@J3E@G zUl3u24<&`DuSky>4*A&`!=n+a1oUI+!Wh4Dr7p1z74*J;7W!zyg=@c>MUkoW?} zhMI5p9l)!Vw4XrYw=Jy>ow@E*TKkqU(2)woE1kIn*sMJzx4s^?dfKwmbK>@%kXPQ2a!u zMT=N8Oa{^A{uA5!wO0C7*wRtc(|W21jOmdu4{OkwQMUR)LUhWv4T&6Kwxgc`m7Z5W zdyqSoIp~}y8ce3aKC$qXXJ795R$t&IcG?5LFnO+6U3_e_uHsUj*sszH*!dpV)b10t zgteySs%o*`rZ=U^bl4j8Cd`%2ZY-1O?#{yv^UN&e2cR>@64=yD@NZNMClY%bsXgfW zY9TC*{*XOas~$AIhdF%qhMNFb8cwiWhROSGj$(h@+t@9RF6F!itRWm0L(i44s5TLg!7JTIm?~iLJk|xHbofX@l zwJvWlYrPdXS9SHi`-Y3FGw?`ZA!nN&9(HM!401$AbZiQ}w&Ch8>Hi(0LV*|G;D?B3 z&qplwJsH0|jk~evnXt4VD+BZ>p1X7?UM0{Zy&V6@T8tuS*!&sTbS+q2++gIyopb(t zc1+z79YpG`qmH-#-RJ|~+J#nG)yuQn^*3gZqw1P4*=qq69R>&VlT!Y9^0sG<;~!eo zNB*dwPs>ucZ^OAhYC_u79;zC)**+U3Y8`&<#!mnQ0FUF<^Z7BC2*hG{Jl~Eoes!^b zZc7`*N89^&S4yxONs~P8Xhy_&p!w8|Fdqb`K6WW`~r7jQra#r2D4OPB~dqSZ`YOj~QPcGIHtu`o?edGQP zV|52XU5ULqi`f`Q)Rj$}EHA7Fk|I)T5EG=b zErUCoO}Gn>8PRchMH13XOY`ejaiMdpxAdd@&MmD-I0UiQJMp&p zraOna74A_wzGesGRd-LCDO6>=NZI?k2Yyr{FiPIfomyy-s_UHxRt32u#Cz)0i61Cm z4;JPZrJNn->S@0dBj(Z0<9U(BOlidzpR#2BfB{#Q&KfNi8E&SDPU6tGt^ZD!oV>bo z1#fwBuk_Y}Xv%S3aUGAm=ST>OV7plz#%`ZZ=WU*bL&m90d!n<>hl5qC>o1cM1~}=F zRM^X@CysTNOTIc_gUtnp*spzaY^n&$AdpE|d}KFS4GPn?h#vQ2NFr>lf+MxvDpQGT z+b709ELbS_0S@t^?~b9~zMx8?4M)Xf3>qf!T9>X8<4-I;cjP-9E_DK5g{^j82mbWKM6OK?h#4|H4C#JD2Vq?em3J)QM<`Az z?~7&1ivnMFbO@8(7S5BDHPv!V)U&ZxW#kv%H|7}zYBU{#}?_K zdjP6{z#*h-n>+0~1$%I?HMfLZ?V6iSjd+0fb1)A5ftqA^%cm-fd%{*uJ(2#R*e%`u zhEHi%r`QmAMc$Q3bMcIhU)90RX5*WWGbhT0-Qk^OPH95mMklt5z1KP??HwXMdAq4S`V9|KEQV7*YnyqMgq11WAZSv;KrnIEV3e!svBM{JhwqwuaT$n~SV_o1xDOO>OF zM|$N$qTxL`9L+)2dq092FOzE}%g@B^WY+!d1czz{NOT!r`Ud*(0ZdOLYJZtiymV-$ zS+7WscHpWw;HynxTkd3o36F_bC*S49UUtoMUB}y2HnTlA?H(_qbEy+~?B~-^FVhgQ zzjb|j_&M9vjnMkX%%gS!>4Q!E1Es6n^gh`o`Qb+7HF#&~RgU;?hen-K`5@{cyov!5 z%aTpXlbl{@^Y?3m*&B1-N)riZs~xz)UFwTGTBk|&CWPyDx10d*R5IKVzpMUGScx?D zfC%U-(pNirZ?xhov{5Z$*Dg2xwpozkv+K<|jl6dY&%$A8Grf|@t4|9$!wp$O$bxCk zXoH)$4L(wpJWNVrh(5W~toGx;l_AsNL47fO$AcYD?95Rx>NX+}&JgUi*K(c^1MLgK z%D1Ub;fzT}q-h*64kx*ORbt9dWxY&fR9&_uJG zXS@l?h$d;j8U1 zWh7h*i7a)DGv|9;jJeIv#}#3$PhOdL=RTpKP>@d;ytElu<&t^flOKc*w~AC%+TNGH z@MnUHw0sMYV_?_dAZx+(iO(SMW$WZle5|YF=W?w__?`4UJtt_cm0vo2_MwH~)rc?08laQF?douUlsQ((k z{k?KQ%X}{HbpV+1yzRLXvq>H3IGEg(TDfa$2v{&BW|qsOSvBHXFhN2GqQAmv6ia5U zx;qn(bMp*;=#^EMa%#U2OpW{mTbRW~g*p*xx|9LPTei40kVR}$2@ + + + + database-plugins-parent + io.cdap.plugin + 1.11.1-SNAPSHOT + + + Amazon Redshift plugin + amazon-redshift-plugin + 4.0.0 + + + 2.1.0.18 + + + + + redshift + http://redshift-maven-repository.s3-website-us-east-1.amazonaws.com/release + + + + + + io.cdap.cdap + cdap-etl-api + + + io.cdap.plugin + database-commons + ${project.version} + + + io.cdap.plugin + hydrator-common + + + com.google.guava + guava + + + + + com.amazon.redshift + redshift-jdbc42 + ${redshift-jdbc.version} + test + + + io.cdap.plugin + database-commons + ${project.version} + test-jar + test + + + io.cdap.cdap + hydrator-test + + + io.cdap.cdap + cdap-data-pipeline3_2.12 + + + junit + junit + + + org.mockito + mockito-core + test + + + io.cdap.cdap + cdap-api + provided + + + org.jetbrains + annotations + RELEASE + compile + + + + + + io.cdap + cdap-maven-plugin + + + org.apache.felix + maven-bundle-plugin + 5.1.2 + true + + + <_exportcontents> + io.cdap.plugin.amazon.redshift.*; + io.cdap.plugin.db.source.*; + org.apache.commons.lang; + org.apache.commons.logging.*; + org.codehaus.jackson.* + + *;inline=false;scope=compile + true + lib + + + + + package + + bundle + + + + + + + diff --git a/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftConnector.java b/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftConnector.java new file mode 100644 index 000000000..fb8cac4a7 --- /dev/null +++ b/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftConnector.java @@ -0,0 +1,117 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.amazon.redshift; + +import io.cdap.cdap.api.annotation.Category; +import io.cdap.cdap.api.annotation.Description; +import io.cdap.cdap.api.annotation.Name; +import io.cdap.cdap.api.annotation.Plugin; +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.etl.api.batch.BatchSource; +import io.cdap.cdap.etl.api.connector.Connector; +import io.cdap.cdap.etl.api.connector.ConnectorSpec; +import io.cdap.cdap.etl.api.connector.ConnectorSpecRequest; +import io.cdap.cdap.etl.api.connector.PluginSpec; +import io.cdap.plugin.common.Constants; +import io.cdap.plugin.common.ReferenceNames; +import io.cdap.plugin.common.db.DBConnectorPath; +import io.cdap.plugin.common.db.DBPath; +import io.cdap.plugin.db.SchemaReader; +import io.cdap.plugin.db.connector.AbstractDBSpecificConnector; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.mapreduce.lib.db.DBWritable; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +/** + * Amazon Redshift Database Connector that connects to Amazon Redshift database via JDBC. + */ +@Plugin(type = Connector.PLUGIN_TYPE) +@Name(RedshiftConnector.NAME) +@Description("Connection to access data in Amazon Redshift using JDBC.") +@Category("Database") +public class RedshiftConnector extends AbstractDBSpecificConnector { + public static final String NAME = RedshiftConstants.PLUGIN_NAME; + private final RedshiftConnectorConfig config; + + public RedshiftConnector(RedshiftConnectorConfig config) { + super(config); + this.config = config; + } + + @Override + protected DBConnectorPath getDBConnectorPath(String path) throws IOException { + return new DBPath(path, true); + } + + @Override + public boolean supportSchema() { + return true; + } + + @Override + protected Class getDBRecordType() { + return RedshiftDBRecord.class; + } + + @Override + public StructuredRecord transform(LongWritable longWritable, RedshiftDBRecord redshiftDBRecord) { + return redshiftDBRecord.getRecord(); + } + + @Override + protected SchemaReader getSchemaReader(String sessionID) { + return new RedshiftSchemaReader(sessionID); + } + + @Override + protected String getTableName(String database, String schema, String table) { + return String.format("\"%s\".\"%s\"", schema, table); + } + + @Override + protected String getRandomQuery(String tableName, int limit) { + return String.format("SELECT * FROM %s\n" + + "TABLESAMPLE BERNOULLI (100.0 * %d / (SELECT COUNT(*) FROM %s))", + tableName, limit, tableName); + } + + @Override + protected void setConnectorSpec(ConnectorSpecRequest request, DBConnectorPath path, + ConnectorSpec.Builder builder) { + Map sourceProperties = new HashMap<>(); + setConnectionProperties(sourceProperties, request); + builder + .addRelatedPlugin(new PluginSpec(RedshiftConstants.PLUGIN_NAME, + BatchSource.PLUGIN_TYPE, sourceProperties)); + + String schema = path.getSchema(); + sourceProperties.put(RedshiftSource.RedshiftSourceConfig.NUM_SPLITS, "1"); + sourceProperties.put(RedshiftSource.RedshiftSourceConfig.FETCH_SIZE, + RedshiftSource.RedshiftSourceConfig.DEFAULT_FETCH_SIZE); + String table = path.getTable(); + if (table == null) { + return; + } + sourceProperties.put(RedshiftSource.RedshiftSourceConfig.IMPORT_QUERY, + getTableQuery(path.getDatabase(), schema, table)); + sourceProperties.put(Constants.Reference.REFERENCE_NAME, ReferenceNames.cleanseReferenceName(table)); + } + +} diff --git a/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftConnectorConfig.java b/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftConnectorConfig.java new file mode 100644 index 000000000..f05f26d10 --- /dev/null +++ b/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftConnectorConfig.java @@ -0,0 +1,87 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.amazon.redshift; + +import io.cdap.cdap.api.annotation.Description; +import io.cdap.cdap.api.annotation.Macro; +import io.cdap.cdap.api.annotation.Name; +import io.cdap.plugin.db.ConnectionConfig; +import io.cdap.plugin.db.connector.AbstractDBConnectorConfig; + +import javax.annotation.Nullable; + +/** + * Configuration for Redshift connector + */ +public class RedshiftConnectorConfig extends AbstractDBConnectorConfig { + + @Name(ConnectionConfig.HOST) + @Description( + "The endpoint of the Amazon Redshift cluster.") + @Macro + private String host; + + @Name(ConnectionConfig.PORT) + @Description("Database port number") + @Macro + @Nullable + private Integer port; + + @Name(ConnectionConfig.DATABASE) + @Description("Database name to connect to") + @Macro + private String database; + + public RedshiftConnectorConfig(String username, String password, String jdbcPluginName, + String connectionArguments, String host, + String database, @Nullable Integer port) { + this.user = username; + this.password = password; + this.jdbcPluginName = jdbcPluginName; + this.connectionArguments = connectionArguments; + this.host = host; + this.database = database; + this.port = port; + } + + public String getDatabase() { + return database; + } + + public String getHost() { + return host; + } + + public int getPort() { + return port == null ? 5439 : port; + } + + @Override + public String getConnectionString() { + return String.format( + RedshiftConstants.REDSHIFT_CONNECTION_STRING_FORMAT, + host, + getPort(), + database); + } + + @Override + public boolean canConnect() { + return super.canConnect() && !containsMacro(ConnectionConfig.HOST) && + !containsMacro(ConnectionConfig.PORT) && !containsMacro(ConnectionConfig.DATABASE); + } +} diff --git a/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftConstants.java b/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftConstants.java new file mode 100644 index 000000000..081052fb1 --- /dev/null +++ b/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftConstants.java @@ -0,0 +1,27 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.amazon.redshift; + +/** Amazon Redshift constants. */ +public final class RedshiftConstants { + + private RedshiftConstants() { + } + + public static final String PLUGIN_NAME = "Redshift"; + public static final String REDSHIFT_CONNECTION_STRING_FORMAT = "jdbc:redshift://%s:%s/%s"; +} diff --git a/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftDBRecord.java b/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftDBRecord.java new file mode 100644 index 000000000..38e9140d8 --- /dev/null +++ b/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftDBRecord.java @@ -0,0 +1,129 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.amazon.redshift; + +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.db.DBRecord; +import io.cdap.plugin.db.SchemaReader; +import io.cdap.plugin.util.DBUtils; + +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.sql.Types; +import java.time.OffsetDateTime; +import java.time.ZoneId; +import java.time.ZonedDateTime; + +/** + * Writable class for Redshift Source + */ +public class RedshiftDBRecord extends DBRecord { + + /** + * Used in map-reduce. Do not remove. + */ + @SuppressWarnings("unused") + public RedshiftDBRecord() { + } + + @Override + protected void handleField(ResultSet resultSet, StructuredRecord.Builder recordBuilder, Schema.Field field, + int columnIndex, int sqlType, int sqlPrecision, int sqlScale) throws SQLException { + ResultSetMetaData metadata = resultSet.getMetaData(); + String columnTypeName = metadata.getColumnTypeName(columnIndex); + if (isUseSchema(metadata, columnIndex)) { + setFieldAccordingToSchema(resultSet, recordBuilder, field, columnIndex); + return; + } + + // HandleTimestamp + if (sqlType == Types.TIMESTAMP && columnTypeName.equalsIgnoreCase("timestamp")) { + Timestamp timestamp = resultSet.getTimestamp(columnIndex, DBUtils.PURE_GREGORIAN_CALENDAR); + if (timestamp != null) { + ZonedDateTime zonedDateTime = OffsetDateTime.of(timestamp.toLocalDateTime(), OffsetDateTime.now().getOffset()) + .atZoneSameInstant(ZoneId.of("UTC")); + Schema nonNullableSchema = field.getSchema().isNullable() ? + field.getSchema().getNonNullable() : field.getSchema(); + setZonedDateTimeBasedOnOutputSchema(recordBuilder, nonNullableSchema.getLogicalType(), + field.getName(), zonedDateTime); + } else { + recordBuilder.set(field.getName(), null); + } + return; + } + + // HandleTimestampTZ + if (sqlType == Types.TIMESTAMP && columnTypeName.equalsIgnoreCase("timestamptz")) { + OffsetDateTime timestamp = resultSet.getObject(columnIndex, OffsetDateTime.class); + if (timestamp != null) { + recordBuilder.setTimestamp(field.getName(), timestamp.atZoneSameInstant(ZoneId.of("UTC"))); + } else { + recordBuilder.set(field.getName(), null); + } + return; + } + + // HandleNumeric + int columnType = metadata.getColumnType(columnIndex); + if (columnType == Types.NUMERIC) { + Schema nonNullableSchema = field.getSchema().isNullable() ? + field.getSchema().getNonNullable() : field.getSchema(); + int precision = metadata.getPrecision(columnIndex); + if (precision == 0 && Schema.Type.STRING.equals(nonNullableSchema.getType())) { + // When output schema is set to String for precision less numbers + recordBuilder.set(field.getName(), resultSet.getString(columnIndex)); + } else if (Schema.LogicalType.DECIMAL.equals(nonNullableSchema.getLogicalType())) { + BigDecimal originalDecimalValue = resultSet.getBigDecimal(columnIndex); + if (originalDecimalValue != null) { + BigDecimal newDecimalValue = new BigDecimal(originalDecimalValue.toPlainString()) + .setScale(nonNullableSchema.getScale(), RoundingMode.HALF_EVEN); + recordBuilder.setDecimal(field.getName(), newDecimalValue); + } + } + return; + } + setField(resultSet, recordBuilder, field, columnIndex, sqlType, sqlPrecision, sqlScale); + } + + private void setZonedDateTimeBasedOnOutputSchema(StructuredRecord.Builder recordBuilder, + Schema.LogicalType logicalType, + String fieldName, + ZonedDateTime zonedDateTime) { + if (Schema.LogicalType.DATETIME.equals(logicalType)) { + recordBuilder.setDateTime(fieldName, zonedDateTime.toLocalDateTime()); + } else if (Schema.LogicalType.TIMESTAMP_MICROS.equals(logicalType)) { + recordBuilder.setTimestamp(fieldName, zonedDateTime); + } + } + + private static boolean isUseSchema(ResultSetMetaData metadata, int columnIndex) throws SQLException { + String columnTypeName = metadata.getColumnTypeName(columnIndex); + // If the column Type Name is present in the String mapped Redshift types then return true. + return RedshiftSchemaReader.STRING_MAPPED_REDSHIFT_TYPES_NAMES.contains(columnTypeName); + } + + @Override + protected SchemaReader getSchemaReader() { + return new RedshiftSchemaReader(); + } + +} diff --git a/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftSchemaReader.java b/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftSchemaReader.java new file mode 100644 index 000000000..df9938a45 --- /dev/null +++ b/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftSchemaReader.java @@ -0,0 +1,117 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.amazon.redshift; + +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Lists; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.db.CommonSchemaReader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Types; +import java.util.List; +import java.util.Set; + +/** + * Redshift Schema Reader class + */ +public class RedshiftSchemaReader extends CommonSchemaReader { + + private static final Logger LOG = LoggerFactory.getLogger(RedshiftSchemaReader.class); + + public static final Set STRING_MAPPED_REDSHIFT_TYPES_NAMES = ImmutableSet.of( + "timetz", "money" + ); + + private final String sessionID; + + public RedshiftSchemaReader() { + this(null); + } + + public RedshiftSchemaReader(String sessionID) { + super(); + this.sessionID = sessionID; + } + + @Override + public Schema getSchema(ResultSetMetaData metadata, int index) throws SQLException { + String typeName = metadata.getColumnTypeName(index); + int columnType = metadata.getColumnType(index); + + if (STRING_MAPPED_REDSHIFT_TYPES_NAMES.contains(typeName)) { + return Schema.of(Schema.Type.STRING); + } + if (typeName.equalsIgnoreCase("INT")) { + return Schema.of(Schema.Type.INT); + } + if (typeName.equalsIgnoreCase("BIGINT")) { + return Schema.of(Schema.Type.LONG); + } + + // If it is a numeric type without precision then use the Schema of String to avoid any precision loss + if (Types.NUMERIC == columnType) { + int precision = metadata.getPrecision(index); + if (precision == 0) { + LOG.warn(String.format("Field '%s' is a %s type without precision and scale, " + + "converting into STRING type to avoid any precision loss.", + metadata.getColumnName(index), + metadata.getColumnTypeName(index))); + return Schema.of(Schema.Type.STRING); + } + } + + if (typeName.equalsIgnoreCase("timestamp")) { + return Schema.of(Schema.LogicalType.DATETIME); + } + + return super.getSchema(metadata, index); + } + + @Override + public boolean shouldIgnoreColumn(ResultSetMetaData metadata, int index) throws SQLException { + if (sessionID == null) { + return false; + } + return metadata.getColumnName(index).equals("c_" + sessionID) || + metadata.getColumnName(index).equals("sqn_" + sessionID); + } + + @Override + public List getSchemaFields(ResultSet resultSet) throws SQLException { + List schemaFields = Lists.newArrayList(); + ResultSetMetaData metadata = resultSet.getMetaData(); + // ResultSetMetadata columns are numbered starting with 1 + for (int i = 1; i <= metadata.getColumnCount(); i++) { + if (shouldIgnoreColumn(metadata, i)) { + continue; + } + String columnName = metadata.getColumnName(i); + Schema columnSchema = getSchema(metadata, i); + // Setting up schema as nullable as cdata driver doesn't provide proper information about isNullable. + columnSchema = Schema.nullableOf(columnSchema); + Schema.Field field = Schema.Field.of(columnName, columnSchema); + schemaFields.add(field); + } + return schemaFields; + } + +} diff --git a/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftSource.java b/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftSource.java new file mode 100644 index 000000000..1b5894de9 --- /dev/null +++ b/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftSource.java @@ -0,0 +1,128 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.amazon.redshift; + +import io.cdap.cdap.api.annotation.Description; +import io.cdap.cdap.api.annotation.Macro; +import io.cdap.cdap.api.annotation.Metadata; +import io.cdap.cdap.api.annotation.MetadataProperty; +import io.cdap.cdap.api.annotation.Name; +import io.cdap.cdap.api.annotation.Plugin; +import io.cdap.cdap.etl.api.FailureCollector; +import io.cdap.cdap.etl.api.batch.BatchSource; +import io.cdap.cdap.etl.api.batch.BatchSourceContext; +import io.cdap.cdap.etl.api.connector.Connector; +import io.cdap.plugin.common.Asset; +import io.cdap.plugin.common.ConfigUtil; +import io.cdap.plugin.common.LineageRecorder; +import io.cdap.plugin.db.SchemaReader; +import io.cdap.plugin.db.config.AbstractDBSpecificSourceConfig; +import io.cdap.plugin.db.source.AbstractDBSource; +import io.cdap.plugin.util.DBUtils; +import org.apache.hadoop.mapreduce.lib.db.DBWritable; + +import java.util.Collections; +import java.util.Map; +import javax.annotation.Nullable; + +/** + * Batch source to read from an Amazon Redshift database. + */ +@Plugin(type = BatchSource.PLUGIN_TYPE) +@Name(RedshiftConstants.PLUGIN_NAME) +@Description( + "Reads from a Amazon Redshift database table(s) using a configurable SQL query." + + " Outputs one record for each row returned by the query.") +@Metadata(properties = {@MetadataProperty(key = Connector.PLUGIN_TYPE, value = RedshiftConnector.NAME)}) +public class RedshiftSource + extends AbstractDBSource { + + private final RedshiftSourceConfig redshiftSourceConfig; + + public RedshiftSource(RedshiftSourceConfig redshiftSourceConfig) { + super(redshiftSourceConfig); + this.redshiftSourceConfig = redshiftSourceConfig; + } + + @Override + protected SchemaReader getSchemaReader() { + return new RedshiftSchemaReader(); + } + + @Override + protected Class getDBRecordType() { + return RedshiftDBRecord.class; + } + + @Override + protected String createConnectionString() { + return String.format( + RedshiftConstants.REDSHIFT_CONNECTION_STRING_FORMAT, + redshiftSourceConfig.connection.getHost(), + redshiftSourceConfig.connection.getPort(), + redshiftSourceConfig.connection.getDatabase()); + } + + @Override + protected LineageRecorder getLineageRecorder(BatchSourceContext context) { + String fqn = DBUtils.constructFQN("redshift", redshiftSourceConfig.getConnection().getHost(), + redshiftSourceConfig.getConnection().getPort(), + redshiftSourceConfig.getConnection().getDatabase(), + redshiftSourceConfig.getReferenceName()); + Asset.Builder assetBuilder = Asset.builder(redshiftSourceConfig.getReferenceName()).setFqn(fqn); + return new LineageRecorder(context, assetBuilder.build()); + } + + /** + * Redshift source config. + */ + public static class RedshiftSourceConfig extends AbstractDBSpecificSourceConfig { + + @Name(ConfigUtil.NAME_USE_CONNECTION) + @Nullable + @Description("Whether to use an existing connection.") + private Boolean useConnection; + + @Name(ConfigUtil.NAME_CONNECTION) + @Macro + @Nullable + @Description("The existing connection to use.") + private RedshiftConnectorConfig connection; + + @Override + public Map getDBSpecificArguments() { + return Collections.emptyMap(); + } + + @Override + public Integer getFetchSize() { + Integer fetchSize = super.getFetchSize(); + return fetchSize == null ? Integer.parseInt(DEFAULT_FETCH_SIZE) : fetchSize; + } + + @Override + protected RedshiftConnectorConfig getConnection() { + return connection; + } + + @Override + public void validate(FailureCollector collector) { + ConfigUtil.validateConnection(this, useConnection, connection, collector); + super.validate(collector); + } + } +} diff --git a/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftConnectorTest.java b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftConnectorTest.java new file mode 100644 index 000000000..a43eb4302 --- /dev/null +++ b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftConnectorTest.java @@ -0,0 +1,39 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.amazon.redshift; + +import io.cdap.plugin.db.connector.DBSpecificConnectorBaseTest; +import org.junit.Test; + +import java.io.IOException; + +/** + * Unit tests for {@link RedshiftConnector} + */ +public class RedshiftConnectorTest extends DBSpecificConnectorBaseTest { + + private static final String JDBC_DRIVER_CLASS_NAME = "com.amazon.redshift.Driver"; + + @Test + public void test() throws IOException, ClassNotFoundException, InstantiationException, IllegalAccessException { + test(new RedshiftConnector( + new RedshiftConnectorConfig(username, password, JDBC_PLUGIN_NAME, connectionArguments, host, database, + port)), + JDBC_DRIVER_CLASS_NAME, RedshiftConstants.PLUGIN_NAME); + } +} + diff --git a/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftConnectorUnitTest.java b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftConnectorUnitTest.java new file mode 100644 index 000000000..39579cb60 --- /dev/null +++ b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftConnectorUnitTest.java @@ -0,0 +1,55 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.amazon.redshift; + +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +/** + * Unit tests for {@link RedshiftConnector} + */ +public class RedshiftConnectorUnitTest { + @Rule + public ExpectedException expectedEx = ExpectedException.none(); + + private static final RedshiftConnector CONNECTOR = new RedshiftConnector(null); + + /** + * Unit test for getTableName() + */ + @Test + public void getTableNameTest() { + Assert.assertEquals("\"schema\".\"table\"", + CONNECTOR.getTableName("db", "schema", "table")); + } + + /** + * Unit tests for getTableQuery() + */ + @Test + public void getTableQueryTest() { + String tableName = CONNECTOR.getTableName("db", "schema", "table"); + + // random query + Assert.assertEquals(String.format("SELECT * FROM %s\n" + + "TABLESAMPLE BERNOULLI (100.0 * %d / (SELECT COUNT(*) FROM %s))", + tableName, 100, tableName), + CONNECTOR.getRandomQuery(tableName, 100)); + } +} diff --git a/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftDBRecordUnitTest.java b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftDBRecordUnitTest.java new file mode 100644 index 000000000..4d11004e4 --- /dev/null +++ b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftDBRecordUnitTest.java @@ -0,0 +1,155 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.amazon.redshift; + +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.util.DBUtils; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import java.math.BigDecimal; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.sql.Types; +import java.time.OffsetDateTime; +import java.time.ZoneId; +import java.time.ZoneOffset; + +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.when; + +/** + * Unit Test class for the PostgresDBRecord + */ +@RunWith(MockitoJUnitRunner.class) +public class RedshiftDBRecordUnitTest { + + private static final int DEFAULT_PRECISION = 38; + + /** + * Validate the precision less Numbers handling against following use cases. + * 1. Ensure that the numeric type with [p,s] set as [38,4] detect as BigDecimal(38,4) in cdap. + * 2. Ensure that the numeric type without [p,s] detect as String type in cdap. + * + * @throws Exception + */ + @Test + public void validatePrecisionLessDecimalParsing() throws Exception { + Schema.Field field1 = Schema.Field.of("ID1", Schema.decimalOf(DEFAULT_PRECISION, 4)); + Schema.Field field2 = Schema.Field.of("ID2", Schema.of(Schema.Type.STRING)); + + Schema schema = Schema.recordOf( + "dbRecord", + field1, + field2 + ); + + ResultSetMetaData resultSetMetaData = Mockito.mock(ResultSetMetaData.class); + Mockito.when(resultSetMetaData.getColumnType(Mockito.eq(1))).thenReturn(Types.NUMERIC); + Mockito.when(resultSetMetaData.getPrecision(Mockito.eq(1))).thenReturn(DEFAULT_PRECISION); + Mockito.when(resultSetMetaData.getColumnType(eq(2))).thenReturn(Types.NUMERIC); + when(resultSetMetaData.getPrecision(eq(2))).thenReturn(0); + + ResultSet resultSet = Mockito.mock(ResultSet.class); + + when(resultSet.getMetaData()).thenReturn(resultSetMetaData); + when(resultSet.getBigDecimal(eq(1))).thenReturn(BigDecimal.valueOf(123.4568)); + when(resultSet.getString(eq(2))).thenReturn("123.4568"); + + StructuredRecord.Builder builder = StructuredRecord.builder(schema); + RedshiftDBRecord dbRecord = new RedshiftDBRecord(); + dbRecord.handleField(resultSet, builder, field1, 1, Types.NUMERIC, DEFAULT_PRECISION, 4); + dbRecord.handleField(resultSet, builder, field2, 2, Types.NUMERIC, 0, -127); + + StructuredRecord record = builder.build(); + Assert.assertTrue(record.getDecimal("ID1") instanceof BigDecimal); + Assert.assertEquals(record.getDecimal("ID1"), BigDecimal.valueOf(123.4568)); + Assert.assertTrue(record.get("ID2") instanceof String); + Assert.assertEquals(record.get("ID2"), "123.4568"); + } + + @Test + public void validateTimestampType() throws SQLException { + OffsetDateTime offsetDateTime = OffsetDateTime.of(2023, 1, 1, 1, 0, 0, 0, ZoneOffset.UTC); + ResultSetMetaData metaData = Mockito.mock(ResultSetMetaData.class); + when(metaData.getColumnTypeName(eq(0))).thenReturn("timestamp"); + + ResultSet resultSet = Mockito.mock(ResultSet.class); + when(resultSet.getMetaData()).thenReturn(metaData); + when(resultSet.getTimestamp(eq(0), eq(DBUtils.PURE_GREGORIAN_CALENDAR))) + .thenReturn(Timestamp.from(offsetDateTime.toInstant())); + + Schema.Field field1 = Schema.Field.of("field1", Schema.of(Schema.LogicalType.DATETIME)); + Schema schema = Schema.recordOf( + "dbRecord", + field1 + ); + StructuredRecord.Builder builder = StructuredRecord.builder(schema); + + RedshiftDBRecord dbRecord = new RedshiftDBRecord(); + dbRecord.handleField(resultSet, builder, field1, 0, Types.TIMESTAMP, 0, 0); + StructuredRecord record = builder.build(); + Assert.assertNotNull(record); + Assert.assertNotNull(record.getDateTime("field1")); + Assert.assertEquals(record.getDateTime("field1").toInstant(ZoneOffset.UTC), offsetDateTime.toInstant()); + + // Validate backward compatibility + + field1 = Schema.Field.of("field1", Schema.of(Schema.LogicalType.TIMESTAMP_MICROS)); + schema = Schema.recordOf( + "dbRecord", + field1 + ); + builder = StructuredRecord.builder(schema); + dbRecord.handleField(resultSet, builder, field1, 0, Types.TIMESTAMP, 0, 0); + record = builder.build(); + Assert.assertNotNull(record); + Assert.assertNotNull(record.getTimestamp("field1")); + Assert.assertEquals(record.getTimestamp("field1").toInstant(), offsetDateTime.toInstant()); + } + + @Test + public void validateTimestampTZType() throws SQLException { + OffsetDateTime offsetDateTime = OffsetDateTime.of(2023, 1, 1, 1, 0, 0, 0, ZoneOffset.UTC); + ResultSetMetaData metaData = Mockito.mock(ResultSetMetaData.class); + when(metaData.getColumnTypeName(eq(0))).thenReturn("timestamptz"); + + ResultSet resultSet = Mockito.mock(ResultSet.class); + when(resultSet.getMetaData()).thenReturn(metaData); + when(resultSet.getObject(eq(0), eq(OffsetDateTime.class))).thenReturn(offsetDateTime); + + Schema.Field field1 = Schema.Field.of("field1", Schema.of(Schema.LogicalType.TIMESTAMP_MICROS)); + Schema schema = Schema.recordOf( + "dbRecord", + field1 + ); + StructuredRecord.Builder builder = StructuredRecord.builder(schema); + + RedshiftDBRecord dbRecord = new RedshiftDBRecord(); + dbRecord.handleField(resultSet, builder, field1, 0, Types.TIMESTAMP, 0, 0); + StructuredRecord record = builder.build(); + Assert.assertNotNull(record); + Assert.assertNotNull(record.getTimestamp("field1", ZoneId.of("UTC"))); + Assert.assertEquals(record.getTimestamp("field1", ZoneId.of("UTC")).toInstant(), offsetDateTime.toInstant()); + } +} diff --git a/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftFailedConnectionTest.java b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftFailedConnectionTest.java new file mode 100644 index 000000000..2d21c4478 --- /dev/null +++ b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftFailedConnectionTest.java @@ -0,0 +1,38 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.amazon.redshift; + +import io.cdap.plugin.db.connector.DBSpecificFailedConnectionTest; +import org.junit.Test; + +import java.io.IOException; + +public class RedshiftFailedConnectionTest extends DBSpecificFailedConnectionTest { + private static final String JDBC_DRIVER_CLASS_NAME = "com.amazon.redshift.Driver"; + + @Test + public void test() throws ClassNotFoundException, IOException { + + RedshiftConnector connector = new RedshiftConnector( + new RedshiftConnectorConfig("username", "password", "jdbc", "", "localhost", "db", 5432)); + + super.test(JDBC_DRIVER_CLASS_NAME, connector, "Failed to create connection to database via connection string: " + + "jdbc:redshift://localhost:5432/db and arguments: " + + "{user=username}. Error: ConnectException: Connection refused " + + "(Connection refused)."); + } +} diff --git a/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftPluginTestBase.java b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftPluginTestBase.java new file mode 100644 index 000000000..5df4fb300 --- /dev/null +++ b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftPluginTestBase.java @@ -0,0 +1,218 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.amazon.redshift; + +import com.google.common.base.Charsets; +import com.google.common.base.Throwables; +import com.google.common.collect.Sets; +import io.cdap.cdap.api.artifact.ArtifactSummary; +import io.cdap.cdap.api.plugin.PluginClass; +import io.cdap.cdap.datapipeline.DataPipelineApp; +import io.cdap.cdap.proto.id.ArtifactId; +import io.cdap.cdap.proto.id.NamespaceId; +import io.cdap.plugin.db.ConnectionConfig; +import io.cdap.plugin.db.DBRecord; +import io.cdap.plugin.db.batch.DatabasePluginTestBase; +import io.cdap.plugin.db.sink.ETLDBOutputFormat; +import io.cdap.plugin.db.source.DataDrivenETLDBInputFormat; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.math.BigDecimal; +import java.sql.Connection; +import java.sql.Date; +import java.sql.Driver; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.Time; +import java.sql.Timestamp; +import java.util.Arrays; +import java.util.Calendar; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.TimeZone; + +/** + * Base test class for Redshift plugins. + */ +public abstract class RedshiftPluginTestBase extends DatabasePluginTestBase { + private static final Logger LOGGER = LoggerFactory.getLogger(RedshiftPluginTestBase.class); + protected static final ArtifactId DATAPIPELINE_ARTIFACT_ID = NamespaceId.DEFAULT.artifact("data-pipeline", "3.2.0"); + protected static final ArtifactSummary DATAPIPELINE_ARTIFACT = new ArtifactSummary("data-pipeline", "3.2.0"); + protected static final long CURRENT_TS = System.currentTimeMillis(); + + protected static final String JDBC_DRIVER_NAME = "redshift"; + protected static final Map BASE_PROPS = new HashMap<>(); + + protected static String connectionUrl; + protected static int year; + protected static final int PRECISION = 10; + protected static final int SCALE = 6; + private static int startCount; + + @BeforeClass + public static void setupTest() throws Exception { + if (startCount++ > 0) { + return; + } + + getProperties(); + + Calendar calendar = Calendar.getInstance(); + calendar.setTime(new Date(CURRENT_TS)); + year = calendar.get(Calendar.YEAR); + + setupBatchArtifacts(DATAPIPELINE_ARTIFACT_ID, DataPipelineApp.class); + + addPluginArtifact(NamespaceId.DEFAULT.artifact(JDBC_DRIVER_NAME, "1.0.0"), + DATAPIPELINE_ARTIFACT_ID, + RedshiftSource.class, DBRecord.class, + ETLDBOutputFormat.class, DataDrivenETLDBInputFormat.class, DBRecord.class); + + // add mysql 3rd party plugin + PluginClass mysqlDriver = new PluginClass(ConnectionConfig.JDBC_PLUGIN_TYPE, JDBC_DRIVER_NAME, + "redshift driver class", Driver.class.getName(), + null, Collections.emptyMap()); + addPluginArtifact(NamespaceId.DEFAULT.artifact("redshift-jdbc-connector", "1.0.0"), + DATAPIPELINE_ARTIFACT_ID, + Sets.newHashSet(mysqlDriver), Driver.class); + + TimeZone.setDefault(TimeZone.getTimeZone("UTC")); + + connectionUrl = "jdbc:redshift://" + BASE_PROPS.get(ConnectionConfig.HOST) + ":" + + BASE_PROPS.get(ConnectionConfig.PORT) + "/" + BASE_PROPS.get(ConnectionConfig.DATABASE); + Connection conn = createConnection(); + createTestTables(conn); + prepareTestData(conn); + } + + private static void getProperties() { + BASE_PROPS.put(ConnectionConfig.HOST, getPropertyOrSkip("redshift.clusterEndpoint")); + BASE_PROPS.put(ConnectionConfig.PORT, getPropertyOrSkip("redshift.port")); + BASE_PROPS.put(ConnectionConfig.DATABASE, getPropertyOrSkip("redshift.database")); + BASE_PROPS.put(ConnectionConfig.USER, getPropertyOrSkip("redshift.username")); + BASE_PROPS.put(ConnectionConfig.PASSWORD, getPropertyOrSkip("redshift.password")); + BASE_PROPS.put(ConnectionConfig.JDBC_PLUGIN_NAME, JDBC_DRIVER_NAME); + } + + protected static void createTestTables(Connection conn) throws SQLException { + try (Statement stmt = conn.createStatement()) { + // create a table that the action will truncate at the end of the run + stmt.execute("CREATE TABLE \"dbActionTest\" (x int, day varchar(10))"); + // create a table that the action will truncate at the end of the run + stmt.execute("CREATE TABLE \"postActionTest\" (x int, day varchar(10))"); + + stmt.execute("CREATE TABLE my_table" + + "(" + + "\"ID\" INT NOT NULL," + + "\"NAME\" VARCHAR(40) NOT NULL," + + "\"SCORE\" REAL," + + "\"GRADUATED\" BOOLEAN," + + "\"NOT_IMPORTED\" VARCHAR(30)," + + "\"SMALLINT_COL\" SMALLINT," + + "\"BIG\" BIGINT," + + "\"NUMERIC_COL\" NUMERIC(" + PRECISION + "," + SCALE + ")," + + "\"DECIMAL_COL\" DECIMAL(" + PRECISION + "," + SCALE + ")," + + "\"DOUBLE_PREC_COL\" DOUBLE PRECISION," + + "\"DATE_COL\" DATE," + + "\"TIME_COL\" TIME," + + "\"TIMESTAMP_COL\" TIMESTAMP(3)," + + "\"TEXT_COL\" TEXT," + + "\"CHAR_COL\" CHAR(100)," + + "\"BYTEA_COL\" BYTEA" + + ")"); + stmt.execute("CREATE TABLE \"MY_DEST_TABLE\" AS " + + "SELECT * FROM my_table"); + stmt.execute("CREATE TABLE your_table AS " + + "SELECT * FROM my_table"); + } + } + + protected static void prepareTestData(Connection conn) throws SQLException { + try ( + Statement stmt = conn.createStatement(); + PreparedStatement pStmt1 = + conn.prepareStatement("INSERT INTO my_table " + + "VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?," + + " ?, ?, ?, ?, ?, ?)"); + PreparedStatement pStmt2 = + conn.prepareStatement("INSERT INTO your_table " + + "VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?," + + " ?, ?, ?, ?, ?, ?)")) { + + stmt.execute("insert into \"dbActionTest\" values (1, '1970-01-01')"); + stmt.execute("insert into \"postActionTest\" values (1, '1970-01-01')"); + + populateData(pStmt1, pStmt2); + } + } + + private static void populateData(PreparedStatement... stmts) throws SQLException { + // insert the same data into both tables: my_table and your_table + for (PreparedStatement pStmt : stmts) { + for (int i = 1; i <= 5; i++) { + String name = "user" + i; + pStmt.setInt(1, i); + pStmt.setString(2, name); + pStmt.setDouble(3, 123.45 + i); + pStmt.setBoolean(4, (i % 2 == 0)); + pStmt.setString(5, "random" + i); + pStmt.setShort(6, (short) i); + pStmt.setLong(7, (long) i); + pStmt.setBigDecimal(8, new BigDecimal("123.45").add(new BigDecimal(i))); + pStmt.setBigDecimal(9, new BigDecimal("123.45").add(new BigDecimal(i))); + pStmt.setDouble(10, 123.45 + i); + pStmt.setDate(11, new Date(CURRENT_TS)); + pStmt.setTime(12, new Time(CURRENT_TS)); + pStmt.setTimestamp(13, new Timestamp(CURRENT_TS)); + pStmt.setString(14, name); + pStmt.setString(15, "char" + i); + pStmt.setBytes(16, name.getBytes(Charsets.UTF_8)); + pStmt.executeUpdate(); + } + } + } + + public static Connection createConnection() { + try { + Class.forName(Driver.class.getCanonicalName()); + return DriverManager.getConnection(connectionUrl, BASE_PROPS.get(ConnectionConfig.USER), + BASE_PROPS.get(ConnectionConfig.PASSWORD)); + } catch (Exception e) { + throw Throwables.propagate(e); + } + } + + @AfterClass + public static void tearDownDB() { + try (Connection conn = createConnection(); + Statement stmt = conn.createStatement()) { + executeCleanup(Arrays.asList(() -> stmt.execute("DROP TABLE my_table"), + () -> stmt.execute("DROP TABLE your_table"), + () -> stmt.execute("DROP TABLE postActionTest"), + () -> stmt.execute("DROP TABLE dbActionTest"), + () -> stmt.execute("DROP TABLE MY_DEST_TABLE")), LOGGER); + } catch (Exception e) { + LOGGER.warn("Fail to tear down.", e); + } + } +} diff --git a/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftPluginTestSuite.java b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftPluginTestSuite.java new file mode 100644 index 000000000..95ad0938b --- /dev/null +++ b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftPluginTestSuite.java @@ -0,0 +1,31 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.amazon.redshift; + +import io.cdap.cdap.common.test.TestSuite; +import org.junit.runner.RunWith; +import org.junit.runners.Suite; + +/** + * This is a test suite that runs all the tests for Redshift plugins. + */ +@RunWith(TestSuite.class) +@Suite.SuiteClasses({ + RedshiftSourceTestRun.class, +}) +public class RedshiftPluginTestSuite extends RedshiftPluginTestBase { +} diff --git a/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftSourceTestRun.java b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftSourceTestRun.java new file mode 100644 index 000000000..1ac41bcd0 --- /dev/null +++ b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftSourceTestRun.java @@ -0,0 +1,332 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.amazon.redshift; + +import com.google.common.collect.ImmutableMap; +import io.cdap.cdap.api.common.Bytes; +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.dataset.table.Table; +import io.cdap.cdap.etl.api.batch.BatchSource; +import io.cdap.cdap.etl.mock.batch.MockSink; +import io.cdap.cdap.etl.proto.v2.ETLBatchConfig; +import io.cdap.cdap.etl.proto.v2.ETLPlugin; +import io.cdap.cdap.etl.proto.v2.ETLStage; +import io.cdap.cdap.proto.artifact.AppRequest; +import io.cdap.cdap.proto.id.ApplicationId; +import io.cdap.cdap.proto.id.NamespaceId; +import io.cdap.cdap.test.ApplicationManager; +import io.cdap.cdap.test.DataSetManager; +import io.cdap.plugin.common.Constants; +import io.cdap.plugin.db.ConnectionConfig; +import io.cdap.plugin.db.DBConfig; +import io.cdap.plugin.db.source.AbstractDBSource; +import org.junit.Assert; +import org.junit.Test; + +import java.math.BigDecimal; +import java.math.MathContext; +import java.nio.ByteBuffer; +import java.sql.Date; +import java.sql.Time; +import java.text.SimpleDateFormat; +import java.time.LocalDate; +import java.time.LocalTime; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Test for Redshift source plugin. + */ +public class RedshiftSourceTestRun extends RedshiftPluginTestBase { + + @Test + @SuppressWarnings("ConstantConditions") + public void testDBMacroSupport() throws Exception { + String importQuery = "SELECT * FROM my_table WHERE \"DATE_COL\" <= '${logicalStartTime(yyyy-MM-dd,1d)}' " + + "AND $CONDITIONS"; + String boundingQuery = "SELECT MIN(ID),MAX(ID) from my_table"; + String splitBy = "ID"; + + ImmutableMap sourceProps = ImmutableMap.builder() + .putAll(BASE_PROPS) + .put(AbstractDBSource.DBSourceConfig.IMPORT_QUERY, importQuery) + .put(AbstractDBSource.DBSourceConfig.BOUNDING_QUERY, boundingQuery) + .put(AbstractDBSource.DBSourceConfig.SPLIT_BY, splitBy) + .put(Constants.Reference.REFERENCE_NAME, "DBTestSource").build(); + + ETLPlugin sourceConfig = new ETLPlugin( + RedshiftConstants.PLUGIN_NAME, + BatchSource.PLUGIN_TYPE, + sourceProps + ); + + ETLPlugin sinkConfig = MockSink.getPlugin("macroOutputTable"); + + ApplicationManager appManager = deployETL(sourceConfig, sinkConfig, + DATAPIPELINE_ARTIFACT, "testDBMacro"); + runETLOnce(appManager, ImmutableMap.of("logical.start.time", String.valueOf(CURRENT_TS))); + + DataSetManager outputManager = getDataset("macroOutputTable"); + Assert.assertTrue(MockSink.readOutput(outputManager).isEmpty()); + } + + @Test + @SuppressWarnings("ConstantConditions") + public void testDBSource() throws Exception { + String importQuery = "SELECT \"ID\", \"NAME\", \"SCORE\", \"GRADUATED\", \"SMALLINT_COL\", \"BIG\", " + + "\"NUMERIC_COL\", \"CHAR_COL\", \"DECIMAL_COL\", \"BYTEA_COL\", \"DATE_COL\", \"TIME_COL\", \"TIMESTAMP_COL\", " + + "\"TEXT_COL\", \"DOUBLE_PREC_COL\" FROM my_table " + + "WHERE \"ID\" < 3 AND $CONDITIONS"; + String boundingQuery = "SELECT MIN(\"ID\"),MAX(\"ID\") from my_table"; + String splitBy = "ID"; + ETLPlugin sourceConfig = new ETLPlugin( + RedshiftConstants.PLUGIN_NAME, + BatchSource.PLUGIN_TYPE, + ImmutableMap.builder() + .putAll(BASE_PROPS) + .put(AbstractDBSource.DBSourceConfig.IMPORT_QUERY, importQuery) + .put(AbstractDBSource.DBSourceConfig.BOUNDING_QUERY, boundingQuery) + .put(AbstractDBSource.DBSourceConfig.SPLIT_BY, splitBy) + .put(Constants.Reference.REFERENCE_NAME, "DBSourceTest") + .build(), + null + ); + + String outputDatasetName = "output-dbsourcetest"; + ETLPlugin sinkConfig = MockSink.getPlugin(outputDatasetName); + + ApplicationManager appManager = deployETL(sourceConfig, sinkConfig, + DATAPIPELINE_ARTIFACT, "testDBSource"); + runETLOnce(appManager); + + DataSetManager
outputManager = getDataset(outputDatasetName); + List outputRecords = MockSink.readOutput(outputManager); + + Assert.assertEquals(2, outputRecords.size()); + String userid = outputRecords.get(0).get("NAME"); + StructuredRecord row1 = "user1".equals(userid) ? outputRecords.get(0) : outputRecords.get(1); + StructuredRecord row2 = "user1".equals(userid) ? outputRecords.get(1) : outputRecords.get(0); + + // Verify data + Assert.assertEquals("user1", row1.get("NAME")); + Assert.assertEquals("user2", row2.get("NAME")); + Assert.assertEquals("user1", row1.get("TEXT_COL")); + Assert.assertEquals("user2", row2.get("TEXT_COL")); + Assert.assertEquals("char1", ((String) row1.get("CHAR_COL")).trim()); + Assert.assertEquals("char2", ((String) row2.get("CHAR_COL")).trim()); + Assert.assertEquals(124.45f, ((Float) row1.get("SCORE")).doubleValue(), 0.000001); + Assert.assertEquals(125.45f, ((Float) row2.get("SCORE")).doubleValue(), 0.000001); + Assert.assertEquals(false, row1.get("GRADUATED")); + Assert.assertEquals(true, row2.get("GRADUATED")); + Assert.assertNull(row1.get("NOT_IMPORTED")); + Assert.assertNull(row2.get("NOT_IMPORTED")); + + Assert.assertEquals(1, (int) row1.get("SMALLINT_COL")); + Assert.assertEquals(2, (int) row2.get("SMALLINT_COL")); + Assert.assertEquals(1, (long) row1.get("BIG")); + Assert.assertEquals(2, (long) row2.get("BIG")); + + Assert.assertEquals(new BigDecimal("124.45", new MathContext(PRECISION)).setScale(SCALE), + row1.getDecimal("NUMERIC_COL")); + Assert.assertEquals(new BigDecimal("125.45", new MathContext(PRECISION)).setScale(SCALE), + row2.getDecimal("NUMERIC_COL")); + Assert.assertEquals(new BigDecimal("124.45", new MathContext(PRECISION)).setScale(SCALE), + row1.getDecimal("DECIMAL_COL")); + + Assert.assertEquals(124.45, (double) row1.get("DOUBLE_PREC_COL"), 0.000001); + Assert.assertEquals(125.45, (double) row2.get("DOUBLE_PREC_COL"), 0.000001); + // Verify time columns + java.util.Date date = new java.util.Date(CURRENT_TS); + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); + LocalDate expectedDate = Date.valueOf(sdf.format(date)).toLocalDate(); + sdf = new SimpleDateFormat("H:mm:ss"); + LocalTime expectedTime = Time.valueOf(sdf.format(date)).toLocalTime(); + ZonedDateTime expectedTs = date.toInstant().atZone(ZoneId.ofOffset("UTC", ZoneOffset.UTC)); + Assert.assertEquals(expectedDate, row1.getDate("DATE_COL")); + Assert.assertEquals(expectedTime, row1.getTime("TIME_COL")); + Assert.assertEquals(expectedTs, row1.getTimestamp("TIMESTAMP_COL", ZoneId.ofOffset("UTC", ZoneOffset.UTC))); + + // verify binary columns + Assert.assertEquals("user1", Bytes.toString(((ByteBuffer) row1.get("BYTEA_COL")).array(), 0, 5)); + Assert.assertEquals("user2", Bytes.toString(((ByteBuffer) row2.get("BYTEA_COL")).array(), 0, 5)); + } + + @Test + public void testDbSourceMultipleTables() throws Exception { + String importQuery = "SELECT \"my_table\".\"ID\", \"your_table\".\"NAME\" FROM \"my_table\", \"your_table\"" + + "WHERE \"my_table\".\"ID\" < 3 and \"my_table\".\"ID\" = \"your_table\".\"ID\" and $CONDITIONS"; + String boundingQuery = "SELECT MIN(MIN(\"my_table\".\"ID\"), MIN(\"your_table\".\"ID\")), " + + "MAX(MAX(\"my_table\".\"ID\"), MAX(\"your_table\".\"ID\"))"; + String splitBy = "\"my_table\".\"ID\""; + ETLPlugin sourceConfig = new ETLPlugin( + RedshiftConstants.PLUGIN_NAME, + BatchSource.PLUGIN_TYPE, + ImmutableMap.builder() + .putAll(BASE_PROPS) + .put(AbstractDBSource.DBSourceConfig.IMPORT_QUERY, importQuery) + .put(AbstractDBSource.DBSourceConfig.BOUNDING_QUERY, boundingQuery) + .put(AbstractDBSource.DBSourceConfig.SPLIT_BY, splitBy) + .put(Constants.Reference.REFERENCE_NAME, "DBMultipleTest") + .build(), + null + ); + + String outputDatasetName = "output-multitabletest"; + ETLPlugin sinkConfig = MockSink.getPlugin(outputDatasetName); + + ApplicationManager appManager = deployETL(sourceConfig, sinkConfig, + DATAPIPELINE_ARTIFACT, "testDBSourceWithMultipleTables"); + runETLOnce(appManager); + + // records should be written + DataSetManager
outputManager = getDataset(outputDatasetName); + List outputRecords = MockSink.readOutput(outputManager); + Assert.assertEquals(2, outputRecords.size()); + String userid = outputRecords.get(0).get("NAME"); + StructuredRecord row1 = "user1".equals(userid) ? outputRecords.get(0) : outputRecords.get(1); + StructuredRecord row2 = "user1".equals(userid) ? outputRecords.get(1) : outputRecords.get(0); + // Verify data + Assert.assertEquals("user1", row1.get("NAME")); + Assert.assertEquals("user2", row2.get("NAME")); + Assert.assertEquals(1, row1.get("ID").intValue()); + Assert.assertEquals(2, row2.get("ID").intValue()); + } + + @Test + public void testUserNamePasswordCombinations() throws Exception { + String importQuery = "SELECT * FROM \"my_table\" WHERE $CONDITIONS"; + String boundingQuery = "SELECT MIN(\"ID\"),MAX(\"ID\") from \"my_table\""; + String splitBy = "\"ID\""; + + ETLPlugin sinkConfig = MockSink.getPlugin("outputTable"); + + Map baseSourceProps = ImmutableMap.builder() + .put(ConnectionConfig.HOST, BASE_PROPS.get(ConnectionConfig.HOST)) + .put(ConnectionConfig.PORT, BASE_PROPS.get(ConnectionConfig.PORT)) + .put(ConnectionConfig.DATABASE, BASE_PROPS.get(ConnectionConfig.DATABASE)) + .put(ConnectionConfig.JDBC_PLUGIN_NAME, JDBC_DRIVER_NAME) + .put(AbstractDBSource.DBSourceConfig.IMPORT_QUERY, importQuery) + .put(AbstractDBSource.DBSourceConfig.BOUNDING_QUERY, boundingQuery) + .put(AbstractDBSource.DBSourceConfig.SPLIT_BY, splitBy) + .put(Constants.Reference.REFERENCE_NAME, "UserPassDBTest") + .build(); + + ApplicationId appId = NamespaceId.DEFAULT.app("dbTest"); + + // null user name, null password. Should succeed. + // as source + ETLPlugin dbConfig = new ETLPlugin(RedshiftConstants.PLUGIN_NAME, BatchSource.PLUGIN_TYPE, + baseSourceProps, null); + ETLStage table = new ETLStage("uniqueTableSink", sinkConfig); + ETLStage database = new ETLStage("databaseSource", dbConfig); + ETLBatchConfig etlConfig = ETLBatchConfig.builder() + .addStage(database) + .addStage(table) + .addConnection(database.getName(), table.getName()) + .build(); + AppRequest appRequest = new AppRequest<>(DATAPIPELINE_ARTIFACT, etlConfig); + deployApplication(appId, appRequest); + + // null user name, non-null password. Should fail. + // as source + Map noUser = new HashMap<>(baseSourceProps); + noUser.put(DBConfig.PASSWORD, "password"); + database = new ETLStage("databaseSource", new ETLPlugin(RedshiftConstants.PLUGIN_NAME, + BatchSource.PLUGIN_TYPE, noUser, null)); + etlConfig = ETLBatchConfig.builder() + .addStage(database) + .addStage(table) + .addConnection(database.getName(), table.getName()) + .build(); + assertDeploymentFailure(appId, etlConfig, DATAPIPELINE_ARTIFACT, + "Deploying DB Source with null username but non-null password should have failed."); + + // non-null username, non-null, but empty password. Should succeed. + // as source + Map emptyPassword = new HashMap<>(baseSourceProps); + emptyPassword.put(DBConfig.USER, "root"); + emptyPassword.put(DBConfig.PASSWORD, ""); + database = new ETLStage("databaseSource", new ETLPlugin(RedshiftConstants.PLUGIN_NAME, + BatchSource.PLUGIN_TYPE, emptyPassword, null)); + etlConfig = ETLBatchConfig.builder() + .addStage(database) + .addStage(table) + .addConnection(database.getName(), table.getName()) + .build(); + appRequest = new AppRequest<>(DATAPIPELINE_ARTIFACT, etlConfig); + deployApplication(appId, appRequest); + } + + @Test + public void testNonExistentDBTable() throws Exception { + // source + String importQuery = "SELECT \"ID\", \"NAME\" FROM \"dummy\" WHERE ID < 3 AND $CONDITIONS"; + String boundingQuery = "SELECT MIN(\"ID\"),MAX(\"ID\") FROM \"dummy\""; + String splitBy = "\"ID\""; + ETLPlugin sinkConfig = MockSink.getPlugin("table"); + ETLPlugin sourceBadNameConfig = new ETLPlugin( + RedshiftConstants.PLUGIN_NAME, + BatchSource.PLUGIN_TYPE, + ImmutableMap.builder() + .putAll(BASE_PROPS) + .put(AbstractDBSource.DBSourceConfig.IMPORT_QUERY, importQuery) + .put(AbstractDBSource.DBSourceConfig.BOUNDING_QUERY, boundingQuery) + .put(AbstractDBSource.DBSourceConfig.SPLIT_BY, splitBy) + .put(Constants.Reference.REFERENCE_NAME, "DBNonExistentTest") + .build(), + null); + ETLStage sink = new ETLStage("sink", sinkConfig); + ETLStage sourceBadName = new ETLStage("sourceBadName", sourceBadNameConfig); + + ETLBatchConfig etlConfig = ETLBatchConfig.builder() + .addStage(sourceBadName) + .addStage(sink) + .addConnection(sourceBadName.getName(), sink.getName()) + .build(); + ApplicationId appId = NamespaceId.DEFAULT.app("dbSourceNonExistingTest"); + assertDeployAppFailure(appId, etlConfig, DATAPIPELINE_ARTIFACT); + + // Bad connection + ETLPlugin sourceBadConnConfig = new ETLPlugin( + RedshiftConstants.PLUGIN_NAME, + BatchSource.PLUGIN_TYPE, + ImmutableMap.builder() + .put(ConnectionConfig.HOST, BASE_PROPS.get(ConnectionConfig.HOST)) + .put(ConnectionConfig.PORT, BASE_PROPS.get(ConnectionConfig.PORT)) + .put(ConnectionConfig.DATABASE, "dumDB") + .put(ConnectionConfig.USER, BASE_PROPS.get(ConnectionConfig.USER)) + .put(ConnectionConfig.PASSWORD, BASE_PROPS.get(ConnectionConfig.PASSWORD)) + .put(ConnectionConfig.JDBC_PLUGIN_NAME, JDBC_DRIVER_NAME) + .put(AbstractDBSource.DBSourceConfig.IMPORT_QUERY, importQuery) + .put(AbstractDBSource.DBSourceConfig.BOUNDING_QUERY, boundingQuery) + .put(AbstractDBSource.DBSourceConfig.SPLIT_BY, splitBy) + .put(Constants.Reference.REFERENCE_NAME, "RedshiftTest") + .build(), + null); + ETLStage sourceBadConn = new ETLStage("sourceBadConn", sourceBadConnConfig); + etlConfig = ETLBatchConfig.builder() + .addStage(sourceBadConn) + .addStage(sink) + .addConnection(sourceBadConn.getName(), sink.getName()) + .build(); + assertDeployAppFailure(appId, etlConfig, DATAPIPELINE_ARTIFACT); + } +} diff --git a/amazon-redshift-plugin/widgets/Redshift-batchsource.json b/amazon-redshift-plugin/widgets/Redshift-batchsource.json new file mode 100644 index 000000000..91e860ee9 --- /dev/null +++ b/amazon-redshift-plugin/widgets/Redshift-batchsource.json @@ -0,0 +1,240 @@ +{ + "metadata": { + "spec-version": "1.5" + }, + "display-name": "Redshift", + "configuration-groups": [ + { + "label": "Connection", + "properties": [ + { + "widget-type": "toggle", + "label": "Use connection", + "name": "useConnection", + "widget-attributes": { + "on": { + "value": "true", + "label": "YES" + }, + "off": { + "value": "false", + "label": "NO" + }, + "default": "false" + } + }, + { + "widget-type": "connection-select", + "label": "Connection", + "name": "connection", + "widget-attributes": { + "connectionType": "Redshift" + } + }, + { + "widget-type": "plugin-list", + "label": "JDBC Driver name", + "name": "jdbcPluginName", + "widget-attributes": { + "plugin-type": "jdbc" + } + }, + { + "widget-type": "textbox", + "label": "Host", + "name": "host", + "widget-attributes": { + "placeholder": "Redshift endpoint host name." + } + }, + { + "widget-type": "number", + "label": "Port", + "name": "port", + "widget-attributes": { + "default": "5439" + } + }, + { + "widget-type": "textbox", + "label": "Username", + "name": "user" + }, + { + "widget-type": "password", + "label": "Password", + "name": "password" + }, + { + "widget-type": "keyvalue", + "label": "Connection Arguments", + "name": "connectionArguments", + "widget-attributes": { + "showDelimiter": "false", + "key-placeholder": "Key", + "value-placeholder": "Value", + "kv-delimiter" : "=", + "delimiter" : ";" + } + } + ] + }, + { + "label": "Basic", + "properties": [ + { + "widget-type": "textbox", + "label": "Reference Name", + "name": "referenceName", + "widget-attributes": { + "placeholder": "Name used to identify this source for lineage. Typically, the name of the table/view." + } + }, + { + "widget-type": "textbox", + "label": "Database", + "name": "database" + }, + { + "widget-type": "connection-browser", + "widget-category": "plugin", + "widget-attributes": { + "connectionType": "Redshift", + "label": "Browse Database" + } + } + ] + }, + { + "label": "SQL Query", + "properties": [ + { + "widget-type": "textarea", + "label": "Import Query", + "name": "importQuery", + "widget-attributes": { + "rows": "4" + } + }, + { + "widget-type": "get-schema", + "widget-category": "plugin" + } + ] + }, + { + "label": "Advanced", + "properties": [ + { + "widget-type": "textarea", + "label": "Bounding Query", + "name": "boundingQuery", + "widget-attributes": { + "rows": "4" + } + }, + { + "widget-type": "textbox", + "label": "Split Column", + "name": "splitBy" + }, + { + "widget-type": "textbox", + "label": "Number of Splits", + "name": "numSplits", + "widget-attributes": { + "default": "1" + } + }, + { + "widget-type": "number", + "label": "Fetch Size", + "name": "fetchSize", + "widget-attributes": { + "default": "1000", + "minimum": "0" + } + } + ] + } + ], + "outputs": [ + { + "name": "schema", + "widget-type": "schema", + "widget-attributes": { + "schema-types": [ + "boolean", + "int", + "long", + "float", + "double", + "bytes", + "string" + ], + "schema-default-type": "string" + } + } + ], + "filters": [ + { + "name": "showConnectionProperties ", + "condition": { + "expression": "useConnection == false" + }, + "show": [ + { + "type": "property", + "name": "jdbcPluginName" + }, + { + "type": "property", + "name": "instanceType" + }, + { + "type": "property", + "name": "host" + }, + { + "type": "property", + "name": "port" + }, + { + "type": "property", + "name": "user" + }, + { + "type": "property", + "name": "password" + }, + { + "type": "property", + "name": "database" + }, + { + "type": "property", + "name": "connectionArguments" + } + ] + }, + { + "name": "showConnectionId", + "condition": { + "expression": "useConnection == true" + }, + "show": [ + { + "type": "property", + "name": "connection" + } + ] + }, + ], + "jump-config": { + "datasets": [ + { + "ref-property-name": "referenceName" + } + ] + } +} diff --git a/amazon-redshift-plugin/widgets/Redshift-connector.json b/amazon-redshift-plugin/widgets/Redshift-connector.json new file mode 100644 index 000000000..3a2af8e01 --- /dev/null +++ b/amazon-redshift-plugin/widgets/Redshift-connector.json @@ -0,0 +1,75 @@ +{ + "metadata": { + "spec-version": "1.0" + }, + "display-name": "Redshift", + "configuration-groups": [ + { + "label": "Basic", + "properties": [ + { + "widget-type": "plugin-list", + "label": "JDBC Driver name", + "name": "jdbcPluginName", + "widget-attributes": { + "plugin-type": "jdbc" + } + }, + { + "widget-type": "textbox", + "label": "Host", + "name": "host", + "widget-attributes": { + "default": "localhost" + } + }, + { + "widget-type": "number", + "label": "Port", + "name": "port", + "widget-attributes": { + "default": "5439" + } + }, + { + "widget-type": "textbox", + "label": "Database", + "name": "database" + } + ] + }, + { + "label": "Credentials", + "properties": [ + { + "widget-type": "textbox", + "label": "Username", + "name": "user" + }, + { + "widget-type": "password", + "label": "Password", + "name": "password" + } + ] + }, + { + "label": "Advanced", + "properties": [ + { + "widget-type": "keyvalue", + "label": "Connection Arguments", + "name": "connectionArguments", + "widget-attributes": { + "showDelimiter": "false", + "key-placeholder": "Key", + "value-placeholder": "Value", + "kv-delimiter": "=", + "delimiter": ";" + } + } + ] + } + ], + "outputs": [] +} diff --git a/pom.xml b/pom.xml index 08928d5c2..ced39e5e1 100644 --- a/pom.xml +++ b/pom.xml @@ -44,6 +44,7 @@ cloudsql-postgresql-plugin teradata-plugin generic-db-argument-setter + amazon-redshift-plugin From a164c7b3313dfe7b23e688f57decbd780a0fd46e Mon Sep 17 00:00:00 2001 From: Shubhangi-cs Date: Fri, 23 Feb 2024 10:21:55 +0530 Subject: [PATCH 05/52] Version Bumpup 1.11.1 --- amazon-redshift-plugin/pom.xml | 2 +- aurora-mysql-plugin/pom.xml | 2 +- aurora-postgresql-plugin/pom.xml | 2 +- cloudsql-mysql-plugin/pom.xml | 4 ++-- cloudsql-postgresql-plugin/pom.xml | 2 +- database-commons/pom.xml | 2 +- db2-plugin/pom.xml | 2 +- generic-database-plugin/pom.xml | 2 +- generic-db-argument-setter/pom.xml | 2 +- mariadb-plugin/pom.xml | 2 +- memsql-plugin/pom.xml | 2 +- mssql-plugin/pom.xml | 2 +- mysql-plugin/pom.xml | 2 +- netezza-plugin/pom.xml | 2 +- oracle-plugin/pom.xml | 2 +- pom.xml | 2 +- postgresql-plugin/pom.xml | 2 +- saphana-plugin/pom.xml | 2 +- teradata-plugin/pom.xml | 2 +- 19 files changed, 20 insertions(+), 20 deletions(-) diff --git a/amazon-redshift-plugin/pom.xml b/amazon-redshift-plugin/pom.xml index 25e837e68..c22cc1249 100644 --- a/amazon-redshift-plugin/pom.xml +++ b/amazon-redshift-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.1-SNAPSHOT + 1.11.1 Amazon Redshift plugin diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index 4399d5276..ed233e6fd 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.1-SNAPSHOT + 1.11.1 Aurora DB MySQL plugin diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index 81494de40..1f3ff23b2 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.1-SNAPSHOT + 1.11.1 Aurora DB PostgreSQL plugin diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index 10c662ed2..f5c4a63b1 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.1-SNAPSHOT + 1.11.1 CloudSQL MySQL plugin @@ -45,7 +45,7 @@ io.cdap.plugin mysql-plugin - 1.11.1-SNAPSHOT + ${project.version} diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index 614b55a6e..dc55b5992 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.1-SNAPSHOT + 1.11.1 CloudSQL PostgreSQL plugin diff --git a/database-commons/pom.xml b/database-commons/pom.xml index 262e0b695..e48fc2789 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.1-SNAPSHOT + 1.11.1 Database Commons diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index 77f64ce0c..21602c531 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.1-SNAPSHOT + 1.11.1 IBM DB2 plugin diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index 218a1e71b..f460057fe 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.1-SNAPSHOT + 1.11.1 Generic database plugin diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index 1fd9d97ef..5131d1988 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.1-SNAPSHOT + 1.11.1 Generic database argument setter plugin diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index 5ba0cad4e..872bb8ae6 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.1-SNAPSHOT + 1.11.1 Maria DB plugin diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index 88e491000..a8e6392ee 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.1-SNAPSHOT + 1.11.1 Memsql plugin diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index 26575edf8..ccf4b4fb5 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.1-SNAPSHOT + 1.11.1 Microsoft SQL Server plugin diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index efb2cce40..6da8aa630 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.1-SNAPSHOT + 1.11.1 Mysql plugin diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index 74100cc01..27fc28887 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.1-SNAPSHOT + 1.11.1 Netezza plugin diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index e701b6a78..f61a48942 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.1-SNAPSHOT + 1.11.1 Oracle plugin diff --git a/pom.xml b/pom.xml index ced39e5e1..9cbe24119 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin database-plugins-parent - 1.11.1-SNAPSHOT + 1.11.1 pom Database Plugins Collection of database plugins diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index c4bec7485..3c413b4ba 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.1-SNAPSHOT + 1.11.1 PostgreSQL plugin diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index 10902c00d..7e1558d94 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.1-SNAPSHOT + 1.11.1 SAP HANA plugin diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index 38741b6a3..a38775943 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -21,7 +21,7 @@ database-plugins-parent io.cdap.plugin - 1.11.1-SNAPSHOT + 1.11.1 teradata-plugin From d2a74a351af5e1c0c2e11ed2db774649edaa53db Mon Sep 17 00:00:00 2001 From: Rahul Sharma <112750762+MrRahulSharma@users.noreply.github.com> Date: Mon, 29 Apr 2024 13:00:27 +0000 Subject: [PATCH 06/52] Fix/cherry picks (#494) * ConnectionString calculation refactoring (#476) * Fixed Typo in connection function call. (#477) --- .../io/cdap/plugin/oracle/OracleAction.java | 9 +------ .../cdap/plugin/oracle/OracleConnector.java | 11 ++------ .../plugin/oracle/OracleConnectorConfig.java | 8 +----- .../cdap/plugin/oracle/OracleConstants.java | 26 +++++++++++++++++++ .../cdap/plugin/oracle/OraclePostAction.java | 8 +----- .../io/cdap/plugin/oracle/OracleSource.java | 11 ++------ 6 files changed, 33 insertions(+), 40 deletions(-) diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleAction.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleAction.java index 81604858a..9b5331d11 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleAction.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleAction.java @@ -57,16 +57,9 @@ public static class OracleActionConfig extends DBSpecificQueryConfig { @Override public String getConnectionString() { - if (OracleConstants.TNS_CONNECTION_TYPE.equals(this.connectionType)) { - return String.format(OracleConstants.ORACLE_CONNECTION_STRING_TNS_FORMAT, database); - } else if (OracleConstants.SERVICE_CONNECTION_TYPE.equals(this.connectionType)) { - return String.format(OracleConstants.ORACLE_CONNECTION_STRING_SERVICE_NAME_FORMAT, host, port, database); - } else { - return String.format(OracleConstants.ORACLE_CONNECTION_STRING_SID_FORMAT, host, port, database); - } + return OracleConstants.getConnectionString(this.connectionType, host, port, database); } - @Override protected Map getDBSpecificArguments() { return ImmutableMap.of(OracleConstants.DEFAULT_BATCH_VALUE, String.valueOf(defaultBatchValue)); diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java index fde72c8ad..bc7907b26 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java @@ -125,15 +125,8 @@ protected String getConnectionString(@Nullable String database) { if (database == null) { return config.getConnectionString(); } - if (OracleConstants.TNS_CONNECTION_TYPE.equals(config.getConnectionType())) { - return String.format(OracleConstants.ORACLE_CONNECTION_STRING_TNS_FORMAT, database); - } else if (OracleConstants.SERVICE_CONNECTION_TYPE.equals(config.getConnectionType())) { - return String.format(OracleConstants.ORACLE_CONNECTION_STRING_SERVICE_NAME_FORMAT, config.getHost(), - config.getPort(), database); - } else { - return String.format(OracleConstants.ORACLE_CONNECTION_STRING_SID_FORMAT, - config.getHost(), config.getPort(), database); - } + return OracleConstants.getConnectionString(config.getConnectionType(), + config.getHost(), config.getPort(), database); } @Override diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java index 73b005243..a60476bd5 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java @@ -63,13 +63,7 @@ public OracleConnectorConfig(String host, int port, String user, String password @Override public String getConnectionString() { - if (OracleConstants.TNS_CONNECTION_TYPE.equals(getConnectionType())) { - return String.format(OracleConstants.ORACLE_CONNECTION_STRING_TNS_FORMAT, database); - } else if (OracleConstants.SERVICE_CONNECTION_TYPE.equals(getConnectionType())) { - return String.format(OracleConstants.ORACLE_CONNECTION_STRING_SERVICE_NAME_FORMAT, host, getPort(), database); - } else { - return String.format(OracleConstants.ORACLE_CONNECTION_STRING_SID_FORMAT, host, getPort(), database); - } + return OracleConstants.getConnectionString(connectionType, host, getPort(), database); } @Name(OracleConstants.CONNECTION_TYPE) diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java index 040780a89..ec44d7b94 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java @@ -16,6 +16,8 @@ package io.cdap.plugin.oracle; +import javax.annotation.Nullable; + /** * Oracle Constants. */ @@ -36,4 +38,28 @@ private OracleConstants() { public static final String NAME_DATABASE = "database"; public static final String TNS_CONNECTION_TYPE = "TNS"; public static final String TRANSACTION_ISOLATION_LEVEL = "transactionIsolationLevel"; + + /** + * Returns the Connection String for the given ConnectionType. + * + * @param connectionType TNS/Service/SID + * @param host Host name of the oracle server + * @param port Port of the oracle server + * @param database Database to connect to + * @return Connection String based on the given ConnectionType + */ + public static String getConnectionString(String connectionType, + @Nullable String host, + @Nullable int port, + String database) { + if (OracleConstants.TNS_CONNECTION_TYPE.equalsIgnoreCase(connectionType)) { + return String.format(OracleConstants.ORACLE_CONNECTION_STRING_TNS_FORMAT, database); + } + if (OracleConstants.SERVICE_CONNECTION_TYPE.equalsIgnoreCase(connectionType)) { + return String.format(OracleConstants.ORACLE_CONNECTION_STRING_SERVICE_NAME_FORMAT, + host, port, database); + } + return String.format(OracleConstants.ORACLE_CONNECTION_STRING_SID_FORMAT, + host, port, database); + } } diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OraclePostAction.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OraclePostAction.java index f8ebd9ac2..4862aebfa 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OraclePostAction.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OraclePostAction.java @@ -57,13 +57,7 @@ public static class OracleQueryActionConfig extends DBSpecificQueryActionConfig @Override public String getConnectionString() { - if (OracleConstants.TNS_CONNECTION_TYPE.equals(this.connectionType)) { - return String.format(OracleConstants.ORACLE_CONNECTION_STRING_TNS_FORMAT, database); - } else if (OracleConstants.SERVICE_CONNECTION_TYPE.equals(this.connectionType)) { - return String.format(OracleConstants.ORACLE_CONNECTION_STRING_SERVICE_NAME_FORMAT, host, port, database); - } else { - return String.format(OracleConstants.ORACLE_CONNECTION_STRING_SID_FORMAT, host, port, database); - } + return OracleConstants.getConnectionString(this.connectionType, host, port, database); } @Override diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java index 9a554a4a4..eca7e2532 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java @@ -131,15 +131,8 @@ public OracleSourceConfig(String host, int port, String user, String password, S @Override public String getConnectionString() { - if (OracleConstants.TNS_CONNECTION_TYPE.equals(connection.getConnectionType())) { - return String.format(OracleConstants.ORACLE_CONNECTION_STRING_TNS_FORMAT, connection.getDatabase()); - } else if (OracleConstants.SERVICE_CONNECTION_TYPE.equals(connection.getConnectionType())) { - return String.format(OracleConstants.ORACLE_CONNECTION_STRING_SERVICE_NAME_FORMAT, connection.getHost(), - connection.getPort(), connection.getDatabase()); - } else { - return String.format(OracleConstants.ORACLE_CONNECTION_STRING_SID_FORMAT, - connection.getHost(), connection.getPort(), connection.getDatabase()); - } + return OracleConstants.getConnectionString(connection.getConnectionType(), connection.getHost(), + connection.getPort(), connection.getDatabase()); } @Override From b06236d09096ab20fdfcaba4ced5b55387a14db8 Mon Sep 17 00:00:00 2001 From: Rahul Sharma <112750762+MrRahulSharma@users.noreply.github.com> Date: Mon, 29 Apr 2024 14:35:08 +0000 Subject: [PATCH 07/52] Upgrading to next release version (#495) --- amazon-redshift-plugin/pom.xml | 2 +- aurora-mysql-plugin/pom.xml | 2 +- aurora-postgresql-plugin/pom.xml | 2 +- cloudsql-mysql-plugin/pom.xml | 2 +- cloudsql-postgresql-plugin/pom.xml | 2 +- database-commons/pom.xml | 2 +- db2-plugin/pom.xml | 2 +- generic-database-plugin/pom.xml | 2 +- generic-db-argument-setter/pom.xml | 2 +- mariadb-plugin/pom.xml | 2 +- memsql-plugin/pom.xml | 2 +- mssql-plugin/pom.xml | 2 +- mysql-plugin/pom.xml | 2 +- netezza-plugin/pom.xml | 2 +- oracle-plugin/pom.xml | 2 +- pom.xml | 2 +- postgresql-plugin/pom.xml | 2 +- saphana-plugin/pom.xml | 2 +- teradata-plugin/pom.xml | 2 +- 19 files changed, 19 insertions(+), 19 deletions(-) diff --git a/amazon-redshift-plugin/pom.xml b/amazon-redshift-plugin/pom.xml index c22cc1249..1fd272ab4 100644 --- a/amazon-redshift-plugin/pom.xml +++ b/amazon-redshift-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.1 + 1.11.2 Amazon Redshift plugin diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index ed233e6fd..64eb9e58a 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.1 + 1.11.2 Aurora DB MySQL plugin diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index 1f3ff23b2..a6d009e10 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.1 + 1.11.2 Aurora DB PostgreSQL plugin diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index f5c4a63b1..cbec989cf 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.1 + 1.11.2 CloudSQL MySQL plugin diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index dc55b5992..f798f09bd 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.1 + 1.11.2 CloudSQL PostgreSQL plugin diff --git a/database-commons/pom.xml b/database-commons/pom.xml index e48fc2789..baa21ea44 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.1 + 1.11.2 Database Commons diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index 21602c531..d6586f23d 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.1 + 1.11.2 IBM DB2 plugin diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index f460057fe..5e0d423dc 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.1 + 1.11.2 Generic database plugin diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index 5131d1988..2f6e5d39c 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.1 + 1.11.2 Generic database argument setter plugin diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index 872bb8ae6..7b6ffb250 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.1 + 1.11.2 Maria DB plugin diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index a8e6392ee..59db3a3ca 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.1 + 1.11.2 Memsql plugin diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index ccf4b4fb5..fb52a379d 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.1 + 1.11.2 Microsoft SQL Server plugin diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index 6da8aa630..4e3a91bc5 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.1 + 1.11.2 Mysql plugin diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index 27fc28887..77d8060e9 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.1 + 1.11.2 Netezza plugin diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index f61a48942..13bf68a3d 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.1 + 1.11.2 Oracle plugin diff --git a/pom.xml b/pom.xml index 9cbe24119..d2cc6b05e 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin database-plugins-parent - 1.11.1 + 1.11.2 pom Database Plugins Collection of database plugins diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index 3c413b4ba..9169d5754 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.1 + 1.11.2 PostgreSQL plugin diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index 7e1558d94..015302914 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.1 + 1.11.2 SAP HANA plugin diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index a38775943..843472286 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -21,7 +21,7 @@ database-plugins-parent io.cdap.plugin - 1.11.1 + 1.11.2 teradata-plugin From 61ce420f93a505048ced756e496ad02f4d6baad9 Mon Sep 17 00:00:00 2001 From: vikasrathee-cs Date: Thu, 20 Jun 2024 16:58:20 +0530 Subject: [PATCH 08/52] bump up to 1.11.3-SNAPSHOT bump up to 1.11.3-SNAPSHOT and e2e workflow added for release --- .github/workflows/e2e.yml | 4 ++-- amazon-redshift-plugin/pom.xml | 2 +- aurora-mysql-plugin/pom.xml | 2 +- aurora-postgresql-plugin/pom.xml | 2 +- cloudsql-mysql-plugin/pom.xml | 2 +- cloudsql-postgresql-plugin/pom.xml | 2 +- database-commons/pom.xml | 2 +- db2-plugin/pom.xml | 2 +- generic-database-plugin/pom.xml | 2 +- generic-db-argument-setter/pom.xml | 2 +- mariadb-plugin/pom.xml | 2 +- memsql-plugin/pom.xml | 2 +- mssql-plugin/pom.xml | 2 +- mysql-plugin/pom.xml | 2 +- netezza-plugin/pom.xml | 2 +- oracle-plugin/pom.xml | 2 +- pom.xml | 2 +- postgresql-plugin/pom.xml | 2 +- saphana-plugin/pom.xml | 2 +- teradata-plugin/pom.xml | 2 +- 20 files changed, 21 insertions(+), 21 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 1ced579cb..b142ed3b1 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -16,9 +16,9 @@ name: Build e2e tests on: push: - branches: [ develop ] + branches: [ develop, release/** ] pull_request: - branches: [ develop ] + branches: [ develop, release/** ] types: [ opened, synchronize, reopened, labeled ] workflow_dispatch: diff --git a/amazon-redshift-plugin/pom.xml b/amazon-redshift-plugin/pom.xml index 1fd272ab4..fb8e91604 100644 --- a/amazon-redshift-plugin/pom.xml +++ b/amazon-redshift-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.2 + 1.11.3-SNAPSHOT Amazon Redshift plugin diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index 64eb9e58a..0113d9fb4 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.2 + 1.11.3-SNAPSHOT Aurora DB MySQL plugin diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index a6d009e10..54d9e041d 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.2 + 1.11.3-SNAPSHOT Aurora DB PostgreSQL plugin diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index cbec989cf..66f0d4df1 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.2 + 1.11.3-SNAPSHOT CloudSQL MySQL plugin diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index f798f09bd..46bf075e9 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.2 + 1.11.3-SNAPSHOT CloudSQL PostgreSQL plugin diff --git a/database-commons/pom.xml b/database-commons/pom.xml index baa21ea44..45fc6cbde 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.2 + 1.11.3-SNAPSHOT Database Commons diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index d6586f23d..cdc3c709f 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.2 + 1.11.3-SNAPSHOT IBM DB2 plugin diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index 5e0d423dc..a74c2a250 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.2 + 1.11.3-SNAPSHOT Generic database plugin diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index 2f6e5d39c..372a0261d 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.2 + 1.11.3-SNAPSHOT Generic database argument setter plugin diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index 7b6ffb250..58273afdc 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.2 + 1.11.3-SNAPSHOT Maria DB plugin diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index 59db3a3ca..8dbd76a03 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.2 + 1.11.3-SNAPSHOT Memsql plugin diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index fb52a379d..39c0323df 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.2 + 1.11.3-SNAPSHOT Microsoft SQL Server plugin diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index 4e3a91bc5..9fc01b436 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.2 + 1.11.3-SNAPSHOT Mysql plugin diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index 77d8060e9..a0ea7c088 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.2 + 1.11.3-SNAPSHOT Netezza plugin diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index 13bf68a3d..7e391541f 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.2 + 1.11.3-SNAPSHOT Oracle plugin diff --git a/pom.xml b/pom.xml index d2cc6b05e..2be1cec2b 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin database-plugins-parent - 1.11.2 + 1.11.3-SNAPSHOT pom Database Plugins Collection of database plugins diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index 9169d5754..dc11a831c 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.2 + 1.11.3-SNAPSHOT PostgreSQL plugin diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index 015302914..d6ab5da70 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.2 + 1.11.3-SNAPSHOT SAP HANA plugin diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index 843472286..a4d4afc88 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -21,7 +21,7 @@ database-plugins-parent io.cdap.plugin - 1.11.2 + 1.11.3-SNAPSHOT teradata-plugin From 1772dc905d4d767d322efa67a2cde9ee48fa904c Mon Sep 17 00:00:00 2001 From: vikasrathee-cs Date: Fri, 14 Jun 2024 17:25:48 +0530 Subject: [PATCH 09/52] oracle issue commit --- .../io/cdap/plugin/oracle/OracleSinkDBRecord.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSinkDBRecord.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSinkDBRecord.java index 7bbd25f22..01b9a8247 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSinkDBRecord.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSinkDBRecord.java @@ -17,9 +17,12 @@ package io.cdap.plugin.oracle; import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; import io.cdap.plugin.db.ColumnType; import io.cdap.plugin.db.SchemaReader; +import java.sql.PreparedStatement; +import java.sql.SQLException; import java.util.List; /** @@ -37,4 +40,14 @@ public OracleSinkDBRecord(StructuredRecord record, List columnTypes) protected SchemaReader getSchemaReader() { return new OracleSinkSchemaReader(); } + + @Override + protected void insertOperation(PreparedStatement stmt) throws SQLException { + for (int fieldIndex = 0; fieldIndex < columnTypes.size(); fieldIndex++) { + ColumnType columnType = columnTypes.get(fieldIndex); + // Get the field from the schema using the column name with ignoring case. + Schema.Field field = record.getSchema().getField(columnType.getName(), true); + writeToDB(stmt, field, fieldIndex); + } + } } From be384f0b30aa9a6d8b4f76d883a283772cf1de71 Mon Sep 17 00:00:00 2001 From: vikasrathee-cs Date: Tue, 18 Jun 2024 16:57:28 +0530 Subject: [PATCH 10/52] Add e2e test case for small case schema. Add e2e test case for small case schema. --- .../features/sink/OracleRunTime.feature | 50 +++++++++++++++++++ .../java/io.cdap.plugin/BQValidation.java | 13 +++-- .../common.stepsdesign/TestSetupHooks.java | 14 ++++++ .../oracle/stepsdesign/Oracle.java | 21 +++++++- .../resources/pluginParameters.properties | 4 ++ .../BigQueryCreateTableQuerySmallCase.txt | 1 + .../BigQueryInsertDataQuerySmallCase.txt | 6 +++ 7 files changed, 103 insertions(+), 6 deletions(-) create mode 100644 oracle-plugin/src/e2e-test/resources/testdata/BigQuery/BigQueryCreateTableQuerySmallCase.txt create mode 100644 oracle-plugin/src/e2e-test/resources/testdata/BigQuery/BigQueryInsertDataQuerySmallCase.txt diff --git a/oracle-plugin/src/e2e-test/features/sink/OracleRunTime.feature b/oracle-plugin/src/e2e-test/features/sink/OracleRunTime.feature index c2b56e8b7..67293700b 100644 --- a/oracle-plugin/src/e2e-test/features/sink/OracleRunTime.feature +++ b/oracle-plugin/src/e2e-test/features/sink/OracleRunTime.feature @@ -117,3 +117,53 @@ Feature: Oracle - Verify data transfer from BigQuery source to Oracle sink Then Verify the pipeline status is "Succeeded" Then Validate records transferred to target table with record counts of BigQuery table Then Validate the values of records transferred to target Oracle table is equal to the values from source BigQuery table + + @BQ_SOURCE_TEST_SMALL_CASE @ORACLE_TEST_TABLE + Scenario: To verify data is getting transferred from BigQuery source to Oracle sink successfully when schema is coming in small case + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "BigQuery" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "Oracle" from the plugins list as: "Sink" + Then Connect plugins: "BigQuery" and "Oracle" to establish connection + Then Navigate to the properties page of plugin: "BigQuery" + Then Replace input plugin property: "project" with value: "projectId" + Then Enter input plugin property: "datasetProject" with value: "projectId" + Then Enter input plugin property: "referenceName" with value: "BQReferenceName" + Then Enter input plugin property: "dataset" with value: "dataset" + Then Enter input plugin property: "table" with value: "bqSourceTable" + Then Click on the Get Schema button + Then Verify the Output Schema matches the Expected Schema: "bqOutputDatatypesSchemaSmallCase" + Then Validate "BigQuery" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "Oracle" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "dbSchemaName" with value: "schema" + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Validate "Oracle" plugin properties + Then Close the Plugin Properties page + Then Save the pipeline + Then Preview and run the pipeline + Then Verify the preview of pipeline is "success" + Then Click on preview data for Oracle sink + Then Close the preview data + Then Deploy the pipeline + Then Run the Pipeline in Runtime + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Validate records transferred to target table with record counts of BigQuery table + Then Validate the values of records transferred to target Oracle table is equal to the values from source BigQuery table with case diff --git a/oracle-plugin/src/e2e-test/java/io.cdap.plugin/BQValidation.java b/oracle-plugin/src/e2e-test/java/io.cdap.plugin/BQValidation.java index 6edfcc8fd..b7d93c80a 100644 --- a/oracle-plugin/src/e2e-test/java/io.cdap.plugin/BQValidation.java +++ b/oracle-plugin/src/e2e-test/java/io.cdap.plugin/BQValidation.java @@ -68,11 +68,12 @@ public static boolean validateDBToBQRecordValues(String schema, String sourceTab ResultSet.HOLD_CURSORS_OVER_COMMIT); ResultSet rsSource = statement1.executeQuery(getSourceQuery); - return compareResultSetAndJsonData(rsSource, jsonResponse); + return compareResultSetAndJsonData(rsSource, jsonResponse, false); } } - public static boolean validateBQToDBRecordValues(String schema, String sourceTable, String targetTable) + public static boolean validateBQToDBRecordValues(String schema, String sourceTable, String targetTable, + boolean isSchemaSmallCase) throws SQLException, ClassNotFoundException, ParseException, IOException, InterruptedException { List jsonResponse = new ArrayList<>(); List bigQueryRows = new ArrayList<>(); @@ -88,7 +89,7 @@ public static boolean validateBQToDBRecordValues(String schema, String sourceTab ResultSet.HOLD_CURSORS_OVER_COMMIT); ResultSet rsTarget = statement1.executeQuery(getTargetQuery); - return compareResultSetAndJsonData(rsTarget, jsonResponse); + return compareResultSetAndJsonData(rsTarget, jsonResponse, isSchemaSmallCase); } } @@ -119,7 +120,8 @@ private static void getBigQueryTableData(String table, List bigQueryRows * @throws ParseException If an error occurs while parsing the data. */ - public static boolean compareResultSetAndJsonData(ResultSet rsSource, List bigQueryData) + public static boolean compareResultSetAndJsonData(ResultSet rsSource, List bigQueryData, + boolean isSchemaSmallCase) throws SQLException, ParseException { ResultSetMetaData mdSource = rsSource.getMetaData(); boolean result = false; @@ -146,7 +148,8 @@ public static boolean compareResultSetAndJsonData(ResultSet rsSource, List Date: Mon, 2 Sep 2024 19:41:05 +0530 Subject: [PATCH 11/52] Run build with unit tests without elevated permissions --- .github/workflows/build-report.yml | 51 ++++++++++++++++++++++++++++++ .github/workflows/build.yml | 41 +++++++++++------------- .github/workflows/trigger.yml | 46 --------------------------- 3 files changed, 70 insertions(+), 68 deletions(-) create mode 100644 .github/workflows/build-report.yml delete mode 100644 .github/workflows/trigger.yml diff --git a/.github/workflows/build-report.yml b/.github/workflows/build-report.yml new file mode 100644 index 000000000..691236ceb --- /dev/null +++ b/.github/workflows/build-report.yml @@ -0,0 +1,51 @@ +# Copyright © 2024 Cask Data, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +# This workflow will build a Java project with Maven +# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven +# Note: Any changes to this workflow would be used only after merging into develop +name: Build Unit Tests Report + +on: + workflow_run: + workflows: + - Build with unit tests + types: + - completed + +jobs: + build: + runs-on: ubuntu-latest + + if: ${{ github.event.workflow_run.conclusion != 'skipped' }} + + steps: + # Pinned 1.0.0 version + - uses: marocchino/action-workflow_run-status@54b6e87d6cb552fc5f36dbe9a722a6048725917a + + - name: Download artifact + uses: actions/download-artifact@v4 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + run-id: ${{ github.event.workflow_run.id }} + path: artifacts/ + + - name: Surefire Report + # Pinned 3.5.2 version + uses: mikepenz/action-junit-report@16a9560bd02f11e7e3bf6b3e2ef6bba6c9d07c32 + if: always() + with: + report_paths: '**/target/surefire-reports/TEST-*.xml' + github_token: ${{ secrets.GITHUB_TOKEN }} + detailed_summary: true + commit: ${{ github.event.workflow_run.head_sha }} + check_name: Build Test Report + diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6ce0eb526..1fb6ac78a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,22 +15,28 @@ name: Build with unit tests on: - workflow_run: - workflows: - - Trigger build - types: - - completed + push: + branches: [ develop, release/** ] + pull_request: + branches: [ develop, release/** ] + types: [opened, synchronize, reopened, labeled] jobs: build: runs-on: k8s-runner-build - if: ${{ github.event.workflow_run.conclusion != 'skipped' }} - + # We allow builds: + # 1) When it's a merge into a branch + # 2) For PRs that are labeled as build and + # - It's a code change + # - A build label was just added + # A bit complex, but prevents builds when other labels are manipulated + if: > + github.event_name == 'push' + || (contains(github.event.pull_request.labels.*.name, 'build') + && (github.event.action != 'labeled' || github.event.label.name == 'build') + ) steps: - # Pinned 1.0.0 version - - uses: haya14busa/action-workflow_run-status@967ed83efa565c257675ed70cfe5231f062ddd94 - - uses: actions/checkout@v3 with: ref: ${{ github.event.workflow_run.head_sha }} @@ -47,21 +53,12 @@ jobs: run: mvn clean test -fae -T 2 -B -V -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 - name: Archive build artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() with: - name: Build debug files + name: reports-${{ github.run_id }} path: | **/target/rat.txt **/target/surefire-reports/* - - name: Surefire Report - # Pinned 3.5.2 version - uses: mikepenz/action-junit-report@16a9560bd02f11e7e3bf6b3e2ef6bba6c9d07c32 - if: always() - with: - report_paths: '**/target/surefire-reports/TEST-*.xml' - github_token: ${{ secrets.GITHUB_TOKEN }} - detailed_summary: true - commit: ${{ github.event.workflow_run.head_sha }} - check_name: Test Report \ No newline at end of file + diff --git a/.github/workflows/trigger.yml b/.github/workflows/trigger.yml deleted file mode 100644 index 11db8ac25..000000000 --- a/.github/workflows/trigger.yml +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright © 2022 Cask Data, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. - -# This workflow will trigger build.yml only when needed. -# This way we don't flood main workflow run list -# Note that build.yml from develop will be used even for PR builds -# Also it will have access to the proper GITHUB_SECRET - -name: Trigger build - -on: - push: - branches: [ develop, release/** ] - pull_request: - branches: [ develop, release/** ] - types: [opened, synchronize, reopened, labeled] - workflow_dispatch: - -jobs: - trigger: - runs-on: ubuntu-latest - - # We allow builds: - # 1) When triggered manually - # 2) When it's a merge into a branch - # 3) For PRs that are labeled as build and - # - It's a code change - # - A build label was just added - # A bit complex, but prevents builds when other labels are manipulated - if: > - github.event_name == 'workflow_dispatch' - || github.event_name == 'push' - || (contains(github.event.pull_request.labels.*.name, 'build') - && (github.event.action != 'labeled' || github.event.label.name == 'build') - ) - steps: - - name: Trigger build - run: echo Maven build will be triggered now \ No newline at end of file From 415268544321b8182edfd3ab7bba28d4c107da98 Mon Sep 17 00:00:00 2001 From: vipinbhatt Date: Mon, 7 Oct 2024 11:10:38 +0530 Subject: [PATCH 12/52] Enable TLS Support (DTS - Specific) (cherry picked from commit e9dd995453cc614607156f869afa5333de9214ed) --- .../io/cdap/plugin/oracle/OracleAction.java | 2 +- .../cdap/plugin/oracle/OracleConnector.java | 2 +- .../plugin/oracle/OracleConnectorConfig.java | 17 +++- .../cdap/plugin/oracle/OracleConstants.java | 86 +++++++++++++++++-- .../cdap/plugin/oracle/OraclePostAction.java | 2 +- .../io/cdap/plugin/oracle/OracleSource.java | 6 +- .../oracle/OracleFailedConnectionTest.java | 3 +- oracle-plugin/widgets/Oracle-batchsink.json | 20 +++++ oracle-plugin/widgets/Oracle-batchsource.json | 20 +++++ oracle-plugin/widgets/Oracle-connector.json | 20 +++++ 10 files changed, 159 insertions(+), 19 deletions(-) diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleAction.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleAction.java index 9b5331d11..d698e17da 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleAction.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleAction.java @@ -57,7 +57,7 @@ public static class OracleActionConfig extends DBSpecificQueryConfig { @Override public String getConnectionString() { - return OracleConstants.getConnectionString(this.connectionType, host, port, database); + return OracleConstants.getConnectionString(this.connectionType, host, port, database, null); } @Override diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java index bc7907b26..3d2f7399a 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java @@ -126,7 +126,7 @@ protected String getConnectionString(@Nullable String database) { return config.getConnectionString(); } return OracleConstants.getConnectionString(config.getConnectionType(), - config.getHost(), config.getPort(), database); + config.getHost(), config.getPort(), database, config.getSSlMode()); } @Override diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java index a60476bd5..10022364a 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java @@ -43,12 +43,12 @@ public OracleConnectorConfig(String host, int port, String user, String password public OracleConnectorConfig(String host, int port, String user, String password, String jdbcPluginName, String connectionArguments, String connectionType, String database) { - this(host, port, user, password, jdbcPluginName, connectionArguments, connectionType, database, null); + this(host, port, user, password, jdbcPluginName, connectionArguments, connectionType, database, null, null); } public OracleConnectorConfig(String host, int port, String user, String password, String jdbcPluginName, String connectionArguments, String connectionType, String database, - String role) { + String role, Boolean useSSL) { this.host = host; this.port = port; @@ -59,11 +59,12 @@ public OracleConnectorConfig(String host, int port, String user, String password this.connectionType = connectionType; this.database = database; this.role = role; + this.useSSL = useSSL; } @Override public String getConnectionString() { - return OracleConstants.getConnectionString(connectionType, host, getPort(), database); + return OracleConstants.getConnectionString(connectionType, host, getPort(), database, useSSL); } @Name(OracleConstants.CONNECTION_TYPE) @@ -86,6 +87,11 @@ public String getConnectionString() { @Nullable private String transactionIsolationLevel; + @Name(OracleConstants.USE_SSL) + @Description("Turns on SSL encryption. Connection will fail if SSL is not available") + @Nullable + public Boolean useSSL; + @Override protected int getDefaultPort() { return 1521; @@ -103,6 +109,11 @@ public String getDatabase() { return database; } + public Boolean getSSlMode() { + // return false if useSSL is null, otherwise return its value + return useSSL != null && useSSL; + } + @Override public Properties getConnectionArgumentsProperties() { Properties prop = super.getConnectionArgumentsProperties(); diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java index ec44d7b94..dc38f80ac 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java @@ -29,6 +29,10 @@ private OracleConstants() { public static final String PLUGIN_NAME = "Oracle"; public static final String ORACLE_CONNECTION_STRING_SID_FORMAT = "jdbc:oracle:thin:@%s:%s:%s"; public static final String ORACLE_CONNECTION_STRING_SERVICE_NAME_FORMAT = "jdbc:oracle:thin:@//%s:%s/%s"; + // Connection formats to accept protocol (e.g., jdbc:oracle:thin:@://:/) + public static final String ORACLE_CONNECTION_STRING_SID_FORMAT_WITH_PROTOCOL = "jdbc:oracle:thin:@%s:%s:%s/%s"; + public static final String ORACLE_CONNECTION_STRING_SERVICE_NAME_FORMAT_WITH_PROTOCOL = + "jdbc:oracle:thin:@%s://%s:%s/%s"; public static final String ORACLE_CONNECTION_STRING_TNS_FORMAT = "jdbc:oracle:thin:@%s"; public static final String DEFAULT_BATCH_VALUE = "defaultBatchValue"; public static final String DEFAULT_ROW_PREFETCH = "defaultRowPrefetch"; @@ -36,28 +40,92 @@ private OracleConstants() { public static final String CONNECTION_TYPE = "connectionType"; public static final String ROLE = "role"; public static final String NAME_DATABASE = "database"; - public static final String TNS_CONNECTION_TYPE = "TNS"; + public static final String TNS_CONNECTION_TYPE = "tns"; public static final String TRANSACTION_ISOLATION_LEVEL = "transactionIsolationLevel"; + public static final String USE_SSL = "useSSL"; /** - * Returns the Connection String for the given ConnectionType. + * Constructs the Oracle connection string based on the provided connection type, host, port, and database. + * If SSL is enabled, the connection protocol will be "tcps" instead of "tcp". * * @param connectionType TNS/Service/SID * @param host Host name of the oracle server * @param port Port of the oracle server * @param database Database to connect to - * @return Connection String based on the given ConnectionType + * @param useSSL Whether SSL/TLS is required(YES/NO) + * @return Connection String based on the given parameters and connection type. */ public static String getConnectionString(String connectionType, @Nullable String host, @Nullable int port, - String database) { - if (OracleConstants.TNS_CONNECTION_TYPE.equalsIgnoreCase(connectionType)) { - return String.format(OracleConstants.ORACLE_CONNECTION_STRING_TNS_FORMAT, database); + String database, + @Nullable Boolean useSSL) { + // Use protocol as "tcps" when SSL is requested or else use "tcp". + String connectionProtocol; + boolean isSSLEnabled = false; + if (useSSL != null && useSSL) { + connectionProtocol = "tcps"; + isSSLEnabled = true; + } else { + connectionProtocol = "tcp"; } - if (OracleConstants.SERVICE_CONNECTION_TYPE.equalsIgnoreCase(connectionType)) { - return String.format(OracleConstants.ORACLE_CONNECTION_STRING_SERVICE_NAME_FORMAT, - host, port, database); + + switch (connectionType.toLowerCase()) { + case OracleConstants.TNS_CONNECTION_TYPE: + // TNS connection doesn't require protocol + return String.format(OracleConstants.ORACLE_CONNECTION_STRING_TNS_FORMAT, database); + case OracleConstants.SERVICE_CONNECTION_TYPE: + // Create connection string for SERVICE type. + return getConnectionStringWithService(host, port, database, connectionProtocol, isSSLEnabled); + default: + // Default to SID format if no matching case is found. + return getConnectionStringWithSID(host, port, database, connectionProtocol, isSSLEnabled); + } + } + + /** + * Constructs the connection string for a SERVICE connection type. + * + * @param host Host name of the Oracle server. + * @param port Port of the Oracle server. + * @param database Database name to connect to. + * @param connectionProtocol Protocol to use for the connection ("tcp" or "tcps"). + * @param isSSLEnabled Indicates if SSL is enabled. + * @return Formatted connection string for a SERVICE connection. + */ + private static String getConnectionStringWithService(@Nullable String host, + @Nullable int port, + String database, + String connectionProtocol, + boolean isSSLEnabled) { + // Choose the appropriate format based on whether SSL is enabled. + if (isSSLEnabled) { + return String.format(OracleConstants.ORACLE_CONNECTION_STRING_SERVICE_NAME_FORMAT_WITH_PROTOCOL, + connectionProtocol, host, port, database); + } + return String.format(OracleConstants.ORACLE_CONNECTION_STRING_SERVICE_NAME_FORMAT, + host, port, database); + } + + /** + * Constructs the connection string for a SID connection type. + * + * @param host Host name of the Oracle server. + * @param port Port of the Oracle server. + * @param database Database name to connect to. + * @param connectionProtocol Protocol to use for the connection ("tcp" or "tcps"). + * @param isSSLEnabled Indicates if SSL is enabled. + * @return Formatted connection string for a SID connection. + */ + private static String getConnectionStringWithSID(@Nullable String host, + @Nullable int port, + String database, + String connectionProtocol, + boolean isSSLEnabled) { + // Choose the appropriate format based on whether SSL is enabled. + if (isSSLEnabled) { + return String.format(OracleConstants.ORACLE_CONNECTION_STRING_SID_FORMAT_WITH_PROTOCOL, + connectionProtocol, host, port, database); } return String.format(OracleConstants.ORACLE_CONNECTION_STRING_SID_FORMAT, host, port, database); diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OraclePostAction.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OraclePostAction.java index 4862aebfa..e11e455c1 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OraclePostAction.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OraclePostAction.java @@ -57,7 +57,7 @@ public static class OracleQueryActionConfig extends DBSpecificQueryActionConfig @Override public String getConnectionString() { - return OracleConstants.getConnectionString(this.connectionType, host, port, database); + return OracleConstants.getConnectionString(this.connectionType, host, port, database, null); } @Override diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java index eca7e2532..6df62e63e 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java @@ -117,9 +117,9 @@ public OracleSourceConfig(String host, int port, String user, String password, S String connectionArguments, String connectionType, String database, String role, int defaultBatchValue, int defaultRowPrefetch, String importQuery, Integer numSplits, int fetchSize, - String boundingQuery, String splitBy) { + String boundingQuery, String splitBy, Boolean useSSL) { this.connection = new OracleConnectorConfig(host, port, user, password, jdbcPluginName, connectionArguments, - connectionType, database, role); + connectionType, database, role, useSSL); this.defaultBatchValue = defaultBatchValue; this.defaultRowPrefetch = defaultRowPrefetch; this.fetchSize = fetchSize; @@ -132,7 +132,7 @@ public OracleSourceConfig(String host, int port, String user, String password, S @Override public String getConnectionString() { return OracleConstants.getConnectionString(connection.getConnectionType(), connection.getHost(), - connection.getPort(), connection.getDatabase()); + connection.getPort(), connection.getDatabase(), connection.getSSlMode()); } @Override diff --git a/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleFailedConnectionTest.java b/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleFailedConnectionTest.java index a2c9bcd5e..7ec6f3844 100644 --- a/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleFailedConnectionTest.java +++ b/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleFailedConnectionTest.java @@ -28,7 +28,8 @@ public class OracleFailedConnectionTest extends DBSpecificFailedConnectionTest { public void test() throws ClassNotFoundException, IOException { OracleConnector connector = new OracleConnector( - new OracleConnectorConfig("localhost", 1521, "username", "password", "jdbc", "", "database")); + new OracleConnectorConfig("localhost", 1521, "username", "password", "jdbc", "", + "SID", "database")); super.test(JDBC_DRIVER_CLASS_NAME, connector, "Failed to create connection to database via connection string:" + " jdbc:oracle:thin:@localhost:1521:database and arguments: " + diff --git a/oracle-plugin/widgets/Oracle-batchsink.json b/oracle-plugin/widgets/Oracle-batchsink.json index 30d5b345f..8d6168780 100644 --- a/oracle-plugin/widgets/Oracle-batchsink.json +++ b/oracle-plugin/widgets/Oracle-batchsink.json @@ -100,6 +100,26 @@ "default": "TRANSACTION_SERIALIZABLE" } }, + { + "widget-type": "hidden", + "label": "TLS Encryption", + "name": "useSSL", + "description": "Enable TLS encryption (true/false)", + "widget-attributes": { + "layout": "inline", + "default": "false", + "options": [ + { + "id": "true", + "label": "true" + }, + { + "id": "false", + "label": "false" + } + ] + } + }, { "name": "connectionType", "label": "Connection Type", diff --git a/oracle-plugin/widgets/Oracle-batchsource.json b/oracle-plugin/widgets/Oracle-batchsource.json index 0fc0a5285..5eca20cc4 100644 --- a/oracle-plugin/widgets/Oracle-batchsource.json +++ b/oracle-plugin/widgets/Oracle-batchsource.json @@ -100,6 +100,26 @@ "default": "TRANSACTION_SERIALIZABLE" } }, + { + "widget-type": "hidden", + "label": "TLS Encryption", + "name": "useSSL", + "description": "Enable TLS encryption (true/false)", + "widget-attributes": { + "layout": "inline", + "default": "false", + "options": [ + { + "id": "true", + "label": "true" + }, + { + "id": "false", + "label": "false" + } + ] + } + }, { "name": "connectionType", "label": "Connection Type", diff --git a/oracle-plugin/widgets/Oracle-connector.json b/oracle-plugin/widgets/Oracle-connector.json index 46f006c9c..628027caf 100644 --- a/oracle-plugin/widgets/Oracle-connector.json +++ b/oracle-plugin/widgets/Oracle-connector.json @@ -109,6 +109,26 @@ ], "default": "TRANSACTION_SERIALIZABLE" } + }, + { + "widget-type": "hidden", + "label": "TLS Encryption", + "name": "useSSL", + "description": "Enable TLS encryption (true/false)", + "widget-attributes": { + "layout": "inline", + "default": "false", + "options": [ + { + "id": "true", + "label": "true" + }, + { + "id": "false", + "label": "false" + } + ] + } } ] }, From 5df04fde274eada88617ab6b7db6cf11e4576f52 Mon Sep 17 00:00:00 2001 From: vipinbhatt Date: Tue, 22 Oct 2024 12:46:57 +0530 Subject: [PATCH 13/52] Upgrading to release version 1.11.3 --- amazon-redshift-plugin/pom.xml | 2 +- aurora-mysql-plugin/pom.xml | 2 +- aurora-postgresql-plugin/pom.xml | 2 +- cloudsql-mysql-plugin/pom.xml | 2 +- cloudsql-postgresql-plugin/pom.xml | 2 +- database-commons/pom.xml | 2 +- db2-plugin/pom.xml | 2 +- generic-database-plugin/pom.xml | 2 +- generic-db-argument-setter/pom.xml | 2 +- mariadb-plugin/pom.xml | 2 +- memsql-plugin/pom.xml | 2 +- mssql-plugin/pom.xml | 2 +- mysql-plugin/pom.xml | 2 +- netezza-plugin/pom.xml | 2 +- oracle-plugin/pom.xml | 2 +- pom.xml | 2 +- postgresql-plugin/pom.xml | 2 +- saphana-plugin/pom.xml | 2 +- teradata-plugin/pom.xml | 2 +- 19 files changed, 19 insertions(+), 19 deletions(-) diff --git a/amazon-redshift-plugin/pom.xml b/amazon-redshift-plugin/pom.xml index fb8e91604..80f6ea472 100644 --- a/amazon-redshift-plugin/pom.xml +++ b/amazon-redshift-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.3-SNAPSHOT + 1.11.3 Amazon Redshift plugin diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index 0113d9fb4..d5d9798b8 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.3-SNAPSHOT + 1.11.3 Aurora DB MySQL plugin diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index 54d9e041d..a1d370667 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.3-SNAPSHOT + 1.11.3 Aurora DB PostgreSQL plugin diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index 66f0d4df1..38b821b68 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.3-SNAPSHOT + 1.11.3 CloudSQL MySQL plugin diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index 46bf075e9..cfa5c9e76 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.3-SNAPSHOT + 1.11.3 CloudSQL PostgreSQL plugin diff --git a/database-commons/pom.xml b/database-commons/pom.xml index 45fc6cbde..87f9b9546 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.3-SNAPSHOT + 1.11.3 Database Commons diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index cdc3c709f..779a60880 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.3-SNAPSHOT + 1.11.3 IBM DB2 plugin diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index a74c2a250..1756f441e 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.3-SNAPSHOT + 1.11.3 Generic database plugin diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index 372a0261d..1e0685e54 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.3-SNAPSHOT + 1.11.3 Generic database argument setter plugin diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index 58273afdc..ea36e3706 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.3-SNAPSHOT + 1.11.3 Maria DB plugin diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index 8dbd76a03..6fa66978c 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.3-SNAPSHOT + 1.11.3 Memsql plugin diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index 39c0323df..5e7506eaa 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.3-SNAPSHOT + 1.11.3 Microsoft SQL Server plugin diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index 9fc01b436..16d2e4f7f 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.3-SNAPSHOT + 1.11.3 Mysql plugin diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index a0ea7c088..2ec28d2cf 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.3-SNAPSHOT + 1.11.3 Netezza plugin diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index 7e391541f..082b678f9 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.3-SNAPSHOT + 1.11.3 Oracle plugin diff --git a/pom.xml b/pom.xml index 2be1cec2b..81b93410a 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin database-plugins-parent - 1.11.3-SNAPSHOT + 1.11.3 pom Database Plugins Collection of database plugins diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index dc11a831c..084080223 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.3-SNAPSHOT + 1.11.3 PostgreSQL plugin diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index d6ab5da70..d2d25e3cf 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.3-SNAPSHOT + 1.11.3 SAP HANA plugin diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index a4d4afc88..0efe51840 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -21,7 +21,7 @@ database-plugins-parent io.cdap.plugin - 1.11.3-SNAPSHOT + 1.11.3 teradata-plugin From 170df0463af8e199e07f49b1c0682e0274e5afd9 Mon Sep 17 00:00:00 2001 From: vikasrathee-cs Date: Wed, 23 Oct 2024 07:53:22 +0530 Subject: [PATCH 14/52] Bump up to 1.11.4-SNAPSHOT --- amazon-redshift-plugin/pom.xml | 2 +- aurora-mysql-plugin/pom.xml | 2 +- aurora-postgresql-plugin/pom.xml | 2 +- cloudsql-mysql-plugin/pom.xml | 2 +- cloudsql-postgresql-plugin/pom.xml | 2 +- database-commons/pom.xml | 2 +- db2-plugin/pom.xml | 2 +- generic-database-plugin/pom.xml | 2 +- generic-db-argument-setter/pom.xml | 2 +- mariadb-plugin/pom.xml | 2 +- memsql-plugin/pom.xml | 2 +- mssql-plugin/pom.xml | 2 +- mysql-plugin/pom.xml | 2 +- netezza-plugin/pom.xml | 2 +- oracle-plugin/pom.xml | 2 +- pom.xml | 2 +- postgresql-plugin/pom.xml | 2 +- saphana-plugin/pom.xml | 2 +- teradata-plugin/pom.xml | 2 +- 19 files changed, 19 insertions(+), 19 deletions(-) diff --git a/amazon-redshift-plugin/pom.xml b/amazon-redshift-plugin/pom.xml index 80f6ea472..db99429ff 100644 --- a/amazon-redshift-plugin/pom.xml +++ b/amazon-redshift-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.3 + 1.11.4-SNAPSHOT Amazon Redshift plugin diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index d5d9798b8..f37390c37 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.3 + 1.11.4-SNAPSHOT Aurora DB MySQL plugin diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index a1d370667..a3fe5bed9 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.3 + 1.11.4-SNAPSHOT Aurora DB PostgreSQL plugin diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index 38b821b68..cd3b7e1e1 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.3 + 1.11.4-SNAPSHOT CloudSQL MySQL plugin diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index cfa5c9e76..5d97d547b 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.3 + 1.11.4-SNAPSHOT CloudSQL PostgreSQL plugin diff --git a/database-commons/pom.xml b/database-commons/pom.xml index 87f9b9546..c51034cb5 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.3 + 1.11.4-SNAPSHOT Database Commons diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index 779a60880..406a2614a 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.3 + 1.11.4-SNAPSHOT IBM DB2 plugin diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index 1756f441e..58d7600f9 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.3 + 1.11.4-SNAPSHOT Generic database plugin diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index 1e0685e54..b2db66d2e 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.3 + 1.11.4-SNAPSHOT Generic database argument setter plugin diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index ea36e3706..ed6d20f19 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.3 + 1.11.4-SNAPSHOT Maria DB plugin diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index 6fa66978c..757ca5ee2 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.3 + 1.11.4-SNAPSHOT Memsql plugin diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index 5e7506eaa..4e8fb3048 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.3 + 1.11.4-SNAPSHOT Microsoft SQL Server plugin diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index 16d2e4f7f..34ec642c6 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.3 + 1.11.4-SNAPSHOT Mysql plugin diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index 2ec28d2cf..a11b1d01d 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.3 + 1.11.4-SNAPSHOT Netezza plugin diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index 082b678f9..4c3902c3a 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.3 + 1.11.4-SNAPSHOT Oracle plugin diff --git a/pom.xml b/pom.xml index 81b93410a..ff9f58a58 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin database-plugins-parent - 1.11.3 + 1.11.4-SNAPSHOT pom Database Plugins Collection of database plugins diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index 084080223..2a9a28cfa 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.3 + 1.11.4-SNAPSHOT PostgreSQL plugin diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index d2d25e3cf..6805f04af 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.3 + 1.11.4-SNAPSHOT SAP HANA plugin diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index 0efe51840..166564699 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -21,7 +21,7 @@ database-plugins-parent io.cdap.plugin - 1.11.3 + 1.11.4-SNAPSHOT teradata-plugin From c94601ba52761297163d1aeea5ecdcb6a0894f55 Mon Sep 17 00:00:00 2001 From: vikasrathee-cs Date: Tue, 15 Oct 2024 23:22:21 +0530 Subject: [PATCH 15/52] Added fix for date datatype Added fix for date datatype (cherry picked from commit ca328a2fea43e0be57ca183546d51010e11e6214) --- .../features/sink/OracleRunTime.feature | 51 +++++++++++++++++++ .../java/io.cdap.plugin/BQValidation.java | 48 ++++++++++++++--- .../java/io.cdap.plugin/OracleClient.java | 10 ++++ .../common.stepsdesign/TestSetupHooks.java | 30 +++++++++++ .../resources/pluginParameters.properties | 6 +++ .../BigQuery/CreateBQTableQueryFileDate.txt | 1 + .../BigQuery/InsertBQDataQueryFileDate.txt | 1 + .../plugin/oracle/OracleSourceDBRecord.java | 4 +- 8 files changed, 142 insertions(+), 9 deletions(-) create mode 100644 oracle-plugin/src/e2e-test/resources/testdata/BigQuery/CreateBQTableQueryFileDate.txt create mode 100644 oracle-plugin/src/e2e-test/resources/testdata/BigQuery/InsertBQDataQueryFileDate.txt diff --git a/oracle-plugin/src/e2e-test/features/sink/OracleRunTime.feature b/oracle-plugin/src/e2e-test/features/sink/OracleRunTime.feature index 67293700b..70b1bdba6 100644 --- a/oracle-plugin/src/e2e-test/features/sink/OracleRunTime.feature +++ b/oracle-plugin/src/e2e-test/features/sink/OracleRunTime.feature @@ -167,3 +167,54 @@ Feature: Oracle - Verify data transfer from BigQuery source to Oracle sink Then Verify the pipeline status is "Succeeded" Then Validate records transferred to target table with record counts of BigQuery table Then Validate the values of records transferred to target Oracle table is equal to the values from source BigQuery table with case + + + @BQ_SOURCE_TEST_DATE @ORACLE_DATE_TABLE + Scenario: To verify data is getting transferred from BigQuery source to Oracle sink successfully when schema is having date and timestamp fields + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "BigQuery" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "Oracle" from the plugins list as: "Sink" + Then Connect plugins: "BigQuery" and "Oracle" to establish connection + Then Navigate to the properties page of plugin: "BigQuery" + Then Replace input plugin property: "project" with value: "projectId" + Then Enter input plugin property: "datasetProject" with value: "projectId" + Then Enter input plugin property: "referenceName" with value: "BQReferenceName" + Then Enter input plugin property: "dataset" with value: "dataset" + Then Enter input plugin property: "table" with value: "bqSourceTable" + Then Click on the Get Schema button + Then Verify the Output Schema matches the Expected Schema: "outputDatatypesDateTimeSchema" + Then Validate "BigQuery" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "Oracle" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "dbSchemaName" with value: "schema" + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Validate "Oracle" plugin properties + Then Close the Plugin Properties page + Then Save the pipeline + Then Preview and run the pipeline + Then Verify the preview of pipeline is "success" + Then Click on preview data for Oracle sink + Then Close the preview data + Then Deploy the pipeline + Then Run the Pipeline in Runtime + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Validate records transferred to target table with record counts of BigQuery table + Then Validate the values of records transferred to target Oracle table is equal to the values from source BigQuery table diff --git a/oracle-plugin/src/e2e-test/java/io.cdap.plugin/BQValidation.java b/oracle-plugin/src/e2e-test/java/io.cdap.plugin/BQValidation.java index b7d93c80a..b5a82e420 100644 --- a/oracle-plugin/src/e2e-test/java/io.cdap.plugin/BQValidation.java +++ b/oracle-plugin/src/e2e-test/java/io.cdap.plugin/BQValidation.java @@ -33,7 +33,12 @@ import java.sql.Types; import java.text.ParseException; import java.text.SimpleDateFormat; +import java.time.LocalDateTime; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Base64; import java.util.Date; import java.util.List; @@ -44,6 +49,13 @@ public class BQValidation { + private static final List TIMESTAMP_DATE_FORMATS = Arrays.asList( + new SimpleDateFormat("yyyy-MM-dd'T'hh:mm:ss"), + new SimpleDateFormat("yyyy-MM-dd")); + private static final List TIMESTAMP_TZ_DATE_FORMATS = Arrays.asList( + DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ssXXX"), + DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSSSSXXX")); + /** * Extracts entire data from source and target tables. * @@ -173,21 +185,43 @@ public static boolean compareResultSetAndJsonData(ResultSet rsSource, List Date: Wed, 23 Oct 2024 12:17:33 +0530 Subject: [PATCH 16/52] Bump up to 1.11.4 --- amazon-redshift-plugin/pom.xml | 2 +- aurora-mysql-plugin/pom.xml | 2 +- aurora-postgresql-plugin/pom.xml | 2 +- cloudsql-mysql-plugin/pom.xml | 2 +- cloudsql-postgresql-plugin/pom.xml | 2 +- database-commons/pom.xml | 2 +- db2-plugin/pom.xml | 2 +- generic-database-plugin/pom.xml | 2 +- generic-db-argument-setter/pom.xml | 2 +- mariadb-plugin/pom.xml | 2 +- memsql-plugin/pom.xml | 2 +- mssql-plugin/pom.xml | 2 +- mysql-plugin/pom.xml | 2 +- netezza-plugin/pom.xml | 2 +- oracle-plugin/pom.xml | 2 +- pom.xml | 2 +- postgresql-plugin/pom.xml | 2 +- saphana-plugin/pom.xml | 2 +- teradata-plugin/pom.xml | 2 +- 19 files changed, 19 insertions(+), 19 deletions(-) diff --git a/amazon-redshift-plugin/pom.xml b/amazon-redshift-plugin/pom.xml index db99429ff..46fbbdf28 100644 --- a/amazon-redshift-plugin/pom.xml +++ b/amazon-redshift-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.4-SNAPSHOT + 1.11.4 Amazon Redshift plugin diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index f37390c37..52bd863c4 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.4-SNAPSHOT + 1.11.4 Aurora DB MySQL plugin diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index a3fe5bed9..0ea3d211a 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.4-SNAPSHOT + 1.11.4 Aurora DB PostgreSQL plugin diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index cd3b7e1e1..6daba90c1 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.4-SNAPSHOT + 1.11.4 CloudSQL MySQL plugin diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index 5d97d547b..185235a04 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.4-SNAPSHOT + 1.11.4 CloudSQL PostgreSQL plugin diff --git a/database-commons/pom.xml b/database-commons/pom.xml index c51034cb5..85de33814 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.4-SNAPSHOT + 1.11.4 Database Commons diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index 406a2614a..f74a73313 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.4-SNAPSHOT + 1.11.4 IBM DB2 plugin diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index 58d7600f9..f60c6e1b4 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.4-SNAPSHOT + 1.11.4 Generic database plugin diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index b2db66d2e..11176aad4 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.4-SNAPSHOT + 1.11.4 Generic database argument setter plugin diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index ed6d20f19..3d45a86f2 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.4-SNAPSHOT + 1.11.4 Maria DB plugin diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index 757ca5ee2..40f541c1d 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.4-SNAPSHOT + 1.11.4 Memsql plugin diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index 4e8fb3048..bc189004a 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.4-SNAPSHOT + 1.11.4 Microsoft SQL Server plugin diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index 34ec642c6..ee353ab20 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.4-SNAPSHOT + 1.11.4 Mysql plugin diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index a11b1d01d..d44e0615b 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.4-SNAPSHOT + 1.11.4 Netezza plugin diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index 4c3902c3a..ee01680b1 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.4-SNAPSHOT + 1.11.4 Oracle plugin diff --git a/pom.xml b/pom.xml index ff9f58a58..7fe284df2 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin database-plugins-parent - 1.11.4-SNAPSHOT + 1.11.4 pom Database Plugins Collection of database plugins diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index 2a9a28cfa..73f16b2f2 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.4-SNAPSHOT + 1.11.4 PostgreSQL plugin diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index 6805f04af..0bcca7c45 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.4-SNAPSHOT + 1.11.4 SAP HANA plugin diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index 166564699..b6b7809d6 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -21,7 +21,7 @@ database-plugins-parent io.cdap.plugin - 1.11.4-SNAPSHOT + 1.11.4 teradata-plugin From 0ffe34f5ae3cd6e8b6fde7547dd1fcc0f03ee57a Mon Sep 17 00:00:00 2001 From: psainics Date: Tue, 5 Nov 2024 09:46:17 +0530 Subject: [PATCH 17/52] Bump up to 1.11.5-SNAPSHOT --- amazon-redshift-plugin/pom.xml | 2 +- aurora-mysql-plugin/pom.xml | 2 +- aurora-postgresql-plugin/pom.xml | 2 +- cloudsql-mysql-plugin/pom.xml | 2 +- cloudsql-postgresql-plugin/pom.xml | 2 +- database-commons/pom.xml | 2 +- db2-plugin/pom.xml | 2 +- generic-database-plugin/pom.xml | 2 +- generic-db-argument-setter/pom.xml | 2 +- mariadb-plugin/pom.xml | 2 +- memsql-plugin/pom.xml | 2 +- mssql-plugin/pom.xml | 2 +- mysql-plugin/pom.xml | 2 +- netezza-plugin/pom.xml | 2 +- oracle-plugin/pom.xml | 2 +- pom.xml | 2 +- postgresql-plugin/pom.xml | 2 +- saphana-plugin/pom.xml | 2 +- teradata-plugin/pom.xml | 2 +- 19 files changed, 19 insertions(+), 19 deletions(-) diff --git a/amazon-redshift-plugin/pom.xml b/amazon-redshift-plugin/pom.xml index 46fbbdf28..f59abf579 100644 --- a/amazon-redshift-plugin/pom.xml +++ b/amazon-redshift-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.4 + 1.11.5-SNAPSHOT Amazon Redshift plugin diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index 52bd863c4..35d21e272 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.4 + 1.11.5-SNAPSHOT Aurora DB MySQL plugin diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index 0ea3d211a..3d4812d84 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.4 + 1.11.5-SNAPSHOT Aurora DB PostgreSQL plugin diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index 6daba90c1..fc475c599 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.4 + 1.11.5-SNAPSHOT CloudSQL MySQL plugin diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index 185235a04..7da42f292 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.4 + 1.11.5-SNAPSHOT CloudSQL PostgreSQL plugin diff --git a/database-commons/pom.xml b/database-commons/pom.xml index 85de33814..bf39b30de 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.4 + 1.11.5-SNAPSHOT Database Commons diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index f74a73313..1124e34f3 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.4 + 1.11.5-SNAPSHOT IBM DB2 plugin diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index f60c6e1b4..b2a915027 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.4 + 1.11.5-SNAPSHOT Generic database plugin diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index 11176aad4..8a7e9c7ac 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.4 + 1.11.5-SNAPSHOT Generic database argument setter plugin diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index 3d45a86f2..d746a5776 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.4 + 1.11.5-SNAPSHOT Maria DB plugin diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index 40f541c1d..f0e332ea6 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.4 + 1.11.5-SNAPSHOT Memsql plugin diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index bc189004a..3ddcfaf39 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.4 + 1.11.5-SNAPSHOT Microsoft SQL Server plugin diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index ee353ab20..1e01d3b84 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.4 + 1.11.5-SNAPSHOT Mysql plugin diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index d44e0615b..7bd519af9 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.4 + 1.11.5-SNAPSHOT Netezza plugin diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index ee01680b1..f777c165f 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.4 + 1.11.5-SNAPSHOT Oracle plugin diff --git a/pom.xml b/pom.xml index 7fe284df2..347d697dc 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin database-plugins-parent - 1.11.4 + 1.11.5-SNAPSHOT pom Database Plugins Collection of database plugins diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index 73f16b2f2..564c759fd 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.4 + 1.11.5-SNAPSHOT PostgreSQL plugin diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index 0bcca7c45..2af7afcf0 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.4 + 1.11.5-SNAPSHOT SAP HANA plugin diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index b6b7809d6..adb8de4d6 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -21,7 +21,7 @@ database-plugins-parent io.cdap.plugin - 1.11.4 + 1.11.5-SNAPSHOT teradata-plugin From 60a4a65492b91a16154d688eaa72eea5941694db Mon Sep 17 00:00:00 2001 From: psainics Date: Wed, 23 Oct 2024 14:59:17 +0530 Subject: [PATCH 18/52] Escape column names --- .../cloudsql/mysql/CloudSQLMySQLSink.java | 25 +++++++++++++ .../cloudsql/mysql/CloudSQLMySQLSinkTest.java | 35 +++++++++++++++++++ .../io/cdap/plugin/mysql/MysqlDBRecord.java | 9 +++++ .../java/io/cdap/plugin/mysql/MysqlSink.java | 24 +++++++++++++ .../io/cdap/plugin/mysql/MysqlSinkTest.java | 35 +++++++++++++++++++ 5 files changed, 128 insertions(+) create mode 100644 cloudsql-mysql-plugin/src/test/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSinkTest.java create mode 100644 mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlSinkTest.java diff --git a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSink.java b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSink.java index 271012f7e..6149c114b 100644 --- a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSink.java +++ b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSink.java @@ -16,6 +16,7 @@ package io.cdap.plugin.cloudsql.mysql; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Strings; import com.google.common.collect.ImmutableMap; import io.cdap.cdap.api.annotation.Description; @@ -25,6 +26,7 @@ import io.cdap.cdap.api.annotation.Name; import io.cdap.cdap.api.annotation.Plugin; import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; import io.cdap.cdap.etl.api.FailureCollector; import io.cdap.cdap.etl.api.PipelineConfigurer; import io.cdap.cdap.etl.api.batch.BatchSink; @@ -40,7 +42,11 @@ import io.cdap.plugin.util.CloudSQLUtil; import io.cdap.plugin.util.DBUtils; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import java.util.Map; +import java.util.StringJoiner; import javax.annotation.Nullable; /** Sink support for a CloudSQL MySQL database. */ @@ -52,6 +58,7 @@ public class CloudSQLMySQLSink extends AbstractDBSink { private final CloudSQLMySQLSinkConfig cloudsqlMysqlSinkConfig; + private static final Character ESCAPE_CHAR = '`'; public CloudSQLMySQLSink(CloudSQLMySQLSinkConfig cloudsqlMysqlSinkConfig) { super(cloudsqlMysqlSinkConfig); @@ -78,6 +85,24 @@ protected DBRecord getDBRecord(StructuredRecord output) { return new MysqlDBRecord(output, columnTypes); } + @Override + protected void setColumnsInfo(List fields) { + List columnsList = new ArrayList<>(); + StringJoiner columnsJoiner = new StringJoiner(","); + for (Schema.Field field : fields) { + columnsList.add(field.getName()); + columnsJoiner.add(ESCAPE_CHAR + field.getName() + ESCAPE_CHAR); + } + + super.columns = Collections.unmodifiableList(columnsList); + super.dbColumns = columnsJoiner.toString(); + } + + @VisibleForTesting + String getDbColumns() { + return dbColumns; + } + @Override protected LineageRecorder getLineageRecorder(BatchSinkContext context) { String host; diff --git a/cloudsql-mysql-plugin/src/test/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSinkTest.java b/cloudsql-mysql-plugin/src/test/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSinkTest.java new file mode 100644 index 000000000..65a14502e --- /dev/null +++ b/cloudsql-mysql-plugin/src/test/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSinkTest.java @@ -0,0 +1,35 @@ +/* + * Copyright © 2024 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.cloudsql.mysql; + +import io.cdap.cdap.api.data.schema.Schema; +import org.junit.Assert; +import org.junit.Test; + +public class CloudSQLMySQLSinkTest { + @Test + public void testSetColumnsInfo() { + Schema outputSchema = Schema.recordOf("output", + Schema.Field.of("id", Schema.of(Schema.Type.INT)), + Schema.Field.of("name", Schema.of(Schema.Type.STRING)), + Schema.Field.of("insert", Schema.of(Schema.Type.STRING))); + CloudSQLMySQLSink cloudSQLMySQLSink = new CloudSQLMySQLSink(new CloudSQLMySQLSink.CloudSQLMySQLSinkConfig()); + Assert.assertNotNull(outputSchema.getFields()); + cloudSQLMySQLSink.setColumnsInfo(outputSchema.getFields()); + Assert.assertEquals("`id`,`name`,`insert`", cloudSQLMySQLSink.getDbColumns()); + } +} diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlDBRecord.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlDBRecord.java index 0560b10c3..94b711786 100644 --- a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlDBRecord.java +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlDBRecord.java @@ -93,4 +93,13 @@ protected void writeNonNullToDB(PreparedStatement stmt, Schema fieldSchema, super.writeNonNullToDB(stmt, fieldSchema, fieldName, fieldIndex); } + + @Override + protected void insertOperation(PreparedStatement stmt) throws SQLException { + for (int fieldIndex = 0; fieldIndex < columnTypes.size(); fieldIndex++) { + ColumnType columnType = columnTypes.get(fieldIndex); + Schema.Field field = record.getSchema().getField(columnType.getName(), true); + writeToDB(stmt, field, fieldIndex); + } + } } diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSink.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSink.java index c839cb12b..f71371026 100644 --- a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSink.java +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSink.java @@ -16,6 +16,7 @@ package io.cdap.plugin.mysql; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Strings; import io.cdap.cdap.api.annotation.Description; import io.cdap.cdap.api.annotation.Macro; @@ -24,6 +25,7 @@ import io.cdap.cdap.api.annotation.Name; import io.cdap.cdap.api.annotation.Plugin; import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; import io.cdap.cdap.etl.api.FailureCollector; import io.cdap.cdap.etl.api.batch.BatchSink; import io.cdap.cdap.etl.api.batch.BatchSinkContext; @@ -39,9 +41,12 @@ import io.cdap.plugin.db.sink.FieldsValidator; import io.cdap.plugin.util.DBUtils; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.StringJoiner; +import java.util.stream.Collectors; import javax.annotation.Nullable; /** @@ -54,6 +59,7 @@ public class MysqlSink extends AbstractDBSink { private final MysqlSinkConfig mysqlSinkConfig; + private static final Character ESCAPE_CHAR = '`'; public MysqlSink(MysqlSinkConfig mysqlSinkConfig) { super(mysqlSinkConfig); @@ -85,6 +91,24 @@ protected SchemaReader getSchemaReader() { return new MysqlSchemaReader(null); } + @Override + protected void setColumnsInfo(List fields) { + List columnsList = new ArrayList<>(); + StringJoiner columnsJoiner = new StringJoiner(","); + for (Schema.Field field : fields) { + columnsList.add(field.getName()); + columnsJoiner.add(ESCAPE_CHAR + field.getName() + ESCAPE_CHAR); + } + + super.columns = Collections.unmodifiableList(columnsList); + super.dbColumns = columnsJoiner.toString(); + } + + @VisibleForTesting + String getDbColumns() { + return dbColumns; + } + /** * MySQL action configuration. */ diff --git a/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlSinkTest.java b/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlSinkTest.java new file mode 100644 index 000000000..1dd4e809e --- /dev/null +++ b/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlSinkTest.java @@ -0,0 +1,35 @@ +/* + * Copyright © 2024 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.mysql; + +import io.cdap.cdap.api.data.schema.Schema; +import org.junit.Assert; +import org.junit.Test; + +public class MysqlSinkTest { + @Test + public void testSetColumnsInfo() { + Schema outputSchema = Schema.recordOf("output", + Schema.Field.of("id", Schema.of(Schema.Type.INT)), + Schema.Field.of("name", Schema.of(Schema.Type.STRING)), + Schema.Field.of("insert", Schema.of(Schema.Type.STRING))); + MysqlSink mySQLSink = new MysqlSink(new MysqlSink.MysqlSinkConfig()); + Assert.assertNotNull(outputSchema.getFields()); + mySQLSink.setColumnsInfo(outputSchema.getFields()); + Assert.assertEquals("`id`,`name`,`insert`", mySQLSink.getDbColumns()); + } +} From dc23cb080e8fc5b8c06241f7fce6a0eb430c5682 Mon Sep 17 00:00:00 2001 From: psainics Date: Wed, 6 Nov 2024 10:40:57 +0530 Subject: [PATCH 19/52] Remove SNAPSHOT for release v1.11.5 --- amazon-redshift-plugin/pom.xml | 2 +- aurora-mysql-plugin/pom.xml | 2 +- aurora-postgresql-plugin/pom.xml | 2 +- cloudsql-mysql-plugin/pom.xml | 2 +- cloudsql-postgresql-plugin/pom.xml | 2 +- database-commons/pom.xml | 2 +- db2-plugin/pom.xml | 2 +- generic-database-plugin/pom.xml | 2 +- generic-db-argument-setter/pom.xml | 2 +- mariadb-plugin/pom.xml | 2 +- memsql-plugin/pom.xml | 2 +- mssql-plugin/pom.xml | 2 +- mysql-plugin/pom.xml | 2 +- netezza-plugin/pom.xml | 2 +- oracle-plugin/pom.xml | 2 +- pom.xml | 2 +- postgresql-plugin/pom.xml | 2 +- saphana-plugin/pom.xml | 2 +- teradata-plugin/pom.xml | 2 +- 19 files changed, 19 insertions(+), 19 deletions(-) diff --git a/amazon-redshift-plugin/pom.xml b/amazon-redshift-plugin/pom.xml index f59abf579..d849d4690 100644 --- a/amazon-redshift-plugin/pom.xml +++ b/amazon-redshift-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.5-SNAPSHOT + 1.11.5 Amazon Redshift plugin diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index 35d21e272..3cd712871 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.5-SNAPSHOT + 1.11.5 Aurora DB MySQL plugin diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index 3d4812d84..1bcb7efc5 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.5-SNAPSHOT + 1.11.5 Aurora DB PostgreSQL plugin diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index fc475c599..c186be7ee 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.5-SNAPSHOT + 1.11.5 CloudSQL MySQL plugin diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index 7da42f292..70e586580 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.5-SNAPSHOT + 1.11.5 CloudSQL PostgreSQL plugin diff --git a/database-commons/pom.xml b/database-commons/pom.xml index bf39b30de..55e869595 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.5-SNAPSHOT + 1.11.5 Database Commons diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index 1124e34f3..c1ee5cdc7 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.5-SNAPSHOT + 1.11.5 IBM DB2 plugin diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index b2a915027..4120109f3 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.5-SNAPSHOT + 1.11.5 Generic database plugin diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index 8a7e9c7ac..d988040a0 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.5-SNAPSHOT + 1.11.5 Generic database argument setter plugin diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index d746a5776..9209850e0 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.5-SNAPSHOT + 1.11.5 Maria DB plugin diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index f0e332ea6..f58a07b8e 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.5-SNAPSHOT + 1.11.5 Memsql plugin diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index 3ddcfaf39..d956def9b 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.5-SNAPSHOT + 1.11.5 Microsoft SQL Server plugin diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index 1e01d3b84..261e18cb8 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.5-SNAPSHOT + 1.11.5 Mysql plugin diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index 7bd519af9..0219ab04a 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.5-SNAPSHOT + 1.11.5 Netezza plugin diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index f777c165f..5e6b99500 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.5-SNAPSHOT + 1.11.5 Oracle plugin diff --git a/pom.xml b/pom.xml index 347d697dc..e1c82414e 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin database-plugins-parent - 1.11.5-SNAPSHOT + 1.11.5 pom Database Plugins Collection of database plugins diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index 564c759fd..f8a2372a9 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.5-SNAPSHOT + 1.11.5 PostgreSQL plugin diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index 2af7afcf0..e44ee8202 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.5-SNAPSHOT + 1.11.5 SAP HANA plugin diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index adb8de4d6..d622812fa 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -21,7 +21,7 @@ database-plugins-parent io.cdap.plugin - 1.11.5-SNAPSHOT + 1.11.5 teradata-plugin From 932de430afc808f9cdad80a9172dde392df34247 Mon Sep 17 00:00:00 2001 From: psainics Date: Wed, 2 Apr 2025 13:08:01 +0530 Subject: [PATCH 20/52] sync .github develop --- .github/workflows/build-report.yml | 6 ++++++ .github/workflows/build.yml | 4 ++-- .github/workflows/e2e.yml | 21 +++++++++------------ 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build-report.yml b/.github/workflows/build-report.yml index 691236ceb..22f7ea248 100644 --- a/.github/workflows/build-report.yml +++ b/.github/workflows/build-report.yml @@ -21,6 +21,12 @@ on: types: - completed +permissions: + actions: read # Allows reading workflow run information + statuses: write # Required if the action updates commit statuses + checks: write # Required if it updates GitHub Checks API + + jobs: build: runs-on: ubuntu-latest diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1fb6ac78a..55cd4617e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -37,12 +37,12 @@ jobs: && (github.event.action != 'labeled' || github.event.label.name == 'build') ) steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: ref: ${{ github.event.workflow_run.head_sha }} - name: Cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/.m2/repository key: ${{ runner.os }}-maven-${{ github.workflow }}-${{ hashFiles('**/pom.xml') }} diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index b142ed3b1..c710ef929 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -45,7 +45,7 @@ jobs: steps: # Pinned 1.0.0 version - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: path: plugin submodules: 'recursive' @@ -61,13 +61,13 @@ jobs: - '${{ matrix.module }}/**/e2e-test/**' - name: Checkout e2e test repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: cdapio/cdap-e2e-tests path: e2e - name: Cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/.m2/repository key: ${{ runner.os }}-maven-${{ github.workflow }}-${{ hashFiles('**/pom.xml') }} @@ -156,24 +156,21 @@ jobs: CLOUDSQL_MYSQL_PASSWORD: ${{ steps.secrets.outputs.CLOUDSQL_MYSQL_PASSWORD }} CLOUDSQL_MYSQL_CONNECTION_NAME: ${{ steps.secrets.outputs.CLOUDSQL_MYSQL_CONNECTION_NAME }} - - name: Upload report - uses: actions/upload-artifact@v3 - if: always() - with: - name: Cucumber report - ${{ matrix.module }} - path: ./**/target/cucumber-reports - - name: Upload debug files - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() with: name: Debug files - ${{ matrix.module }} path: ./**/target/e2e-debug - name: Upload files to GCS - uses: google-github-actions/upload-cloud-storage@v0 + uses: google-github-actions/upload-cloud-storage@v2 if: always() with: path: ./plugin destination: e2e-tests-cucumber-reports/${{ github.event.repository.name }}/${{ github.ref }} glob: '**/target/cucumber-reports/**' + + - name: Cucumber Report URL + if: always() + run: echo "https://storage.googleapis.com/e2e-tests-cucumber-reports/${{ github.event.repository.name }}/${{ github.ref }}/plugin/${{ matrix.module }}/target/cucumber-reports/advanced-reports/cucumber-html-reports/overview-features.html" From 702cfc8e85c154313a15db2881aabe87d17ed58d Mon Sep 17 00:00:00 2001 From: psainics Date: Wed, 2 Apr 2025 12:52:56 +0530 Subject: [PATCH 21/52] Bump 1.15.6-SNAPSHOT --- amazon-redshift-plugin/pom.xml | 2 +- aurora-mysql-plugin/pom.xml | 2 +- aurora-postgresql-plugin/pom.xml | 2 +- cloudsql-mysql-plugin/pom.xml | 2 +- cloudsql-postgresql-plugin/pom.xml | 2 +- database-commons/pom.xml | 2 +- db2-plugin/pom.xml | 2 +- generic-database-plugin/pom.xml | 2 +- generic-db-argument-setter/pom.xml | 2 +- mariadb-plugin/pom.xml | 2 +- memsql-plugin/pom.xml | 2 +- mssql-plugin/pom.xml | 2 +- mysql-plugin/pom.xml | 2 +- netezza-plugin/pom.xml | 2 +- oracle-plugin/pom.xml | 2 +- pom.xml | 2 +- postgresql-plugin/pom.xml | 2 +- saphana-plugin/pom.xml | 2 +- teradata-plugin/pom.xml | 2 +- 19 files changed, 19 insertions(+), 19 deletions(-) diff --git a/amazon-redshift-plugin/pom.xml b/amazon-redshift-plugin/pom.xml index d849d4690..573765c78 100644 --- a/amazon-redshift-plugin/pom.xml +++ b/amazon-redshift-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.5 + 1.11.6-SNAPSHOT Amazon Redshift plugin diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index 3cd712871..3614b2ebf 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.5 + 1.11.6-SNAPSHOT Aurora DB MySQL plugin diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index 1bcb7efc5..0b1d0724e 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.5 + 1.11.6-SNAPSHOT Aurora DB PostgreSQL plugin diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index c186be7ee..96f19944c 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.5 + 1.11.6-SNAPSHOT CloudSQL MySQL plugin diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index 70e586580..90566bf65 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.5 + 1.11.6-SNAPSHOT CloudSQL PostgreSQL plugin diff --git a/database-commons/pom.xml b/database-commons/pom.xml index 55e869595..ceb3f8208 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.5 + 1.11.6-SNAPSHOT Database Commons diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index c1ee5cdc7..f25f6fe15 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.5 + 1.11.6-SNAPSHOT IBM DB2 plugin diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index 4120109f3..593bff82f 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.5 + 1.11.6-SNAPSHOT Generic database plugin diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index d988040a0..af5fc943d 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.5 + 1.11.6-SNAPSHOT Generic database argument setter plugin diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index 9209850e0..b82cc7c24 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.5 + 1.11.6-SNAPSHOT Maria DB plugin diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index f58a07b8e..40918105e 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.5 + 1.11.6-SNAPSHOT Memsql plugin diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index d956def9b..57d4874ac 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.5 + 1.11.6-SNAPSHOT Microsoft SQL Server plugin diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index 261e18cb8..1266823b7 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.5 + 1.11.6-SNAPSHOT Mysql plugin diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index 0219ab04a..bdfab69b8 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.5 + 1.11.6-SNAPSHOT Netezza plugin diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index 5e6b99500..45873a221 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.5 + 1.11.6-SNAPSHOT Oracle plugin diff --git a/pom.xml b/pom.xml index e1c82414e..ec91f1faa 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin database-plugins-parent - 1.11.5 + 1.11.6-SNAPSHOT pom Database Plugins Collection of database plugins diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index f8a2372a9..d55da093a 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.5 + 1.11.6-SNAPSHOT PostgreSQL plugin diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index e44ee8202..9823860cc 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.5 + 1.11.6-SNAPSHOT SAP HANA plugin diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index d622812fa..0b315c273 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -21,7 +21,7 @@ database-plugins-parent io.cdap.plugin - 1.11.5 + 1.11.6-SNAPSHOT teradata-plugin From 0d585b3957401ac57de1dab00f20f5d0b34fd560 Mon Sep 17 00:00:00 2001 From: "danish.amjad" Date: Fri, 15 Mar 2024 23:09:25 +0100 Subject: [PATCH 22/52] Fix non-nullable datetime when zeroDateTimeBehavior is CONVERT_TO_NULL. --- .../mysql/CloudSQLMySQLConnector.java | 3 +- .../cloudsql/mysql/CloudSQLMySQLSource.java | 7 +++ .../io/cdap/plugin/mysql/MysqlConnector.java | 3 +- .../io/cdap/plugin/mysql/MysqlConstants.java | 1 + .../cdap/plugin/mysql/MysqlSchemaReader.java | 34 ++++++++++ .../io/cdap/plugin/mysql/MysqlSource.java | 2 +- .../java/io/cdap/plugin/mysql/MysqlUtil.java | 17 +++++ .../mysql/MysqlSchemaReaderUnitTest.java | 33 ++++++++++ .../cdap/plugin/mysql/MysqlUtilUnitTest.java | 62 +++++++++++++++++++ 9 files changed, 159 insertions(+), 3 deletions(-) create mode 100644 mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlUtilUnitTest.java diff --git a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnector.java b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnector.java index a5ee68787..b4b87c81b 100644 --- a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnector.java +++ b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnector.java @@ -16,6 +16,7 @@ package io.cdap.plugin.cloudsql.mysql; +import com.google.common.collect.Maps; import io.cdap.cdap.api.annotation.Category; import io.cdap.cdap.api.annotation.Description; import io.cdap.cdap.api.annotation.Name; @@ -75,7 +76,7 @@ public StructuredRecord transform(LongWritable longWritable, MysqlDBRecord mysql @Override protected SchemaReader getSchemaReader(String sessionID) { - return new MysqlSchemaReader(sessionID); + return new MysqlSchemaReader(sessionID, Maps.fromProperties(config.getConnectionArgumentsProperties())); } @Override diff --git a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSource.java b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSource.java index b8b6fbf27..b0bea9e7a 100644 --- a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSource.java +++ b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSource.java @@ -31,9 +31,11 @@ import io.cdap.plugin.common.Asset; import io.cdap.plugin.common.ConfigUtil; import io.cdap.plugin.common.LineageRecorder; +import io.cdap.plugin.db.SchemaReader; import io.cdap.plugin.db.config.AbstractDBSpecificSourceConfig; import io.cdap.plugin.db.source.AbstractDBSource; import io.cdap.plugin.mysql.MysqlDBRecord; +import io.cdap.plugin.mysql.MysqlSchemaReader; import io.cdap.plugin.util.CloudSQLUtil; import io.cdap.plugin.util.DBUtils; import org.apache.hadoop.mapreduce.lib.db.DBWritable; @@ -120,6 +122,11 @@ protected LineageRecorder getLineageRecorder(BatchSourceContext context) { return new LineageRecorder(context, assetBuilder.build()); } + @Override + protected SchemaReader getSchemaReader() { + return new MysqlSchemaReader(null, cloudsqlMysqlSourceConfig.getConnectionArguments()); + } + /** CloudSQL MySQL source config. */ public static class CloudSQLMySQLSourceConfig extends AbstractDBSpecificSourceConfig { diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConnector.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConnector.java index 3dede5d49..e7e935135 100644 --- a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConnector.java +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConnector.java @@ -16,6 +16,7 @@ package io.cdap.plugin.mysql; +import com.google.common.collect.Maps; import io.cdap.cdap.api.annotation.Category; import io.cdap.cdap.api.annotation.Description; import io.cdap.cdap.api.annotation.Name; @@ -62,7 +63,7 @@ public boolean supportSchema() { @Override protected SchemaReader getSchemaReader(String sessionID) { - return new MysqlSchemaReader(sessionID); + return new MysqlSchemaReader(sessionID, Maps.fromProperties(config.getConnectionArgumentsProperties())); } @Override diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConstants.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConstants.java index 39c0b8d08..54593f580 100644 --- a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConstants.java +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConstants.java @@ -39,6 +39,7 @@ private MysqlConstants() { public static final String TRUST_CERT_KEYSTORE_PASSWORD = "trustCertificateKeyStorePassword"; public static final String MYSQL_CONNECTION_STRING_FORMAT = "jdbc:mysql://%s:%s/%s"; public static final String USE_CURSOR_FETCH = "useCursorFetch"; + public static final String ZERO_DATE_TIME_BEHAVIOR = "zeroDateTimeBehavior"; /** * Query to set SQL_MODE system variable. diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSchemaReader.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSchemaReader.java index a842ba568..50907c063 100644 --- a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSchemaReader.java +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSchemaReader.java @@ -16,12 +16,16 @@ package io.cdap.plugin.mysql; +import com.google.common.collect.Lists; import io.cdap.cdap.api.data.schema.Schema; import io.cdap.plugin.db.CommonSchemaReader; +import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.sql.Types; +import java.util.List; +import java.util.Map; /** * Schema reader for mapping Mysql DB type @@ -31,12 +35,42 @@ public class MysqlSchemaReader extends CommonSchemaReader { public static final String YEAR_TYPE_NAME = "YEAR"; public static final String MEDIUMINT_UNSIGNED_TYPE_NAME = "MEDIUMINT UNSIGNED"; private final String sessionID; + private boolean zeroDateTimeToNull; public MysqlSchemaReader(String sessionID) { super(); this.sessionID = sessionID; } + public MysqlSchemaReader(String sessionID, Map connectionArguments) { + super(); + this.sessionID = sessionID; + this.zeroDateTimeToNull = MysqlUtil.isZeroDateTimeToNull(connectionArguments); + } + + @Override + public List getSchemaFields(ResultSet resultSet) throws SQLException { + List schemaFields = Lists.newArrayList(); + ResultSetMetaData metadata = resultSet.getMetaData(); + // ResultSetMetadata columns are numbered starting with 1 + for (int i = 1; i <= metadata.getColumnCount(); i++) { + if (shouldIgnoreColumn(metadata, i)) { + continue; + } + + String columnName = metadata.getColumnName(i); + Schema columnSchema = getSchema(metadata, i); + + if (ResultSetMetaData.columnNullable == metadata.isNullable(i) + || (zeroDateTimeToNull && MysqlUtil.isDateTimeLikeType(metadata.getColumnType(i)))) { + columnSchema = Schema.nullableOf(columnSchema); + } + Schema.Field field = Schema.Field.of(columnName, columnSchema); + schemaFields.add(field); + } + return schemaFields; + } + @Override public boolean shouldIgnoreColumn(ResultSetMetaData metadata, int index) throws SQLException { return metadata.getColumnName(index).equals("c_" + sessionID) || diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java index 71f113436..a91139196 100644 --- a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java @@ -81,7 +81,7 @@ protected LineageRecorder getLineageRecorder(BatchSourceContext context) { @Override protected SchemaReader getSchemaReader() { - return new MysqlSchemaReader(null); + return new MysqlSchemaReader(null, mysqlSourceConfig.getConnectionArguments()); } /** diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlUtil.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlUtil.java index c1c770c06..abb4aa27b 100644 --- a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlUtil.java +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlUtil.java @@ -18,6 +18,7 @@ import com.google.common.collect.ImmutableMap; +import java.sql.Types; import java.util.Map; /** @@ -91,4 +92,20 @@ public static Map composeDbSpecificArgumentsMap(Boolean autoReco public static String getConnectionString(String host, Integer port, String database) { return String.format(MysqlConstants.MYSQL_CONNECTION_STRING_FORMAT, host, port, database); } + + public static boolean isDateTimeLikeType(int columnType) { + int[] dateTimeLikeTypes = new int[]{Types.TIMESTAMP, Types.TIMESTAMP_WITH_TIMEZONE, Types.DATE}; + + for (int dttType : dateTimeLikeTypes) { + if (dttType == columnType) { + return true; + } + } + return false; + } + + public static boolean isZeroDateTimeToNull(Map connectionArguments) { + String argValue = connectionArguments.getOrDefault(MysqlConstants.ZERO_DATE_TIME_BEHAVIOR, ""); + return argValue.equals("CONVERT_TO_NULL") || argValue.equals("convertToNull"); + } } diff --git a/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlSchemaReaderUnitTest.java b/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlSchemaReaderUnitTest.java index 28582bc3b..fa7029c8f 100644 --- a/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlSchemaReaderUnitTest.java +++ b/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlSchemaReaderUnitTest.java @@ -21,9 +21,13 @@ import org.junit.Test; import org.mockito.Mockito; +import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.sql.Types; +import java.util.HashMap; +import java.util.List; +import java.util.Map; public class MysqlSchemaReaderUnitTest { @@ -37,4 +41,33 @@ public void validateYearTypeToStringTypeConversion() throws SQLException { Schema schema = schemaReader.getSchema(metadata, 1); Assert.assertTrue(Schema.of(Schema.Type.INT).equals(schema)); } + + @Test + public void validateZeroDateTimeBehavior() throws SQLException { + ResultSet resultSet = Mockito.mock(ResultSet.class); + ResultSetMetaData metadata = Mockito.mock(ResultSetMetaData.class); + Mockito.when(resultSet.getMetaData()).thenReturn(metadata); + + Mockito.when(metadata.getColumnCount()).thenReturn(1); + Mockito.when(metadata.getColumnName(Mockito.eq(1))).thenReturn("some_date"); + + Mockito.when(metadata.getColumnType(Mockito.eq(1))).thenReturn(Types.DATE); + Mockito.when(metadata.getColumnTypeName(Mockito.eq(1))).thenReturn(MysqlSchemaReader.YEAR_TYPE_NAME); + + // non-nullable column + Mockito.when(metadata.isNullable(Mockito.eq(1))).thenReturn(0); + + // test that non-nullable date remains non-nullable when no conn arg is present + MysqlSchemaReader schemaReader = new MysqlSchemaReader(null); + List schemaFields = schemaReader.getSchemaFields(resultSet); + Assert.assertFalse(schemaFields.get(0).getSchema().isNullable()); + + // test that it converts non-nullable date column to nullable when zeroDateTimeBehavior is convert to null + Map connectionArguments = new HashMap<>(); + connectionArguments.put("zeroDateTimeBehavior", "CONVERT_TO_NULL"); + + schemaReader = new MysqlSchemaReader(null, connectionArguments); + schemaFields = schemaReader.getSchemaFields(resultSet); + Assert.assertTrue(schemaFields.get(0).getSchema().isNullable()); + } } diff --git a/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlUtilUnitTest.java b/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlUtilUnitTest.java new file mode 100644 index 000000000..9481068f1 --- /dev/null +++ b/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlUtilUnitTest.java @@ -0,0 +1,62 @@ + +/* + * Copyright © 2024 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.mysql; + +import org.junit.Test; + +import java.sql.Types; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class MysqlUtilUnitTest { + + @Test + public void testIsZeroDateTimeToNull() { + Map connArgsMap = new HashMap<>(1); + + connArgsMap.put("zeroDateTimeBehavior", ""); + assertFalse(MysqlUtil.isZeroDateTimeToNull(connArgsMap)); + + connArgsMap.put("zeroDateTimeBehavior", "ROUND"); + assertFalse(MysqlUtil.isZeroDateTimeToNull(connArgsMap)); + + connArgsMap.put("zeroDateTimeBehavior", "CONVERT_TO_NULL"); + assertTrue(MysqlUtil.isZeroDateTimeToNull(connArgsMap)); + + connArgsMap.put("zeroDateTimeBehavior", "convertToNull"); + assertTrue(MysqlUtil.isZeroDateTimeToNull(connArgsMap)); + } + + @Test + public void testIsDateTimeLikeType() { + int dateType = Types.DATE; + int timestampType = Types.TIMESTAMP; + int timestampWithTimezoneType = Types.TIMESTAMP_WITH_TIMEZONE; + int timeType = Types.TIME; + int stringType = Types.VARCHAR; + + assertTrue(MysqlUtil.isDateTimeLikeType(dateType)); + assertTrue(MysqlUtil.isDateTimeLikeType(timestampType)); + assertTrue(MysqlUtil.isDateTimeLikeType(timestampWithTimezoneType)); + assertFalse(MysqlUtil.isDateTimeLikeType(timeType)); + assertFalse(MysqlUtil.isDateTimeLikeType(stringType)); + } +} From a5076a3049dce54c836a9d64174aa8851ec23fb3 Mon Sep 17 00:00:00 2001 From: psainics Date: Mon, 14 Apr 2025 16:16:58 +0530 Subject: [PATCH 23/52] Add MariadbDBRecord implementations UPDATE DOCS --- mariadb-plugin/docs/Mariadb-batchsink.md | 68 +++++++++--------- mariadb-plugin/docs/Mariadb-batchsource.md | 70 +++++++++---------- mariadb-plugin/pom.xml | 5 ++ .../cdap/plugin/mariadb/MariadbDBRecord.java | 40 +++++++++++ .../plugin/mariadb/MariadbSchemaReader.java | 36 ++++++++++ .../io/cdap/plugin/mariadb/MariadbSink.java | 15 ++++ .../io/cdap/plugin/mariadb/MariadbSource.java | 46 ++++++++++++ 7 files changed, 208 insertions(+), 72 deletions(-) create mode 100644 mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbDBRecord.java create mode 100644 mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSchemaReader.java diff --git a/mariadb-plugin/docs/Mariadb-batchsink.md b/mariadb-plugin/docs/Mariadb-batchsink.md index 11176c0db..e4541fe67 100644 --- a/mariadb-plugin/docs/Mariadb-batchsink.md +++ b/mariadb-plugin/docs/Mariadb-batchsink.md @@ -60,41 +60,39 @@ connections. Data Types Mapping ---------- - +--------------------------------+-----------------------+------------------------------------+ - | MariaDB Data Type | CDAP Schema Data Type | Comment | - +--------------------------------+-----------------------+------------------------------------+ - | TINYINT | int | | - | BOOLEAN, BOOL | boolean | | - | SMALLINT | int | | - | MEDIUMINT | int | | - | INT, INTEGER | int | | - | BIGINT | long | | - | DECIMAL, DEC, NUMERIC, FIXED | decimal | | - | FLOAT | float | | - | DOUBLE, DOUBLE PRECISION, REAL | decimal | | - | BIT | boolean | | - | CHAR | string | | - | VARCHAR | string | | - | BINARY | bytes | | - | CHAR BYTE | bytes | | - | VARBINARY | bytes | | - | TINYBLOB | bytes | | - | BLOB | bytes | | - | MEDIUMBLOB | bytes | | - | LONGBLOB | bytes | | - | TINYTEXT | string | | - | TEXT | string | | - | MEDIUMTEXT | string | | - | LONGTEXT | string | | - | JSON | string | In MariaDB it is alias to LONGTEXT | - | ENUM | string | Mapping to String by default | - | SET | string | | - | DATE | date | | - | TIME | time_micros | | - | DATETIME | timestamp_micros | | - | TIMESTAMP | timestamp_micros | | - | YEAR | date | | - +--------------------------------+-----------------------+------------------------------------+ + | MariaDB Data Type | CDAP Schema Data Type | Comment | + |--------------------------------|-----------------------|---------------------------------------------------------| + | TINYINT | int | | + | BOOLEAN, BOOL | boolean | | + | SMALLINT | int | | + | MEDIUMINT | int | | + | INT, INTEGER | int | | + | BIGINT | long | | + | DECIMAL, DEC, NUMERIC, FIXED | decimal | | + | FLOAT | float | | + | DOUBLE, DOUBLE PRECISION, REAL | decimal | | + | BIT | boolean | | + | CHAR | string | | + | VARCHAR | string | | + | BINARY | bytes | | + | CHAR BYTE | bytes | | + | VARBINARY | bytes | | + | TINYBLOB | bytes | | + | BLOB | bytes | | + | MEDIUMBLOB | bytes | | + | LONGBLOB | bytes | | + | TINYTEXT | string | | + | TEXT | string | | + | MEDIUMTEXT | string | | + | LONGTEXT | string | | + | JSON | string | In MariaDB it is alias to LONGTEXT | + | ENUM | string | Mapping to String by default | + | SET | string | | + | DATE | date | | + | TIME | time_micros | | + | DATETIME | timestamp_micros | | + | TIMESTAMP | timestamp_micros | | + | YEAR | int | Users can manually set output schema to map it to Date. | Example ------- diff --git a/mariadb-plugin/docs/Mariadb-batchsource.md b/mariadb-plugin/docs/Mariadb-batchsource.md index 2b1fe3944..713af2ee8 100644 --- a/mariadb-plugin/docs/Mariadb-batchsource.md +++ b/mariadb-plugin/docs/Mariadb-batchsource.md @@ -78,43 +78,39 @@ with the tradeoff of higher memory usage. Data Types Mapping ---------- - - +--------------------------------+-----------------------+------------------------------------+ - | MariaDB Data Type | CDAP Schema Data Type | Comment | - +--------------------------------+-----------------------+------------------------------------+ - | TINYINT | int | | - | BOOLEAN, BOOL | boolean | | - | SMALLINT | int | | - | MEDIUMINT | int | | - | INT, INTEGER | int | | - | BIGINT | long | | - | DECIMAL, DEC, NUMERIC, FIXED | decimal | | - | FLOAT | float | | - | DOUBLE, DOUBLE PRECISION, REAL | decimal | | - | BIT | boolean | | - | CHAR | string | | - | VARCHAR | string | | - | BINARY | bytes | | - | CHAR BYTE | bytes | | - | VARBINARY | bytes | | - | TINYBLOB | bytes | | - | BLOB | bytes | | - | MEDIUMBLOB | bytes | | - | LONGBLOB | bytes | | - | TINYTEXT | string | | - | TEXT | string | | - | MEDIUMTEXT | string | | - | LONGTEXT | string | | - | JSON | string | In MariaDB it is alias to LONGTEXT | - | ENUM | string | Mapping to String by default | - | SET | string | | - | DATE | date | | - | TIME | time_micros | | - | DATETIME | timestamp_micros | | - | TIMESTAMP | timestamp_micros | | - | YEAR | date | | - +--------------------------------+-----------------------+------------------------------------+ - + | MariaDB Data Type | CDAP Schema Data Type | Comment | + |--------------------------------|-----------------------|---------------------------------------------------------| + | TINYINT | int | | + | BOOLEAN, BOOL | boolean | | + | SMALLINT | int | | + | MEDIUMINT | int | | + | INT, INTEGER | int | | + | BIGINT | long | | + | DECIMAL, DEC, NUMERIC, FIXED | decimal | | + | FLOAT | float | | + | DOUBLE, DOUBLE PRECISION, REAL | decimal | | + | BIT | boolean | | + | CHAR | string | | + | VARCHAR | string | | + | BINARY | bytes | | + | CHAR BYTE | bytes | | + | VARBINARY | bytes | | + | TINYBLOB | bytes | | + | BLOB | bytes | | + | MEDIUMBLOB | bytes | | + | LONGBLOB | bytes | | + | TINYTEXT | string | | + | TEXT | string | | + | MEDIUMTEXT | string | | + | LONGTEXT | string | | + | JSON | string | In MariaDB it is alias to LONGTEXT | + | ENUM | string | Mapping to String by default | + | SET | string | | + | DATE | date | | + | TIME | time_micros | | + | DATETIME | timestamp_micros | | + | TIMESTAMP | timestamp_micros | | + | YEAR | int | Users can manually set output schema to map it to Date. | Example ------ diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index b82cc7c24..e95253277 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -83,6 +83,11 @@ RELEASE compile + + io.cdap.plugin + mysql-plugin + ${project.version} + diff --git a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbDBRecord.java b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbDBRecord.java new file mode 100644 index 000000000..94498c787 --- /dev/null +++ b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbDBRecord.java @@ -0,0 +1,40 @@ +/* + * Copyright © 2025 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.mariadb; + +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.plugin.db.ColumnType; +import io.cdap.plugin.mysql.MysqlDBRecord; +import java.util.List; + +/** + * Writable class for MariaDB Source/Sink. + */ +public class MariadbDBRecord extends MysqlDBRecord { + + /** + * Used in map-reduce. Do not remove. + */ + @SuppressWarnings("unused") + public MariadbDBRecord() { + // Required by Hadoop DBRecordReader to create an instance + } + + public MariadbDBRecord(StructuredRecord record, List columnTypes) { + super(record, columnTypes); + } +} diff --git a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSchemaReader.java b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSchemaReader.java new file mode 100644 index 000000000..37ac12a93 --- /dev/null +++ b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSchemaReader.java @@ -0,0 +1,36 @@ +/* + * Copyright © 2025 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.mariadb; + + +import io.cdap.plugin.mysql.MysqlSchemaReader; +import java.util.Map; + +/** + * Schema reader for mapping Maria DB type + */ +public class MariadbSchemaReader extends MysqlSchemaReader { + + public MariadbSchemaReader (String sessionID) { + super(sessionID); + } + + public MariadbSchemaReader (String sessionID, Map connectionArguments) { + super(sessionID, connectionArguments); + } + +} diff --git a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSink.java b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSink.java index ab20f3c5d..9dbf0c7d4 100644 --- a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSink.java +++ b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSink.java @@ -19,10 +19,14 @@ import io.cdap.cdap.api.annotation.Description; import io.cdap.cdap.api.annotation.Name; import io.cdap.cdap.api.annotation.Plugin; +import io.cdap.cdap.api.data.format.StructuredRecord; import io.cdap.cdap.etl.api.batch.BatchSink; +import io.cdap.plugin.db.DBRecord; +import io.cdap.plugin.db.SchemaReader; import io.cdap.plugin.db.config.DBSpecificSinkConfig; import io.cdap.plugin.db.sink.AbstractDBSink; +import io.cdap.plugin.mysql.MysqlDBRecord; import java.util.Map; import javax.annotation.Nullable; @@ -45,6 +49,17 @@ public MariadbSink(MariadbSinkConfig mariadbSinkConfig) { this.mariadbSinkConfig = mariadbSinkConfig; } + @Override + protected DBRecord getDBRecord(StructuredRecord output) { + return new MariadbDBRecord(output, columnTypes); + } + + @Override + protected SchemaReader getSchemaReader() { + return new MariadbSchemaReader(null); + } + + /** * MariaDB Sink Config. */ diff --git a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSource.java b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSource.java index d5ffcb290..4a4d689bb 100644 --- a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSource.java +++ b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSource.java @@ -20,9 +20,16 @@ import io.cdap.cdap.api.annotation.Name; import io.cdap.cdap.api.annotation.Plugin; import io.cdap.cdap.etl.api.batch.BatchSource; +import io.cdap.cdap.etl.api.batch.BatchSourceContext; +import io.cdap.plugin.common.Asset; +import io.cdap.plugin.common.LineageRecorder; +import io.cdap.plugin.db.SchemaReader; import io.cdap.plugin.db.config.DBSpecificSourceConfig; import io.cdap.plugin.db.source.AbstractDBSource; +import io.cdap.plugin.util.DBUtils; +import org.apache.hadoop.mapreduce.lib.db.DBWritable; +import java.util.HashMap; import java.util.List; import java.util.Map; import javax.annotation.Nullable; @@ -53,10 +60,36 @@ protected String createConnectionString() { mariadbSourceConfig.host, mariadbSourceConfig.port, mariadbSourceConfig.database); } + @Override + protected Class getDBRecordType() { + return MariadbDBRecord.class; + } + + @Override + protected LineageRecorder getLineageRecorder(BatchSourceContext context) { + String fqn = DBUtils.constructFQN("mariadb", + mariadbSourceConfig.host, + mariadbSourceConfig.port, + mariadbSourceConfig.database, + mariadbSourceConfig.getReferenceName()); + Asset asset = Asset.builder(mariadbSourceConfig.getReferenceName()).setFqn(fqn).build(); + return new LineageRecorder(context, asset); + } + + @Override + protected SchemaReader getSchemaReader() { + return new MariadbSchemaReader(null, mariadbSourceConfig.getConnectionArguments()); + } + /** * MaraiDB source mariadbSourceConfig. */ public static class MariadbSourceConfig extends DBSpecificSourceConfig { + private static final String JDBC_PROPERTY_CONNECT_TIMEOUT = "connectTimeout"; + private static final String JDBC_PROPERTY_SOCKET_TIMEOUT = "socketTimeout"; + private static final String JDBC_REWRITE_BATCHED_STATEMENTS = "rewriteBatchedStatements"; + + private static final String MARIADB_TINYINT1_IS_BIT = "tinyInt1isBit"; @Name(MariadbConstants.AUTO_RECONNECT) @Description("Should the driver try to re-establish stale and/or dead connections") @@ -116,5 +149,18 @@ public Map getDBSpecificArguments() { public List getInitQueries() { return MariadbUtil.composeDbInitQueries(useAnsiQuotes); } + + @Override + public Map getConnectionArguments() { + Map arguments = new HashMap<>(super.getConnectionArguments()); + // the unit below is millisecond + arguments.putIfAbsent(JDBC_PROPERTY_CONNECT_TIMEOUT, "20000"); + arguments.putIfAbsent(JDBC_PROPERTY_SOCKET_TIMEOUT, "20000"); + arguments.putIfAbsent(JDBC_REWRITE_BATCHED_STATEMENTS, "true"); + // MariaDB property to ensure that TINYINT(1) type data is not converted to MariaDB Bit/Boolean type in the + // ResultSet. + arguments.putIfAbsent(MARIADB_TINYINT1_IS_BIT, "false"); + return arguments; + } } } From 948e481c151dc3b5bc91c8c2ce2ce8676951e14a Mon Sep 17 00:00:00 2001 From: psainics Date: Wed, 16 Apr 2025 13:56:47 +0530 Subject: [PATCH 24/52] Release version 1.11.6 for database plugins --- amazon-redshift-plugin/pom.xml | 2 +- aurora-mysql-plugin/pom.xml | 2 +- aurora-postgresql-plugin/pom.xml | 2 +- cloudsql-mysql-plugin/pom.xml | 2 +- cloudsql-postgresql-plugin/pom.xml | 2 +- database-commons/pom.xml | 2 +- db2-plugin/pom.xml | 2 +- generic-database-plugin/pom.xml | 2 +- generic-db-argument-setter/pom.xml | 2 +- mariadb-plugin/pom.xml | 2 +- memsql-plugin/pom.xml | 2 +- mssql-plugin/pom.xml | 2 +- mysql-plugin/pom.xml | 2 +- netezza-plugin/pom.xml | 2 +- oracle-plugin/pom.xml | 2 +- pom.xml | 2 +- postgresql-plugin/pom.xml | 2 +- saphana-plugin/pom.xml | 2 +- teradata-plugin/pom.xml | 2 +- 19 files changed, 19 insertions(+), 19 deletions(-) diff --git a/amazon-redshift-plugin/pom.xml b/amazon-redshift-plugin/pom.xml index 573765c78..5d0b788d1 100644 --- a/amazon-redshift-plugin/pom.xml +++ b/amazon-redshift-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.6-SNAPSHOT + 1.11.6 Amazon Redshift plugin diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index 3614b2ebf..cc927e8e5 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.6-SNAPSHOT + 1.11.6 Aurora DB MySQL plugin diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index 0b1d0724e..dc9c0091a 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.6-SNAPSHOT + 1.11.6 Aurora DB PostgreSQL plugin diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index 96f19944c..fde73c2d4 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.6-SNAPSHOT + 1.11.6 CloudSQL MySQL plugin diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index 90566bf65..ee09065c0 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.6-SNAPSHOT + 1.11.6 CloudSQL PostgreSQL plugin diff --git a/database-commons/pom.xml b/database-commons/pom.xml index ceb3f8208..3c07e1b5f 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.6-SNAPSHOT + 1.11.6 Database Commons diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index f25f6fe15..407c140f5 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.6-SNAPSHOT + 1.11.6 IBM DB2 plugin diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index 593bff82f..dc3c1db58 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.6-SNAPSHOT + 1.11.6 Generic database plugin diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index af5fc943d..eafa25291 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.6-SNAPSHOT + 1.11.6 Generic database argument setter plugin diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index e95253277..c2600591e 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.6-SNAPSHOT + 1.11.6 Maria DB plugin diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index 40918105e..ea5105e45 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.6-SNAPSHOT + 1.11.6 Memsql plugin diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index 57d4874ac..1877cafc2 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.6-SNAPSHOT + 1.11.6 Microsoft SQL Server plugin diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index 1266823b7..fe3941273 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.6-SNAPSHOT + 1.11.6 Mysql plugin diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index bdfab69b8..700c32da3 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.6-SNAPSHOT + 1.11.6 Netezza plugin diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index 45873a221..172878e10 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.6-SNAPSHOT + 1.11.6 Oracle plugin diff --git a/pom.xml b/pom.xml index ec91f1faa..a29f6efe8 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin database-plugins-parent - 1.11.6-SNAPSHOT + 1.11.6 pom Database Plugins Collection of database plugins diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index d55da093a..8815eeb92 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.6-SNAPSHOT + 1.11.6 PostgreSQL plugin diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index 9823860cc..65d8590b5 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.6-SNAPSHOT + 1.11.6 SAP HANA plugin diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index 0b315c273..2def186b9 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -21,7 +21,7 @@ database-plugins-parent io.cdap.plugin - 1.11.6-SNAPSHOT + 1.11.6 teradata-plugin From a9ccbfde514682578d24f9f0153c83c90f31c47c Mon Sep 17 00:00:00 2001 From: Sanchit Garg Date: Fri, 2 May 2025 15:34:25 +0530 Subject: [PATCH 25/52] Bump version from 1.11.6 to 1.11.7-SNAPSHOT --- amazon-redshift-plugin/pom.xml | 2 +- aurora-mysql-plugin/pom.xml | 2 +- aurora-postgresql-plugin/pom.xml | 2 +- cloudsql-mysql-plugin/pom.xml | 2 +- cloudsql-postgresql-plugin/pom.xml | 2 +- database-commons/pom.xml | 2 +- db2-plugin/pom.xml | 2 +- generic-database-plugin/pom.xml | 2 +- generic-db-argument-setter/pom.xml | 2 +- mariadb-plugin/pom.xml | 2 +- memsql-plugin/pom.xml | 2 +- mssql-plugin/pom.xml | 2 +- mysql-plugin/pom.xml | 2 +- netezza-plugin/pom.xml | 2 +- oracle-plugin/pom.xml | 2 +- pom.xml | 2 +- postgresql-plugin/pom.xml | 2 +- saphana-plugin/pom.xml | 2 +- teradata-plugin/pom.xml | 2 +- 19 files changed, 19 insertions(+), 19 deletions(-) diff --git a/amazon-redshift-plugin/pom.xml b/amazon-redshift-plugin/pom.xml index 5d0b788d1..41ab5084b 100644 --- a/amazon-redshift-plugin/pom.xml +++ b/amazon-redshift-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.6 + 1.11.7-SNAPSHOT Amazon Redshift plugin diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index cc927e8e5..cdb407152 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.6 + 1.11.7-SNAPSHOT Aurora DB MySQL plugin diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index dc9c0091a..e5ebd788e 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.6 + 1.11.7-SNAPSHOT Aurora DB PostgreSQL plugin diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index fde73c2d4..047637657 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.6 + 1.11.7-SNAPSHOT CloudSQL MySQL plugin diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index ee09065c0..8a709d399 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.6 + 1.11.7-SNAPSHOT CloudSQL PostgreSQL plugin diff --git a/database-commons/pom.xml b/database-commons/pom.xml index 3c07e1b5f..c96954258 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.6 + 1.11.7-SNAPSHOT Database Commons diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index 407c140f5..fff1f51e0 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.6 + 1.11.7-SNAPSHOT IBM DB2 plugin diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index dc3c1db58..fcd4de36a 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.6 + 1.11.7-SNAPSHOT Generic database plugin diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index eafa25291..54868c1f3 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.6 + 1.11.7-SNAPSHOT Generic database argument setter plugin diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index c2600591e..e0f9090b0 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.6 + 1.11.7-SNAPSHOT Maria DB plugin diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index ea5105e45..6b3bc281a 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.6 + 1.11.7-SNAPSHOT Memsql plugin diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index 1877cafc2..bbe53e7df 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.6 + 1.11.7-SNAPSHOT Microsoft SQL Server plugin diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index fe3941273..f04564a42 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.6 + 1.11.7-SNAPSHOT Mysql plugin diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index 700c32da3..2647adf76 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.6 + 1.11.7-SNAPSHOT Netezza plugin diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index 172878e10..a53d72fcf 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.6 + 1.11.7-SNAPSHOT Oracle plugin diff --git a/pom.xml b/pom.xml index a29f6efe8..6370371b7 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin database-plugins-parent - 1.11.6 + 1.11.7-SNAPSHOT pom Database Plugins Collection of database plugins diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index 8815eeb92..5f720a5c3 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.6 + 1.11.7-SNAPSHOT PostgreSQL plugin diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index 65d8590b5..1862b86d1 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.6 + 1.11.7-SNAPSHOT SAP HANA plugin diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index 2def186b9..d8a71d087 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -21,7 +21,7 @@ database-plugins-parent io.cdap.plugin - 1.11.6 + 1.11.7-SNAPSHOT teradata-plugin From 2171550536a7d68cd643769081b4dfaba7a58040 Mon Sep 17 00:00:00 2001 From: psainics Date: Fri, 2 May 2025 08:33:23 +0530 Subject: [PATCH 26/52] Skip field validation for compatiblity --- .../plugin/db/source/AbstractDBSource.java | 24 ++++++++++------- .../db/source/AbstractDBSourceTest.java | 12 ++++++--- .../mariadb/MariadbFieldsValidator.java | 25 +++++++++++++++++ .../io/cdap/plugin/mariadb/MariadbSink.java | 7 ++++- .../io/cdap/plugin/mariadb/MariadbSource.java | 27 +++++++++++++++++++ 5 files changed, 82 insertions(+), 13 deletions(-) create mode 100644 mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbFieldsValidator.java diff --git a/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java b/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java index 987b5cc17..8eeb4a155 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java @@ -484,7 +484,7 @@ public void validateSchema(Schema actualSchema, FailureCollector collector) { } @VisibleForTesting - static void validateSchema(Schema actualSchema, Schema configSchema, FailureCollector collector) { + void validateSchema(Schema actualSchema, Schema configSchema, FailureCollector collector) { if (configSchema == null) { collector.addFailure("Schema should not be null or empty.", null) .withConfigProperty(SCHEMA); @@ -505,14 +505,20 @@ static void validateSchema(Schema actualSchema, Schema configSchema, FailureColl Schema expectedFieldSchema = field.getSchema().isNullable() ? field.getSchema().getNonNullable() : field.getSchema(); - if (actualFieldSchema.getType() != expectedFieldSchema.getType() || - actualFieldSchema.getLogicalType() != expectedFieldSchema.getLogicalType()) { - collector.addFailure( - String.format("Schema field '%s' has type '%s but found '%s'.", - field.getName(), expectedFieldSchema.getDisplayName(), - actualFieldSchema.getDisplayName()), null) - .withOutputSchemaField(field.getName()); - } + validateField(collector, field, actualFieldSchema, expectedFieldSchema); + } + } + + protected void validateField(FailureCollector collector, Schema.Field field, Schema actualFieldSchema, + Schema expectedFieldSchema) { + if (actualFieldSchema.getType() != expectedFieldSchema.getType() || + actualFieldSchema.getLogicalType() != expectedFieldSchema.getLogicalType()) { + collector.addFailure( + String.format("Schema field '%s' is expected to have type '%s but found '%s'.", field.getName(), + expectedFieldSchema.getDisplayName(), actualFieldSchema.getDisplayName()), + String.format("Change the data type of field %s to %s.", field.getName(), + actualFieldSchema.getDisplayName())) + .withOutputSchemaField(field.getName()); } } diff --git a/database-commons/src/test/java/io/cdap/plugin/db/source/AbstractDBSourceTest.java b/database-commons/src/test/java/io/cdap/plugin/db/source/AbstractDBSourceTest.java index 3dc7a2d1c..a8be38b46 100644 --- a/database-commons/src/test/java/io/cdap/plugin/db/source/AbstractDBSourceTest.java +++ b/database-commons/src/test/java/io/cdap/plugin/db/source/AbstractDBSourceTest.java @@ -43,11 +43,17 @@ public class AbstractDBSourceTest { Schema.Field.of("double_column", Schema.nullableOf(Schema.of(Schema.Type.DOUBLE))), Schema.Field.of("boolean_column", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN))) ); + private static final AbstractDBSource.DBSourceConfig TEST_CONFIG = new AbstractDBSource.DBSourceConfig() { + @Override + public String getConnectionString() { + return ""; + } + }; @Test public void testValidateSourceSchemaCorrectSchema() { MockFailureCollector collector = new MockFailureCollector(MOCK_STAGE); - AbstractDBSource.DBSourceConfig.validateSchema(SCHEMA, SCHEMA, collector); + TEST_CONFIG.validateSchema(SCHEMA, SCHEMA, collector); Assert.assertEquals(0, collector.getValidationFailures().size()); } @@ -65,7 +71,7 @@ public void testValidateSourceSchemaMismatchFields() { ); MockFailureCollector collector = new MockFailureCollector(MOCK_STAGE); - AbstractDBSource.DBSourceConfig.validateSchema(actualSchema, SCHEMA, collector); + TEST_CONFIG.validateSchema(actualSchema, SCHEMA, collector); assertPropertyValidationFailed(collector, "boolean_column"); } @@ -84,7 +90,7 @@ public void testValidateSourceSchemaInvalidFieldType() { ); MockFailureCollector collector = new MockFailureCollector(MOCK_STAGE); - AbstractDBSource.DBSourceConfig.validateSchema(actualSchema, SCHEMA, collector); + TEST_CONFIG.validateSchema(actualSchema, SCHEMA, collector); assertPropertyValidationFailed(collector, "boolean_column"); } diff --git a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbFieldsValidator.java b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbFieldsValidator.java new file mode 100644 index 000000000..71ccb0d06 --- /dev/null +++ b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbFieldsValidator.java @@ -0,0 +1,25 @@ +/* + * Copyright © 2025 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.mariadb; + +import io.cdap.plugin.mysql.MysqlFieldsValidator; + +/** + * Field validator for maraidb + */ +public class MariadbFieldsValidator extends MysqlFieldsValidator { +} diff --git a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSink.java b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSink.java index 9dbf0c7d4..57455cbdc 100644 --- a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSink.java +++ b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSink.java @@ -25,8 +25,8 @@ import io.cdap.plugin.db.SchemaReader; import io.cdap.plugin.db.config.DBSpecificSinkConfig; import io.cdap.plugin.db.sink.AbstractDBSink; +import io.cdap.plugin.db.sink.FieldsValidator; -import io.cdap.plugin.mysql.MysqlDBRecord; import java.util.Map; import javax.annotation.Nullable; @@ -60,6 +60,11 @@ protected SchemaReader getSchemaReader() { } + @Override + protected FieldsValidator getFieldsValidator() { + return new MariadbFieldsValidator(); + } + /** * MariaDB Sink Config. */ diff --git a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSource.java b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSource.java index 4a4d689bb..3a473dca7 100644 --- a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSource.java +++ b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSource.java @@ -19,6 +19,8 @@ import io.cdap.cdap.api.annotation.Description; import io.cdap.cdap.api.annotation.Name; import io.cdap.cdap.api.annotation.Plugin; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.cdap.etl.api.FailureCollector; import io.cdap.cdap.etl.api.batch.BatchSource; import io.cdap.cdap.etl.api.batch.BatchSourceContext; import io.cdap.plugin.common.Asset; @@ -162,5 +164,30 @@ public Map getConnectionArguments() { arguments.putIfAbsent(MARIADB_TINYINT1_IS_BIT, "false"); return arguments; } + + @Override + protected void validateField(FailureCollector collector, + Schema.Field field, + Schema actualFieldSchema, + Schema expectedFieldSchema) { + // Backward compatibility changes to support MySQL YEAR to Date type conversion + if (Schema.LogicalType.DATE.equals(expectedFieldSchema.getLogicalType()) + && Schema.Type.INT.equals(actualFieldSchema.getType())) { + return; + } + + // Backward compatibility change to support MySQL MEDIUMINT UNSIGNED to Long type conversion + if (Schema.Type.LONG.equals(expectedFieldSchema.getType()) + && Schema.Type.INT.equals(actualFieldSchema.getType())) { + return; + } + + // Backward compatibility change to support MySQL TINYINT(1) to Bool type conversion + if (Schema.Type.BOOLEAN.equals(expectedFieldSchema.getType()) + && Schema.Type.INT.equals(actualFieldSchema.getType())) { + return; + } + super.validateField(collector, field, actualFieldSchema, expectedFieldSchema); + } } } From 7959157ea9da4e36ba51d2145912dbac730dd477 Mon Sep 17 00:00:00 2001 From: psainics Date: Mon, 21 Apr 2025 15:27:42 +0530 Subject: [PATCH 27/52] Set TRANSACTION_ISOLATION_LEVEL config in MySQL, PostgreSQL & MSSQL plugins Add transaction isolation level to PostgreSQL, MYSQL and MSSQL Plugins. --- .../io/cdap/plugin/db/ConnectionConfig.java | 1 + .../AbstractDBSpecificConnectorConfig.java | 26 ++++++++++++++++++- mssql-plugin/docs/SQL Server-connector.md | 8 ++++++ mssql-plugin/docs/SqlServer-batchsink.md | 8 ++++++ mssql-plugin/docs/SqlServer-batchsource.md | 8 ++++++ .../io/cdap/plugin/mssql/SqlServerSink.java | 5 ++++ .../io/cdap/plugin/mssql/SqlServerSource.java | 5 ++++ .../widgets/SQL Server-connector.json | 14 ++++++++++ mssql-plugin/widgets/SqlServer-batchsink.json | 18 +++++++++++++ .../widgets/SqlServer-batchsource.json | 18 +++++++++++++ mysql-plugin/docs/MySQL-connector.md | 8 ++++++ mysql-plugin/docs/Mysql-batchsink.md | 8 ++++++ mysql-plugin/docs/Mysql-batchsource.md | 8 ++++++ .../java/io/cdap/plugin/mysql/MysqlSink.java | 5 ++++ .../io/cdap/plugin/mysql/MysqlSource.java | 5 ++++ mysql-plugin/widgets/MySQL-connector.json | 14 ++++++++++ mysql-plugin/widgets/Mysql-batchsink.json | 18 +++++++++++++ mysql-plugin/widgets/Mysql-batchsource.json | 18 +++++++++++++ .../plugin/oracle/OracleConnectorConfig.java | 18 ++----------- .../docs/PostgreSQL-connector.md | 8 ++++++ postgresql-plugin/docs/Postgres-batchsink.md | 8 ++++++ .../docs/Postgres-batchsource.md | 8 ++++++ .../io/cdap/plugin/postgres/PostgresSink.java | 5 ++++ .../cdap/plugin/postgres/PostgresSource.java | 5 ++++ .../widgets/PostgreSQL-connector.json | 13 ++++++++++ .../widgets/Postgres-batchsink.json | 17 ++++++++++++ .../widgets/Postgres-batchsource.json | 17 ++++++++++++ 27 files changed, 277 insertions(+), 17 deletions(-) diff --git a/database-commons/src/main/java/io/cdap/plugin/db/ConnectionConfig.java b/database-commons/src/main/java/io/cdap/plugin/db/ConnectionConfig.java index 588ed78b8..c5320e25e 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/ConnectionConfig.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/ConnectionConfig.java @@ -45,6 +45,7 @@ public abstract class ConnectionConfig extends PluginConfig implements DatabaseC public static final String CONNECTION_ARGUMENTS = "connectionArguments"; public static final String JDBC_PLUGIN_NAME = "jdbcPluginName"; public static final String JDBC_PLUGIN_TYPE = "jdbc"; + public static final String TRANSACTION_ISOLATION_LEVEL = "transactionIsolationLevel"; @Name(JDBC_PLUGIN_NAME) @Description("Name of the JDBC driver to use. This is the value of the 'jdbcPluginName' key defined in the JSON " + diff --git a/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnectorConfig.java b/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnectorConfig.java index 5c6b08031..8de0e4d70 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnectorConfig.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnectorConfig.java @@ -20,8 +20,9 @@ import io.cdap.cdap.api.annotation.Macro; import io.cdap.cdap.api.annotation.Name; import io.cdap.plugin.db.ConnectionConfig; +import io.cdap.plugin.db.TransactionIsolationLevel; -import java.util.Collections; +import java.util.HashMap; import java.util.Map; import javax.annotation.Nullable; @@ -42,6 +43,12 @@ public abstract class AbstractDBSpecificConnectorConfig extends AbstractDBConnec @Nullable protected Integer port; + @Name(ConnectionConfig.TRANSACTION_ISOLATION_LEVEL) + @Description("The transaction isolation level for the database session.") + @Macro + @Nullable + protected String transactionIsolationLevel; + public String getHost() { return host; } @@ -55,4 +62,21 @@ public int getPort() { public boolean canConnect() { return super.canConnect() && !containsMacro(ConnectionConfig.HOST) && !containsMacro(ConnectionConfig.PORT); } + + @Override + public Map getAdditionalArguments() { + Map additonalArguments = new HashMap<>(); + if (getTransactionIsolationLevel() != null) { + additonalArguments.put(TransactionIsolationLevel.CONF_KEY, getTransactionIsolationLevel()); + } + return additonalArguments; + } + + public String getTransactionIsolationLevel() { + if (transactionIsolationLevel == null) { + return null; + } + return TransactionIsolationLevel.Level.valueOf(transactionIsolationLevel).name(); + } } + diff --git a/mssql-plugin/docs/SQL Server-connector.md b/mssql-plugin/docs/SQL Server-connector.md index cb72161f5..6f0038715 100644 --- a/mssql-plugin/docs/SQL Server-connector.md +++ b/mssql-plugin/docs/SQL Server-connector.md @@ -22,6 +22,14 @@ authentication. Optional for databases that do not require authentication. **Password:** Password to use to connect to the specified database. +**Transaction Isolation Level** The transaction isolation level of the database connection +- TRANSACTION_READ_COMMITTED: No dirty reads. Non-repeatable reads and phantom reads are possible. +- TRANSACTION_SERIALIZABLE: No dirty reads. Non-repeatable and phantom reads are prevented. +- TRANSACTION_REPEATABLE_READ: No dirty reads. Prevents non-repeatable reads, but phantom reads are still possible. +- TRANSACTION_READ_UNCOMMITTED: Allows dirty reads (reading uncommitted changes from other transactions). Non-repeatable reads and phantom reads are possible. + +For more details on the Transaction Isolation Levels supported in SQL Server, refer to the [SQL Server documentation](https://learn.microsoft.com/en-us/sql/t-sql/statements/set-transaction-isolation-level-transact-sql?view=sql-server-ver16) + **Authentication Type:** Indicates which authentication method will be used for the connection. Use 'SQL Login'. to connect to a SQL Server using username and password properties. Use 'Active Directory Password' to connect to an Azure SQL Database/Data Warehouse using an Azure AD principal name and password. diff --git a/mssql-plugin/docs/SqlServer-batchsink.md b/mssql-plugin/docs/SqlServer-batchsink.md index 5d10b4bb6..b4ca1cbc5 100644 --- a/mssql-plugin/docs/SqlServer-batchsink.md +++ b/mssql-plugin/docs/SqlServer-batchsink.md @@ -46,6 +46,14 @@ an Azure SQL Database/Data Warehouse using an Azure AD principal name and passwo **Password:** Password to use to connect to the specified database. +**Transaction Isolation Level** The transaction isolation level of the database connection +- TRANSACTION_READ_COMMITTED: No dirty reads. Non-repeatable reads and phantom reads are possible. +- TRANSACTION_SERIALIZABLE: No dirty reads. Non-repeatable and phantom reads are prevented. +- TRANSACTION_REPEATABLE_READ: No dirty reads. Prevents non-repeatable reads, but phantom reads are still possible. +- TRANSACTION_READ_UNCOMMITTED: Allows dirty reads (reading uncommitted changes from other transactions). Non-repeatable reads and phantom reads are possible. + +For more details on the Transaction Isolation Levels supported in SQL Server, refer to the [SQL Server documentation](https://learn.microsoft.com/en-us/sql/t-sql/statements/set-transaction-isolation-level-transact-sql?view=sql-server-ver16) + **Instance Name:** SQL Server instance name to connect to. When it is not specified, a connection is made to the default instance. For the case where both the instanceName and port are specified, see the notes for port. If you specify a Virtual Network Name in the Server connection property, you cannot diff --git a/mssql-plugin/docs/SqlServer-batchsource.md b/mssql-plugin/docs/SqlServer-batchsource.md index c8e30f77e..5c917621c 100644 --- a/mssql-plugin/docs/SqlServer-batchsource.md +++ b/mssql-plugin/docs/SqlServer-batchsource.md @@ -56,6 +56,14 @@ an Azure SQL Database/Data Warehouse using an Azure AD principal name and passwo **Password:** Password to use to connect to the specified database. +**Transaction Isolation Level** The transaction isolation level of the database connection +- TRANSACTION_READ_COMMITTED: No dirty reads. Non-repeatable reads and phantom reads are possible. +- TRANSACTION_SERIALIZABLE: No dirty reads. Non-repeatable and phantom reads are prevented. +- TRANSACTION_REPEATABLE_READ: No dirty reads. Prevents non-repeatable reads, but phantom reads are still possible. +- TRANSACTION_READ_UNCOMMITTED: Allows dirty reads (reading uncommitted changes from other transactions). Non-repeatable reads and phantom reads are possible. + +For more details on the Transaction Isolation Levels supported in SQL Server, refer to the [SQL Server documentation](https://learn.microsoft.com/en-us/sql/t-sql/statements/set-transaction-isolation-level-transact-sql?view=sql-server-ver16) + **Instance Name:** SQL Server instance name to connect to. When it is not specified, a connection is made to the default instance. For the case where both the instanceName and port are specified, see the notes for port. If you specify a Virtual Network Name in the Server connection property, you cannot diff --git a/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSink.java b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSink.java index 0fa8991c5..7b749cdc5 100644 --- a/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSink.java +++ b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSink.java @@ -167,6 +167,11 @@ public Map getDBSpecificArguments() { packetSize, queryTimeout); } + @Override + public String getTransactionIsolationLevel() { + return connection.getTransactionIsolationLevel(); + } + @Override public String getConnectionString() { return String.format(SqlServerConstants.SQL_SERVER_CONNECTION_STRING_FORMAT, diff --git a/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSource.java b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSource.java index 9603b24db..a76ed732d 100644 --- a/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSource.java +++ b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSource.java @@ -188,6 +188,11 @@ public List getInitQueries() { return Collections.emptyList(); } + @Override + public String getTransactionIsolationLevel() { + return connection.getTransactionIsolationLevel(); + } + @Override public void validate(FailureCollector collector) { ConfigUtil.validateConnection(this, useConnection, connection, collector); diff --git a/mssql-plugin/widgets/SQL Server-connector.json b/mssql-plugin/widgets/SQL Server-connector.json index 171076295..c326cd81d 100644 --- a/mssql-plugin/widgets/SQL Server-connector.json +++ b/mssql-plugin/widgets/SQL Server-connector.json @@ -64,6 +64,20 @@ "widget-type": "password", "label": "Password", "name": "password" + }, + { + "widget-type": "select", + "label": "Transaction Isolation Level", + "name": "transactionIsolationLevel", + "widget-attributes": { + "values": [ + "TRANSACTION_READ_UNCOMMITTED", + "TRANSACTION_READ_COMMITTED", + "TRANSACTION_REPEATABLE_READ", + "TRANSACTION_SERIALIZABLE" + ], + "default": "TRANSACTION_SERIALIZABLE" + } } ] }, diff --git a/mssql-plugin/widgets/SqlServer-batchsink.json b/mssql-plugin/widgets/SqlServer-batchsink.json index 260c66259..fb20cad9d 100644 --- a/mssql-plugin/widgets/SqlServer-batchsink.json +++ b/mssql-plugin/widgets/SqlServer-batchsink.json @@ -84,6 +84,20 @@ "label": "Password", "name": "password" }, + { + "widget-type": "select", + "label": "Transaction Isolation Level", + "name": "transactionIsolationLevel", + "widget-attributes": { + "values": [ + "TRANSACTION_READ_UNCOMMITTED", + "TRANSACTION_READ_COMMITTED", + "TRANSACTION_REPEATABLE_READ", + "TRANSACTION_SERIALIZABLE" + ], + "default": "TRANSACTION_SERIALIZABLE" + } + }, { "widget-type": "keyvalue", "label": "Connection Arguments", @@ -280,6 +294,10 @@ { "type": "property", "name": "connectionArguments" + }, + { + "type": "property", + "name": "transactionIsolationLevel" } ] }, diff --git a/mssql-plugin/widgets/SqlServer-batchsource.json b/mssql-plugin/widgets/SqlServer-batchsource.json index dad5f4708..b3494e485 100644 --- a/mssql-plugin/widgets/SqlServer-batchsource.json +++ b/mssql-plugin/widgets/SqlServer-batchsource.json @@ -84,6 +84,20 @@ "label": "Password", "name": "password" }, + { + "widget-type": "select", + "label": "Transaction Isolation Level", + "name": "transactionIsolationLevel", + "widget-attributes": { + "values": [ + "TRANSACTION_READ_UNCOMMITTED", + "TRANSACTION_READ_COMMITTED", + "TRANSACTION_REPEATABLE_READ", + "TRANSACTION_SERIALIZABLE" + ], + "default": "TRANSACTION_SERIALIZABLE" + } + }, { "widget-type": "keyvalue", "label": "Connection Arguments", @@ -316,6 +330,10 @@ { "type": "property", "name": "connectionArguments" + }, + { + "type": "property", + "name": "transactionIsolationLevel" } ] }, diff --git a/mysql-plugin/docs/MySQL-connector.md b/mysql-plugin/docs/MySQL-connector.md index fb5c1fbb8..f586084c1 100644 --- a/mysql-plugin/docs/MySQL-connector.md +++ b/mysql-plugin/docs/MySQL-connector.md @@ -22,6 +22,14 @@ authentication. Optional for databases that do not require authentication. **Password:** Password to use to connect to the specified database. +**Transaction Isolation Level** The transaction isolation level of the databse connection +- TRANSACTION_READ_COMMITTED: No dirty reads. Non-repeatable reads and phantom reads are possible. +- TRANSACTION_SERIALIZABLE: No dirty reads. Non-repeatable and phantom reads are prevented. +- TRANSACTION_REPEATABLE_READ: No dirty reads. Prevents non-repeatable reads, but phantom reads are still possible. +- TRANSACTION_READ_UNCOMMITTED: Allows dirty reads (reading uncommitted changes from other transactions). Non-repeatable reads and phantom reads are possible. + +For more details on the Transaction Isolation Levels supported in MySQL, refer to the [MySQL documentation](https://dev.mysql.com/doc/refman/8.4/en/innodb-transaction-isolation-levels.html) + **Connection Arguments:** A list of arbitrary string tag/value pairs as connection arguments. These arguments will be passed to the JDBC driver, as connection arguments, for JDBC drivers that may need additional configurations. This is a semicolon-separated list of key-value pairs, where each pair is separated by a equals '=' and specifies diff --git a/mysql-plugin/docs/Mysql-batchsink.md b/mysql-plugin/docs/Mysql-batchsink.md index b28a28618..46a763f9d 100644 --- a/mysql-plugin/docs/Mysql-batchsink.md +++ b/mysql-plugin/docs/Mysql-batchsink.md @@ -39,6 +39,14 @@ You also can use the macro function ${conn(connection-name)}. **Password:** Password to use to connect to the specified database. +**Transaction Isolation Level** The transaction isolation level of the databse connection +- TRANSACTION_READ_COMMITTED: No dirty reads. Non-repeatable reads and phantom reads are possible. +- TRANSACTION_SERIALIZABLE: No dirty reads. Non-repeatable and phantom reads are prevented. +- TRANSACTION_REPEATABLE_READ: No dirty reads. Prevents non-repeatable reads, but phantom reads are still possible. +- TRANSACTION_READ_UNCOMMITTED: Allows dirty reads (reading uncommitted changes from other transactions). Non-repeatable reads and phantom reads are possible. + +For more details on the Transaction Isolation Levels supported in MySQL, refer to the [MySQL documentation](https://dev.mysql.com/doc/refman/8.4/en/innodb-transaction-isolation-levels.html) + **Connection Arguments:** A list of arbitrary string key/value pairs as connection arguments. These arguments will be passed to the JDBC driver as connection arguments for JDBC drivers that may need additional configurations. diff --git a/mysql-plugin/docs/Mysql-batchsource.md b/mysql-plugin/docs/Mysql-batchsource.md index 010e08216..552bb5504 100644 --- a/mysql-plugin/docs/Mysql-batchsource.md +++ b/mysql-plugin/docs/Mysql-batchsource.md @@ -49,6 +49,14 @@ For example, 'SELECT MIN(id),MAX(id) FROM table'. Not required if numSplits is s **Password:** Password to use to connect to the specified database. +**Transaction Isolation Level** The transaction isolation level of the database connection +- TRANSACTION_READ_COMMITTED: No dirty reads. Non-repeatable reads and phantom reads are possible. +- TRANSACTION_SERIALIZABLE: No dirty reads. Non-repeatable and phantom reads are prevented. +- TRANSACTION_REPEATABLE_READ: No dirty reads. Prevents non-repeatable reads, but phantom reads are still possible. +- TRANSACTION_READ_UNCOMMITTED: Allows dirty reads (reading uncommitted changes from other transactions). Non-repeatable reads and phantom reads are possible. + +For more details on the Transaction Isolation Levels supported in MySQL, refer to the [MySQL documentation](https://dev.mysql.com/doc/refman/8.4/en/innodb-transaction-isolation-levels.html) + **Connection Arguments:** A list of arbitrary string key/value pairs as connection arguments. These arguments will be passed to the JDBC driver as connection arguments for JDBC drivers that may need additional configurations. diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSink.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSink.java index f71371026..bf3b6fe5b 100644 --- a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSink.java +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSink.java @@ -184,6 +184,11 @@ public Map getDBSpecificArguments() { trustCertificateKeyStorePassword, false); } + @Override + public String getTransactionIsolationLevel() { + return connection.getTransactionIsolationLevel(); + } + @Override public MysqlConnectorConfig getConnection() { return connection; diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java index a91139196..b8fd3975c 100644 --- a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java @@ -187,6 +187,11 @@ public MysqlConnectorConfig getConnection() { return connection; } + @Override + public String getTransactionIsolationLevel() { + return connection.getTransactionIsolationLevel(); + } + @Override public void validate(FailureCollector collector) { ConfigUtil.validateConnection(this, useConnection, connection, collector); diff --git a/mysql-plugin/widgets/MySQL-connector.json b/mysql-plugin/widgets/MySQL-connector.json index 9064d1bf6..f60f5526f 100644 --- a/mysql-plugin/widgets/MySQL-connector.json +++ b/mysql-plugin/widgets/MySQL-connector.json @@ -30,6 +30,20 @@ "widget-attributes": { "default": "3306" } + }, + { + "widget-type": "select", + "label": "Transaction Isolation Level", + "name": "transactionIsolationLevel", + "widget-attributes": { + "values": [ + "TRANSACTION_READ_UNCOMMITTED", + "TRANSACTION_READ_COMMITTED", + "TRANSACTION_REPEATABLE_READ", + "TRANSACTION_SERIALIZABLE" + ], + "default": "TRANSACTION_SERIALIZABLE" + } } ] }, diff --git a/mysql-plugin/widgets/Mysql-batchsink.json b/mysql-plugin/widgets/Mysql-batchsink.json index c525ead40..58596aae2 100644 --- a/mysql-plugin/widgets/Mysql-batchsink.json +++ b/mysql-plugin/widgets/Mysql-batchsink.json @@ -65,6 +65,20 @@ "label": "Password", "name": "password" }, + { + "widget-type": "select", + "label": "Transaction Isolation Level", + "name": "transactionIsolationLevel", + "widget-attributes": { + "values": [ + "TRANSACTION_READ_UNCOMMITTED", + "TRANSACTION_READ_COMMITTED", + "TRANSACTION_REPEATABLE_READ", + "TRANSACTION_SERIALIZABLE" + ], + "default": "TRANSACTION_SERIALIZABLE" + } + }, { "widget-type": "keyvalue", "label": "Connection Arguments", @@ -225,6 +239,10 @@ "type": "property", "name": "password" }, + { + "type": "property", + "name": "transactionIsolationLevel" + }, { "type": "property", "name": "host" diff --git a/mysql-plugin/widgets/Mysql-batchsource.json b/mysql-plugin/widgets/Mysql-batchsource.json index 9175bd5ed..506e837f7 100644 --- a/mysql-plugin/widgets/Mysql-batchsource.json +++ b/mysql-plugin/widgets/Mysql-batchsource.json @@ -65,6 +65,20 @@ "label": "Password", "name": "password" }, + { + "widget-type": "select", + "label": "Transaction Isolation Level", + "name": "transactionIsolationLevel", + "widget-attributes": { + "values": [ + "TRANSACTION_READ_UNCOMMITTED", + "TRANSACTION_READ_COMMITTED", + "TRANSACTION_REPEATABLE_READ", + "TRANSACTION_SERIALIZABLE" + ], + "default": "TRANSACTION_SERIALIZABLE" + } + }, { "widget-type": "keyvalue", "label": "Connection Arguments", @@ -277,6 +291,10 @@ "type": "property", "name": "password" }, + { + "type": "property", + "name": "transactionIsolationLevel" + }, { "type": "property", "name": "host" diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java index 10022364a..c3ce051e5 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java @@ -81,12 +81,6 @@ public String getConnectionString() { @Macro private String database; - @Name(OracleConstants.TRANSACTION_ISOLATION_LEVEL) - @Description("The transaction isolation level for the database session.") - @Macro - @Nullable - private String transactionIsolationLevel; - @Name(OracleConstants.USE_SSL) @Description("Turns on SSL encryption. Connection will fail if SSL is not available") @Nullable @@ -124,6 +118,7 @@ public Properties getConnectionArgumentsProperties() { return prop; } + @Override public String getTransactionIsolationLevel() { //if null default to the highest isolation level possible if (transactionIsolationLevel == null) { @@ -133,16 +128,7 @@ public String getTransactionIsolationLevel() { //This ensures that the role is mapped to the right serialization level, even w/ incorrect user input //if role is SYSDBA or SYSOP it will map to read_committed. else serialized return (!getRole().equals(ROLE_NORMAL)) ? TransactionIsolationLevel.Level.TRANSACTION_READ_COMMITTED.name() : - TransactionIsolationLevel.Level.valueOf(transactionIsolationLevel).name(); - } - - @Override - public Map getAdditionalArguments() { - Map additonalArguments = new HashMap<>(); - if (getTransactionIsolationLevel() != null) { - additonalArguments.put(TransactionIsolationLevel.CONF_KEY, getTransactionIsolationLevel()); - } - return additonalArguments; + TransactionIsolationLevel.Level.valueOf(transactionIsolationLevel).name(); } @Override diff --git a/postgresql-plugin/docs/PostgreSQL-connector.md b/postgresql-plugin/docs/PostgreSQL-connector.md index 739c678e3..fe442cbf1 100644 --- a/postgresql-plugin/docs/PostgreSQL-connector.md +++ b/postgresql-plugin/docs/PostgreSQL-connector.md @@ -22,6 +22,14 @@ authentication. Optional for databases that do not require authentication. **Password:** Password to use to connect to the specified database. +**Transaction Isolation Level** The transaction isolation level of the databse connection +- TRANSACTION_READ_COMMITTED: No dirty reads. Non-repeatable reads and phantom reads are possible. +- TRANSACTION_SERIALIZABLE: No dirty reads. Non-repeatable and phantom reads are prevented. +- TRANSACTION_REPEATABLE_READ: No dirty reads. Prevents non-repeatable reads, but phantom reads are still possible. +- Note: PostgreSQL does not implement `TRANSACTION_READ_UNCOMMITTED` as a distinct isolation level. Instead, this mode behaves identically to`TRANSACTION_READ_COMMITTED`, which is why it is not exposed as a separate option. + +For more details on the Transaction Isolation Levels supported in PostgreSQL, refer to the [PostgreSQL documentation](https://www.postgresql.org/docs/current/transaction-iso.html#TRANSACTION-ISO) + **Database:** The name of the database to connect to. **Connection Arguments:** A list of arbitrary string tag/value pairs as connection arguments. These arguments diff --git a/postgresql-plugin/docs/Postgres-batchsink.md b/postgresql-plugin/docs/Postgres-batchsink.md index b8a996463..82065e0fd 100644 --- a/postgresql-plugin/docs/Postgres-batchsink.md +++ b/postgresql-plugin/docs/Postgres-batchsink.md @@ -39,6 +39,14 @@ You also can use the macro function ${conn(connection-name)}. **Password:** Password to use to connect to the specified database. +**Transaction Isolation Level** The transaction isolation level of the databse connection +- TRANSACTION_READ_COMMITTED: No dirty reads. Non-repeatable reads and phantom reads are possible. +- TRANSACTION_SERIALIZABLE: No dirty reads. Non-repeatable and phantom reads are prevented. +- TRANSACTION_REPEATABLE_READ: No dirty reads. Prevents non-repeatable reads, but phantom reads are still possible. +- Note: PostgreSQL does not implement `TRANSACTION_READ_UNCOMMITTED` as a distinct isolation level. Instead, this mode behaves identically to`TRANSACTION_READ_COMMITTED`, which is why it is not exposed as a separate option. + +For more details on the Transaction Isolation Levels supported in PostgreSQL, refer to the [PostgreSQL documentation](https://www.postgresql.org/docs/current/transaction-iso.html#TRANSACTION-ISO) + **Connection Arguments:** A list of arbitrary string key/value pairs as connection arguments. These arguments will be passed to the JDBC driver as connection arguments for JDBC drivers that may need additional configurations. diff --git a/postgresql-plugin/docs/Postgres-batchsource.md b/postgresql-plugin/docs/Postgres-batchsource.md index af359022d..559723526 100644 --- a/postgresql-plugin/docs/Postgres-batchsource.md +++ b/postgresql-plugin/docs/Postgres-batchsource.md @@ -49,6 +49,14 @@ For example, 'SELECT MIN(id),MAX(id) FROM table'. Not required if numSplits is s **Password:** Password to use to connect to the specified database. +**Transaction Isolation Level** The transaction isolation level of the databse connection +- TRANSACTION_READ_COMMITTED: No dirty reads. Non-repeatable reads and phantom reads are possible. +- TRANSACTION_SERIALIZABLE: No dirty reads. Non-repeatable and phantom reads are prevented. +- TRANSACTION_REPEATABLE_READ: No dirty reads. Prevents non-repeatable reads, but phantom reads are still possible. +- Note: PostgreSQL does not implement `TRANSACTION_READ_UNCOMMITTED` as a distinct isolation level. Instead, this mode behaves identically to`TRANSACTION_READ_COMMITTED`, which is why it is not exposed as a separate option. + +For more details on the Transaction Isolation Levels supported in PostgreSQL, refer to the [PostgreSQL documentation](https://www.postgresql.org/docs/current/transaction-iso.html#TRANSACTION-ISO) + **Connection Arguments:** A list of arbitrary string key/value pairs as connection arguments. These arguments will be passed to the JDBC driver as connection arguments for JDBC drivers that may need additional configurations. diff --git a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSink.java b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSink.java index 8fd91cc63..6525a3dfa 100644 --- a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSink.java +++ b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSink.java @@ -165,6 +165,11 @@ public Map getDBSpecificArguments() { return ImmutableMap.of(PostgresConstants.CONNECTION_TIMEOUT, String.valueOf(connectionTimeout)); } + @Override + public String getTransactionIsolationLevel() { + return connection.getTransactionIsolationLevel(); + } + @Override protected PostgresConnectorConfig getConnection() { return connection; diff --git a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSource.java b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSource.java index d6677884f..ccef4078e 100644 --- a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSource.java +++ b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSource.java @@ -133,6 +133,11 @@ protected PostgresConnectorConfig getConnection() { return connection; } + @Override + public String getTransactionIsolationLevel() { + return connection.getTransactionIsolationLevel(); + } + @Override public void validate(FailureCollector collector) { ConfigUtil.validateConnection(this, useConnection, connection, collector); diff --git a/postgresql-plugin/widgets/PostgreSQL-connector.json b/postgresql-plugin/widgets/PostgreSQL-connector.json index 091afc972..9a7a02e14 100644 --- a/postgresql-plugin/widgets/PostgreSQL-connector.json +++ b/postgresql-plugin/widgets/PostgreSQL-connector.json @@ -31,6 +31,19 @@ "default": "5432" } }, + { + "widget-type": "select", + "label": "Transaction Isolation Level", + "name": "transactionIsolationLevel", + "widget-attributes": { + "values": [ + "TRANSACTION_READ_COMMITTED", + "TRANSACTION_REPEATABLE_READ", + "TRANSACTION_SERIALIZABLE" + ], + "default": "TRANSACTION_SERIALIZABLE" + } + }, { "widget-type": "textbox", "label": "Database", diff --git a/postgresql-plugin/widgets/Postgres-batchsink.json b/postgresql-plugin/widgets/Postgres-batchsink.json index 6aa2dad8a..14e6f8154 100644 --- a/postgresql-plugin/widgets/Postgres-batchsink.json +++ b/postgresql-plugin/widgets/Postgres-batchsink.json @@ -65,6 +65,19 @@ "label": "Password", "name": "password" }, + { + "widget-type": "select", + "label": "Transaction Isolation Level", + "name": "transactionIsolationLevel", + "widget-attributes": { + "values": [ + "TRANSACTION_READ_COMMITTED", + "TRANSACTION_REPEATABLE_READ", + "TRANSACTION_SERIALIZABLE" + ], + "default": "TRANSACTION_SERIALIZABLE" + } + }, { "widget-type": "keyvalue", "label": "Connection Arguments", @@ -186,6 +199,10 @@ "type": "property", "name": "port" }, + { + "type": "property", + "name": "transactionIsolationLevel" + }, { "type": "property", "name": "database" diff --git a/postgresql-plugin/widgets/Postgres-batchsource.json b/postgresql-plugin/widgets/Postgres-batchsource.json index 0e4ba28c1..60de4725f 100644 --- a/postgresql-plugin/widgets/Postgres-batchsource.json +++ b/postgresql-plugin/widgets/Postgres-batchsource.json @@ -65,6 +65,19 @@ "label": "Password", "name": "password" }, + { + "widget-type": "select", + "label": "Transaction Isolation Level", + "name": "transactionIsolationLevel", + "widget-attributes": { + "values": [ + "TRANSACTION_READ_COMMITTED", + "TRANSACTION_REPEATABLE_READ", + "TRANSACTION_SERIALIZABLE" + ], + "default": "TRANSACTION_SERIALIZABLE" + } + }, { "widget-type": "keyvalue", "label": "Connection Arguments", @@ -206,6 +219,10 @@ "type": "property", "name": "port" }, + { + "type": "property", + "name": "transactionIsolationLevel" + }, { "type": "property", "name": "database" From 8d80ebd196e1127216493d8056b4100b702ba3b2 Mon Sep 17 00:00:00 2001 From: Sanchit Garg Date: Tue, 6 May 2025 12:47:12 +0530 Subject: [PATCH 28/52] Remove Snapshot for release 1.11.7 Prepare release for database-plugins 1.11.7 --- amazon-redshift-plugin/pom.xml | 2 +- aurora-mysql-plugin/pom.xml | 2 +- aurora-postgresql-plugin/pom.xml | 2 +- cloudsql-mysql-plugin/pom.xml | 2 +- cloudsql-postgresql-plugin/pom.xml | 2 +- database-commons/pom.xml | 2 +- db2-plugin/pom.xml | 2 +- generic-database-plugin/pom.xml | 2 +- generic-db-argument-setter/pom.xml | 2 +- mariadb-plugin/pom.xml | 2 +- memsql-plugin/pom.xml | 2 +- mssql-plugin/pom.xml | 2 +- mysql-plugin/pom.xml | 2 +- netezza-plugin/pom.xml | 2 +- oracle-plugin/pom.xml | 2 +- pom.xml | 2 +- postgresql-plugin/pom.xml | 2 +- saphana-plugin/pom.xml | 2 +- teradata-plugin/pom.xml | 2 +- 19 files changed, 19 insertions(+), 19 deletions(-) diff --git a/amazon-redshift-plugin/pom.xml b/amazon-redshift-plugin/pom.xml index 41ab5084b..eb25da493 100644 --- a/amazon-redshift-plugin/pom.xml +++ b/amazon-redshift-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.7-SNAPSHOT + 1.11.7 Amazon Redshift plugin diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index cdb407152..ce6fc8052 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.7-SNAPSHOT + 1.11.7 Aurora DB MySQL plugin diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index e5ebd788e..2fe68b64e 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.7-SNAPSHOT + 1.11.7 Aurora DB PostgreSQL plugin diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index 047637657..706502fc9 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.7-SNAPSHOT + 1.11.7 CloudSQL MySQL plugin diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index 8a709d399..66d8a05f0 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.7-SNAPSHOT + 1.11.7 CloudSQL PostgreSQL plugin diff --git a/database-commons/pom.xml b/database-commons/pom.xml index c96954258..bd5de4128 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.7-SNAPSHOT + 1.11.7 Database Commons diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index fff1f51e0..3a600eaaf 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.7-SNAPSHOT + 1.11.7 IBM DB2 plugin diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index fcd4de36a..3db66ca8f 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.7-SNAPSHOT + 1.11.7 Generic database plugin diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index 54868c1f3..af9ce1797 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.7-SNAPSHOT + 1.11.7 Generic database argument setter plugin diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index e0f9090b0..a37b89019 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.7-SNAPSHOT + 1.11.7 Maria DB plugin diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index 6b3bc281a..6c0e4304e 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.7-SNAPSHOT + 1.11.7 Memsql plugin diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index bbe53e7df..bda544b24 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.7-SNAPSHOT + 1.11.7 Microsoft SQL Server plugin diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index f04564a42..238ff2170 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.7-SNAPSHOT + 1.11.7 Mysql plugin diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index 2647adf76..415450aa2 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.7-SNAPSHOT + 1.11.7 Netezza plugin diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index a53d72fcf..7074373c4 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.7-SNAPSHOT + 1.11.7 Oracle plugin diff --git a/pom.xml b/pom.xml index 6370371b7..f09b52aad 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin database-plugins-parent - 1.11.7-SNAPSHOT + 1.11.7 pom Database Plugins Collection of database plugins diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index 5f720a5c3..90debbed9 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.7-SNAPSHOT + 1.11.7 PostgreSQL plugin diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index 1862b86d1..a740d7c15 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.7-SNAPSHOT + 1.11.7 SAP HANA plugin diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index d8a71d087..7c9d7d3c3 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -21,7 +21,7 @@ database-plugins-parent io.cdap.plugin - 1.11.7-SNAPSHOT + 1.11.7 teradata-plugin From 7be0f1c78457fea11e1e9ed95a40937db87e995e Mon Sep 17 00:00:00 2001 From: sahusanket Date: Tue, 3 Jun 2025 15:08:21 +0530 Subject: [PATCH 29/52] [PLUGIN-1893] Adding fields in Oracle source and connector which acts like a flag for Backward compatibility issues for Timestamp and number (precisionless) --- .../cdap/plugin/oracle/OracleConnector.java | 3 +- .../plugin/oracle/OracleConnectorConfig.java | 28 +++++++++-- .../cdap/plugin/oracle/OracleConstants.java | 2 + .../io/cdap/plugin/oracle/OracleSource.java | 13 ++++-- .../oracle/OracleSourceSchemaReader.java | 44 ++++++++++++------ oracle-plugin/widgets/Oracle-batchsource.json | 46 +++++++++++++++++++ oracle-plugin/widgets/Oracle-connector.json | 38 +++++++++++++++ 7 files changed, 153 insertions(+), 21 deletions(-) diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java index 3d2f7399a..16371d5c1 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java @@ -112,7 +112,8 @@ protected DBConnectorPath getDBConnectorPath(String path) { @Override protected SchemaReader getSchemaReader(String sessionID) { - return new OracleSourceSchemaReader(sessionID); + return new OracleSourceSchemaReader(sessionID, config.getTreatAsOldTimestamp(), + config.getTreatPrecisionlessNumAsDeci()); } @Override diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java index c3ce051e5..cbc1e5ed2 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java @@ -22,8 +22,6 @@ import io.cdap.plugin.db.TransactionIsolationLevel; import io.cdap.plugin.db.connector.AbstractDBSpecificConnectorConfig; -import java.util.HashMap; -import java.util.Map; import java.util.Properties; import javax.annotation.Nullable; @@ -43,12 +41,14 @@ public OracleConnectorConfig(String host, int port, String user, String password public OracleConnectorConfig(String host, int port, String user, String password, String jdbcPluginName, String connectionArguments, String connectionType, String database) { - this(host, port, user, password, jdbcPluginName, connectionArguments, connectionType, database, null, null); + this(host, port, user, password, jdbcPluginName, connectionArguments, connectionType, database, null, null, null, + null); } public OracleConnectorConfig(String host, int port, String user, String password, String jdbcPluginName, String connectionArguments, String connectionType, String database, - String role, Boolean useSSL) { + String role, Boolean useSSL, @Nullable Boolean treatAsOldTimestamp, + @Nullable Boolean treatPrecisionlessNumAsDeci) { this.host = host; this.port = port; @@ -60,6 +60,8 @@ public OracleConnectorConfig(String host, int port, String user, String password this.database = database; this.role = role; this.useSSL = useSSL; + this.treatAsOldTimestamp = treatAsOldTimestamp; + this.treatPrecisionlessNumAsDeci = treatPrecisionlessNumAsDeci; } @Override @@ -86,6 +88,16 @@ public String getConnectionString() { @Nullable public Boolean useSSL; + @Name(OracleConstants.TREAT_AS_OLD_TIMESTAMP) + @Description("A hidden field to handle timestamp as CDAP's timestamp micros or string as per old behavior.") + @Nullable + public Boolean treatAsOldTimestamp; + + @Name(OracleConstants.TREAT_PRECISIONLESSNUM_AS_DECI) + @Description("A hidden field to handle precision less number as CDAP's decimal per old behavior.") + @Nullable + public Boolean treatPrecisionlessNumAsDeci; + @Override protected int getDefaultPort() { return 1521; @@ -108,6 +120,14 @@ public Boolean getSSlMode() { return useSSL != null && useSSL; } + public Boolean getTreatAsOldTimestamp() { + return Boolean.TRUE.equals(treatAsOldTimestamp); + } + + public Boolean getTreatPrecisionlessNumAsDeci() { + return Boolean.TRUE.equals(treatPrecisionlessNumAsDeci); + } + @Override public Properties getConnectionArgumentsProperties() { Properties prop = super.getConnectionArgumentsProperties(); diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java index dc38f80ac..cbd411175 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java @@ -43,6 +43,8 @@ private OracleConstants() { public static final String TNS_CONNECTION_TYPE = "tns"; public static final String TRANSACTION_ISOLATION_LEVEL = "transactionIsolationLevel"; public static final String USE_SSL = "useSSL"; + public static final String TREAT_AS_OLD_TIMESTAMP = "treatAsOldTimestamp"; + public static final String TREAT_PRECISIONLESSNUM_AS_DECI = "treatPrecisionlessNumAsDeci"; /** * Constructs the Oracle connection string based on the provided connection type, host, port, and database. diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java index 6df62e63e..583fd686f 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java @@ -63,7 +63,12 @@ protected String createConnectionString() { @Override protected SchemaReader getSchemaReader() { - return new OracleSourceSchemaReader(); + // PLUGIN-1893 : Based on field/properties from Oracle source and Oracle connection we will pass the flag to control + // handle schema to make it backward compatible. + boolean treatAsOldTimestamp = oracleSourceConfig.getConnection().getTreatAsOldTimestamp(); + boolean treatPrecisionlessNumAsDeci = oracleSourceConfig.getConnection().getTreatPrecisionlessNumAsDeci(); + + return new OracleSourceSchemaReader(null, treatAsOldTimestamp, treatPrecisionlessNumAsDeci); } @Override @@ -117,9 +122,11 @@ public OracleSourceConfig(String host, int port, String user, String password, S String connectionArguments, String connectionType, String database, String role, int defaultBatchValue, int defaultRowPrefetch, String importQuery, Integer numSplits, int fetchSize, - String boundingQuery, String splitBy, Boolean useSSL) { + String boundingQuery, String splitBy, Boolean useSSL, Boolean treatAsOldTimestamp, + Boolean treatPrecisionlessNumAsDeci) { this.connection = new OracleConnectorConfig(host, port, user, password, jdbcPluginName, connectionArguments, - connectionType, database, role, useSSL); + connectionType, database, role, useSSL, treatAsOldTimestamp, + treatPrecisionlessNumAsDeci); this.defaultBatchValue = defaultBatchValue; this.defaultRowPrefetch = defaultRowPrefetch; this.fetchSize = fetchSize; diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceSchemaReader.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceSchemaReader.java index 7d35f9bc7..dd17d2e84 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceSchemaReader.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceSchemaReader.java @@ -26,6 +26,7 @@ import java.sql.SQLException; import java.sql.Types; import java.util.Set; +import javax.annotation.Nullable; /** * Oracle Source schema reader. @@ -65,14 +66,17 @@ public class OracleSourceSchemaReader extends CommonSchemaReader { ); private final String sessionID; + private final Boolean isTimestampOldBehavior; + private final Boolean isPrecisionlessNumAsDecimal; public OracleSourceSchemaReader() { - this(null); + this(null, false, false); } - - public OracleSourceSchemaReader(String sessionID) { - super(); + public OracleSourceSchemaReader(@Nullable String sessionID, boolean isTimestampOldBehavior, + boolean isPrecisionlessNumAsDecimal) { this.sessionID = sessionID; + this.isTimestampOldBehavior = isTimestampOldBehavior; + this.isPrecisionlessNumAsDecimal = isPrecisionlessNumAsDecimal; } @Override @@ -81,10 +85,12 @@ public Schema getSchema(ResultSetMetaData metadata, int index) throws SQLExcepti switch (sqlType) { case TIMESTAMP_TZ: - return Schema.of(Schema.LogicalType.TIMESTAMP_MICROS); - case Types.TIMESTAMP: + return isTimestampOldBehavior ? Schema.of(Schema.Type.STRING) : Schema.of(Schema.LogicalType.TIMESTAMP_MICROS); case TIMESTAMP_LTZ: - return Schema.of(Schema.LogicalType.DATETIME); + return isTimestampOldBehavior ? Schema.of(Schema.LogicalType.TIMESTAMP_MICROS) + : Schema.of(Schema.LogicalType.DATETIME); + case Types.TIMESTAMP: + return isTimestampOldBehavior ? super.getSchema(metadata, index) : Schema.of(Schema.LogicalType.DATETIME); case BINARY_FLOAT: return Schema.of(Schema.Type.FLOAT); case BINARY_DOUBLE: @@ -107,12 +113,24 @@ public Schema getSchema(ResultSetMetaData metadata, int index) throws SQLExcepti // For a Number type without specified precision and scale, precision will be 0 and scale will be -127 if (precision == 0) { // reference : https://docs.oracle.com/cd/B28359_01/server.111/b28318/datatype.htm#CNCPT1832 - LOG.warn(String.format("Field '%s' is a %s type without precision and scale, " - + "converting into STRING type to avoid any precision loss.", - metadata.getColumnName(index), - metadata.getColumnTypeName(index), - metadata.getColumnName(index))); - return Schema.of(Schema.Type.STRING); + if (isPrecisionlessNumAsDecimal) { + precision = 38; + scale = 0; + LOG.warn(String.format("%s type with undefined precision and scale is detected, " + + "there may be a precision loss while running the pipeline. " + + "Please define an output precision and scale for field '%s' to avoid " + + "precision loss.", + metadata.getColumnTypeName(index), + metadata.getColumnName(index))); + return Schema.decimalOf(precision, scale); + } else { + LOG.warn(String.format("Field '%s' is a %s type without precision and scale, " + + "converting into STRING type to avoid any precision loss.", + metadata.getColumnName(index), + metadata.getColumnTypeName(index), + metadata.getColumnName(index))); + return Schema.of(Schema.Type.STRING); + } } return Schema.decimalOf(precision, scale); } diff --git a/oracle-plugin/widgets/Oracle-batchsource.json b/oracle-plugin/widgets/Oracle-batchsource.json index 5eca20cc4..404262fb2 100644 --- a/oracle-plugin/widgets/Oracle-batchsource.json +++ b/oracle-plugin/widgets/Oracle-batchsource.json @@ -120,6 +120,44 @@ ] } }, + { + "widget-type": "hidden", + "label": "Treat as old timestamp", + "name": "treatAsOldTimestamp", + "widget-attributes": { + "layout": "inline", + "default": "false", + "options": [ + { + "id": "true", + "label": "true" + }, + { + "id": "false", + "label": "false" + } + ] + } + }, + { + "widget-type": "hidden", + "label": "Treat precision less number as Decimal(old behavior)", + "name": "treatPrecisionlessNumAsDeci", + "widget-attributes": { + "layout": "inline", + "default": "false", + "options": [ + { + "id": "true", + "label": "true" + }, + { + "id": "false", + "label": "false" + } + ] + } + }, { "name": "connectionType", "label": "Connection Type", @@ -326,6 +364,14 @@ { "type": "property", "name": "transactionIsolationLevel" + }, + { + "type": "property", + "name": "getTreatAsOldTimestampConn" + }, + { + "type": "property", + "name": "treatPrecisionlessNumAsDeci" } ] }, diff --git a/oracle-plugin/widgets/Oracle-connector.json b/oracle-plugin/widgets/Oracle-connector.json index 628027caf..013f3b240 100644 --- a/oracle-plugin/widgets/Oracle-connector.json +++ b/oracle-plugin/widgets/Oracle-connector.json @@ -129,6 +129,44 @@ } ] } + }, + { + "widget-type": "hidden", + "label": "Treat as old timestamp", + "name": "treatAsOldTimestamp", + "widget-attributes": { + "layout": "inline", + "default": "false", + "options": [ + { + "id": "true", + "label": "true" + }, + { + "id": "false", + "label": "false" + } + ] + } + }, + { + "widget-type": "hidden", + "label": "Treat precision less number as Decimal(old behavior)", + "name": "treatPrecisionlessNumAsDeci", + "widget-attributes": { + "layout": "inline", + "default": "false", + "options": [ + { + "id": "true", + "label": "true" + }, + { + "id": "false", + "label": "false" + } + ] + } } ] }, From 0234e1560468421e3d37f584918e03a3f3b4ba72 Mon Sep 17 00:00:00 2001 From: sahusanket Date: Tue, 3 Jun 2025 20:29:36 +0530 Subject: [PATCH 30/52] Bumping Snapshot. --- amazon-redshift-plugin/pom.xml | 2 +- aurora-mysql-plugin/pom.xml | 2 +- aurora-postgresql-plugin/pom.xml | 2 +- cloudsql-mysql-plugin/pom.xml | 2 +- cloudsql-postgresql-plugin/pom.xml | 2 +- database-commons/pom.xml | 2 +- db2-plugin/pom.xml | 2 +- generic-database-plugin/pom.xml | 2 +- generic-db-argument-setter/pom.xml | 2 +- mariadb-plugin/pom.xml | 2 +- memsql-plugin/pom.xml | 2 +- mssql-plugin/pom.xml | 2 +- mysql-plugin/pom.xml | 2 +- netezza-plugin/pom.xml | 2 +- oracle-plugin/pom.xml | 2 +- pom.xml | 2 +- postgresql-plugin/pom.xml | 2 +- saphana-plugin/pom.xml | 2 +- teradata-plugin/pom.xml | 2 +- 19 files changed, 19 insertions(+), 19 deletions(-) diff --git a/amazon-redshift-plugin/pom.xml b/amazon-redshift-plugin/pom.xml index eb25da493..110eea394 100644 --- a/amazon-redshift-plugin/pom.xml +++ b/amazon-redshift-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.7 + 1.11.8-SNAPSHOT Amazon Redshift plugin diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index ce6fc8052..4f4ac4907 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.7 + 1.11.8-SNAPSHOT Aurora DB MySQL plugin diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index 2fe68b64e..421f557cc 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.7 + 1.11.8-SNAPSHOT Aurora DB PostgreSQL plugin diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index 706502fc9..60212b67d 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.7 + 1.11.8-SNAPSHOT CloudSQL MySQL plugin diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index 66d8a05f0..bbbda9df1 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.7 + 1.11.8-SNAPSHOT CloudSQL PostgreSQL plugin diff --git a/database-commons/pom.xml b/database-commons/pom.xml index bd5de4128..a59453390 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.7 + 1.11.8-SNAPSHOT Database Commons diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index 3a600eaaf..f1db30f12 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.7 + 1.11.8-SNAPSHOT IBM DB2 plugin diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index 3db66ca8f..e9dea0b93 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.7 + 1.11.8-SNAPSHOT Generic database plugin diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index af9ce1797..ef450a352 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.7 + 1.11.8-SNAPSHOT Generic database argument setter plugin diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index a37b89019..0537b9dbf 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.7 + 1.11.8-SNAPSHOT Maria DB plugin diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index 6c0e4304e..bcd20bf3e 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.7 + 1.11.8-SNAPSHOT Memsql plugin diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index bda544b24..348f8d5c0 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.7 + 1.11.8-SNAPSHOT Microsoft SQL Server plugin diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index 238ff2170..23a5aca38 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.7 + 1.11.8-SNAPSHOT Mysql plugin diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index 415450aa2..9ebe52134 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.7 + 1.11.8-SNAPSHOT Netezza plugin diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index 7074373c4..1db95d3c1 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.7 + 1.11.8-SNAPSHOT Oracle plugin diff --git a/pom.xml b/pom.xml index f09b52aad..45afac8ee 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin database-plugins-parent - 1.11.7 + 1.11.8-SNAPSHOT pom Database Plugins Collection of database plugins diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index 90debbed9..bd66a7eb4 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.7 + 1.11.8-SNAPSHOT PostgreSQL plugin diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index a740d7c15..2459b73c7 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.7 + 1.11.8-SNAPSHOT SAP HANA plugin diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index 7c9d7d3c3..797365865 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -21,7 +21,7 @@ database-plugins-parent io.cdap.plugin - 1.11.7 + 1.11.8-SNAPSHOT teradata-plugin From 2ce0804cbd641a7289ed939a8cc007aa08dfbeb0 Mon Sep 17 00:00:00 2001 From: sahusanket Date: Wed, 4 Jun 2025 16:59:22 +0530 Subject: [PATCH 31/52] removing Snapshot for hub release --- amazon-redshift-plugin/pom.xml | 2 +- aurora-mysql-plugin/pom.xml | 2 +- aurora-postgresql-plugin/pom.xml | 2 +- cloudsql-mysql-plugin/pom.xml | 2 +- cloudsql-postgresql-plugin/pom.xml | 2 +- database-commons/pom.xml | 2 +- db2-plugin/pom.xml | 2 +- generic-database-plugin/pom.xml | 2 +- generic-db-argument-setter/pom.xml | 2 +- mariadb-plugin/pom.xml | 2 +- memsql-plugin/pom.xml | 2 +- mssql-plugin/pom.xml | 2 +- mysql-plugin/pom.xml | 2 +- netezza-plugin/pom.xml | 2 +- oracle-plugin/pom.xml | 2 +- pom.xml | 2 +- postgresql-plugin/pom.xml | 2 +- saphana-plugin/pom.xml | 2 +- teradata-plugin/pom.xml | 2 +- 19 files changed, 19 insertions(+), 19 deletions(-) diff --git a/amazon-redshift-plugin/pom.xml b/amazon-redshift-plugin/pom.xml index 110eea394..d66e009fc 100644 --- a/amazon-redshift-plugin/pom.xml +++ b/amazon-redshift-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.8-SNAPSHOT + 1.11.8 Amazon Redshift plugin diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index 4f4ac4907..d552ce1b4 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.8-SNAPSHOT + 1.11.8 Aurora DB MySQL plugin diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index 421f557cc..316f05f7c 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.8-SNAPSHOT + 1.11.8 Aurora DB PostgreSQL plugin diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index 60212b67d..fd9b742c2 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.8-SNAPSHOT + 1.11.8 CloudSQL MySQL plugin diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index bbbda9df1..b1217665e 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.8-SNAPSHOT + 1.11.8 CloudSQL PostgreSQL plugin diff --git a/database-commons/pom.xml b/database-commons/pom.xml index a59453390..0ceec28ca 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.8-SNAPSHOT + 1.11.8 Database Commons diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index f1db30f12..85f2d2089 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.8-SNAPSHOT + 1.11.8 IBM DB2 plugin diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index e9dea0b93..d7696c84c 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.8-SNAPSHOT + 1.11.8 Generic database plugin diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index ef450a352..ad084bbce 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.8-SNAPSHOT + 1.11.8 Generic database argument setter plugin diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index 0537b9dbf..bdb8cfb2f 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.8-SNAPSHOT + 1.11.8 Maria DB plugin diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index bcd20bf3e..f943e43d4 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.8-SNAPSHOT + 1.11.8 Memsql plugin diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index 348f8d5c0..6d4474256 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.8-SNAPSHOT + 1.11.8 Microsoft SQL Server plugin diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index 23a5aca38..44f904dfe 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.8-SNAPSHOT + 1.11.8 Mysql plugin diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index 9ebe52134..1bc303e9a 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.8-SNAPSHOT + 1.11.8 Netezza plugin diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index 1db95d3c1..c12bb2fde 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.8-SNAPSHOT + 1.11.8 Oracle plugin diff --git a/pom.xml b/pom.xml index 45afac8ee..db2abfec5 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin database-plugins-parent - 1.11.8-SNAPSHOT + 1.11.8 pom Database Plugins Collection of database plugins diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index bd66a7eb4..9a5d33bfa 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.8-SNAPSHOT + 1.11.8 PostgreSQL plugin diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index 2459b73c7..4b15f4e89 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.8-SNAPSHOT + 1.11.8 SAP HANA plugin diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index 797365865..94c47c674 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -21,7 +21,7 @@ database-plugins-parent io.cdap.plugin - 1.11.8-SNAPSHOT + 1.11.8 teradata-plugin From ff6d6f44bb652d0573b78d4e79001af318fd3f38 Mon Sep 17 00:00:00 2001 From: dj-smart Date: Fri, 11 Jul 2025 11:06:13 +0000 Subject: [PATCH 32/52] CDF OSS Migration for CDF 6.10 --- pom.xml | 35 +++++++---------------------------- 1 file changed, 7 insertions(+), 28 deletions(-) diff --git a/pom.xml b/pom.xml index db2abfec5..4c24c7477 100644 --- a/pom.xml +++ b/pom.xml @@ -78,23 +78,12 @@ - - sonatype - https://oss.sonatype.org/content/groups/public - sonatype-snapshots - https://oss.sonatype.org/content/repositories/snapshots + https://central.sonatype.com/repository/maven-snapshots - - - sonatype - https://oss.sonatype.org/content/groups/public/ - - - @@ -349,16 +338,6 @@ - - - sonatype.release - https://oss.sonatype.org/service/local/staging/deploy/maven2 - - - sonatype.snapshots - https://oss.sonatype.org/content/repositories/snapshots - - ${testSourceLocation} @@ -534,14 +513,14 @@ - org.sonatype.plugins - nexus-staging-maven-plugin - 1.6.2 + org.sonatype.central + central-publishing-maven-plugin + 0.8.0 true - https://oss.sonatype.org - sonatype.release - 655dc88dc770c3 + sonatype.release + false + true From 775cf19cfb2d7ed070e7e50b80c29cc2cdb1c29b Mon Sep 17 00:00:00 2001 From: sgarg-CS Date: Fri, 5 Sep 2025 16:42:54 +0530 Subject: [PATCH 33/52] PLUGIN-1925: Introduce SafeBigDecimal Splitter --- .../db/source/DataDrivenETLDBInputFormat.java | 11 +++ .../db/source/SafeBigDecimalSplitter.java | 55 +++++++++++ .../db/source/SafeBigDecimalSplitterTest.java | 98 +++++++++++++++++++ 3 files changed, 164 insertions(+) create mode 100644 database-commons/src/main/java/io/cdap/plugin/db/source/SafeBigDecimalSplitter.java create mode 100644 database-commons/src/test/java/io/cdap/plugin/db/source/SafeBigDecimalSplitterTest.java diff --git a/database-commons/src/main/java/io/cdap/plugin/db/source/DataDrivenETLDBInputFormat.java b/database-commons/src/main/java/io/cdap/plugin/db/source/DataDrivenETLDBInputFormat.java index 3416360a9..1c347dd03 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/source/DataDrivenETLDBInputFormat.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/source/DataDrivenETLDBInputFormat.java @@ -28,6 +28,7 @@ import org.apache.hadoop.mapreduce.TaskAttemptContext; import org.apache.hadoop.mapreduce.lib.db.DBConfiguration; import org.apache.hadoop.mapreduce.lib.db.DBInputFormat; +import org.apache.hadoop.mapreduce.lib.db.DBSplitter; import org.apache.hadoop.mapreduce.lib.db.DBWritable; import org.apache.hadoop.mapreduce.lib.db.DataDrivenDBInputFormat; import org.slf4j.Logger; @@ -39,6 +40,7 @@ import java.sql.DriverManager; import java.sql.SQLException; import java.sql.Statement; +import java.sql.Types; import java.util.Properties; /** @@ -128,6 +130,15 @@ public Connection createConnection() { return getConnection(); } + @Override + protected DBSplitter getSplitter(int sqlDataType) { + // Use SafeBigDecimalSplitter for columns having high precision decimal or numeric columns + if (sqlDataType == Types.NUMERIC || sqlDataType == Types.DECIMAL) { + return new SafeBigDecimalSplitter(); + } + return super.getSplitter(sqlDataType); + } + @Override public RecordReader createDBRecordReader(DBInputSplit split, Configuration conf) throws IOException { final RecordReader dbRecordReader = super.createDBRecordReader(split, conf); diff --git a/database-commons/src/main/java/io/cdap/plugin/db/source/SafeBigDecimalSplitter.java b/database-commons/src/main/java/io/cdap/plugin/db/source/SafeBigDecimalSplitter.java new file mode 100644 index 000000000..8649515e8 --- /dev/null +++ b/database-commons/src/main/java/io/cdap/plugin/db/source/SafeBigDecimalSplitter.java @@ -0,0 +1,55 @@ +/* + * Copyright © 2025 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.db.source; + +import org.apache.hadoop.mapreduce.lib.db.BigDecimalSplitter; +import java.math.BigDecimal; +import java.math.RoundingMode; + +/** + * Safe implementation of {@link BigDecimalSplitter} to ensure precise division of BigDecimal values while calculating + * split points for NUMERIC and DECIMAL types. + * + *

Problem: The default {@link BigDecimalSplitter} implementation may return 0 when the numerator is smaller than the + * denominator (e.g., 1 / 4 = 0), due to the lack of a defined scale for division. Since the result (0) is smaller than + * {@link BigDecimalSplitter#MIN_INCREMENT} (i.e. {@code 10000 * Double.MIN_VALUE}), the split size defaults to + * {@code MIN_INCREMENT}, leading to an excessive number of splits (~10M) and potential OOM errors.

+ * + *

Fix: This implementation derives scale from column metadata, adds a buffer of 5 decimal places, and uses + * {@link RoundingMode#HALF_UP} as the rounding mode.

Note: This class is used by {@link DataDrivenETLDBInputFormat}.

+ */ +public class SafeBigDecimalSplitter extends BigDecimalSplitter { + + /* An additional buffer of +5 digits is applied to preserve accuracy during division. */ + public static final int SCALE_BUFFER = 5; + /** + * Performs safe division with correct scale handling. + * + * @param numerator the dividend (BigDecimal) + * @param denominator the divisor (BigDecimal) + * @return quotient with derived scale + * @throws ArithmeticException if denominator is zero + */ + @Override + protected BigDecimal tryDivide(BigDecimal numerator, BigDecimal denominator) { + // Determine the required scale for the division and add a buffer to ensure accuracy + int effectiveScale = Math.max(numerator.scale(), denominator.scale()) + SCALE_BUFFER; + return numerator.divide(denominator, effectiveScale, RoundingMode.HALF_UP); + } +} diff --git a/database-commons/src/test/java/io/cdap/plugin/db/source/SafeBigDecimalSplitterTest.java b/database-commons/src/test/java/io/cdap/plugin/db/source/SafeBigDecimalSplitterTest.java new file mode 100644 index 000000000..4aff4eac2 --- /dev/null +++ b/database-commons/src/test/java/io/cdap/plugin/db/source/SafeBigDecimalSplitterTest.java @@ -0,0 +1,98 @@ +/* + * Copyright © 2025 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.db.source; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.mapreduce.InputSplit; +import org.apache.hadoop.mapreduce.lib.db.BigDecimalSplitter; +import org.junit.Test; + +import java.math.BigDecimal; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Test class for {@link SafeBigDecimalSplitter} + */ +public class SafeBigDecimalSplitterTest { + private final SafeBigDecimalSplitter splitter = new SafeBigDecimalSplitter(); + + @Test + public void testSmallRangeDivision() { + BigDecimal result = splitter.tryDivide(BigDecimal.ONE, new BigDecimal("4")); + assertEquals(new BigDecimal("0.25000"), result); + } + + @Test + public void testLargePrecision() { + BigDecimal numerator = new BigDecimal("1.0000000000000000001"); + BigDecimal denominator = new BigDecimal("3"); + BigDecimal result = splitter.tryDivide(numerator, denominator); + assertTrue(result.compareTo(BigDecimal.ZERO) > 0); + } + + @Test + public void testDivisionByZero() { + assertThrows(ArithmeticException.class, () -> + splitter.tryDivide(BigDecimal.ONE, BigDecimal.ZERO)); + } + + @Test + public void testDivisionWithZeroNumerator() { + // when minVal == maxVal + BigDecimal result = splitter.tryDivide(BigDecimal.ZERO, BigDecimal.ONE); + assertEquals(0, result.compareTo(BigDecimal.ZERO)); + } + + @Test + public void testSplits() throws SQLException { + BigDecimal minVal = BigDecimal.valueOf(1); + BigDecimal maxVal = BigDecimal.valueOf(2); + int numSplits = 4; + ResultSet resultSet = mock(ResultSet.class); + Configuration conf = mock(Configuration.class); + when(conf.getInt("mapreduce.job.maps", 1)).thenReturn(numSplits); + when(resultSet.getBigDecimal(1)).thenReturn(minVal); + when(resultSet.getBigDecimal(2)).thenReturn(maxVal); + BigDecimalSplitter bigDecimalSplitter = new SafeBigDecimalSplitter(); + List actualSplits = bigDecimalSplitter.split(conf, resultSet, "id"); + assertEquals(numSplits, actualSplits.size()); + } + + @Test + public void testSplitsWithMinValueEqualToMaxValue() throws SQLException { + // when minVal == maxVal + BigDecimal minVal = BigDecimal.valueOf(1); + BigDecimal maxVal = BigDecimal.valueOf(1); + int numSplits = 1; + ResultSet resultSet = mock(ResultSet.class); + Configuration conf = mock(Configuration.class); + when(conf.getInt("mapreduce.job.maps", 1)).thenReturn(numSplits); + when(resultSet.getBigDecimal(1)).thenReturn(minVal); + when(resultSet.getBigDecimal(2)).thenReturn(maxVal); + BigDecimalSplitter bigDecimalSplitter = new SafeBigDecimalSplitter(); + List actualSplits = bigDecimalSplitter.split(conf, resultSet, "id"); + assertEquals(numSplits, actualSplits.size()); + } +} From f31af7e33533cc4c33d8da0eb613f1d128c41797 Mon Sep 17 00:00:00 2001 From: vipinbhatt Date: Wed, 17 Sep 2025 18:09:14 +0000 Subject: [PATCH 34/52] Prevent auto security injection on TCPS URLs --- .../io/cdap/plugin/oracle/OracleConstants.java | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java index cbd411175..7d86ec820 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java @@ -29,10 +29,11 @@ private OracleConstants() { public static final String PLUGIN_NAME = "Oracle"; public static final String ORACLE_CONNECTION_STRING_SID_FORMAT = "jdbc:oracle:thin:@%s:%s:%s"; public static final String ORACLE_CONNECTION_STRING_SERVICE_NAME_FORMAT = "jdbc:oracle:thin:@//%s:%s/%s"; - // Connection formats to accept protocol (e.g., jdbc:oracle:thin:@://:/) - public static final String ORACLE_CONNECTION_STRING_SID_FORMAT_WITH_PROTOCOL = "jdbc:oracle:thin:@%s:%s:%s/%s"; - public static final String ORACLE_CONNECTION_STRING_SERVICE_NAME_FORMAT_WITH_PROTOCOL = - "jdbc:oracle:thin:@%s://%s:%s/%s"; + // Connection formats using TNS DESCRIPTOR to accept protocol + public static final String ORACLE_SERVICE_NAME_FORMAT_TNS_DESCRIPTOR_WITH_PROTOCOL = + "jdbc:oracle:thin:@(DESCRIPTION=(ADDRESS=(PROTOCOL=%s)(HOST=%s)(PORT=%s))(CONNECT_DATA=(SERVICE_NAME=%s)))"; + public static final String ORACLE_SID_FORMAT_TNS_DESCRIPTOR_WITH_PROTOCOL = + "jdbc:oracle:thin:@(DESCRIPTION=(ADDRESS=(PROTOCOL=%s)(HOST=%s)(PORT=%s))(CONNECT_DATA=(SID=%s)))"; public static final String ORACLE_CONNECTION_STRING_TNS_FORMAT = "jdbc:oracle:thin:@%s"; public static final String DEFAULT_BATCH_VALUE = "defaultBatchValue"; public static final String DEFAULT_ROW_PREFETCH = "defaultRowPrefetch"; @@ -102,7 +103,8 @@ private static String getConnectionStringWithService(@Nullable String host, boolean isSSLEnabled) { // Choose the appropriate format based on whether SSL is enabled. if (isSSLEnabled) { - return String.format(OracleConstants.ORACLE_CONNECTION_STRING_SERVICE_NAME_FORMAT_WITH_PROTOCOL, + // Use the TNS descriptor format for TCPS to prevent automatic security injection. + return String.format(ORACLE_SERVICE_NAME_FORMAT_TNS_DESCRIPTOR_WITH_PROTOCOL, connectionProtocol, host, port, database); } return String.format(OracleConstants.ORACLE_CONNECTION_STRING_SERVICE_NAME_FORMAT, @@ -126,7 +128,8 @@ private static String getConnectionStringWithSID(@Nullable String host, boolean isSSLEnabled) { // Choose the appropriate format based on whether SSL is enabled. if (isSSLEnabled) { - return String.format(OracleConstants.ORACLE_CONNECTION_STRING_SID_FORMAT_WITH_PROTOCOL, + // Use the TNS descriptor format for TCPS to prevent automatic security injection. + return String.format(ORACLE_SID_FORMAT_TNS_DESCRIPTOR_WITH_PROTOCOL, connectionProtocol, host, port, database); } return String.format(OracleConstants.ORACLE_CONNECTION_STRING_SID_FORMAT, From 2de8137dd62a9caa93c084bbfb5962b9d677fd88 Mon Sep 17 00:00:00 2001 From: sgarg-CS Date: Thu, 18 Sep 2025 21:50:40 +0530 Subject: [PATCH 35/52] Disable verify goal and e2e error message changes --- .../src/e2e-test/resources/errorMessage.properties | 6 +++--- .../src/e2e-test/resources/errorMessage.properties | 2 +- .../src/e2e-test/resources/errorMessage.properties | 8 ++++---- .../src/e2e-test/resources/errorMessage.properties | 4 ++-- .../src/e2e-test/features/source/OracleRunTime.feature | 2 +- .../src/e2e-test/resources/errorMessage.properties | 9 ++++----- pom.xml | 5 +++-- .../src/e2e-test/resources/errorMessage.properties | 4 ++-- 8 files changed, 20 insertions(+), 20 deletions(-) diff --git a/cloudsql-mysql-plugin/src/e2e-test/resources/errorMessage.properties b/cloudsql-mysql-plugin/src/e2e-test/resources/errorMessage.properties index 5ff3357f2..96321fb4f 100644 --- a/cloudsql-mysql-plugin/src/e2e-test/resources/errorMessage.properties +++ b/cloudsql-mysql-plugin/src/e2e-test/resources/errorMessage.properties @@ -1,9 +1,9 @@ -errorMessageInvalidSourceDatabase=SQL error while getting query schema +errorMessageInvalidSourceDatabase=SQL Error occurred, sqlState: '42000', errorCode: '1049' errorMessageInvalidImportQuery=Import Query select must contain the string '$CONDITIONS'. if Number of Splits is not set\ \ to 1. Include '$CONDITIONS' in the Import Query errorMessageCloudMySqlInvalidReferenceName=Invalid reference name errorMessageBlankUsername=Username is required when password is given. -errorMessageBlankPassword=SQL error while getting query schema +errorMessageBlankPassword=SQL Error occurred, sqlState: '28000', errorCode: '1045', errorMessage: SQL Exception occurred: errorMessageInvalidFetchSize=Invalid fetch size. Fetch size must be a positive integer. errorMessageBlankSplitBy=Split-By Field Name must be specified if Number of Splits is not set to 1. Specify the Split-by Field Name. errorMessageInvalidNumberOfSplits=Invalid value for Number of Splits '0'. Must be at least 1. Specify a Number of Splits no less than 1. @@ -12,7 +12,7 @@ errorMessageNumberOfSplits=Split-By Field Name must be specified if Number of Sp errorMessageBoundingQuery=Bounding Query must be specified if Number of Splits is not set to 1. Specify the Bounding Query. errorMessageInvalidSinkDatabase=Error encountered while configuring the stage: 'URLDecoder: Illegal hex characters in escape (%) pattern - For input string: "$^"' errorMessageInvalidTableName=Table 'Invalidtable' does not exist. Ensure table 'Invalidtable' is set correctly and -errorMessageConnectionName=Connection Name must be in the format :: to connect to a public CloudSQL PostgreSQL instance. +errorMessageConnectionName=Connection Name must be in the format :: to connect to a public CloudSQL MySQL instance. validationSuccessMessage=No errors found. validationErrorMessage=COUNT ERROR found errorLogsMessageInvalidTableName=Spark program 'phase-1' failed with error: Errors were encountered during validation. \ diff --git a/cloudsql-postgresql-plugin/src/e2e-test/resources/errorMessage.properties b/cloudsql-postgresql-plugin/src/e2e-test/resources/errorMessage.properties index 1d094aedd..887cfe44a 100644 --- a/cloudsql-postgresql-plugin/src/e2e-test/resources/errorMessage.properties +++ b/cloudsql-postgresql-plugin/src/e2e-test/resources/errorMessage.properties @@ -15,7 +15,7 @@ errorMessageBlankPassword=SQL error while getting query schema: errorMessageInvalidPassword=SQL error while getting query schema: FATAL: password authentication failed for user errorMessageInvalidSourceHost=SQL error while getting query schema: The connection attempt failed. errorMessageInvalidTableName=Table 'table' does not exist. Ensure table '"table"' is set correctly and that the -errorMessageInvalidSinkDatabase=Exception while trying to validate schema of database table +errorMessageInvalidSinkDatabase=Error encountered while configuring the stage: 'SQL Error occurred, sqlState: '3D000', errorCode: '0', errorLogsMessageInvalidBoundingQuery=Spark program 'phase-1' failed with error: The column index is out of range: 1, \ number of columns: 0.. Please check the system logs for more details. errorMessageConnectionName=Connection Name must be in the format :: to connect to \ diff --git a/mssql-plugin/src/e2e-test/resources/errorMessage.properties b/mssql-plugin/src/e2e-test/resources/errorMessage.properties index 00721f148..9b5ba1a06 100644 --- a/mssql-plugin/src/e2e-test/resources/errorMessage.properties +++ b/mssql-plugin/src/e2e-test/resources/errorMessage.properties @@ -1,18 +1,18 @@ validationSuccessMessage=No errors found. -errorMessageInvalidSourceDatabase=SQL error while getting query schema +errorMessageInvalidSourceDatabase=SQL Error occurred, sqlState: 'S0001', errorCode: '4060', errorMessageInvalidImportQuery=Import Query select must contain the string '$CONDITIONS'. errorMessageMssqlInvalidReferenceName=Invalid reference name 'invalidRef&^*&&*'. errorMessageBlankUsername=Username is required when password is given. -errorMessageBlankPassword=SQL error while getting query schema +errorMessageBlankPassword=SQL Error occurred, sqlState: 'S0001', errorCode: '18456', errorMessageInvalidFetchSize=Invalid fetch size. Fetch size must be a positive integer. errorMessageBlankSplitBy=Split-By Field Name must be specified if Number of Splits is not set to 1. errorMessageInvalidNumberOfSplits=Invalid value for Number of Splits '0'. Must be at least 1. errorMessageNumberOfSplitNotNumber=Unable to create config for batchsource errorMessageBoundingQuery=Bounding Query must be specified if Number of Splits is not set to 1. errorMessagenumofSplit=Split-By Field Name must be specified if Number of Splits is not set to 1. -errorMessageInvalidSinkDatabase=Exception while trying to validate schema of database table +errorMessageInvalidSinkDatabase=Error encountered while configuring the stage: 'SQL Error occurred, sqlState: 'S0001', errorCode: '4060', errorMessageInvalidSinkTableName=Table 'Table123@' does not exist. -errormessageBlankHost=Exception while trying to validate schema of database table +errormessageBlankHost=Error encountered while configuring the stage: 'SQL Error occurred, sqlState: '08S01', errorCode: '0', errorMessageInvalidTableName=Spark program 'phase-1' failed with error: Errors were encountered during validation. \ Table 'Table123@' does not exist.. Please check the system logs for more details. errorMessageInvalidCredentials=Spark program 'phase-1' failed with error: Unable to create config for batchsink SqlServer \ diff --git a/mysql-plugin/src/e2e-test/resources/errorMessage.properties b/mysql-plugin/src/e2e-test/resources/errorMessage.properties index 0c1ac0400..adba2f7c6 100644 --- a/mysql-plugin/src/e2e-test/resources/errorMessage.properties +++ b/mysql-plugin/src/e2e-test/resources/errorMessage.properties @@ -2,12 +2,12 @@ validationSuccessMessage=No errors found. invalid.username.message=SQL error while getting query schema: invalidtableName.error.message=Table '123#' does not exist. Ensure table '123#' is set correctly invalidreferenceName.error.message=Invalid reference name -invalid.host.message=Exception while trying to validate schema of database table +invalid.host.message=Error encountered while configuring the stage: 'SQL Error occurred, sqlState: '08S01', errorCode: '0', invalid.password.message=SQL error while getting query schema: invalid.databasename.message=SQL error while getting query schema: invalid.query.message=SQL error while getting query schema: numberofsplits.error.message=Unable to create config for batchsource Mysql 'numSplits' is invalid boundingQuery.error.message=Bounding Query must be specified if Number of Splits is not set to 1. splitfield.error.message=Split-By Field Name must be specified if Number of Splits is not set to 1 -invalid.sink.database.message=Exception while trying to validate schema +invalid.sink.database.message=Error encountered while configuring the stage: 'SQL Error occurred, sqlState: '42000', errorCode: '1049', blank.username.message=Username is required when password is given. diff --git a/oracle-plugin/src/e2e-test/features/source/OracleRunTime.feature b/oracle-plugin/src/e2e-test/features/source/OracleRunTime.feature index 2d1ca9ad1..007215089 100644 --- a/oracle-plugin/src/e2e-test/features/source/OracleRunTime.feature +++ b/oracle-plugin/src/e2e-test/features/source/OracleRunTime.feature @@ -296,7 +296,7 @@ Feature: Oracle - Verify data transfer from Oracle source to BigQuery sink Then Close the Plugin Properties page Then Save the pipeline Then Preview and run the pipeline - Then Wait till pipeline preview is in running state + Then Wait till pipeline preview is in running state and check if any error occurs Then Verify the preview run status of pipeline in the logs is "failed" @ORACLE_SOURCE_TEST @BQ_SINK_TEST diff --git a/oracle-plugin/src/e2e-test/resources/errorMessage.properties b/oracle-plugin/src/e2e-test/resources/errorMessage.properties index 12588ac61..6e3a23c3d 100644 --- a/oracle-plugin/src/e2e-test/resources/errorMessage.properties +++ b/oracle-plugin/src/e2e-test/resources/errorMessage.properties @@ -7,13 +7,12 @@ errorMessageInvalidNumberOfSplits=Invalid value for Number of Splits '0'. Must b errorMessageNumberOfSplitNotNumber=Unable to create config for batchsource Oracle 'numSplits' is invalid: Value of field\ \ class io.cdap.plugin.db.config.AbstractDBSpecificSourceConfig.numSplits is expected to be a number. errorMessageInvalidFetchSize=Invalid fetch size. Fetch size must be a positive integer. -errorMessageInvalidSourceDatabase=SQL error while getting query schema: Listener refused the connection with the following \ - error: ORA-12514, TNS:listener does not currently know of service requested in connect descriptor +errorMessageInvalidSourceDatabase=SQL Error occurred, sqlState: '08006', errorCode: '12514', errorMessageInvalidImportQuery=Import Query select must contain the string '$CONDITIONS'. if Number of Splits is not set\ \ to 1. Include '$CONDITIONS' in the Import Query errorMessageBlankUsername=Username is required when password is given. -errorMessageInvalidTableName=Exception while trying to validate schema of database table '"table"' for connection -errorMessageInvalidSinkDatabase=Exception while trying to validate schema of database table '"TARGETTABLE_ -errorMessageInvalidHost=Exception while trying to validate schema of database table '"table"' for connection +errorMessageInvalidTableName=Error encountered while configuring the stage: 'SQL Error occurred, sqlState: '08006', errorCode: '17002', +errorMessageInvalidSinkDatabase=Error encountered while configuring the stage: 'SQL Error occurred, sqlState: '08006', errorCode: '17002' +errorMessageInvalidHost=Error encountered while configuring the stage: 'SQL Error occurred, sqlState: '08006', errorCode: '17002', errorLogsMessageInvalidBoundingQuery=Spark program 'phase-1' failed with error: ORA-00936: missing expression . \ Please check the system logs for more details. diff --git a/pom.xml b/pom.xml index 4c24c7477..98b055cd9 100644 --- a/pom.xml +++ b/pom.xml @@ -650,7 +650,8 @@ integration-test - verify + + @@ -705,7 +706,7 @@ io.cdap.tests.e2e cdap-e2e-framework - 0.3.0 + 0.5.0-SNAPSHOT test diff --git a/postgresql-plugin/src/e2e-test/resources/errorMessage.properties b/postgresql-plugin/src/e2e-test/resources/errorMessage.properties index 6e1929245..e59464785 100644 --- a/postgresql-plugin/src/e2e-test/resources/errorMessage.properties +++ b/postgresql-plugin/src/e2e-test/resources/errorMessage.properties @@ -16,7 +16,7 @@ errorMessageBlankPassword=SQL error while getting query schema: The server reque errorMessageInvalidPassword=SQL error while getting query schema: FATAL: password authentication failed for user errorMessageInvalidSourceHost=SQL error while getting query schema: The connection attempt failed. errorMessageInvalidTableName=Table 'table' does not exist. Ensure table '"table"' is set correctly and that the -errorMessageInvalidSinkDatabase=Exception while trying to validate schema of database table '"targettable_ -errorMessageInvalidHost=Exception while trying to validate schema of database table '"table"' for connection +errorMessageInvalidSinkDatabase=Error encountered while configuring the stage: 'SQL Error occurred, sqlState: '3D000', errorCode: '0', +errorMessageInvalidHost=Error encountered while configuring the stage: 'SQL Error occurred, sqlState: '08001', errorCode: '0', errorLogsMessageInvalidBoundingQuery=Spark program 'phase-1' failed with error: The column index is out of range: 1, \ number of columns: 0.. Please check the system logs for more details. From e1e36e05db4d8abb806fcf76de4400b5dbb27ba1 Mon Sep 17 00:00:00 2001 From: sgarg-CS Date: Fri, 19 Sep 2025 16:14:17 +0530 Subject: [PATCH 36/52] Revert "Disable verify goal and e2e error message changes" This reverts commit 2de8137dd62a9caa93c084bbfb5962b9d677fd88. --- .../src/e2e-test/resources/errorMessage.properties | 6 +++--- .../src/e2e-test/resources/errorMessage.properties | 2 +- .../src/e2e-test/resources/errorMessage.properties | 8 ++++---- .../src/e2e-test/resources/errorMessage.properties | 4 ++-- .../src/e2e-test/features/source/OracleRunTime.feature | 2 +- .../src/e2e-test/resources/errorMessage.properties | 9 +++++---- pom.xml | 5 ++--- .../src/e2e-test/resources/errorMessage.properties | 4 ++-- 8 files changed, 20 insertions(+), 20 deletions(-) diff --git a/cloudsql-mysql-plugin/src/e2e-test/resources/errorMessage.properties b/cloudsql-mysql-plugin/src/e2e-test/resources/errorMessage.properties index 96321fb4f..5ff3357f2 100644 --- a/cloudsql-mysql-plugin/src/e2e-test/resources/errorMessage.properties +++ b/cloudsql-mysql-plugin/src/e2e-test/resources/errorMessage.properties @@ -1,9 +1,9 @@ -errorMessageInvalidSourceDatabase=SQL Error occurred, sqlState: '42000', errorCode: '1049' +errorMessageInvalidSourceDatabase=SQL error while getting query schema errorMessageInvalidImportQuery=Import Query select must contain the string '$CONDITIONS'. if Number of Splits is not set\ \ to 1. Include '$CONDITIONS' in the Import Query errorMessageCloudMySqlInvalidReferenceName=Invalid reference name errorMessageBlankUsername=Username is required when password is given. -errorMessageBlankPassword=SQL Error occurred, sqlState: '28000', errorCode: '1045', errorMessage: SQL Exception occurred: +errorMessageBlankPassword=SQL error while getting query schema errorMessageInvalidFetchSize=Invalid fetch size. Fetch size must be a positive integer. errorMessageBlankSplitBy=Split-By Field Name must be specified if Number of Splits is not set to 1. Specify the Split-by Field Name. errorMessageInvalidNumberOfSplits=Invalid value for Number of Splits '0'. Must be at least 1. Specify a Number of Splits no less than 1. @@ -12,7 +12,7 @@ errorMessageNumberOfSplits=Split-By Field Name must be specified if Number of Sp errorMessageBoundingQuery=Bounding Query must be specified if Number of Splits is not set to 1. Specify the Bounding Query. errorMessageInvalidSinkDatabase=Error encountered while configuring the stage: 'URLDecoder: Illegal hex characters in escape (%) pattern - For input string: "$^"' errorMessageInvalidTableName=Table 'Invalidtable' does not exist. Ensure table 'Invalidtable' is set correctly and -errorMessageConnectionName=Connection Name must be in the format :: to connect to a public CloudSQL MySQL instance. +errorMessageConnectionName=Connection Name must be in the format :: to connect to a public CloudSQL PostgreSQL instance. validationSuccessMessage=No errors found. validationErrorMessage=COUNT ERROR found errorLogsMessageInvalidTableName=Spark program 'phase-1' failed with error: Errors were encountered during validation. \ diff --git a/cloudsql-postgresql-plugin/src/e2e-test/resources/errorMessage.properties b/cloudsql-postgresql-plugin/src/e2e-test/resources/errorMessage.properties index 887cfe44a..1d094aedd 100644 --- a/cloudsql-postgresql-plugin/src/e2e-test/resources/errorMessage.properties +++ b/cloudsql-postgresql-plugin/src/e2e-test/resources/errorMessage.properties @@ -15,7 +15,7 @@ errorMessageBlankPassword=SQL error while getting query schema: errorMessageInvalidPassword=SQL error while getting query schema: FATAL: password authentication failed for user errorMessageInvalidSourceHost=SQL error while getting query schema: The connection attempt failed. errorMessageInvalidTableName=Table 'table' does not exist. Ensure table '"table"' is set correctly and that the -errorMessageInvalidSinkDatabase=Error encountered while configuring the stage: 'SQL Error occurred, sqlState: '3D000', errorCode: '0', +errorMessageInvalidSinkDatabase=Exception while trying to validate schema of database table errorLogsMessageInvalidBoundingQuery=Spark program 'phase-1' failed with error: The column index is out of range: 1, \ number of columns: 0.. Please check the system logs for more details. errorMessageConnectionName=Connection Name must be in the format :: to connect to \ diff --git a/mssql-plugin/src/e2e-test/resources/errorMessage.properties b/mssql-plugin/src/e2e-test/resources/errorMessage.properties index 9b5ba1a06..00721f148 100644 --- a/mssql-plugin/src/e2e-test/resources/errorMessage.properties +++ b/mssql-plugin/src/e2e-test/resources/errorMessage.properties @@ -1,18 +1,18 @@ validationSuccessMessage=No errors found. -errorMessageInvalidSourceDatabase=SQL Error occurred, sqlState: 'S0001', errorCode: '4060', +errorMessageInvalidSourceDatabase=SQL error while getting query schema errorMessageInvalidImportQuery=Import Query select must contain the string '$CONDITIONS'. errorMessageMssqlInvalidReferenceName=Invalid reference name 'invalidRef&^*&&*'. errorMessageBlankUsername=Username is required when password is given. -errorMessageBlankPassword=SQL Error occurred, sqlState: 'S0001', errorCode: '18456', +errorMessageBlankPassword=SQL error while getting query schema errorMessageInvalidFetchSize=Invalid fetch size. Fetch size must be a positive integer. errorMessageBlankSplitBy=Split-By Field Name must be specified if Number of Splits is not set to 1. errorMessageInvalidNumberOfSplits=Invalid value for Number of Splits '0'. Must be at least 1. errorMessageNumberOfSplitNotNumber=Unable to create config for batchsource errorMessageBoundingQuery=Bounding Query must be specified if Number of Splits is not set to 1. errorMessagenumofSplit=Split-By Field Name must be specified if Number of Splits is not set to 1. -errorMessageInvalidSinkDatabase=Error encountered while configuring the stage: 'SQL Error occurred, sqlState: 'S0001', errorCode: '4060', +errorMessageInvalidSinkDatabase=Exception while trying to validate schema of database table errorMessageInvalidSinkTableName=Table 'Table123@' does not exist. -errormessageBlankHost=Error encountered while configuring the stage: 'SQL Error occurred, sqlState: '08S01', errorCode: '0', +errormessageBlankHost=Exception while trying to validate schema of database table errorMessageInvalidTableName=Spark program 'phase-1' failed with error: Errors were encountered during validation. \ Table 'Table123@' does not exist.. Please check the system logs for more details. errorMessageInvalidCredentials=Spark program 'phase-1' failed with error: Unable to create config for batchsink SqlServer \ diff --git a/mysql-plugin/src/e2e-test/resources/errorMessage.properties b/mysql-plugin/src/e2e-test/resources/errorMessage.properties index adba2f7c6..0c1ac0400 100644 --- a/mysql-plugin/src/e2e-test/resources/errorMessage.properties +++ b/mysql-plugin/src/e2e-test/resources/errorMessage.properties @@ -2,12 +2,12 @@ validationSuccessMessage=No errors found. invalid.username.message=SQL error while getting query schema: invalidtableName.error.message=Table '123#' does not exist. Ensure table '123#' is set correctly invalidreferenceName.error.message=Invalid reference name -invalid.host.message=Error encountered while configuring the stage: 'SQL Error occurred, sqlState: '08S01', errorCode: '0', +invalid.host.message=Exception while trying to validate schema of database table invalid.password.message=SQL error while getting query schema: invalid.databasename.message=SQL error while getting query schema: invalid.query.message=SQL error while getting query schema: numberofsplits.error.message=Unable to create config for batchsource Mysql 'numSplits' is invalid boundingQuery.error.message=Bounding Query must be specified if Number of Splits is not set to 1. splitfield.error.message=Split-By Field Name must be specified if Number of Splits is not set to 1 -invalid.sink.database.message=Error encountered while configuring the stage: 'SQL Error occurred, sqlState: '42000', errorCode: '1049', +invalid.sink.database.message=Exception while trying to validate schema blank.username.message=Username is required when password is given. diff --git a/oracle-plugin/src/e2e-test/features/source/OracleRunTime.feature b/oracle-plugin/src/e2e-test/features/source/OracleRunTime.feature index 007215089..2d1ca9ad1 100644 --- a/oracle-plugin/src/e2e-test/features/source/OracleRunTime.feature +++ b/oracle-plugin/src/e2e-test/features/source/OracleRunTime.feature @@ -296,7 +296,7 @@ Feature: Oracle - Verify data transfer from Oracle source to BigQuery sink Then Close the Plugin Properties page Then Save the pipeline Then Preview and run the pipeline - Then Wait till pipeline preview is in running state and check if any error occurs + Then Wait till pipeline preview is in running state Then Verify the preview run status of pipeline in the logs is "failed" @ORACLE_SOURCE_TEST @BQ_SINK_TEST diff --git a/oracle-plugin/src/e2e-test/resources/errorMessage.properties b/oracle-plugin/src/e2e-test/resources/errorMessage.properties index 6e3a23c3d..12588ac61 100644 --- a/oracle-plugin/src/e2e-test/resources/errorMessage.properties +++ b/oracle-plugin/src/e2e-test/resources/errorMessage.properties @@ -7,12 +7,13 @@ errorMessageInvalidNumberOfSplits=Invalid value for Number of Splits '0'. Must b errorMessageNumberOfSplitNotNumber=Unable to create config for batchsource Oracle 'numSplits' is invalid: Value of field\ \ class io.cdap.plugin.db.config.AbstractDBSpecificSourceConfig.numSplits is expected to be a number. errorMessageInvalidFetchSize=Invalid fetch size. Fetch size must be a positive integer. -errorMessageInvalidSourceDatabase=SQL Error occurred, sqlState: '08006', errorCode: '12514', +errorMessageInvalidSourceDatabase=SQL error while getting query schema: Listener refused the connection with the following \ + error: ORA-12514, TNS:listener does not currently know of service requested in connect descriptor errorMessageInvalidImportQuery=Import Query select must contain the string '$CONDITIONS'. if Number of Splits is not set\ \ to 1. Include '$CONDITIONS' in the Import Query errorMessageBlankUsername=Username is required when password is given. -errorMessageInvalidTableName=Error encountered while configuring the stage: 'SQL Error occurred, sqlState: '08006', errorCode: '17002', -errorMessageInvalidSinkDatabase=Error encountered while configuring the stage: 'SQL Error occurred, sqlState: '08006', errorCode: '17002' -errorMessageInvalidHost=Error encountered while configuring the stage: 'SQL Error occurred, sqlState: '08006', errorCode: '17002', +errorMessageInvalidTableName=Exception while trying to validate schema of database table '"table"' for connection +errorMessageInvalidSinkDatabase=Exception while trying to validate schema of database table '"TARGETTABLE_ +errorMessageInvalidHost=Exception while trying to validate schema of database table '"table"' for connection errorLogsMessageInvalidBoundingQuery=Spark program 'phase-1' failed with error: ORA-00936: missing expression . \ Please check the system logs for more details. diff --git a/pom.xml b/pom.xml index 98b055cd9..4c24c7477 100644 --- a/pom.xml +++ b/pom.xml @@ -650,8 +650,7 @@ integration-test - - + verify @@ -706,7 +705,7 @@ io.cdap.tests.e2e cdap-e2e-framework - 0.5.0-SNAPSHOT + 0.3.0 test diff --git a/postgresql-plugin/src/e2e-test/resources/errorMessage.properties b/postgresql-plugin/src/e2e-test/resources/errorMessage.properties index e59464785..6e1929245 100644 --- a/postgresql-plugin/src/e2e-test/resources/errorMessage.properties +++ b/postgresql-plugin/src/e2e-test/resources/errorMessage.properties @@ -16,7 +16,7 @@ errorMessageBlankPassword=SQL error while getting query schema: The server reque errorMessageInvalidPassword=SQL error while getting query schema: FATAL: password authentication failed for user errorMessageInvalidSourceHost=SQL error while getting query schema: The connection attempt failed. errorMessageInvalidTableName=Table 'table' does not exist. Ensure table '"table"' is set correctly and that the -errorMessageInvalidSinkDatabase=Error encountered while configuring the stage: 'SQL Error occurred, sqlState: '3D000', errorCode: '0', -errorMessageInvalidHost=Error encountered while configuring the stage: 'SQL Error occurred, sqlState: '08001', errorCode: '0', +errorMessageInvalidSinkDatabase=Exception while trying to validate schema of database table '"targettable_ +errorMessageInvalidHost=Exception while trying to validate schema of database table '"table"' for connection errorLogsMessageInvalidBoundingQuery=Spark program 'phase-1' failed with error: The column index is out of range: 1, \ number of columns: 0.. Please check the system logs for more details. From 078ebb5a15ed74434c0ce4f548e93e05c4ce7c90 Mon Sep 17 00:00:00 2001 From: sgarg-CS Date: Thu, 18 Sep 2025 12:10:27 +0530 Subject: [PATCH 37/52] Bump version to 1.11.9-SNAPSHOT --- amazon-redshift-plugin/pom.xml | 2 +- aurora-mysql-plugin/pom.xml | 2 +- aurora-postgresql-plugin/pom.xml | 2 +- cloudsql-mysql-plugin/pom.xml | 2 +- cloudsql-postgresql-plugin/pom.xml | 2 +- database-commons/pom.xml | 2 +- db2-plugin/pom.xml | 2 +- generic-database-plugin/pom.xml | 2 +- generic-db-argument-setter/pom.xml | 2 +- mariadb-plugin/pom.xml | 2 +- memsql-plugin/pom.xml | 2 +- mssql-plugin/pom.xml | 2 +- mysql-plugin/pom.xml | 2 +- netezza-plugin/pom.xml | 2 +- oracle-plugin/pom.xml | 2 +- pom.xml | 2 +- postgresql-plugin/pom.xml | 2 +- saphana-plugin/pom.xml | 2 +- teradata-plugin/pom.xml | 2 +- 19 files changed, 19 insertions(+), 19 deletions(-) diff --git a/amazon-redshift-plugin/pom.xml b/amazon-redshift-plugin/pom.xml index d66e009fc..b46ce1d1c 100644 --- a/amazon-redshift-plugin/pom.xml +++ b/amazon-redshift-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.8 + 1.11.9-SNAPSHOT Amazon Redshift plugin diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index d552ce1b4..4a6e53465 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.8 + 1.11.9-SNAPSHOT Aurora DB MySQL plugin diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index 316f05f7c..92dbbe043 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.8 + 1.11.9-SNAPSHOT Aurora DB PostgreSQL plugin diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index fd9b742c2..7473d9c41 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.8 + 1.11.9-SNAPSHOT CloudSQL MySQL plugin diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index b1217665e..c0bb52c22 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.8 + 1.11.9-SNAPSHOT CloudSQL PostgreSQL plugin diff --git a/database-commons/pom.xml b/database-commons/pom.xml index 0ceec28ca..9572f58af 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.8 + 1.11.9-SNAPSHOT Database Commons diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index 85f2d2089..5935a527d 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.8 + 1.11.9-SNAPSHOT IBM DB2 plugin diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index d7696c84c..4957989ae 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.8 + 1.11.9-SNAPSHOT Generic database plugin diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index ad084bbce..faac8a750 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.8 + 1.11.9-SNAPSHOT Generic database argument setter plugin diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index bdb8cfb2f..ec755d055 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.8 + 1.11.9-SNAPSHOT Maria DB plugin diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index f943e43d4..cff460db3 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.8 + 1.11.9-SNAPSHOT Memsql plugin diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index 6d4474256..87a67fe23 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.8 + 1.11.9-SNAPSHOT Microsoft SQL Server plugin diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index 44f904dfe..a4874994a 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.8 + 1.11.9-SNAPSHOT Mysql plugin diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index 1bc303e9a..7098b5a25 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.8 + 1.11.9-SNAPSHOT Netezza plugin diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index c12bb2fde..ea8c5baec 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.8 + 1.11.9-SNAPSHOT Oracle plugin diff --git a/pom.xml b/pom.xml index 4c24c7477..c7ec1800e 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin database-plugins-parent - 1.11.8 + 1.11.9-SNAPSHOT pom Database Plugins Collection of database plugins diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index 9a5d33bfa..830581c89 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.8 + 1.11.9-SNAPSHOT PostgreSQL plugin diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index 4b15f4e89..8179732b2 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.8 + 1.11.9-SNAPSHOT SAP HANA plugin diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index 94c47c674..0984b94a8 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -21,7 +21,7 @@ database-plugins-parent io.cdap.plugin - 1.11.8 + 1.11.9-SNAPSHOT teradata-plugin From c367b42e4c09f83f20292d8fe2070649f46d02b6 Mon Sep 17 00:00:00 2001 From: sgarg-CS Date: Fri, 19 Sep 2025 17:19:46 +0530 Subject: [PATCH 38/52] Remove -SNAPSHOT for release 1.11.9 --- amazon-redshift-plugin/pom.xml | 2 +- aurora-mysql-plugin/pom.xml | 2 +- aurora-postgresql-plugin/pom.xml | 2 +- cloudsql-mysql-plugin/pom.xml | 2 +- cloudsql-postgresql-plugin/pom.xml | 2 +- database-commons/pom.xml | 2 +- db2-plugin/pom.xml | 2 +- generic-database-plugin/pom.xml | 2 +- generic-db-argument-setter/pom.xml | 2 +- mariadb-plugin/pom.xml | 2 +- memsql-plugin/pom.xml | 2 +- mssql-plugin/pom.xml | 2 +- mysql-plugin/pom.xml | 2 +- netezza-plugin/pom.xml | 2 +- oracle-plugin/pom.xml | 2 +- pom.xml | 2 +- postgresql-plugin/pom.xml | 2 +- saphana-plugin/pom.xml | 2 +- teradata-plugin/pom.xml | 2 +- 19 files changed, 19 insertions(+), 19 deletions(-) diff --git a/amazon-redshift-plugin/pom.xml b/amazon-redshift-plugin/pom.xml index b46ce1d1c..3833e521c 100644 --- a/amazon-redshift-plugin/pom.xml +++ b/amazon-redshift-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.9-SNAPSHOT + 1.11.9 Amazon Redshift plugin diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index 4a6e53465..34c1636fa 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.9-SNAPSHOT + 1.11.9 Aurora DB MySQL plugin diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index 92dbbe043..e0c4e4c95 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.9-SNAPSHOT + 1.11.9 Aurora DB PostgreSQL plugin diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index 7473d9c41..81f256e9d 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.9-SNAPSHOT + 1.11.9 CloudSQL MySQL plugin diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index c0bb52c22..01f43c916 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.9-SNAPSHOT + 1.11.9 CloudSQL PostgreSQL plugin diff --git a/database-commons/pom.xml b/database-commons/pom.xml index 9572f58af..52e9d4dea 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.9-SNAPSHOT + 1.11.9 Database Commons diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index 5935a527d..d7b4442c0 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.9-SNAPSHOT + 1.11.9 IBM DB2 plugin diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index 4957989ae..95604ba00 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.9-SNAPSHOT + 1.11.9 Generic database plugin diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index faac8a750..38fad86d8 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.9-SNAPSHOT + 1.11.9 Generic database argument setter plugin diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index ec755d055..92986fe21 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.9-SNAPSHOT + 1.11.9 Maria DB plugin diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index cff460db3..9dace510e 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.9-SNAPSHOT + 1.11.9 Memsql plugin diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index 87a67fe23..8ac5131c7 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.9-SNAPSHOT + 1.11.9 Microsoft SQL Server plugin diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index a4874994a..664011e92 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.9-SNAPSHOT + 1.11.9 Mysql plugin diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index 7098b5a25..b39641535 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.9-SNAPSHOT + 1.11.9 Netezza plugin diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index ea8c5baec..542971f84 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.9-SNAPSHOT + 1.11.9 Oracle plugin diff --git a/pom.xml b/pom.xml index c7ec1800e..821b218d8 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin database-plugins-parent - 1.11.9-SNAPSHOT + 1.11.9 pom Database Plugins Collection of database plugins diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index 830581c89..6a4097742 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.9-SNAPSHOT + 1.11.9 PostgreSQL plugin diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index 8179732b2..a626b6c2a 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.9-SNAPSHOT + 1.11.9 SAP HANA plugin diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index 0984b94a8..10e2b8938 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -21,7 +21,7 @@ database-plugins-parent io.cdap.plugin - 1.11.9-SNAPSHOT + 1.11.9 teradata-plugin From 0278b35aca0c61afa2c46f72a1d7ab6224ee01b3 Mon Sep 17 00:00:00 2001 From: AMit-Cloudsufi Date: Tue, 30 Sep 2025 06:23:06 +0000 Subject: [PATCH 39/52] Add hidden treatTimestampLTZAsTimestamp --- .../cdap/plugin/oracle/OracleConnector.java | 3 +- .../plugin/oracle/OracleConnectorConfig.java | 15 ++- .../cdap/plugin/oracle/OracleConstants.java | 1 + .../io/cdap/plugin/oracle/OracleSource.java | 8 +- .../oracle/OracleSourceSchemaReader.java | 16 +++- .../plugin/oracle/OracleSchemaReaderTest.java | 94 +++++++++++++++++++ oracle-plugin/widgets/Oracle-batchsource.json | 19 ++++ oracle-plugin/widgets/Oracle-connector.json | 19 ++++ 8 files changed, 165 insertions(+), 10 deletions(-) create mode 100644 oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleSchemaReaderTest.java diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java index 16371d5c1..3094e7152 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java @@ -113,7 +113,8 @@ protected DBConnectorPath getDBConnectorPath(String path) { @Override protected SchemaReader getSchemaReader(String sessionID) { return new OracleSourceSchemaReader(sessionID, config.getTreatAsOldTimestamp(), - config.getTreatPrecisionlessNumAsDeci()); + config.getTreatPrecisionlessNumAsDeci(), + config.getTreatTimestampLTZAsTimestamp()); } @Override diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java index cbc1e5ed2..79d14215b 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java @@ -42,13 +42,14 @@ public OracleConnectorConfig(String host, int port, String user, String password public OracleConnectorConfig(String host, int port, String user, String password, String jdbcPluginName, String connectionArguments, String connectionType, String database) { this(host, port, user, password, jdbcPluginName, connectionArguments, connectionType, database, null, null, null, - null); + null, null); } public OracleConnectorConfig(String host, int port, String user, String password, String jdbcPluginName, String connectionArguments, String connectionType, String database, String role, Boolean useSSL, @Nullable Boolean treatAsOldTimestamp, - @Nullable Boolean treatPrecisionlessNumAsDeci) { + @Nullable Boolean treatPrecisionlessNumAsDeci, + @Nullable Boolean treatTimestampLTZAsTimestamp) { this.host = host; this.port = port; @@ -62,6 +63,7 @@ public OracleConnectorConfig(String host, int port, String user, String password this.useSSL = useSSL; this.treatAsOldTimestamp = treatAsOldTimestamp; this.treatPrecisionlessNumAsDeci = treatPrecisionlessNumAsDeci; + this.treatTimestampLTZAsTimestamp = treatTimestampLTZAsTimestamp; } @Override @@ -98,6 +100,11 @@ public String getConnectionString() { @Nullable public Boolean treatPrecisionlessNumAsDeci; + @Name(OracleConstants.TREAT_TIMESTAMP_LTZ_AS_TIMESTAMP) + @Description("A hidden field to handle mapping of Oracle Timestamp_LTZ data type to BQ Timestamp.") + @Nullable + public Boolean treatTimestampLTZAsTimestamp; + @Override protected int getDefaultPort() { return 1521; @@ -128,6 +135,10 @@ public Boolean getTreatPrecisionlessNumAsDeci() { return Boolean.TRUE.equals(treatPrecisionlessNumAsDeci); } + public Boolean getTreatTimestampLTZAsTimestamp() { + return Boolean.TRUE.equals(treatTimestampLTZAsTimestamp); + } + @Override public Properties getConnectionArgumentsProperties() { Properties prop = super.getConnectionArgumentsProperties(); diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java index 7d86ec820..a3560e969 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java @@ -46,6 +46,7 @@ private OracleConstants() { public static final String USE_SSL = "useSSL"; public static final String TREAT_AS_OLD_TIMESTAMP = "treatAsOldTimestamp"; public static final String TREAT_PRECISIONLESSNUM_AS_DECI = "treatPrecisionlessNumAsDeci"; + public static final String TREAT_TIMESTAMP_LTZ_AS_TIMESTAMP = "treatTimestampLTZAsTimestamp"; /** * Constructs the Oracle connection string based on the provided connection type, host, port, and database. diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java index 583fd686f..7cde9a8e9 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java @@ -67,8 +67,10 @@ protected SchemaReader getSchemaReader() { // handle schema to make it backward compatible. boolean treatAsOldTimestamp = oracleSourceConfig.getConnection().getTreatAsOldTimestamp(); boolean treatPrecisionlessNumAsDeci = oracleSourceConfig.getConnection().getTreatPrecisionlessNumAsDeci(); + boolean treatTimestampLTZAsTimestamp = oracleSourceConfig.getConnection().getTreatTimestampLTZAsTimestamp(); - return new OracleSourceSchemaReader(null, treatAsOldTimestamp, treatPrecisionlessNumAsDeci); + return new OracleSourceSchemaReader(null, treatAsOldTimestamp, treatPrecisionlessNumAsDeci, + treatTimestampLTZAsTimestamp); } @Override @@ -123,10 +125,10 @@ public OracleSourceConfig(String host, int port, String user, String password, S int defaultBatchValue, int defaultRowPrefetch, String importQuery, Integer numSplits, int fetchSize, String boundingQuery, String splitBy, Boolean useSSL, Boolean treatAsOldTimestamp, - Boolean treatPrecisionlessNumAsDeci) { + Boolean treatPrecisionlessNumAsDeci, Boolean treatTimestampLTZAsTimestamp) { this.connection = new OracleConnectorConfig(host, port, user, password, jdbcPluginName, connectionArguments, connectionType, database, role, useSSL, treatAsOldTimestamp, - treatPrecisionlessNumAsDeci); + treatPrecisionlessNumAsDeci, treatTimestampLTZAsTimestamp); this.defaultBatchValue = defaultBatchValue; this.defaultRowPrefetch = defaultRowPrefetch; this.fetchSize = fetchSize; diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceSchemaReader.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceSchemaReader.java index dd17d2e84..208b70410 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceSchemaReader.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceSchemaReader.java @@ -19,6 +19,7 @@ import com.google.common.collect.ImmutableSet; import io.cdap.cdap.api.data.schema.Schema; import io.cdap.plugin.db.CommonSchemaReader; +import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -68,15 +69,17 @@ public class OracleSourceSchemaReader extends CommonSchemaReader { private final String sessionID; private final Boolean isTimestampOldBehavior; private final Boolean isPrecisionlessNumAsDecimal; + private final Boolean isTimestampLtzFieldTimestamp; public OracleSourceSchemaReader() { - this(null, false, false); + this(null, false, false, false); } public OracleSourceSchemaReader(@Nullable String sessionID, boolean isTimestampOldBehavior, - boolean isPrecisionlessNumAsDecimal) { + boolean isPrecisionlessNumAsDecimal, boolean isTimestampLtzFieldTimestamp) { this.sessionID = sessionID; this.isTimestampOldBehavior = isTimestampOldBehavior; this.isPrecisionlessNumAsDecimal = isPrecisionlessNumAsDecimal; + this.isTimestampLtzFieldTimestamp = isTimestampLtzFieldTimestamp; } @Override @@ -87,8 +90,7 @@ public Schema getSchema(ResultSetMetaData metadata, int index) throws SQLExcepti case TIMESTAMP_TZ: return isTimestampOldBehavior ? Schema.of(Schema.Type.STRING) : Schema.of(Schema.LogicalType.TIMESTAMP_MICROS); case TIMESTAMP_LTZ: - return isTimestampOldBehavior ? Schema.of(Schema.LogicalType.TIMESTAMP_MICROS) - : Schema.of(Schema.LogicalType.DATETIME); + return getTimestampLtzSchema(); case Types.TIMESTAMP: return isTimestampOldBehavior ? super.getSchema(metadata, index) : Schema.of(Schema.LogicalType.DATETIME); case BINARY_FLOAT: @@ -139,6 +141,12 @@ public Schema getSchema(ResultSetMetaData metadata, int index) throws SQLExcepti } } + private @NotNull Schema getTimestampLtzSchema() { + return isTimestampOldBehavior || isTimestampLtzFieldTimestamp + ? Schema.of(Schema.LogicalType.TIMESTAMP_MICROS) + : Schema.of(Schema.LogicalType.DATETIME); + } + @Override public boolean shouldIgnoreColumn(ResultSetMetaData metadata, int index) throws SQLException { if (sessionID == null) { diff --git a/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleSchemaReaderTest.java b/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleSchemaReaderTest.java new file mode 100644 index 000000000..1ff77c533 --- /dev/null +++ b/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleSchemaReaderTest.java @@ -0,0 +1,94 @@ +/* + * Copyright © 2025 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.oracle; + +import com.google.common.collect.Lists; +import io.cdap.cdap.api.data.schema.Schema; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.util.List; + +public class OracleSchemaReaderTest { + + @Test + public void getSchema_timestampLTZFieldTrue_returnTimestamp() throws SQLException { + OracleSourceSchemaReader schemaReader = new OracleSourceSchemaReader(null, false, false, true); + + ResultSet resultSet = Mockito.mock(ResultSet.class); + ResultSetMetaData metadata = Mockito.mock(ResultSetMetaData.class); + + Mockito.when(resultSet.getMetaData()).thenReturn(metadata); + + Mockito.when(metadata.getColumnCount()).thenReturn(2); + // -101 is for TIMESTAMP_TZ + Mockito.when(metadata.getColumnType(1)).thenReturn(-101); + Mockito.when(metadata.getColumnName(1)).thenReturn("column1"); + + // -102 is for TIMESTAMP_LTZ + Mockito.when(metadata.getColumnType(2)).thenReturn(-102); + Mockito.when(metadata.getColumnName(2)).thenReturn("column2"); + + List expectedSchemaFields = Lists.newArrayList(); + expectedSchemaFields.add(Schema.Field.of("column1", Schema.of(Schema.LogicalType.TIMESTAMP_MICROS))); + expectedSchemaFields.add(Schema.Field.of("column2", Schema.of(Schema.LogicalType.TIMESTAMP_MICROS))); + + List actualSchemaFields = schemaReader.getSchemaFields(resultSet); + + Assert.assertEquals(expectedSchemaFields.get(0).getName(), actualSchemaFields.get(0).getName()); + Assert.assertEquals(expectedSchemaFields.get(0).getSchema(), actualSchemaFields.get(0).getSchema()); + Assert.assertEquals(expectedSchemaFields.get(1).getName(), actualSchemaFields.get(1).getName()); + Assert.assertEquals(expectedSchemaFields.get(1).getSchema(), actualSchemaFields.get(1).getSchema()); + + } + + @Test + public void getSchema_timestampLTZFieldFalse_returnDatetime() throws SQLException { + OracleSourceSchemaReader schemaReader = new OracleSourceSchemaReader(null, false, false, false); + + ResultSet resultSet = Mockito.mock(ResultSet.class); + ResultSetMetaData metadata = Mockito.mock(ResultSetMetaData.class); + + Mockito.when(resultSet.getMetaData()).thenReturn(metadata); + + Mockito.when(metadata.getColumnCount()).thenReturn(2); + // -101 is for TIMESTAMP_TZ + Mockito.when(metadata.getColumnType(1)).thenReturn(-101); + Mockito.when(metadata.getColumnName(1)).thenReturn("column1"); + + // -102 is for TIMESTAMP_LTZ + Mockito.when(metadata.getColumnType(2)).thenReturn(-102); + Mockito.when(metadata.getColumnName(2)).thenReturn("column2"); + + List expectedSchemaFields = Lists.newArrayList(); + expectedSchemaFields.add(Schema.Field.of("column1", Schema.of(Schema.LogicalType.TIMESTAMP_MICROS))); + expectedSchemaFields.add(Schema.Field.of("column2", Schema.of(Schema.LogicalType.DATETIME))); + + List actualSchemaFields = schemaReader.getSchemaFields(resultSet); + + Assert.assertEquals(expectedSchemaFields.get(0).getName(), actualSchemaFields.get(0).getName()); + Assert.assertEquals(expectedSchemaFields.get(0).getSchema(), actualSchemaFields.get(0).getSchema()); + Assert.assertEquals(expectedSchemaFields.get(1).getName(), actualSchemaFields.get(1).getName()); + Assert.assertEquals(expectedSchemaFields.get(1).getSchema(), actualSchemaFields.get(1).getSchema()); + } +} diff --git a/oracle-plugin/widgets/Oracle-batchsource.json b/oracle-plugin/widgets/Oracle-batchsource.json index 404262fb2..ab35f3e8c 100644 --- a/oracle-plugin/widgets/Oracle-batchsource.json +++ b/oracle-plugin/widgets/Oracle-batchsource.json @@ -158,6 +158,25 @@ ] } }, + { + "widget-type": "hidden", + "label": "Treat Timestamp_LTZ as Timestamp", + "name": "treatTimestampLTZAsTimestamp", + "widget-attributes": { + "layout": "inline", + "default": "false", + "options": [ + { + "id": "true", + "label": "true" + }, + { + "id": "false", + "label": "false" + } + ] + } + }, { "name": "connectionType", "label": "Connection Type", diff --git a/oracle-plugin/widgets/Oracle-connector.json b/oracle-plugin/widgets/Oracle-connector.json index 013f3b240..005c3ffbd 100644 --- a/oracle-plugin/widgets/Oracle-connector.json +++ b/oracle-plugin/widgets/Oracle-connector.json @@ -167,6 +167,25 @@ } ] } + }, + { + "widget-type": "hidden", + "label": "Treat Timestamp_LTZ as Timestamp", + "name": "treatTimestampLTZAsTimestamp", + "widget-attributes": { + "layout": "inline", + "default": "false", + "options": [ + { + "id": "true", + "label": "true" + }, + { + "id": "false", + "label": "false" + } + ] + } } ] }, From c6b0604b30b1bd230467583f7c7ad4260da2f818 Mon Sep 17 00:00:00 2001 From: Vikas Date: Fri, 3 Oct 2025 12:42:08 +0000 Subject: [PATCH 40/52] Add hidden treatTimestampLTZAsTimestamp and bump up PR --- amazon-redshift-plugin/pom.xml | 2 +- aurora-mysql-plugin/pom.xml | 2 +- aurora-postgresql-plugin/pom.xml | 2 +- cloudsql-mysql-plugin/pom.xml | 2 +- cloudsql-postgresql-plugin/pom.xml | 2 +- database-commons/pom.xml | 2 +- db2-plugin/pom.xml | 2 +- generic-database-plugin/pom.xml | 2 +- generic-db-argument-setter/pom.xml | 2 +- mariadb-plugin/pom.xml | 2 +- memsql-plugin/pom.xml | 2 +- mssql-plugin/pom.xml | 2 +- mysql-plugin/pom.xml | 2 +- netezza-plugin/pom.xml | 2 +- oracle-plugin/pom.xml | 2 +- pom.xml | 2 +- postgresql-plugin/pom.xml | 2 +- saphana-plugin/pom.xml | 2 +- teradata-plugin/pom.xml | 2 +- 19 files changed, 19 insertions(+), 19 deletions(-) diff --git a/amazon-redshift-plugin/pom.xml b/amazon-redshift-plugin/pom.xml index 3833e521c..2de3903c0 100644 --- a/amazon-redshift-plugin/pom.xml +++ b/amazon-redshift-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.9 + 1.11.10-SNAPSHOT Amazon Redshift plugin diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index 34c1636fa..6c3e49a38 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.9 + 1.11.10-SNAPSHOT Aurora DB MySQL plugin diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index e0c4e4c95..d91ba199b 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.9 + 1.11.10-SNAPSHOT Aurora DB PostgreSQL plugin diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index 81f256e9d..9e12bdede 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.9 + 1.11.10-SNAPSHOT CloudSQL MySQL plugin diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index 01f43c916..3940d6f90 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.9 + 1.11.10-SNAPSHOT CloudSQL PostgreSQL plugin diff --git a/database-commons/pom.xml b/database-commons/pom.xml index 52e9d4dea..b55529032 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.9 + 1.11.10-SNAPSHOT Database Commons diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index d7b4442c0..a997ddb99 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.9 + 1.11.10-SNAPSHOT IBM DB2 plugin diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index 95604ba00..6cab09bd2 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.9 + 1.11.10-SNAPSHOT Generic database plugin diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index 38fad86d8..207bcf123 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.9 + 1.11.10-SNAPSHOT Generic database argument setter plugin diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index 92986fe21..8a4d52891 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.9 + 1.11.10-SNAPSHOT Maria DB plugin diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index 9dace510e..e395f069c 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.9 + 1.11.10-SNAPSHOT Memsql plugin diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index 8ac5131c7..1b7af5e05 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.9 + 1.11.10-SNAPSHOT Microsoft SQL Server plugin diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index 664011e92..849a4db82 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.9 + 1.11.10-SNAPSHOT Mysql plugin diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index b39641535..63767439e 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.9 + 1.11.10-SNAPSHOT Netezza plugin diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index 542971f84..e9beb6285 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.9 + 1.11.10-SNAPSHOT Oracle plugin diff --git a/pom.xml b/pom.xml index 821b218d8..145357db7 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin database-plugins-parent - 1.11.9 + 1.11.10-SNAPSHOT pom Database Plugins Collection of database plugins diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index 6a4097742..c34511070 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.9 + 1.11.10-SNAPSHOT PostgreSQL plugin diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index a626b6c2a..b45975b2b 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.9 + 1.11.10-SNAPSHOT SAP HANA plugin diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index 10e2b8938..e5ba120b5 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -21,7 +21,7 @@ database-plugins-parent io.cdap.plugin - 1.11.9 + 1.11.10-SNAPSHOT teradata-plugin From 4ed12bfc272d25b638315725438da92a385bb933 Mon Sep 17 00:00:00 2001 From: psainics Date: Mon, 6 Oct 2025 15:37:44 +0530 Subject: [PATCH 41/52] Remove snapshot to release 1.11.10 --- amazon-redshift-plugin/pom.xml | 2 +- aurora-mysql-plugin/pom.xml | 2 +- aurora-postgresql-plugin/pom.xml | 2 +- cloudsql-mysql-plugin/pom.xml | 2 +- cloudsql-postgresql-plugin/pom.xml | 2 +- database-commons/pom.xml | 2 +- db2-plugin/pom.xml | 2 +- generic-database-plugin/pom.xml | 2 +- generic-db-argument-setter/pom.xml | 2 +- mariadb-plugin/pom.xml | 2 +- memsql-plugin/pom.xml | 2 +- mssql-plugin/pom.xml | 2 +- mysql-plugin/pom.xml | 2 +- netezza-plugin/pom.xml | 2 +- oracle-plugin/pom.xml | 2 +- pom.xml | 2 +- postgresql-plugin/pom.xml | 2 +- saphana-plugin/pom.xml | 2 +- teradata-plugin/pom.xml | 2 +- 19 files changed, 19 insertions(+), 19 deletions(-) diff --git a/amazon-redshift-plugin/pom.xml b/amazon-redshift-plugin/pom.xml index 2de3903c0..d05b37bcd 100644 --- a/amazon-redshift-plugin/pom.xml +++ b/amazon-redshift-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.10-SNAPSHOT + 1.11.10 Amazon Redshift plugin diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index 6c3e49a38..f37eea1f8 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.10-SNAPSHOT + 1.11.10 Aurora DB MySQL plugin diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index d91ba199b..416b3aff6 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.10-SNAPSHOT + 1.11.10 Aurora DB PostgreSQL plugin diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index 9e12bdede..abc690733 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.10-SNAPSHOT + 1.11.10 CloudSQL MySQL plugin diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index 3940d6f90..e6a73bc3b 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.10-SNAPSHOT + 1.11.10 CloudSQL PostgreSQL plugin diff --git a/database-commons/pom.xml b/database-commons/pom.xml index b55529032..12db73014 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.10-SNAPSHOT + 1.11.10 Database Commons diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index a997ddb99..330794adb 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.10-SNAPSHOT + 1.11.10 IBM DB2 plugin diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index 6cab09bd2..98f5a0599 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.10-SNAPSHOT + 1.11.10 Generic database plugin diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index 207bcf123..654a8fd77 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.10-SNAPSHOT + 1.11.10 Generic database argument setter plugin diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index 8a4d52891..c87f59a85 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.10-SNAPSHOT + 1.11.10 Maria DB plugin diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index e395f069c..981240c53 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.10-SNAPSHOT + 1.11.10 Memsql plugin diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index 1b7af5e05..3abf98fbe 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.10-SNAPSHOT + 1.11.10 Microsoft SQL Server plugin diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index 849a4db82..9a4f1f8f0 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.10-SNAPSHOT + 1.11.10 Mysql plugin diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index 63767439e..2b70b1f65 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.10-SNAPSHOT + 1.11.10 Netezza plugin diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index e9beb6285..9b3946806 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.10-SNAPSHOT + 1.11.10 Oracle plugin diff --git a/pom.xml b/pom.xml index 145357db7..bb39c5a96 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin database-plugins-parent - 1.11.10-SNAPSHOT + 1.11.10 pom Database Plugins Collection of database plugins diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index c34511070..6a3a95880 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.10-SNAPSHOT + 1.11.10 PostgreSQL plugin diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index b45975b2b..0cd76ee65 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.10-SNAPSHOT + 1.11.10 SAP HANA plugin diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index e5ba120b5..ef8bb98f6 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -21,7 +21,7 @@ database-plugins-parent io.cdap.plugin - 1.11.10-SNAPSHOT + 1.11.10 teradata-plugin From 33ad355c25652eeb8e69c38181d235eeb298b0b2 Mon Sep 17 00:00:00 2001 From: anupksingh Date: Thu, 18 Dec 2025 17:48:35 +0000 Subject: [PATCH 42/52] Bump version to 1.11.11-SNAPSHOT. --- amazon-redshift-plugin/pom.xml | 2 +- aurora-mysql-plugin/pom.xml | 2 +- aurora-postgresql-plugin/pom.xml | 2 +- cloudsql-mysql-plugin/pom.xml | 2 +- cloudsql-postgresql-plugin/pom.xml | 2 +- database-commons/pom.xml | 2 +- db2-plugin/pom.xml | 2 +- generic-database-plugin/pom.xml | 2 +- generic-db-argument-setter/pom.xml | 2 +- mariadb-plugin/pom.xml | 2 +- memsql-plugin/pom.xml | 2 +- mssql-plugin/pom.xml | 2 +- mysql-plugin/pom.xml | 2 +- netezza-plugin/pom.xml | 2 +- oracle-plugin/pom.xml | 2 +- pom.xml | 5 +++-- postgresql-plugin/pom.xml | 2 +- saphana-plugin/pom.xml | 2 +- teradata-plugin/pom.xml | 2 +- 19 files changed, 21 insertions(+), 20 deletions(-) diff --git a/amazon-redshift-plugin/pom.xml b/amazon-redshift-plugin/pom.xml index d05b37bcd..99e54060f 100644 --- a/amazon-redshift-plugin/pom.xml +++ b/amazon-redshift-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.10 + 1.11.11-SNAPSHOT Amazon Redshift plugin diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index f37eea1f8..09790c093 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.10 + 1.11.11-SNAPSHOT Aurora DB MySQL plugin diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index 416b3aff6..b18e948ab 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.10 + 1.11.11-SNAPSHOT Aurora DB PostgreSQL plugin diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index abc690733..b621831b6 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.10 + 1.11.11-SNAPSHOT CloudSQL MySQL plugin diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index e6a73bc3b..bd7c9691a 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.10 + 1.11.11-SNAPSHOT CloudSQL PostgreSQL plugin diff --git a/database-commons/pom.xml b/database-commons/pom.xml index 12db73014..02a3862e3 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.10 + 1.11.11-SNAPSHOT Database Commons diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index 330794adb..4d9240f83 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.10 + 1.11.11-SNAPSHOT IBM DB2 plugin diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index 98f5a0599..054255ad2 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.10 + 1.11.11-SNAPSHOT Generic database plugin diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index 654a8fd77..0f3e1838c 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.10 + 1.11.11-SNAPSHOT Generic database argument setter plugin diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index c87f59a85..6f7e8cb72 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.10 + 1.11.11-SNAPSHOT Maria DB plugin diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index 981240c53..7b625112d 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.10 + 1.11.11-SNAPSHOT Memsql plugin diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index 3abf98fbe..d501be7be 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.10 + 1.11.11-SNAPSHOT Microsoft SQL Server plugin diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index 9a4f1f8f0..ae7fef214 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.10 + 1.11.11-SNAPSHOT Mysql plugin diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index 2b70b1f65..1f3f31c0c 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.10 + 1.11.11-SNAPSHOT Netezza plugin diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index 9b3946806..7cac61582 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.10 + 1.11.11-SNAPSHOT Oracle plugin diff --git a/pom.xml b/pom.xml index bb39c5a96..cb6afe99d 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin database-plugins-parent - 1.11.10 + 1.11.11-SNAPSHOT pom Database Plugins Collection of database plugins @@ -650,7 +650,8 @@ integration-test - verify + + diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index 6a3a95880..611176612 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.10 + 1.11.11-SNAPSHOT PostgreSQL plugin diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index 0cd76ee65..d5aa55001 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.10 + 1.11.11-SNAPSHOT SAP HANA plugin diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index ef8bb98f6..1389f2314 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -21,7 +21,7 @@ database-plugins-parent io.cdap.plugin - 1.11.10 + 1.11.11-SNAPSHOT teradata-plugin From 24668e3f1e80b7cfd7bfbe01e4ad5f9e26cba315 Mon Sep 17 00:00:00 2001 From: anupksingh Date: Tue, 16 Dec 2025 17:34:18 +0000 Subject: [PATCH 43/52] PLUGIN-1939: Adding nullable split support in DB plugins. --- .../db/source/DataDrivenETLDBInputFormat.java | 57 +++++++++ .../DataDrivenETLDBInputFormatTest.java | 121 ++++++++++++++++++ 2 files changed, 178 insertions(+) create mode 100644 database-commons/src/test/java/io/cdap/plugin/db/source/DataDrivenETLDBInputFormatTest.java diff --git a/database-commons/src/main/java/io/cdap/plugin/db/source/DataDrivenETLDBInputFormat.java b/database-commons/src/main/java/io/cdap/plugin/db/source/DataDrivenETLDBInputFormat.java index 1c347dd03..1a67f09ea 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/source/DataDrivenETLDBInputFormat.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/source/DataDrivenETLDBInputFormat.java @@ -16,7 +16,10 @@ package io.cdap.plugin.db.source; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Strings; import com.google.common.base.Throwables; +import com.google.common.collect.ImmutableList; import io.cdap.plugin.db.ConnectionConfigAccessor; import io.cdap.plugin.db.JDBCDriverShim; import io.cdap.plugin.db.NoOpCommitConnection; @@ -24,6 +27,7 @@ import io.cdap.plugin.util.DBUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapreduce.InputSplit; +import org.apache.hadoop.mapreduce.JobContext; import org.apache.hadoop.mapreduce.RecordReader; import org.apache.hadoop.mapreduce.TaskAttemptContext; import org.apache.hadoop.mapreduce.lib.db.DBConfiguration; @@ -41,6 +45,7 @@ import java.sql.SQLException; import java.sql.Statement; import java.sql.Types; +import java.util.List; import java.util.Properties; /** @@ -64,6 +69,58 @@ public static void setInput(Configuration conf, new ConnectionConfigAccessor(conf).setAutoCommitEnabled(enableAutoCommit); } + /** + * A wrapper around the superclass getSplits method. This is required for testing, + * as mocking a method that calls its superclass implementation is challenging. + * This wrapper allows unit tests to override this method and mock the behavior of + * the superclass call, isolating the logic within the overridden getSplits method. + * + * @param job The job context. + * @return The list of input splits generated by the parent class. + * @throws IOException If an error occurs while getting the splits. + */ + @VisibleForTesting + List getBaseSplits(JobContext job) throws IOException { + return super.getSplits(job); + } + + @Override + public List getSplits(JobContext job) throws IOException { + List splits = getBaseSplits(job); + + // Handle the edge case where the base implementation returns no splits. In this scenario, + // create a single, all-encompassing split to ensure the job can proceed. + if (splits == null || splits.isEmpty()) { + return ImmutableList.builder() + .add(new DataDrivenDBInputSplit("1=1", "1=1")) + .build(); + } + + // Check if a split for NULL values or an all-encompassing split ("1=1") already exists. + boolean shouldAddNullSplit = splits.stream() + .map(DataDrivenDBInputFormat.DataDrivenDBInputSplit.class::cast) + .map(DataDrivenDBInputFormat.DataDrivenDBInputSplit::getLowerClause) + .noneMatch(lowerClause -> + !Strings.isNullOrEmpty(lowerClause) + && (lowerClause.contains("IS NULL") || lowerClause.contains("1=1"))); + + if (shouldAddNullSplit) { + // Create a dedicated split to handle potential NULL values in the split-by column. + String splitByColumn = getDBConf().getInputOrderBy(); + LOG.debug("No split found for NULL values. Adding a split for '{} IS NULL'.", splitByColumn); + String clause = String.format("%s IS NULL", splitByColumn); + InputSplit nullSplit = + new DataDrivenDBInputFormat.DataDrivenDBInputSplit(clause, clause); + + return ImmutableList.builder() + .addAll(splits) + .add(nullSplit) + .build(); + } + + return splits; + } + @Override public Connection getConnection() { if (this.connection == null) { diff --git a/database-commons/src/test/java/io/cdap/plugin/db/source/DataDrivenETLDBInputFormatTest.java b/database-commons/src/test/java/io/cdap/plugin/db/source/DataDrivenETLDBInputFormatTest.java new file mode 100644 index 000000000..b369d008b --- /dev/null +++ b/database-commons/src/test/java/io/cdap/plugin/db/source/DataDrivenETLDBInputFormatTest.java @@ -0,0 +1,121 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.db.source; + +import com.google.common.collect.ImmutableList; +import org.apache.hadoop.mapreduce.InputSplit; +import org.apache.hadoop.mapreduce.JobContext; +import org.apache.hadoop.mapreduce.lib.db.DBConfiguration; +import org.apache.hadoop.mapreduce.lib.db.DataDrivenDBInputFormat; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.runners.MockitoJUnitRunner; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +@RunWith(MockitoJUnitRunner.class) +public class DataDrivenETLDBInputFormatTest { + + @Mock + private JobContext mockJobContext; + @Mock + private DBConfiguration mockDbConfiguration; + + private DataDrivenETLDBInputFormat inputFormat; + + @Before + public void setUp() { + inputFormat = Mockito.spy(new DataDrivenETLDBInputFormat()); + Mockito.doReturn(mockDbConfiguration).when(inputFormat).getDBConf(); + Mockito.doReturn("id").when(mockDbConfiguration).getInputOrderBy(); + } + + @Test + public void testGetSplitsAddsNullSplit() throws IOException { + DataDrivenDBInputFormat.DataDrivenDBInputSplit existingSplit = + new DataDrivenDBInputFormat.DataDrivenDBInputSplit("id >= 0", "id < 100"); + List initialSplits = ImmutableList.of(existingSplit); + Mockito.doReturn(initialSplits).when(inputFormat).getBaseSplits(mockJobContext); + + List finalSplits = inputFormat.getSplits(mockJobContext); + + Assert.assertEquals("A new split for NULLs should be added", 2, finalSplits.size()); + + DataDrivenDBInputFormat.DataDrivenDBInputSplit nullSplit = + (DataDrivenDBInputFormat.DataDrivenDBInputSplit) finalSplits.get(1); + Assert.assertEquals("id IS NULL", nullSplit.getLowerClause()); + Assert.assertEquals("id IS NULL", nullSplit.getUpperClause()); + } + + @Test + public void testGetSplitsDoesNotAddNullSplitIfPresent() throws IOException { + DataDrivenDBInputFormat.DataDrivenDBInputSplit existingSplit = + new DataDrivenDBInputFormat.DataDrivenDBInputSplit("id >= 0", "id < 100"); + DataDrivenDBInputFormat.DataDrivenDBInputSplit nullSplit = + new DataDrivenDBInputFormat.DataDrivenDBInputSplit("id IS NULL", "id IS NULL"); + List initialSplits = ImmutableList.of(existingSplit, nullSplit); + + Mockito.doReturn(initialSplits).when(inputFormat).getBaseSplits(mockJobContext); + + List finalSplits = inputFormat.getSplits(mockJobContext); + + Assert.assertEquals("Should not add a duplicate NULL split", 2, finalSplits.size()); + } + + @Test + public void testGetSplitsDoesNotAddNullSplitIfSelectAllPresent() throws IOException { + DataDrivenDBInputFormat.DataDrivenDBInputSplit existingSplit = + new DataDrivenDBInputFormat.DataDrivenDBInputSplit("1=1", "1=1"); + List initialSplits = ImmutableList.of(existingSplit); + + Mockito.doReturn(initialSplits).when(inputFormat).getBaseSplits(mockJobContext); + + List finalSplits = inputFormat.getSplits(mockJobContext); + + Assert.assertEquals("Should not add a NULL split", 1, finalSplits.size()); + } + + @Test + public void testGetSplitsDoesNotAddNullSplitIfBaseReturnsNull() throws IOException { + Mockito.doReturn(null).when(inputFormat).getBaseSplits(mockJobContext); + + List finalSplits = inputFormat.getSplits(mockJobContext); + Assert.assertEquals("Should not add a NULL split", 1, finalSplits.size()); + + DataDrivenDBInputFormat.DataDrivenDBInputSplit split = + (DataDrivenDBInputFormat.DataDrivenDBInputSplit) finalSplits.get(0); + Assert.assertEquals("1=1", split.getLowerClause()); + } + + @Test + public void testGetSplitsDoesNotAddNullSplitIfBaseReturnsEmptyList() throws IOException { + Mockito.doReturn(Collections.emptyList()).when(inputFormat).getBaseSplits(mockJobContext); + + List finalSplits = inputFormat.getSplits(mockJobContext); + Assert.assertEquals("Should not add a NULL split", 1, finalSplits.size()); + + DataDrivenDBInputFormat.DataDrivenDBInputSplit split = + (DataDrivenDBInputFormat.DataDrivenDBInputSplit) finalSplits.get(0); + Assert.assertEquals("1=1", split.getLowerClause()); + } +} From dc3b32c5d6b52371f578f072044c285d70787e42 Mon Sep 17 00:00:00 2001 From: anupksingh Date: Thu, 18 Dec 2025 18:25:03 +0000 Subject: [PATCH 44/52] Remove snapshot to release 1.11.11 --- amazon-redshift-plugin/pom.xml | 2 +- aurora-mysql-plugin/pom.xml | 2 +- aurora-postgresql-plugin/pom.xml | 2 +- cloudsql-mysql-plugin/pom.xml | 2 +- cloudsql-postgresql-plugin/pom.xml | 2 +- database-commons/pom.xml | 2 +- db2-plugin/pom.xml | 2 +- generic-database-plugin/pom.xml | 2 +- generic-db-argument-setter/pom.xml | 2 +- mariadb-plugin/pom.xml | 2 +- memsql-plugin/pom.xml | 2 +- mssql-plugin/pom.xml | 2 +- mysql-plugin/pom.xml | 2 +- netezza-plugin/pom.xml | 2 +- oracle-plugin/pom.xml | 2 +- pom.xml | 2 +- postgresql-plugin/pom.xml | 2 +- saphana-plugin/pom.xml | 2 +- teradata-plugin/pom.xml | 2 +- 19 files changed, 19 insertions(+), 19 deletions(-) diff --git a/amazon-redshift-plugin/pom.xml b/amazon-redshift-plugin/pom.xml index 99e54060f..6560a91f6 100644 --- a/amazon-redshift-plugin/pom.xml +++ b/amazon-redshift-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.11-SNAPSHOT + 1.11.11 Amazon Redshift plugin diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index 09790c093..ee105b10d 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.11-SNAPSHOT + 1.11.11 Aurora DB MySQL plugin diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index b18e948ab..8ae556e89 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.11-SNAPSHOT + 1.11.11 Aurora DB PostgreSQL plugin diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index b621831b6..2f2202870 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.11-SNAPSHOT + 1.11.11 CloudSQL MySQL plugin diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index bd7c9691a..c4c7da90e 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.11-SNAPSHOT + 1.11.11 CloudSQL PostgreSQL plugin diff --git a/database-commons/pom.xml b/database-commons/pom.xml index 02a3862e3..a31f7eb6b 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.11-SNAPSHOT + 1.11.11 Database Commons diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index 4d9240f83..a98c38f9e 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.11-SNAPSHOT + 1.11.11 IBM DB2 plugin diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index 054255ad2..ae16aca39 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.11-SNAPSHOT + 1.11.11 Generic database plugin diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index 0f3e1838c..6be45bbb8 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.11-SNAPSHOT + 1.11.11 Generic database argument setter plugin diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index 6f7e8cb72..9341f35bb 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.11-SNAPSHOT + 1.11.11 Maria DB plugin diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index 7b625112d..aaa51a6ce 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.11-SNAPSHOT + 1.11.11 Memsql plugin diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index d501be7be..ccd33b986 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.11-SNAPSHOT + 1.11.11 Microsoft SQL Server plugin diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index ae7fef214..3c7175ade 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.11-SNAPSHOT + 1.11.11 Mysql plugin diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index 1f3f31c0c..96fb52b40 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.11-SNAPSHOT + 1.11.11 Netezza plugin diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index 7cac61582..36e161045 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.11-SNAPSHOT + 1.11.11 Oracle plugin diff --git a/pom.xml b/pom.xml index cb6afe99d..b0a950a6b 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin database-plugins-parent - 1.11.11-SNAPSHOT + 1.11.11 pom Database Plugins Collection of database plugins diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index 611176612..d57f18091 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.11-SNAPSHOT + 1.11.11 PostgreSQL plugin diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index d5aa55001..748ba2a62 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.11-SNAPSHOT + 1.11.11 SAP HANA plugin diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index 1389f2314..7f6291f80 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -21,7 +21,7 @@ database-plugins-parent io.cdap.plugin - 1.11.11-SNAPSHOT + 1.11.11 teradata-plugin From d82f1a519397b7ab6d9326a754766f942f8a8c43 Mon Sep 17 00:00:00 2001 From: anupksingh Date: Mon, 29 Dec 2025 10:55:27 +0000 Subject: [PATCH 45/52] Bump version to 1.11.12-SNAPSHOT. --- amazon-redshift-plugin/pom.xml | 2 +- aurora-mysql-plugin/pom.xml | 2 +- aurora-postgresql-plugin/pom.xml | 2 +- cloudsql-mysql-plugin/pom.xml | 2 +- cloudsql-postgresql-plugin/pom.xml | 2 +- database-commons/pom.xml | 2 +- db2-plugin/pom.xml | 2 +- generic-database-plugin/pom.xml | 2 +- generic-db-argument-setter/pom.xml | 2 +- mariadb-plugin/pom.xml | 2 +- memsql-plugin/pom.xml | 2 +- mssql-plugin/pom.xml | 2 +- mysql-plugin/pom.xml | 2 +- netezza-plugin/pom.xml | 2 +- oracle-plugin/pom.xml | 2 +- pom.xml | 2 +- postgresql-plugin/pom.xml | 2 +- saphana-plugin/pom.xml | 2 +- teradata-plugin/pom.xml | 2 +- 19 files changed, 19 insertions(+), 19 deletions(-) diff --git a/amazon-redshift-plugin/pom.xml b/amazon-redshift-plugin/pom.xml index 6560a91f6..60485ba57 100644 --- a/amazon-redshift-plugin/pom.xml +++ b/amazon-redshift-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.11 + 1.11.12-SNAPSHOT Amazon Redshift plugin diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index ee105b10d..e166b40f9 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.11 + 1.11.12-SNAPSHOT Aurora DB MySQL plugin diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index 8ae556e89..6f2c72236 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.11 + 1.11.12-SNAPSHOT Aurora DB PostgreSQL plugin diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index 2f2202870..a5fde20d1 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.11 + 1.11.12-SNAPSHOT CloudSQL MySQL plugin diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index c4c7da90e..ebc896184 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.11 + 1.11.12-SNAPSHOT CloudSQL PostgreSQL plugin diff --git a/database-commons/pom.xml b/database-commons/pom.xml index a31f7eb6b..d7a4c234a 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.11 + 1.11.12-SNAPSHOT Database Commons diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index a98c38f9e..98fc8954c 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.11 + 1.11.12-SNAPSHOT IBM DB2 plugin diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index ae16aca39..99a2b94f5 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.11 + 1.11.12-SNAPSHOT Generic database plugin diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index 6be45bbb8..e4d41b564 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.11 + 1.11.12-SNAPSHOT Generic database argument setter plugin diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index 9341f35bb..c30ce823e 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.11 + 1.11.12-SNAPSHOT Maria DB plugin diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index aaa51a6ce..df73f319a 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.11 + 1.11.12-SNAPSHOT Memsql plugin diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index ccd33b986..16543c4f9 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.11 + 1.11.12-SNAPSHOT Microsoft SQL Server plugin diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index 3c7175ade..7d3e01ae4 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.11 + 1.11.12-SNAPSHOT Mysql plugin diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index 96fb52b40..755776920 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.11 + 1.11.12-SNAPSHOT Netezza plugin diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index 36e161045..6e8f39e9c 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.11 + 1.11.12-SNAPSHOT Oracle plugin diff --git a/pom.xml b/pom.xml index b0a950a6b..5979f9837 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin database-plugins-parent - 1.11.11 + 1.11.12-SNAPSHOT pom Database Plugins Collection of database plugins diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index d57f18091..aa90bec01 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.11 + 1.11.12-SNAPSHOT PostgreSQL plugin diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index 748ba2a62..5deb75078 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.11 + 1.11.12-SNAPSHOT SAP HANA plugin diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index 7f6291f80..de277829c 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -21,7 +21,7 @@ database-plugins-parent io.cdap.plugin - 1.11.11 + 1.11.12-SNAPSHOT teradata-plugin From 4f2f8ba86421227faae79c8e9e2a665c9ffccba6 Mon Sep 17 00:00:00 2001 From: itsankit-google Date: Wed, 22 Apr 2026 04:56:39 +0000 Subject: [PATCH 46/52] Remove jetrbains dependency from oracle --- amazon-redshift-plugin/pom.xml | 6 ------ aurora-mysql-plugin/pom.xml | 6 ------ aurora-postgresql-plugin/pom.xml | 6 ------ cloudsql-mysql-plugin/pom.xml | 6 ------ cloudsql-postgresql-plugin/pom.xml | 6 ------ db2-plugin/pom.xml | 6 ------ generic-database-plugin/pom.xml | 6 ------ generic-db-argument-setter/pom.xml | 6 ------ mariadb-plugin/pom.xml | 6 ------ memsql-plugin/pom.xml | 6 ------ mssql-plugin/pom.xml | 6 ------ mysql-plugin/pom.xml | 6 ------ netezza-plugin/pom.xml | 6 ------ oracle-plugin/pom.xml | 6 ------ .../io/cdap/plugin/oracle/OracleSourceSchemaReader.java | 3 +-- postgresql-plugin/pom.xml | 6 ------ saphana-plugin/pom.xml | 6 ------ teradata-plugin/pom.xml | 6 ------ 18 files changed, 1 insertion(+), 104 deletions(-) diff --git a/amazon-redshift-plugin/pom.xml b/amazon-redshift-plugin/pom.xml index 60485ba57..db1bacd66 100644 --- a/amazon-redshift-plugin/pom.xml +++ b/amazon-redshift-plugin/pom.xml @@ -93,12 +93,6 @@ cdap-api provided - - org.jetbrains - annotations - RELEASE - compile - diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index e166b40f9..6fb8d4b39 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -77,12 +77,6 @@ cdap-api provided - - org.jetbrains - annotations - RELEASE - compile - diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index 6f2c72236..a39dc275d 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -77,12 +77,6 @@ cdap-api provided - - org.jetbrains - annotations - RELEASE - compile - io.cdap.plugin postgresql-plugin diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index a5fde20d1..491d4d9b7 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -78,12 +78,6 @@ org.mockito mockito-core - - org.jetbrains - annotations - RELEASE - compile - com.google.cloud.sql diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index ebc896184..3749e44b8 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -82,12 +82,6 @@ org.mockito mockito-core - - org.jetbrains - annotations - RELEASE - compile - com.google.cloud.sql diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index 98fc8954c..e4bce9aa8 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -77,12 +77,6 @@ db2jcc4 test - - org.jetbrains - annotations - RELEASE - compile - diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index 99a2b94f5..f9f7be63e 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -76,12 +76,6 @@ cdap-api provided - - org.jetbrains - annotations - RELEASE - compile - diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index e4d41b564..714555c41 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -76,12 +76,6 @@ cdap-api provided - - org.jetbrains - annotations - RELEASE - compile - diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index c30ce823e..0aa611eea 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -77,12 +77,6 @@ 2.7.3 test - - org.jetbrains - annotations - RELEASE - compile - io.cdap.plugin mysql-plugin diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index df73f319a..5990b45a6 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -50,12 +50,6 @@ cdap-api provided - - org.jetbrains - annotations - RELEASE - compile - diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index 16543c4f9..fb9a280e1 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -81,12 +81,6 @@ cdap-api provided - - org.jetbrains - annotations - RELEASE - compile - diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index 7d3e01ae4..c6704802f 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -81,12 +81,6 @@ 8.0.19 test - - org.jetbrains - annotations - RELEASE - compile - ${testSourceLocation} diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index 755776920..310da7114 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -71,12 +71,6 @@ cdap-api provided - - org.jetbrains - annotations - RELEASE - test - diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index 6e8f39e9c..b18d41fd9 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -92,12 +92,6 @@ 1.1.6 test - - org.jetbrains - annotations - RELEASE - compile - diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceSchemaReader.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceSchemaReader.java index 208b70410..fabac2e3a 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceSchemaReader.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceSchemaReader.java @@ -19,7 +19,6 @@ import com.google.common.collect.ImmutableSet; import io.cdap.cdap.api.data.schema.Schema; import io.cdap.plugin.db.CommonSchemaReader; -import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -141,7 +140,7 @@ public Schema getSchema(ResultSetMetaData metadata, int index) throws SQLExcepti } } - private @NotNull Schema getTimestampLtzSchema() { + private Schema getTimestampLtzSchema() { return isTimestampOldBehavior || isTimestampLtzFieldTimestamp ? Schema.of(Schema.LogicalType.TIMESTAMP_MICROS) : Schema.of(Schema.LogicalType.DATETIME); diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index aa90bec01..f466d6eab 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -73,12 +73,6 @@ cdap-api provided - - org.jetbrains - annotations - RELEASE - compile - org.mockito mockito-core diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index 5deb75078..f121e4da3 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -67,12 +67,6 @@ cdap-api provided - - org.jetbrains - annotations - RELEASE - compile - diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index de277829c..356050c9f 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -72,12 +72,6 @@ cdap-api provided - - org.jetbrains - annotations - RELEASE - compile - From 01372880aae77ddacd3a3cbb125f109a149c7ccb Mon Sep 17 00:00:00 2001 From: vanshikaagupta22 Date: Thu, 23 Apr 2026 12:48:22 +0000 Subject: [PATCH 47/52] Bump remove snapshot from 1.11.12 --- amazon-redshift-plugin/pom.xml | 2 +- aurora-mysql-plugin/pom.xml | 2 +- aurora-postgresql-plugin/pom.xml | 2 +- cloudsql-mysql-plugin/pom.xml | 2 +- cloudsql-postgresql-plugin/pom.xml | 2 +- database-commons/pom.xml | 2 +- db2-plugin/pom.xml | 2 +- generic-database-plugin/pom.xml | 2 +- generic-db-argument-setter/pom.xml | 2 +- mariadb-plugin/pom.xml | 2 +- memsql-plugin/pom.xml | 2 +- mssql-plugin/pom.xml | 2 +- mysql-plugin/pom.xml | 2 +- netezza-plugin/pom.xml | 2 +- oracle-plugin/pom.xml | 2 +- pom.xml | 2 +- postgresql-plugin/pom.xml | 2 +- saphana-plugin/pom.xml | 2 +- teradata-plugin/pom.xml | 2 +- 19 files changed, 19 insertions(+), 19 deletions(-) diff --git a/amazon-redshift-plugin/pom.xml b/amazon-redshift-plugin/pom.xml index db1bacd66..1b951bb08 100644 --- a/amazon-redshift-plugin/pom.xml +++ b/amazon-redshift-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.12-SNAPSHOT + 1.11.12 Amazon Redshift plugin diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index 6fb8d4b39..79aab9eb7 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.12-SNAPSHOT + 1.11.12 Aurora DB MySQL plugin diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index a39dc275d..4f6b0e1ad 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.12-SNAPSHOT + 1.11.12 Aurora DB PostgreSQL plugin diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index 491d4d9b7..a50760364 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.12-SNAPSHOT + 1.11.12 CloudSQL MySQL plugin diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index 3749e44b8..2cb4c1a40 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.12-SNAPSHOT + 1.11.12 CloudSQL PostgreSQL plugin diff --git a/database-commons/pom.xml b/database-commons/pom.xml index d7a4c234a..ee7e24bbb 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.12-SNAPSHOT + 1.11.12 Database Commons diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index e4bce9aa8..82dc22c8d 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.12-SNAPSHOT + 1.11.12 IBM DB2 plugin diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index f9f7be63e..51494cd36 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.12-SNAPSHOT + 1.11.12 Generic database plugin diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index 714555c41..14c353462 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.12-SNAPSHOT + 1.11.12 Generic database argument setter plugin diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index 0aa611eea..eee9502dd 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.12-SNAPSHOT + 1.11.12 Maria DB plugin diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index 5990b45a6..4b400c46b 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.12-SNAPSHOT + 1.11.12 Memsql plugin diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index fb9a280e1..e476867e4 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.12-SNAPSHOT + 1.11.12 Microsoft SQL Server plugin diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index c6704802f..dbf8d82bf 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.12-SNAPSHOT + 1.11.12 Mysql plugin diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index 310da7114..ba9885aa5 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.12-SNAPSHOT + 1.11.12 Netezza plugin diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index b18d41fd9..83bd315ee 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.12-SNAPSHOT + 1.11.12 Oracle plugin diff --git a/pom.xml b/pom.xml index 5979f9837..8c3168d5a 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin database-plugins-parent - 1.11.12-SNAPSHOT + 1.11.12 pom Database Plugins Collection of database plugins diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index f466d6eab..2fdf59d19 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.12-SNAPSHOT + 1.11.12 PostgreSQL plugin diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index f121e4da3..10709cf97 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.12-SNAPSHOT + 1.11.12 SAP HANA plugin diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index 356050c9f..9dfe2ca6c 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -21,7 +21,7 @@ database-plugins-parent io.cdap.plugin - 1.11.12-SNAPSHOT + 1.11.12 teradata-plugin From 9ee4d97ac22149039e58541dd864640003d5c7af Mon Sep 17 00:00:00 2001 From: vanshikaagupta22 Date: Fri, 24 Apr 2026 08:42:37 +0000 Subject: [PATCH 48/52] Enable xml type support for oracle for dts-connectors only --- .../cdap/plugin/oracle/OracleConnector.java | 3 +- .../plugin/oracle/OracleConnectorConfig.java | 18 +++++++-- .../cdap/plugin/oracle/OracleConstants.java | 1 + .../io/cdap/plugin/oracle/OracleSource.java | 9 +++-- .../oracle/OracleSourceSchemaReader.java | 11 +++++- .../plugin/oracle/OracleSchemaReaderTest.java | 39 ++++++++++++++++++- oracle-plugin/widgets/Oracle-batchsource.json | 19 +++++++++ oracle-plugin/widgets/Oracle-connector.json | 19 +++++++++ 8 files changed, 108 insertions(+), 11 deletions(-) diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java index 3094e7152..579274e88 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java @@ -114,7 +114,8 @@ protected DBConnectorPath getDBConnectorPath(String path) { protected SchemaReader getSchemaReader(String sessionID) { return new OracleSourceSchemaReader(sessionID, config.getTreatAsOldTimestamp(), config.getTreatPrecisionlessNumAsDeci(), - config.getTreatTimestampLTZAsTimestamp()); + config.getTreatTimestampLTZAsTimestamp(), + config.getXmlTypeEnabled()); } @Override diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java index 79d14215b..03dc56d6d 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java @@ -42,14 +42,15 @@ public OracleConnectorConfig(String host, int port, String user, String password public OracleConnectorConfig(String host, int port, String user, String password, String jdbcPluginName, String connectionArguments, String connectionType, String database) { this(host, port, user, password, jdbcPluginName, connectionArguments, connectionType, database, null, null, null, - null, null); + null, null, null); } public OracleConnectorConfig(String host, int port, String user, String password, String jdbcPluginName, String connectionArguments, String connectionType, String database, String role, Boolean useSSL, @Nullable Boolean treatAsOldTimestamp, @Nullable Boolean treatPrecisionlessNumAsDeci, - @Nullable Boolean treatTimestampLTZAsTimestamp) { + @Nullable Boolean treatTimestampLTZAsTimestamp, + @Nullable Boolean enableXmlType) { this.host = host; this.port = port; @@ -64,6 +65,7 @@ public OracleConnectorConfig(String host, int port, String user, String password this.treatAsOldTimestamp = treatAsOldTimestamp; this.treatPrecisionlessNumAsDeci = treatPrecisionlessNumAsDeci; this.treatTimestampLTZAsTimestamp = treatTimestampLTZAsTimestamp; + this.enableXmlType = enableXmlType; } @Override @@ -101,10 +103,16 @@ public String getConnectionString() { public Boolean treatPrecisionlessNumAsDeci; @Name(OracleConstants.TREAT_TIMESTAMP_LTZ_AS_TIMESTAMP) - @Description("A hidden field to handle mapping of Oracle Timestamp_LTZ data type to BQ Timestamp.") + @Description("A hidden field to handle mapping of Oracle Timestamp_LTZ data type.") @Nullable public Boolean treatTimestampLTZAsTimestamp; + @Name(OracleConstants.ENABLE_XML_TYPE) + @Description("A hidden field to handle mapping of Oracle XML type.") + @Nullable + public Boolean enableXmlType; + + @Override protected int getDefaultPort() { return 1521; @@ -139,6 +147,10 @@ public Boolean getTreatTimestampLTZAsTimestamp() { return Boolean.TRUE.equals(treatTimestampLTZAsTimestamp); } + public Boolean getXmlTypeEnabled() { + return Boolean.TRUE.equals(enableXmlType); + } + @Override public Properties getConnectionArgumentsProperties() { Properties prop = super.getConnectionArgumentsProperties(); diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java index a3560e969..1a9508382 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java @@ -47,6 +47,7 @@ private OracleConstants() { public static final String TREAT_AS_OLD_TIMESTAMP = "treatAsOldTimestamp"; public static final String TREAT_PRECISIONLESSNUM_AS_DECI = "treatPrecisionlessNumAsDeci"; public static final String TREAT_TIMESTAMP_LTZ_AS_TIMESTAMP = "treatTimestampLTZAsTimestamp"; + public static final String ENABLE_XML_TYPE = "enableXmlType"; /** * Constructs the Oracle connection string based on the provided connection type, host, port, and database. diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java index 7cde9a8e9..5df47be12 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java @@ -68,9 +68,10 @@ protected SchemaReader getSchemaReader() { boolean treatAsOldTimestamp = oracleSourceConfig.getConnection().getTreatAsOldTimestamp(); boolean treatPrecisionlessNumAsDeci = oracleSourceConfig.getConnection().getTreatPrecisionlessNumAsDeci(); boolean treatTimestampLTZAsTimestamp = oracleSourceConfig.getConnection().getTreatTimestampLTZAsTimestamp(); + boolean enableXmlType = oracleSourceConfig.getConnection().getXmlTypeEnabled(); return new OracleSourceSchemaReader(null, treatAsOldTimestamp, treatPrecisionlessNumAsDeci, - treatTimestampLTZAsTimestamp); + treatTimestampLTZAsTimestamp, enableXmlType); } @Override @@ -125,10 +126,12 @@ public OracleSourceConfig(String host, int port, String user, String password, S int defaultBatchValue, int defaultRowPrefetch, String importQuery, Integer numSplits, int fetchSize, String boundingQuery, String splitBy, Boolean useSSL, Boolean treatAsOldTimestamp, - Boolean treatPrecisionlessNumAsDeci, Boolean treatTimestampLTZAsTimestamp) { + Boolean treatPrecisionlessNumAsDeci, Boolean treatTimestampLTZAsTimestamp, + Boolean enableXmlType) { this.connection = new OracleConnectorConfig(host, port, user, password, jdbcPluginName, connectionArguments, connectionType, database, role, useSSL, treatAsOldTimestamp, - treatPrecisionlessNumAsDeci, treatTimestampLTZAsTimestamp); + treatPrecisionlessNumAsDeci, treatTimestampLTZAsTimestamp, + enableXmlType); this.defaultBatchValue = defaultBatchValue; this.defaultRowPrefetch = defaultRowPrefetch; this.fetchSize = fetchSize; diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceSchemaReader.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceSchemaReader.java index fabac2e3a..b23dfa031 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceSchemaReader.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceSchemaReader.java @@ -60,6 +60,7 @@ public class OracleSourceSchemaReader extends CommonSchemaReader { BINARY_DOUBLE, BFILE, LONG, + Types.SQLXML, LONG_RAW, Types.NUMERIC, Types.DECIMAL @@ -69,16 +70,19 @@ public class OracleSourceSchemaReader extends CommonSchemaReader { private final Boolean isTimestampOldBehavior; private final Boolean isPrecisionlessNumAsDecimal; private final Boolean isTimestampLtzFieldTimestamp; + private final Boolean isXmlTypeEnabled; public OracleSourceSchemaReader() { - this(null, false, false, false); + this(null, false, false, false, false); } public OracleSourceSchemaReader(@Nullable String sessionID, boolean isTimestampOldBehavior, - boolean isPrecisionlessNumAsDecimal, boolean isTimestampLtzFieldTimestamp) { + boolean isPrecisionlessNumAsDecimal, boolean isTimestampLtzFieldTimestamp, + boolean isXmlTypeEnabled) { this.sessionID = sessionID; this.isTimestampOldBehavior = isTimestampOldBehavior; this.isPrecisionlessNumAsDecimal = isPrecisionlessNumAsDecimal; this.isTimestampLtzFieldTimestamp = isTimestampLtzFieldTimestamp; + this.isXmlTypeEnabled = isXmlTypeEnabled; } @Override @@ -103,6 +107,9 @@ public Schema getSchema(ResultSetMetaData metadata, int index) throws SQLExcepti case INTERVAL_YM: case LONG: return Schema.of(Schema.Type.STRING); + case Types.SQLXML: + // Enabling XML type support for DTS connectors only as it is not in working state in CDAP plugin. + return isXmlTypeEnabled ? Schema.of(Schema.Type.STRING) : super.getSchema(metadata, index); case Types.NUMERIC: case Types.DECIMAL: // FLOAT and REAL are returned as java.sql.Types.NUMERIC but with value that is a java.lang.Double diff --git a/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleSchemaReaderTest.java b/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleSchemaReaderTest.java index 1ff77c533..586ca1141 100644 --- a/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleSchemaReaderTest.java +++ b/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleSchemaReaderTest.java @@ -18,6 +18,7 @@ import com.google.common.collect.Lists; import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.cdap.api.exception.ProgramFailureException; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -27,13 +28,14 @@ import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; +import java.sql.Types; import java.util.List; public class OracleSchemaReaderTest { @Test public void getSchema_timestampLTZFieldTrue_returnTimestamp() throws SQLException { - OracleSourceSchemaReader schemaReader = new OracleSourceSchemaReader(null, false, false, true); + OracleSourceSchemaReader schemaReader = new OracleSourceSchemaReader(null, false, false, true, false); ResultSet resultSet = Mockito.mock(ResultSet.class); ResultSetMetaData metadata = Mockito.mock(ResultSetMetaData.class); @@ -64,7 +66,7 @@ public void getSchema_timestampLTZFieldTrue_returnTimestamp() throws SQLExceptio @Test public void getSchema_timestampLTZFieldFalse_returnDatetime() throws SQLException { - OracleSourceSchemaReader schemaReader = new OracleSourceSchemaReader(null, false, false, false); + OracleSourceSchemaReader schemaReader = new OracleSourceSchemaReader(null, false, false, false, false); ResultSet resultSet = Mockito.mock(ResultSet.class); ResultSetMetaData metadata = Mockito.mock(ResultSetMetaData.class); @@ -91,4 +93,37 @@ public void getSchema_timestampLTZFieldFalse_returnDatetime() throws SQLExceptio Assert.assertEquals(expectedSchemaFields.get(1).getName(), actualSchemaFields.get(1).getName()); Assert.assertEquals(expectedSchemaFields.get(1).getSchema(), actualSchemaFields.get(1).getSchema()); } + + @Test + public void getSchema_xmlField_returnString() throws SQLException { + OracleSourceSchemaReader schemaReader = new OracleSourceSchemaReader(null, false, false, false, true); + ResultSet resultSet = Mockito.mock(ResultSet.class); + ResultSetMetaData metadata = Mockito.mock(ResultSetMetaData.class); + Mockito.when(resultSet.getMetaData()).thenReturn(metadata); + Mockito.when(metadata.getColumnCount()).thenReturn(1); + Mockito.when(metadata.getColumnType(1)).thenReturn(Types.SQLXML); + Mockito.when(metadata.getColumnName(1)).thenReturn("xmlData"); + + List actualSchemaFields = schemaReader.getSchemaFields(resultSet); + + List expectedSchemaFields = Lists.newArrayList(); + expectedSchemaFields.add(Schema.Field.of("xmlData", Schema.of(Schema.Type.STRING))); + Assert.assertEquals(expectedSchemaFields.get(0).getName(), actualSchemaFields.get(0).getName()); + Assert.assertEquals(expectedSchemaFields.get(0).getSchema(), actualSchemaFields.get(0).getSchema()); + } + + @Test + public void getSchema_xmlFieldDisabled_throwsProgramFailureException() throws SQLException { + OracleSourceSchemaReader schemaReader = new OracleSourceSchemaReader(null, + false, false, false, false); + ResultSet resultSet = Mockito.mock(ResultSet.class); + ResultSetMetaData metadata = Mockito.mock(ResultSetMetaData.class); + Mockito.when(resultSet.getMetaData()).thenReturn(metadata); + Mockito.when(metadata.getColumnCount()).thenReturn(1); + Mockito.when(metadata.getColumnType(1)).thenReturn(Types.SQLXML); + Mockito.when(metadata.getColumnName(1)).thenReturn("xmlData"); + + Assert.assertThrows(ProgramFailureException.class, () -> schemaReader.getSchemaFields(resultSet)); + + } } diff --git a/oracle-plugin/widgets/Oracle-batchsource.json b/oracle-plugin/widgets/Oracle-batchsource.json index ab35f3e8c..dba149f4f 100644 --- a/oracle-plugin/widgets/Oracle-batchsource.json +++ b/oracle-plugin/widgets/Oracle-batchsource.json @@ -177,6 +177,25 @@ ] } }, + { + "widget-type": "hidden", + "label": "Enable Xml Type", + "name": "enableXmlType", + "widget-attributes": { + "layout": "inline", + "default": "false", + "options": [ + { + "id": "true", + "label": "true" + }, + { + "id": "false", + "label": "false" + } + ] + } + }, { "name": "connectionType", "label": "Connection Type", diff --git a/oracle-plugin/widgets/Oracle-connector.json b/oracle-plugin/widgets/Oracle-connector.json index 005c3ffbd..c68b95de0 100644 --- a/oracle-plugin/widgets/Oracle-connector.json +++ b/oracle-plugin/widgets/Oracle-connector.json @@ -186,6 +186,25 @@ } ] } + }, + { + "widget-type": "hidden", + "label": "Enable Xml Type", + "name": "enableXmlType", + "widget-attributes": { + "layout": "inline", + "default": "false", + "options": [ + { + "id": "true", + "label": "true" + }, + { + "id": "false", + "label": "false" + } + ] + } } ] }, From bf4a0e2695d94edda1f8151df11aa7b9087ed97b Mon Sep 17 00:00:00 2001 From: vanshikaagupta22 Date: Wed, 29 Apr 2026 07:30:51 +0000 Subject: [PATCH 49/52] Update oracle plugin version to 1.11.13-SNAPSHOT --- amazon-redshift-plugin/pom.xml | 2 +- aurora-mysql-plugin/pom.xml | 2 +- aurora-postgresql-plugin/pom.xml | 2 +- cloudsql-mysql-plugin/pom.xml | 2 +- cloudsql-postgresql-plugin/pom.xml | 2 +- database-commons/pom.xml | 2 +- db2-plugin/pom.xml | 2 +- generic-database-plugin/pom.xml | 2 +- generic-db-argument-setter/pom.xml | 2 +- mariadb-plugin/pom.xml | 2 +- memsql-plugin/pom.xml | 2 +- mssql-plugin/pom.xml | 2 +- mysql-plugin/pom.xml | 2 +- netezza-plugin/pom.xml | 2 +- oracle-plugin/pom.xml | 2 +- .../java/io/cdap/plugin/oracle/OracleSchemaReaderTest.java | 5 ++--- pom.xml | 2 +- postgresql-plugin/pom.xml | 2 +- saphana-plugin/pom.xml | 2 +- teradata-plugin/pom.xml | 2 +- 20 files changed, 21 insertions(+), 22 deletions(-) diff --git a/amazon-redshift-plugin/pom.xml b/amazon-redshift-plugin/pom.xml index 1b951bb08..e187c1ce6 100644 --- a/amazon-redshift-plugin/pom.xml +++ b/amazon-redshift-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.12 + 1.11.13-SNAPSHOT Amazon Redshift plugin diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index 79aab9eb7..bc398a7af 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.12 + 1.11.13-SNAPSHOT Aurora DB MySQL plugin diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index 4f6b0e1ad..0353433b6 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.12 + 1.11.13-SNAPSHOT Aurora DB PostgreSQL plugin diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index a50760364..296110ecd 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.12 + 1.11.13-SNAPSHOT CloudSQL MySQL plugin diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index 2cb4c1a40..56b4e611e 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.12 + 1.11.13-SNAPSHOT CloudSQL PostgreSQL plugin diff --git a/database-commons/pom.xml b/database-commons/pom.xml index ee7e24bbb..559ea3423 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.12 + 1.11.13-SNAPSHOT Database Commons diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index 82dc22c8d..39900c4ae 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.12 + 1.11.13-SNAPSHOT IBM DB2 plugin diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index 51494cd36..daa6f8715 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.12 + 1.11.13-SNAPSHOT Generic database plugin diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index 14c353462..239e7196f 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.12 + 1.11.13-SNAPSHOT Generic database argument setter plugin diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index eee9502dd..fbf3c7518 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.12 + 1.11.13-SNAPSHOT Maria DB plugin diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index 4b400c46b..63bd2fa3b 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.12 + 1.11.13-SNAPSHOT Memsql plugin diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index e476867e4..ce876bbfc 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.12 + 1.11.13-SNAPSHOT Microsoft SQL Server plugin diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index dbf8d82bf..7cbf437ad 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.12 + 1.11.13-SNAPSHOT Mysql plugin diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index ba9885aa5..b14874463 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.12 + 1.11.13-SNAPSHOT Netezza plugin diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index 83bd315ee..7990c492a 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.12 + 1.11.13-SNAPSHOT Oracle plugin diff --git a/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleSchemaReaderTest.java b/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleSchemaReaderTest.java index 586ca1141..6dac3c6dc 100644 --- a/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleSchemaReaderTest.java +++ b/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleSchemaReaderTest.java @@ -18,7 +18,6 @@ import com.google.common.collect.Lists; import io.cdap.cdap.api.data.schema.Schema; -import io.cdap.cdap.api.exception.ProgramFailureException; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -113,7 +112,7 @@ public void getSchema_xmlField_returnString() throws SQLException { } @Test - public void getSchema_xmlFieldDisabled_throwsProgramFailureException() throws SQLException { + public void getSchema_xmlFieldDisabled_throwsSqlException() throws SQLException { OracleSourceSchemaReader schemaReader = new OracleSourceSchemaReader(null, false, false, false, false); ResultSet resultSet = Mockito.mock(ResultSet.class); @@ -123,7 +122,7 @@ public void getSchema_xmlFieldDisabled_throwsProgramFailureException() throws SQ Mockito.when(metadata.getColumnType(1)).thenReturn(Types.SQLXML); Mockito.when(metadata.getColumnName(1)).thenReturn("xmlData"); - Assert.assertThrows(ProgramFailureException.class, () -> schemaReader.getSchemaFields(resultSet)); + Assert.assertThrows(SQLException.class, () -> schemaReader.getSchemaFields(resultSet)); } } diff --git a/pom.xml b/pom.xml index 8c3168d5a..db5f4366b 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin database-plugins-parent - 1.11.12 + 1.11.13-SNAPSHOT pom Database Plugins Collection of database plugins diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index 2fdf59d19..7d19c4163 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.12 + 1.11.13-SNAPSHOT PostgreSQL plugin diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index 10709cf97..2ec2dd0d1 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.12 + 1.11.13-SNAPSHOT SAP HANA plugin diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index 9dfe2ca6c..dfaa8276f 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -21,7 +21,7 @@ database-plugins-parent io.cdap.plugin - 1.11.12 + 1.11.13-SNAPSHOT teradata-plugin From 79939178605114286318fea1b2bc461e69130c02 Mon Sep 17 00:00:00 2001 From: vanshikaagupta22 Date: Wed, 20 May 2026 06:17:47 +0000 Subject: [PATCH 50/52] Update ref branch in github workflow --- .github/workflows/e2e.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index c710ef929..f7d3090db 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -65,6 +65,7 @@ jobs: with: repository: cdapio/cdap-e2e-tests path: e2e + ref: release/6.10 - name: Cache uses: actions/cache@v4 From af045689ce49c0d4a1299ca85f0f46a802318a13 Mon Sep 17 00:00:00 2001 From: vikasrathee-cs Date: Wed, 10 Jun 2026 14:02:46 +0530 Subject: [PATCH 51/52] Bump up e2e framework and update repositories --- pom.xml | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index db5f4366b..3c1ed3fab 100644 --- a/pom.xml +++ b/pom.xml @@ -78,9 +78,26 @@ + + central + Maven Central + https://repo.maven.apache.org/maven2 + + false + + + true + + sonatype-snapshots https://central.sonatype.com/repository/maven-snapshots + + false + + + true + @@ -706,7 +723,7 @@ io.cdap.tests.e2e cdap-e2e-framework - 0.3.0 + 0.3.3 test From 4f69c10dda7521ff21e0a719d5f5a4d86b8bbf54 Mon Sep 17 00:00:00 2001 From: vanshikaagupta22 Date: Wed, 20 May 2026 07:55:34 +0000 Subject: [PATCH 52/52] Bump up version - remove snapshot from 1.11.13 --- amazon-redshift-plugin/pom.xml | 2 +- aurora-mysql-plugin/pom.xml | 2 +- aurora-postgresql-plugin/pom.xml | 2 +- cloudsql-mysql-plugin/pom.xml | 2 +- cloudsql-postgresql-plugin/pom.xml | 2 +- database-commons/pom.xml | 2 +- db2-plugin/pom.xml | 2 +- generic-database-plugin/pom.xml | 2 +- generic-db-argument-setter/pom.xml | 2 +- mariadb-plugin/pom.xml | 2 +- memsql-plugin/pom.xml | 2 +- mssql-plugin/pom.xml | 2 +- mysql-plugin/pom.xml | 2 +- netezza-plugin/pom.xml | 2 +- oracle-plugin/pom.xml | 2 +- pom.xml | 2 +- postgresql-plugin/pom.xml | 2 +- saphana-plugin/pom.xml | 2 +- teradata-plugin/pom.xml | 2 +- 19 files changed, 19 insertions(+), 19 deletions(-) diff --git a/amazon-redshift-plugin/pom.xml b/amazon-redshift-plugin/pom.xml index e187c1ce6..247968541 100644 --- a/amazon-redshift-plugin/pom.xml +++ b/amazon-redshift-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.13-SNAPSHOT + 1.11.13 Amazon Redshift plugin diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index bc398a7af..5c2cefb3f 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.13-SNAPSHOT + 1.11.13 Aurora DB MySQL plugin diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index 0353433b6..22e4ad0f1 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.13-SNAPSHOT + 1.11.13 Aurora DB PostgreSQL plugin diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index 296110ecd..e67b1bdee 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.13-SNAPSHOT + 1.11.13 CloudSQL MySQL plugin diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index 56b4e611e..b3593d7d6 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.13-SNAPSHOT + 1.11.13 CloudSQL PostgreSQL plugin diff --git a/database-commons/pom.xml b/database-commons/pom.xml index 559ea3423..20f8a6975 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.13-SNAPSHOT + 1.11.13 Database Commons diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index 39900c4ae..f11d6aa75 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.13-SNAPSHOT + 1.11.13 IBM DB2 plugin diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index daa6f8715..122f7aa86 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.13-SNAPSHOT + 1.11.13 Generic database plugin diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index 239e7196f..a6788538e 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.13-SNAPSHOT + 1.11.13 Generic database argument setter plugin diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index fbf3c7518..2e291d411 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.13-SNAPSHOT + 1.11.13 Maria DB plugin diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index 63bd2fa3b..8d37d0b32 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.13-SNAPSHOT + 1.11.13 Memsql plugin diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index ce876bbfc..2be35a58a 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.13-SNAPSHOT + 1.11.13 Microsoft SQL Server plugin diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index 7cbf437ad..4d4b3c72c 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.13-SNAPSHOT + 1.11.13 Mysql plugin diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index b14874463..2d87ca056 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.13-SNAPSHOT + 1.11.13 Netezza plugin diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index 7990c492a..2afce153d 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.13-SNAPSHOT + 1.11.13 Oracle plugin diff --git a/pom.xml b/pom.xml index 3c1ed3fab..8c9e59ddb 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin database-plugins-parent - 1.11.13-SNAPSHOT + 1.11.13 pom Database Plugins Collection of database plugins diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index 7d19c4163..bdf2ed706 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.13-SNAPSHOT + 1.11.13 PostgreSQL plugin diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index 2ec2dd0d1..9a1adb038 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.13-SNAPSHOT + 1.11.13 SAP HANA plugin diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index dfaa8276f..738790fde 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -21,7 +21,7 @@ database-plugins-parent io.cdap.plugin - 1.11.13-SNAPSHOT + 1.11.13 teradata-plugin