From b3e1d2baa77f9e93adcef1dc7fc860bdfcd2877f Mon Sep 17 00:00:00 2001 From: Christofer Dutz Date: Fri, 20 Nov 2026 11:52:34 +0100 Subject: [PATCH 1/7] Add a PostgreSQL wire protocol service for the table model PostgreSQL clients (psql, pgJDBC/DBeaver, psycopg, Npgsql, Power BI, Excel) can query the table model, read-only. New built-in external service external-service-impl/postgresql, off by default (enable_postgresql_service): - PostgreSQL protocol 3.0 on Netty: simple and extended query protocol, binary results, portals and SQL cursors, cancel, statement_timeout, streaming with a write timeout, connection limit. - TLS 1.2/1.3 only off loopback (key_store_path); password login through SessionManager, offered only inside TLS or on loopback. - Translation of PostgreSQL SQL into table-model SQL (JSqlParser); constructs without an exact equivalent are refused with SQLSTATE 0A000. - pg_catalog and information_schema emulation as information_schema.pg_* and pg_is_* tables, filtered by the user's privileges. Unit and golden tests over SQL recorded from psql, pgJDBC and psycopg, and integration tests (TableSimpleIT). --- LICENSE-binary | 1 + distribution/pom.xml | 6 + distribution/src/assembly/all.xml | 5 + distribution/src/assembly/edge.xml | 5 + external-service-impl/pom.xml | 1 + external-service-impl/postgresql/README.md | 165 ++ external-service-impl/postgresql/pom.xml | 160 ++ .../iotdb/postgresql/PostgreSqlService.java | 201 ++ .../iotdb/postgresql/catalog/CatalogRows.java | 528 ++++++ .../postgresql/catalog/CatalogSnapshot.java | 292 +++ .../catalog/PgCatalogExtension.java | 54 + .../postgresql/catalog/PgCatalogProvider.java | 137 ++ .../postgresql/catalog/PgCatalogTables.java | 576 ++++++ .../postgresql/catalog/StaticCatalog.java | 920 +++++++++ .../postgresql/config/BindAddresses.java | 98 + .../config/PostgreSqlServiceConfig.java | 208 ++ .../postgresql/execution/ResultCursor.java | 250 +++ .../postgresql/execution/RunningQuery.java | 213 +++ .../postgresql/protocol/BackendMessages.java | 242 +++ .../postgresql/protocol/FrontendMessage.java | 117 ++ .../iotdb/postgresql/protocol/PgErrors.java | 103 + .../postgresql/protocol/PgException.java | 59 + .../postgresql/protocol/PgFrameDecoder.java | 88 + .../iotdb/postgresql/protocol/SqlState.java | 54 + .../server/PgConnectionHandler.java | 405 ++++ .../iotdb/postgresql/server/PgServer.java | 115 ++ .../postgresql/server/PgServerContext.java | 85 + .../postgresql/server/SerialExecutor.java | 69 + .../iotdb/postgresql/server/TlsPolicy.java | 77 + .../postgresql/session/CancelRegistry.java | 93 + .../postgresql/session/CursorCommand.java | 183 ++ .../iotdb/postgresql/session/PgSession.java | 132 ++ .../postgresql/session/QueryProtocol.java | 43 + .../iotdb/postgresql/session/SearchPath.java | 56 + .../postgresql/session/SessionCommands.java | 374 ++++ .../iotdb/postgresql/session/SqlProtocol.java | 930 +++++++++ .../postgresql/session/StatementSplitter.java | 104 + .../postgresql/session/StatementTimeout.java | 88 + .../iotdb/postgresql/sql/ConstructCheck.java | 440 +++++ .../iotdb/postgresql/sql/DateTimeRules.java | 398 ++++ .../iotdb/postgresql/sql/EmptyRelations.java | 388 ++++ .../iotdb/postgresql/sql/Identifiers.java | 49 + .../iotdb/postgresql/sql/OuterConditions.java | 185 ++ .../iotdb/postgresql/sql/OutputNames.java | 113 ++ .../iotdb/postgresql/sql/PgToTableSql.java | 1682 +++++++++++++++++ .../iotdb/postgresql/sql/PgTypeNames.java | 109 ++ .../apache/iotdb/postgresql/sql/PreParse.java | 104 + .../iotdb/postgresql/sql/RegexPatterns.java | 163 ++ .../iotdb/postgresql/sql/StringLiterals.java | 115 ++ .../postgresql/sql/TranslationContext.java | 54 + .../sql/UnsupportedConstructException.java | 41 + .../postgresql/types/ParameterDecoder.java | 257 +++ .../apache/iotdb/postgresql/types/PgType.java | 114 ++ .../postgresql/types/PgValueEncoder.java | 230 +++ .../postgresql/types/TimestampPrecision.java | 63 + ...formationSchema$InformationSchemaExtension | 19 + ...mation.AdditionalInformationSchemaProvider | 19 + ...iotdb.externalservice.api.IExternalService | 20 + .../test/corpus-capture/PgJdbcCapture.java | 83 + .../src/test/corpus-capture/extract.py | 84 + .../test/corpus-capture/psycopg_capture.py | 36 + .../src/test/corpus-capture/setup.sql | 45 + .../postgresql/catalog/CatalogRowsTest.java | 247 +++ .../config/PostgreSqlServiceConfigTest.java | 102 + .../iotdb/postgresql/protocol/CodecTest.java | 171 ++ .../postgresql/protocol/PgErrorsTest.java | 67 + .../iotdb/postgresql/server/StartupTest.java | 191 ++ .../postgresql/session/CursorCommandTest.java | 94 + .../postgresql/session/SessionEndTest.java | 66 + .../session/StatementTimeoutTest.java | 56 + .../postgresql/sql/PgToTableSqlTest.java | 548 ++++++ .../postgresql/sql/TranslatorGoldenTest.java | 217 +++ .../types/ParameterDecoderTest.java | 141 ++ .../postgresql/types/PgValueEncoderTest.java | 173 ++ .../src/test/resources/corpus/README.md | 91 + .../resources/corpus/expected/pgjdbc/1.sql | 19 + .../resources/corpus/expected/pgjdbc/10.sql | 19 + .../resources/corpus/expected/pgjdbc/11.sql | 19 + .../resources/corpus/expected/pgjdbc/12.sql | 19 + .../resources/corpus/expected/pgjdbc/13.sql | 19 + .../resources/corpus/expected/pgjdbc/14.sql | 19 + .../resources/corpus/expected/pgjdbc/15.sql | 19 + .../resources/corpus/expected/pgjdbc/16.sql | 19 + .../resources/corpus/expected/pgjdbc/17.sql | 19 + .../resources/corpus/expected/pgjdbc/18.sql | 19 + .../resources/corpus/expected/pgjdbc/19.sql | 19 + .../resources/corpus/expected/pgjdbc/2.sql | 19 + .../resources/corpus/expected/pgjdbc/20.sql | 19 + .../resources/corpus/expected/pgjdbc/3.sql | 19 + .../resources/corpus/expected/pgjdbc/4.sql | 19 + .../resources/corpus/expected/pgjdbc/5.sql | 19 + .../resources/corpus/expected/pgjdbc/6.sql | 19 + .../resources/corpus/expected/pgjdbc/7.sql | 19 + .../resources/corpus/expected/pgjdbc/8.sql | 19 + .../resources/corpus/expected/pgjdbc/9.sql | 19 + .../test/resources/corpus/expected/psql/1.sql | 19 + .../resources/corpus/expected/psql/10.sql | 19 + .../resources/corpus/expected/psql/11.sql | 19 + .../resources/corpus/expected/psql/12.sql | 19 + .../resources/corpus/expected/psql/13.sql | 19 + .../resources/corpus/expected/psql/14.sql | 19 + .../resources/corpus/expected/psql/15.sql | 19 + .../resources/corpus/expected/psql/16.sql | 19 + .../resources/corpus/expected/psql/17.sql | 19 + .../resources/corpus/expected/psql/18.sql | 19 + .../resources/corpus/expected/psql/19.sql | 19 + .../test/resources/corpus/expected/psql/2.sql | 19 + .../resources/corpus/expected/psql/20.sql | 19 + .../resources/corpus/expected/psql/21.sql | 19 + .../test/resources/corpus/expected/psql/3.sql | 19 + .../test/resources/corpus/expected/psql/4.sql | 19 + .../test/resources/corpus/expected/psql/5.sql | 19 + .../test/resources/corpus/expected/psql/6.sql | 19 + .../test/resources/corpus/expected/psql/7.sql | 19 + .../test/resources/corpus/expected/psql/8.sql | 19 + .../test/resources/corpus/expected/psql/9.sql | 19 + .../resources/corpus/expected/psycopg/1.sql | 19 + .../resources/corpus/expected/psycopg/10.sql | 19 + .../resources/corpus/expected/psycopg/11.sql | 20 + .../resources/corpus/expected/psycopg/12.sql | 19 + .../resources/corpus/expected/psycopg/13.sql | 19 + .../resources/corpus/expected/psycopg/14.sql | 19 + .../resources/corpus/expected/psycopg/2.sql | 19 + .../resources/corpus/expected/psycopg/3.sql | 19 + .../resources/corpus/expected/psycopg/4.sql | 19 + .../resources/corpus/expected/psycopg/5.sql | 19 + .../resources/corpus/expected/psycopg/6.sql | 19 + .../resources/corpus/expected/psycopg/7.sql | 19 + .../resources/corpus/expected/psycopg/8.sql | 19 + .../resources/corpus/expected/psycopg/9.sql | 19 + .../src/test/resources/corpus/pgjdbc.sql | 68 + .../src/test/resources/corpus/psql.sql | 210 ++ .../src/test/resources/corpus/psycopg.sql | 57 + integration-test/pom.xml | 13 + integration-test/src/assembly/mpp-share.xml | 5 + .../env/cluster/config/MppDataNodeConfig.java | 30 + .../remote/config/RemoteDataNodeConfig.java | 25 + .../iotdb/itbase/env/DataNodeConfig.java | 10 + .../it/IoTDBPostgreSqlClientsIT.java | 217 +++ .../it/IoTDBPostgreSqlEngineIT.java | 855 +++++++++ .../it/IoTDBPostgreSqlImportIT.java | 425 +++++ .../it/IoTDBPostgreSqlNoKeyStoreIT.java | 74 + .../it/IoTDBPostgreSqlPortInUseIT.java | 76 + .../it/IoTDBPostgreSqlSecurityIT.java | 124 ++ .../postgresql/it/IoTDBPostgreSqlTlsIT.java | 209 ++ .../iotdb/postgresql/it/PgItSupport.java | 67 + .../org/apache/iotdb/db/conf/IoTDBConfig.java | 11 + .../apache/iotdb/db/conf/IoTDBDescriptor.java | 6 + .../session/ExternalServiceClientSession.java | 134 ++ .../BuiltinExternalServices.java | 6 +- .../conf/iotdb-system.properties.template | 41 + pom.xml | 15 + 152 files changed, 17935 insertions(+), 1 deletion(-) create mode 100644 external-service-impl/postgresql/README.md create mode 100644 external-service-impl/postgresql/pom.xml create mode 100644 external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/PostgreSqlService.java create mode 100644 external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/catalog/CatalogRows.java create mode 100644 external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/catalog/CatalogSnapshot.java create mode 100644 external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/catalog/PgCatalogExtension.java create mode 100644 external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/catalog/PgCatalogProvider.java create mode 100644 external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/catalog/PgCatalogTables.java create mode 100644 external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/catalog/StaticCatalog.java create mode 100644 external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/config/BindAddresses.java create mode 100644 external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/config/PostgreSqlServiceConfig.java create mode 100644 external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/execution/ResultCursor.java create mode 100644 external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/execution/RunningQuery.java create mode 100644 external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/protocol/BackendMessages.java create mode 100644 external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/protocol/FrontendMessage.java create mode 100644 external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/protocol/PgErrors.java create mode 100644 external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/protocol/PgException.java create mode 100644 external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/protocol/PgFrameDecoder.java create mode 100644 external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/protocol/SqlState.java create mode 100644 external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/server/PgConnectionHandler.java create mode 100644 external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/server/PgServer.java create mode 100644 external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/server/PgServerContext.java create mode 100644 external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/server/SerialExecutor.java create mode 100644 external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/server/TlsPolicy.java create mode 100644 external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/session/CancelRegistry.java create mode 100644 external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/session/CursorCommand.java create mode 100644 external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/session/PgSession.java create mode 100644 external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/session/QueryProtocol.java create mode 100644 external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/session/SearchPath.java create mode 100644 external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/session/SessionCommands.java create mode 100644 external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/session/SqlProtocol.java create mode 100644 external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/session/StatementSplitter.java create mode 100644 external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/session/StatementTimeout.java create mode 100644 external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/sql/ConstructCheck.java create mode 100644 external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/sql/DateTimeRules.java create mode 100644 external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/sql/EmptyRelations.java create mode 100644 external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/sql/Identifiers.java create mode 100644 external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/sql/OuterConditions.java create mode 100644 external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/sql/OutputNames.java create mode 100644 external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/sql/PgToTableSql.java create mode 100644 external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/sql/PgTypeNames.java create mode 100644 external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/sql/PreParse.java create mode 100644 external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/sql/RegexPatterns.java create mode 100644 external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/sql/StringLiterals.java create mode 100644 external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/sql/TranslationContext.java create mode 100644 external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/sql/UnsupportedConstructException.java create mode 100644 external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/types/ParameterDecoder.java create mode 100644 external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/types/PgType.java create mode 100644 external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/types/PgValueEncoder.java create mode 100644 external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/types/TimestampPrecision.java create mode 100644 external-service-impl/postgresql/src/main/resources/META-INF/services/org.apache.iotdb.commons.schema.table.InformationSchema$InformationSchemaExtension create mode 100644 external-service-impl/postgresql/src/main/resources/META-INF/services/org.apache.iotdb.db.queryengine.plan.relational.information.AdditionalInformationSchemaProvider create mode 100644 external-service-impl/postgresql/src/main/resources/META-INF/services/org.apache.iotdb.externalservice.api.IExternalService create mode 100644 external-service-impl/postgresql/src/test/corpus-capture/PgJdbcCapture.java create mode 100644 external-service-impl/postgresql/src/test/corpus-capture/extract.py create mode 100644 external-service-impl/postgresql/src/test/corpus-capture/psycopg_capture.py create mode 100644 external-service-impl/postgresql/src/test/corpus-capture/setup.sql create mode 100644 external-service-impl/postgresql/src/test/java/org/apache/iotdb/postgresql/catalog/CatalogRowsTest.java create mode 100644 external-service-impl/postgresql/src/test/java/org/apache/iotdb/postgresql/config/PostgreSqlServiceConfigTest.java create mode 100644 external-service-impl/postgresql/src/test/java/org/apache/iotdb/postgresql/protocol/CodecTest.java create mode 100644 external-service-impl/postgresql/src/test/java/org/apache/iotdb/postgresql/protocol/PgErrorsTest.java create mode 100644 external-service-impl/postgresql/src/test/java/org/apache/iotdb/postgresql/server/StartupTest.java create mode 100644 external-service-impl/postgresql/src/test/java/org/apache/iotdb/postgresql/session/CursorCommandTest.java create mode 100644 external-service-impl/postgresql/src/test/java/org/apache/iotdb/postgresql/session/SessionEndTest.java create mode 100644 external-service-impl/postgresql/src/test/java/org/apache/iotdb/postgresql/session/StatementTimeoutTest.java create mode 100644 external-service-impl/postgresql/src/test/java/org/apache/iotdb/postgresql/sql/PgToTableSqlTest.java create mode 100644 external-service-impl/postgresql/src/test/java/org/apache/iotdb/postgresql/sql/TranslatorGoldenTest.java create mode 100644 external-service-impl/postgresql/src/test/java/org/apache/iotdb/postgresql/types/ParameterDecoderTest.java create mode 100644 external-service-impl/postgresql/src/test/java/org/apache/iotdb/postgresql/types/PgValueEncoderTest.java create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/README.md create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/1.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/10.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/11.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/12.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/13.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/14.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/15.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/16.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/17.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/18.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/19.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/2.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/20.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/3.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/4.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/5.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/6.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/7.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/8.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/9.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/psql/1.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/psql/10.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/psql/11.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/psql/12.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/psql/13.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/psql/14.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/psql/15.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/psql/16.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/psql/17.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/psql/18.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/psql/19.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/psql/2.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/psql/20.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/psql/21.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/psql/3.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/psql/4.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/psql/5.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/psql/6.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/psql/7.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/psql/8.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/psql/9.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/psycopg/1.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/psycopg/10.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/psycopg/11.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/psycopg/12.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/psycopg/13.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/psycopg/14.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/psycopg/2.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/psycopg/3.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/psycopg/4.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/psycopg/5.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/psycopg/6.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/psycopg/7.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/psycopg/8.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/psycopg/9.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/pgjdbc.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/psql.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/psycopg.sql create mode 100644 integration-test/src/test/java/org/apache/iotdb/postgresql/it/IoTDBPostgreSqlClientsIT.java create mode 100644 integration-test/src/test/java/org/apache/iotdb/postgresql/it/IoTDBPostgreSqlEngineIT.java create mode 100644 integration-test/src/test/java/org/apache/iotdb/postgresql/it/IoTDBPostgreSqlImportIT.java create mode 100644 integration-test/src/test/java/org/apache/iotdb/postgresql/it/IoTDBPostgreSqlNoKeyStoreIT.java create mode 100644 integration-test/src/test/java/org/apache/iotdb/postgresql/it/IoTDBPostgreSqlPortInUseIT.java create mode 100644 integration-test/src/test/java/org/apache/iotdb/postgresql/it/IoTDBPostgreSqlSecurityIT.java create mode 100644 integration-test/src/test/java/org/apache/iotdb/postgresql/it/IoTDBPostgreSqlTlsIT.java create mode 100644 integration-test/src/test/java/org/apache/iotdb/postgresql/it/PgItSupport.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/session/ExternalServiceClientSession.java diff --git a/LICENSE-binary b/LICENSE-binary index 6c79ced653d6d..dfafc2ac12bf0 100644 --- a/LICENSE-binary +++ b/LICENSE-binary @@ -278,6 +278,7 @@ jakarta.validation:jakarta.validation-api:3.0.2 net.java.dev.jna:jna:5.14.0 net.java.dev.jna:jna-platform:5.14.0 org.apache.commons:commons-csv:1.10.0 +com.github.jsqlparser:jsqlparser:5.4 (dual-licensed LGPL-2.1 / Apache-2.0, used under the Apache License 2.0) org.apache.commons:commons-lang3:3.18.0 org.apache.commons:commons-pool2:2.11.1 org.apache.ratis:ratis-client:3.3.1 diff --git a/distribution/pom.xml b/distribution/pom.xml index 7eca9e51699c2..5ba989a8a9f10 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -68,6 +68,12 @@ 2.0.13-SNAPSHOT provided + + org.apache.iotdb + postgresql + 2.0.13-SNAPSHOT + provided + diff --git a/distribution/src/assembly/all.xml b/distribution/src/assembly/all.xml index 25662a5fe9570..7385135d085bd 100644 --- a/distribution/src/assembly/all.xml +++ b/distribution/src/assembly/all.xml @@ -120,6 +120,11 @@ ${maven.multiModuleProjectDirectory}/external-service-impl/rest/target/rest-${project.version}-jar-with-dependencies.jar lib + + + ${maven.multiModuleProjectDirectory}/external-service-impl/postgresql/target/postgresql-${project.version}-jar-with-dependencies.jar + lib + common-files.xml diff --git a/distribution/src/assembly/edge.xml b/distribution/src/assembly/edge.xml index 67eed08bec33f..1f18cb8aea323 100644 --- a/distribution/src/assembly/edge.xml +++ b/distribution/src/assembly/edge.xml @@ -151,6 +151,11 @@ ${maven.multiModuleProjectDirectory}/external-service-impl/rest/target/rest-${project.version}-jar-with-dependencies.jar lib + + + ${maven.multiModuleProjectDirectory}/external-service-impl/postgresql/target/postgresql-${project.version}-jar-with-dependencies.jar + lib + common-files.xml diff --git a/external-service-impl/pom.xml b/external-service-impl/pom.xml index 27f6c609719d0..3c71c5f676a19 100644 --- a/external-service-impl/pom.xml +++ b/external-service-impl/pom.xml @@ -33,6 +33,7 @@ mqtt rest rest-openapi + postgresql 17 diff --git a/external-service-impl/postgresql/README.md b/external-service-impl/postgresql/README.md new file mode 100644 index 0000000000000..e07f11ac5d108 --- /dev/null +++ b/external-service-impl/postgresql/README.md @@ -0,0 +1,165 @@ + + +# IoTDB: PostgreSQL wire protocol + +PostgreSQL clients query the **table model**, read-only: Microsoft Power BI (built-in PostgreSQL +connector, Import and DirectQuery, Desktop and on-premises data gateway), Excel, psql, JDBC +(pgJDBC, DBeaver), Python (psycopg), .NET (Npgsql) and others. Every database appears as a +PostgreSQL schema; queries are translated into table-model SQL and run by the database's own +engine, so time filters and aggregations stay in the engine. + +The service is a built-in external service like REST and MQTT (module +`external-service-impl/postgresql`), off by default. + +## Enabling + +The jar ships in `lib/` of the distribution; it must stay there (not in `ext/external_service`), +because the database also loads its catalog tables from it. Set the keys in +`conf/iotdb-system.properties` (all keys and defaults are in the "PostgreSQL Service +Configuration" section of the template): + +```properties +enable_postgresql_service=true +postgresql_service_address=127.0.0.1 # loopback by default +postgresql_service_port=5432 +``` + +Other keys: `postgresql_service_max_connections` (32; more are refused with 53300), +`postgresql_service_idle_timeout_in_ms` (30 minutes), `postgresql_service_startup_timeout_in_ms` +(10 seconds), `postgresql_service_write_timeout_in_ms` (5 minutes: a client that stops reading +results loses its query and its connection). + +## Security + +- **TLS off loopback, no exceptions.** On any address other than loopback the service accepts only + TLS 1.2/1.3 connections with AEAD cipher suites (`sslmode=require` or stricter); a plain + connection gets `FATAL 28000 encryption required`. The certificate comes from `key_store_path` / + `key_store_pwd`. Without a key store the service does not start off loopback and logs why + (naming `postgresql_service_address`). On loopback, plain connections are allowed. +- **Logins** go through the database's normal login (`SessionManager.login`), with its lockout and + error handling; refused logins get `28P01`. The client sends its password with PostgreSQL's + cleartext password authentication, which is only ever offered inside TLS or on loopback. + SCRAM-SHA-256 is not offered: it needs SCRAM verifiers stored for each user, and IoTDB stores a + different password hash. +- A session that the database ends (for example a dropped user) is closed with `FATAL 57P01` at + its next message. + +## Power BI + +### Certificates + +Power BI Desktop and the on-premises data gateway (Npgsql) verify the server certificate: the +certificate must chain to a trusted root, and its subject alternative names must contain the name +used as "Server" in Power BI. + +- Issue the certificate in `key_store_path` for the name clients use (DNS name or IP address), + and import the issuing CA (or the self-signed certificate itself) on the client machines. + +Import on Windows, for Desktop as the user and for the gateway on the gateway machine as the +gateway's service account (or the local machine): `certmgr.msc` (user) or `certlm.msc` (machine) +→ *Trusted Root Certification Authorities* → *Certificates* → *All Tasks* → *Import…*, select the +`.cer`/`.crt` file. Then use exactly one of the certificate's names as server, for example +`plc-17.factory.local:5432`; an IP address works only if the certificate contains it (and with +libpq-based clients only from libpq 15 on, which matches IP-address names). + +### Connecting + +1. Import the certificate as described above, on the Desktop machine and on the gateway machine. +2. *Get Data → PostgreSQL database*: server `` (port 5432 by default), any database name, + Import or DirectQuery, then user and password. +3. The navigator lists the databases you may read as schemas, with their tables and columns. +4. DirectQuery: aggregations, date hierarchy (`date_trunc`), relative date filters and Top N are + translated and run in the database; anything without an exact translation is refused with + SQLSTATE 0A000 naming the construct, never answered approximately. +5. Gateway: add a PostgreSQL data source with the same server name as in Desktop (one of the + certificate's names) and keep certificate validation on. + +## Names and types + +- **Namespaces.** Schema = database: `mydb.sensors` is table `sensors` of database `mydb`. The + `search_path` is empty by default, so unqualified user tables are refused with 42P01 and a hint; + `SET search_path TO mydb` makes them resolvable. `pg_catalog` and `information_schema` are the + emulated catalogs; IoTDB's own `information_schema` stays hidden from PostgreSQL clients. +- **Types.** + + | Table model | PostgreSQL | + |---|---| + | BOOLEAN | `bool` | + | INT32 | `int4` | + | INT64 | `int8` | + | FLOAT | `float4` | + | DOUBLE | `float8` | + | STRING, TEXT | `text` | + | BLOB | `bytea` | + | DATE | `date` | + | TIMESTAMP | `timestamptz` (session time zone; nanosecond precision is truncated to microseconds) | + | OBJECT and others | not shown | + +- **Column categories** (TIME, TAG, ATTRIBUTE, FIELD) are the column comments + (`pg_description`, JDBC `REMARKS`). +- **Catalog tables.** `pg_namespace`, `pg_class`, `pg_attribute`, `pg_type`, `pg_proc`, + `pg_description`, `pg_database` (one database, `iotdb`), `pg_roles`, `pg_settings` and others + exist as `information_schema.pg_*` tables, and PostgreSQL's `information_schema` views as + `information_schema.pg_is_*`. IoTDB clients see them in `SHOW TABLES FROM information_schema`. + They list only what the user may read. + +## Supported SQL + +The rules are documented in `PgToTableSql`, `DateTimeRules` and `ConstructCheck`. In short: + +- `SELECT` with joins, sub-queries, CTEs, `UNION`, `CASE`, aggregates (also `count(DISTINCT x)`), + window functions (`row_number`, `rank`, `lead`, `lag`, ... `OVER (PARTITION BY ... ORDER BY ...)` + without explicit frames), `LIMIT`/`OFFSET`/`FETCH FIRST`, `ILIKE`, casts (`::`), parameters + (`$1`), `E''` strings. +- Dates in the session time zone (`SET TimeZone`): `date_trunc` (day across DST changes, + month/quarter/year calendar-aware; hour/minute/second where the zone's offsets are whole; week + only in zones without DST), `extract`/`date_part` (not `week`: the engine's week is not ISO), + `timestamp ± interval` with fixed-length units (days only in zones without DST, never months or + years), `now()`, `CURRENT_DATE`, `timestamp '...'` literals as PostgreSQL reads them. +- Strings: `lower`, `upper`, `length`, `substring`, `trim`, `replace`, `strpos`, `starts_with`, + `concat`, `||`, `coalesce`, `nullif`, `LIKE`, anchored regular expressions (`~ '^(name)$'`). +- Session commands: `SET`/`RESET`/`SHOW` (`TimeZone`, `search_path`, `statement_timeout`, + `application_name`, ...), `BEGIN`/`COMMIT`/`ROLLBACK` (no effect besides the transaction status), + `DISCARD`, `DEALLOCATE`; SQL cursors (`DECLARE`, `FETCH`/`MOVE` forward, `CLOSE`), as psycopg's + server-side cursors use them. +- Protocol: simple and extended query protocol, binary results, `setFetchSize` (portals), cancel + requests (57014), `statement_timeout`. + +## Limitations + +- Read-only: `INSERT`, `UPDATE`, `DELETE`, DDL and `COPY` are refused with 25006. +- Table model only; tree-model data is visible through table views. +- Constructs without an exact equivalent are refused with 0A000 naming them, among them arrays, + `numeric(p,s)`, `INTERVAL` months/years, explicit window frames, `lpad`/`rpad`. +- Timestamps with nanosecond precision are sent with microsecond precision. +- `length` counts UTF-16 code units (differs from PostgreSQL only for characters outside the + Basic Multilingual Plane). +- psql: `\l`, `\dn`, `\dt`, `\d table` work; `\d+` does not. +- The `iotdb` database name of `pg_database` does not follow the database name a client sends + when connecting (`current_database()` does). + +## Tests + +- Unit and golden tests: `./mvnw -pl external-service-impl/postgresql test`. The client corpus + (statements recorded from psql, pgJDBC and psycopg against PostgreSQL 14) is in + `src/test/resources/corpus`, its expected translations in `corpus/expected`. +- Integration tests: `org.apache.iotdb.postgresql.it.*` in `integration-test`. diff --git a/external-service-impl/postgresql/pom.xml b/external-service-impl/postgresql/pom.xml new file mode 100644 index 0000000000000..2c28bf76a44e8 --- /dev/null +++ b/external-service-impl/postgresql/pom.xml @@ -0,0 +1,160 @@ + + + + 4.0.0 + + org.apache.iotdb + external-service-impl + 2.0.13-SNAPSHOT + + postgresql + IoTDB: External-Service-Impl: PostgreSQL + PostgreSQL wire protocol: lets PostgreSQL clients such as Power BI query the table model, read-only. + + 17 + 17 + UTF-8 + + + + + org.apache.iotdb + external-service-api + 2.0.13-SNAPSHOT + provided + + + org.apache.iotdb + iotdb-server + 2.0.13-SNAPSHOT + provided + + + org.apache.iotdb + node-commons + 2.0.13-SNAPSHOT + provided + + + org.apache.iotdb + service-rpc + 2.0.13-SNAPSHOT + provided + + + org.apache.iotdb + iotdb-thrift-commons + 2.0.13-SNAPSHOT + provided + + + org.apache.iotdb + iotdb-thrift + 2.0.13-SNAPSHOT + provided + + + io.netty + netty-handler + provided + + + org.apache.tsfile + tsfile + ${tsfile.version} + provided + + + io.netty + netty-buffer + provided + + + io.netty + netty-codec + provided + + + io.netty + netty-transport + provided + + + io.netty + netty-common + provided + + + org.apache.tsfile + common + ${tsfile.version} + provided + + + + com.github.jsqlparser + jsqlparser + + + org.slf4j + slf4j-api + provided + + + junit + junit + test + + + + + + maven-assembly-plugin + ${maven.assembly.version} + + + jar-with-dependencies + + + + + make-assembly + + single + + package + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + + org.apache.tsfile:common + + + + + + diff --git a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/PostgreSqlService.java b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/PostgreSqlService.java new file mode 100644 index 0000000000000..3370dd0724f73 --- /dev/null +++ b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/PostgreSqlService.java @@ -0,0 +1,201 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql; + +import org.apache.iotdb.commons.conf.CommonConfig; +import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.iotdb.commons.conf.IoTDBConstant; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.externalservice.api.IExternalService; +import org.apache.iotdb.postgresql.config.PostgreSqlServiceConfig; +import org.apache.iotdb.postgresql.server.PgServer; +import org.apache.iotdb.postgresql.server.PgServerContext; +import org.apache.iotdb.postgresql.session.SqlProtocol; +import org.apache.iotdb.postgresql.types.TimestampPrecision; +import org.apache.iotdb.rpc.RpcSslUtils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.net.ssl.SSLContext; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * The PostgreSQL wire protocol as a built-in external service: off by default ({@code + * enable_postgresql_service}), TLS-only off loopback. + */ +public class PostgreSqlService implements IExternalService { + + private static final Logger LOGGER = LoggerFactory.getLogger(PostgreSqlService.class); + + /** The reported server version: 14, so that clients take their standard paths. */ + static final String SERVER_VERSION = "14.0 (IoTDB " + IoTDBConstant.VERSION + ")"; + + private volatile PgServerContext context; + private PgServer server; + private ExecutorService workers; + + @Override + public synchronized void start() { + PostgreSqlServiceConfig config = PostgreSqlServiceConfig.from(loadProperties()); + if (!config.enabled()) { + LOGGER.debug( + "PostgreSQL endpoint is switched off ({}=false)", PostgreSqlServiceConfig.ENABLED); + return; + } + if (!config.problems().isEmpty()) { + fail("invalid settings: " + config.problems()); + return; + } + SSLContext sslContext; + try { + sslContext = sslContext(config); + } catch (IllegalStateException e) { + fail(e.getMessage()); + return; + } + TimestampPrecision precision = + TimestampPrecision.of( + org.apache.iotdb.commons.conf.CommonDescriptor.getInstance() + .getConfig() + .getTimestampPrecision()); + workers = Executors.newCachedThreadPool(daemonThreads("pg-service-worker")); + context = + new PgServerContext( + config, + sslContext, + workers, + session -> new SqlProtocol(session, config.writeTimeout(), precision), + SERVER_VERSION); + server = new PgServer(context); + try { + server.start(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + fail("interrupted while opening " + config.address() + ":" + config.port()); + } catch (Exception e) { + // also checked exceptions Netty rethrows undeclared (BindException: port in use) + fail( + "cannot open " + + config.address() + + ":" + + config.port() + + " (" + + PostgreSqlServiceConfig.PORT + + "): " + + e.getMessage()); + } + } + + /** + * The TLS context from the core's key store, or {@code null} when none is configured. Off + * loopback a key store is required. + */ + private static SSLContext sslContext(PostgreSqlServiceConfig config) { + CommonConfig common = CommonDescriptor.getInstance().getConfig(); + String keyStore = common.getKeyStorePath(); + boolean configured = keyStore != null && !keyStore.trim().isEmpty(); + if (!configured) { + if (!config.isLoopback()) { + throw new IllegalStateException( + PostgreSqlServiceConfig.ADDRESS + + "=" + + config.address() + + " is not a loopback address and needs TLS, but no key store is configured" + + " (key_store_path)"); + } + return null; + } + if (!Files.isReadable(Paths.get(keyStore))) { + throw new IllegalStateException("the key store " + keyStore + " is not readable"); + } + try { + return RpcSslUtils.createSSLContext(keyStore, common.getKeyStorePwd(), null, null); + } catch (Exception e) { + throw new IllegalStateException( + "the key store " + keyStore + " cannot be used: " + e.getMessage(), e); + } + } + + private void fail(String message) { + LOGGER.error("PostgreSQL endpoint not started: {}", message); + stopServer(); + } + + @Override + public synchronized void stop() { + stopServer(); + } + + private void stopServer() { + if (server != null) { + server.stop(); + server = null; + } + if (workers != null) { + workers.shutdownNow(); + workers = null; + } + context = null; + } + + // ---------------------------------------------------------------- helpers + + private static Map loadProperties() { + Map values = new HashMap<>(); + URL url = IoTDBDescriptor.getPropsUrl(CommonConfig.SYSTEM_CONFIG_NAME); + if (url == null) { + return values; + } + Properties properties = new Properties(); + try (InputStream in = url.openStream()) { + properties.load(in); + } catch (IOException e) { + LOGGER.error("PostgreSQL endpoint: cannot read {}: {}", url, e.toString()); + } + for (String name : properties.stringPropertyNames()) { + if (name.startsWith(PostgreSqlServiceConfig.PREFIX) + || name.equals(PostgreSqlServiceConfig.ENABLED)) { + values.put(name, properties.getProperty(name)); + } + } + return values; + } + + private static java.util.concurrent.ThreadFactory daemonThreads(String prefix) { + AtomicInteger counter = new AtomicInteger(); + return runnable -> { + Thread thread = new Thread(runnable, prefix + "-" + counter.incrementAndGet()); + thread.setDaemon(true); + return thread; + }; + } +} diff --git a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/catalog/CatalogRows.java b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/catalog/CatalogRows.java new file mode 100644 index 0000000000000..8b00d359359f1 --- /dev/null +++ b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/catalog/CatalogRows.java @@ -0,0 +1,528 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.catalog; + +import org.apache.iotdb.postgresql.catalog.CatalogSnapshot.Attribute; +import org.apache.iotdb.postgresql.catalog.CatalogSnapshot.Namespace; +import org.apache.iotdb.postgresql.catalog.CatalogSnapshot.Relation; +import org.apache.iotdb.postgresql.types.PgType; + +import org.apache.tsfile.enums.TSDataType; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Supplier; + +/** + * The rows of the catalog tables for one query: constant rows from {@link StaticCatalog}, the + * others from the user's {@link CatalogSnapshot}. A row maps column names to values; columns + * without a value are NULL. + */ +public final class CatalogRows { + + /** + * The name of the one database a PostgreSQL client sees ({@code pg_database}, {@code + * table_catalog}). Clients name a database at connect time, but it does not select data (spec + * Q2). + */ + public static final String DATABASE = "iotdb"; + + private static final List> TYPES = StaticCatalog.types(); + private static final List> PROCS = StaticCatalog.procs(); + private static final List> ACCESS_METHODS = StaticCatalog.accessMethods(); + private static final List> SETTINGS = StaticCatalog.settings(); + private static final List> COLLATIONS = StaticCatalog.collations(); + + private final String user; + private final Supplier snapshotSource; + private CatalogSnapshot snapshot; + + /** + * @param user the querying user + * @param snapshot the user's snapshot, only built for tables that need it + */ + public CatalogRows(String user, Supplier snapshot) { + this.user = user; + this.snapshotSource = snapshot; + } + + /** The rows of a catalog table, empty for tables without content. */ + public List> rows(String table) { + switch (table) { + case "pg_type": + return TYPES; + case "pg_proc": + return PROCS; + case "pg_am": + return ACCESS_METHODS; + case "pg_settings": + return SETTINGS; + case "pg_collation": + return COLLATIONS; + case "pg_namespace": + return namespaces(); + case "pg_class": + return classes(); + case "pg_attribute": + return attributes(); + case "pg_description": + return descriptions(); + case "pg_database": + return database(); + case "pg_roles": + return roles(); + case "pg_user": + return users(); + case "pg_is_schemata": + return schemata(); + case "pg_is_tables": + return tables(); + case "pg_is_columns": + return columns(); + case "pg_is_views": + return views(); + default: + // pg_range, pg_enum, pg_index, pg_constraint, pg_inherits, pg_attrdef and the + // constraint views: the table model has no ranges, enums, indexes or constraints + return Collections.emptyList(); + } + } + + private CatalogSnapshot snapshot() { + if (snapshot == null) { + snapshot = snapshotSource.get(); + } + return snapshot; + } + + /** The OID of a role name, stable for the same name. */ + static long roleOid(String name) { + return CatalogSnapshot.allocate("u:" + name, new java.util.HashSet<>()); + } + + // ---------------------------------------------------------------------------- pg_catalog + + private List> namespaces() { + List> rows = new ArrayList<>(); + rows.add(namespace(StaticCatalog.PG_CATALOG, "pg_catalog", StaticCatalog.OWNER)); + rows.add( + namespace(StaticCatalog.INFORMATION_SCHEMA, "information_schema", StaticCatalog.OWNER)); + for (Namespace namespace : snapshot().namespaces()) { + rows.add(namespace(namespace.oid(), namespace.name(), StaticCatalog.OWNER)); + } + return rows; + } + + private static Map namespace(long oid, String name, long owner) { + return StaticCatalog.row("oid", oid, "nspname", name, "nspowner", owner); + } + + private List> classes() { + List> rows = new ArrayList<>(); + for (StaticCatalog.SystemRelation relation : StaticCatalog.systemRelations()) { + rows.add( + pgClass( + relation.oid, + relation.name, + relation.namespace, + relation.kind, + PgCatalogTables.tables().get(relation.table).size())); + } + for (Relation relation : snapshot().relations()) { + rows.add( + pgClass( + relation.oid(), + relation.name(), + relation.namespace().oid(), + relation.kind(), + relation.attributes().size())); + } + return rows; + } + + private static Map pgClass( + long oid, String name, long namespace, char kind, int columns) { + Map row = new HashMap<>(); + row.put("oid", oid); + row.put("relname", name); + row.put("relnamespace", namespace); + row.put("reltype", 0L); + row.put("reloftype", 0L); + row.put("relowner", StaticCatalog.OWNER); + row.put("relam", kind == 'r' ? 2L : 0L); + row.put("relfilenode", 0L); + row.put("reltablespace", 0L); + row.put("relpages", 0); + row.put("reltuples", -1.0f); + row.put("relallvisible", 0); + row.put("reltoastrelid", 0L); + row.put("relhasindex", false); + row.put("relisshared", false); + row.put("relpersistence", "p"); + row.put("relkind", String.valueOf(kind)); + row.put("relnatts", columns); + row.put("relchecks", 0); + row.put("relhasrules", false); + row.put("relhastriggers", false); + row.put("relhassubclass", false); + row.put("relrowsecurity", false); + row.put("relforcerowsecurity", false); + row.put("relispopulated", true); + row.put("relreplident", kind == 'r' ? "d" : "n"); + row.put("relispartition", false); + row.put("relrewrite", 0L); + return row; + } + + private List> attributes() { + List> rows = new ArrayList<>(); + for (StaticCatalog.SystemRelation relation : StaticCatalog.systemRelations()) { + int number = 0; + for (PgCatalogTables.Column column : PgCatalogTables.tables().get(relation.table)) { + number++; + PgType type = PgType.of(column.type()).orElse(PgType.TEXT); + rows.add(pgAttribute(relation.oid, column.name(), number, type, false)); + } + } + for (Relation relation : snapshot().relations()) { + for (Attribute attribute : relation.attributes()) { + rows.add( + pgAttribute( + relation.oid(), + attribute.name(), + attribute.number(), + attribute.type(), + attribute.notNull())); + } + } + return rows; + } + + private static Map pgAttribute( + long relation, String name, int number, PgType type, boolean notNull) { + Optional def = StaticCatalog.type(type.oid()); + Map row = new HashMap<>(); + row.put("attrelid", relation); + row.put("attname", name); + row.put("atttypid", (long) type.oid()); + row.put("attstattarget", -1); + row.put("attlen", (int) type.size()); + row.put("attnum", number); + row.put("attndims", 0); + row.put("attcacheoff", -1); + row.put("atttypmod", -1); + row.put("attbyval", def.map(StaticCatalog.TypeDef::byval).orElse(false)); + row.put("attalign", String.valueOf(def.map(StaticCatalog.TypeDef::align).orElse('i'))); + row.put("attstorage", String.valueOf(def.map(StaticCatalog.TypeDef::storage).orElse('p'))); + row.put("attcompression", ""); + row.put("attnotnull", notNull); + row.put("atthasdef", false); + row.put("atthasmissing", false); + row.put("attidentity", ""); + row.put("attgenerated", ""); + row.put("attisdropped", false); + row.put("attislocal", true); + row.put("attinhcount", 0); + row.put("attcollation", def.map(StaticCatalog.TypeDef::collation).orElse(0L)); + return row; + } + + private List> descriptions() { + List> rows = new ArrayList<>(); + for (Relation relation : snapshot().relations()) { + if (relation.description() != null) { + rows.add(description(relation.oid(), 0, relation.description())); + } + for (Attribute attribute : relation.attributes()) { + rows.add(description(relation.oid(), attribute.number(), attribute.description())); + } + } + return rows; + } + + private static Map description(long object, int sub, String text) { + return StaticCatalog.row( + "objoid", object, "classoid", StaticCatalog.PG_CLASS, "objsubid", sub, "description", text); + } + + private static List> database() { + Map row = new HashMap<>(); + row.put("oid", CatalogSnapshot.FIRST_USER_OID - 1); + row.put("datname", DATABASE); + row.put("datdba", StaticCatalog.OWNER); + row.put("encoding", 6); // UTF8 + row.put("datcollate", "C"); + row.put("datctype", "C"); + row.put("datistemplate", false); + row.put("datallowconn", true); + row.put("datconnlimit", -1); + row.put("datlastsysoid", CatalogSnapshot.FIRST_USER_OID - 1); + row.put("dattablespace", 1663L); + return Collections.singletonList(row); + } + + private List> roles() { + Map row = new HashMap<>(); + row.put("oid", roleOid(user)); + row.put("rolname", user); + row.put("rolsuper", false); + row.put("rolinherit", true); + row.put("rolcreaterole", false); + row.put("rolcreatedb", false); + row.put("rolcanlogin", true); + row.put("rolreplication", false); + row.put("rolconnlimit", -1); + row.put("rolpassword", "********"); + row.put("rolbypassrls", false); + return Collections.singletonList(row); + } + + private List> users() { + Map row = new HashMap<>(); + row.put("usename", user); + row.put("usesysid", roleOid(user)); + row.put("usecreatedb", false); + row.put("usesuper", false); + row.put("userepl", false); + row.put("usebypassrls", false); + row.put("passwd", "********"); + return Collections.singletonList(row); + } + + // ---------------------------------------------------------------------------- information_schema + + /** A relation of either kind, as the information_schema views list it. */ + private static final class Listed { + final String schema; + final String name; + final char kind; + final List attributes; + + Listed(String schema, String name, char kind, List attributes) { + this.schema = schema; + this.name = name; + this.kind = kind; + this.attributes = attributes; + } + } + + private List listed() { + List all = new ArrayList<>(); + for (StaticCatalog.SystemRelation relation : StaticCatalog.systemRelations()) { + List attributes = new ArrayList<>(); + for (PgCatalogTables.Column column : PgCatalogTables.tables().get(relation.table)) { + attributes.add( + new Attribute( + column.name(), + attributes.size() + 1, + PgType.of(column.type()).orElse(PgType.TEXT), + false, + null)); + } + all.add( + new Listed( + relation.namespace == StaticCatalog.PG_CATALOG ? "pg_catalog" : "information_schema", + relation.name, + relation.kind, + attributes)); + } + for (Relation relation : snapshot().relations()) { + all.add( + new Listed( + relation.namespace().name(), + relation.name(), + relation.kind(), + relation.attributes())); + } + return all; + } + + private List> schemata() { + List> rows = new ArrayList<>(); + List names = new ArrayList<>(); + names.add("pg_catalog"); + names.add("information_schema"); + for (Namespace namespace : snapshot().namespaces()) { + names.add(namespace.name()); + } + for (String name : names) { + rows.add( + StaticCatalog.row( + "catalog_name", + DATABASE, + "schema_name", + name, + "schema_owner", + name.equals("pg_catalog") || name.equals("information_schema") ? "postgres" : user)); + } + return rows; + } + + private List> tables() { + List> rows = new ArrayList<>(); + for (Listed relation : listed()) { + Map row = new HashMap<>(); + row.put("table_catalog", DATABASE); + row.put("table_schema", relation.schema); + row.put("table_name", relation.name); + row.put("table_type", relation.kind == 'v' ? "VIEW" : "BASE TABLE"); + row.put("is_insertable_into", "NO"); + row.put("is_typed", "NO"); + rows.add(row); + } + return rows; + } + + private List> views() { + List> rows = new ArrayList<>(); + for (Listed relation : listed()) { + if (relation.kind != 'v') { + continue; + } + Map row = new HashMap<>(); + row.put("table_catalog", DATABASE); + row.put("table_schema", relation.schema); + row.put("table_name", relation.name); + row.put("check_option", "NONE"); + row.put("is_updatable", "NO"); + row.put("is_insertable_into", "NO"); + row.put("is_trigger_updatable", "NO"); + row.put("is_trigger_deletable", "NO"); + row.put("is_trigger_insertable_into", "NO"); + rows.add(row); + } + return rows; + } + + private List> columns() { + List> rows = new ArrayList<>(); + for (Listed relation : listed()) { + for (Attribute attribute : relation.attributes) { + PgType type = attribute.type(); + Map row = new HashMap<>(); + row.put("table_catalog", DATABASE); + row.put("table_schema", relation.schema); + row.put("table_name", relation.name); + row.put("column_name", attribute.name()); + row.put("ordinal_position", attribute.number()); + row.put("is_nullable", attribute.notNull() ? "NO" : "YES"); + row.put("data_type", sqlTypeName(type)); + switch (type) { + case INT2: + numeric(row, 16, 2, 0); + break; + case INT4: + numeric(row, 32, 2, 0); + break; + case INT8: + numeric(row, 64, 2, 0); + break; + case FLOAT4: + numeric(row, 24, 2, null); + break; + case FLOAT8: + numeric(row, 53, 2, null); + break; + case DATE: + row.put("datetime_precision", 0); + break; + case TIMESTAMP: + case TIMESTAMPTZ: + row.put("datetime_precision", 6); + break; + case TEXT: + case VARCHAR: + row.put("character_octet_length", 1073741824); + break; + default: + break; + } + row.put("udt_catalog", DATABASE); + row.put("udt_schema", "pg_catalog"); + row.put("udt_name", type.typeName()); + row.put("dtd_identifier", String.valueOf(attribute.number())); + row.put("is_self_referencing", "NO"); + row.put("is_identity", "NO"); + row.put("identity_cycle", "NO"); + row.put("is_generated", "NEVER"); + row.put("is_updatable", "NO"); + rows.add(row); + } + } + return rows; + } + + private static void numeric(Map row, int precision, int radix, Integer scale) { + row.put("numeric_precision", precision); + row.put("numeric_precision_radix", radix); + row.put("numeric_scale", scale); + } + + /** The SQL-standard type name that {@code information_schema.columns.data_type} shows. */ + static String sqlTypeName(PgType type) { + switch (type) { + case BOOL: + return "boolean"; + case INT2: + return "smallint"; + case INT4: + return "integer"; + case INT8: + return "bigint"; + case FLOAT4: + return "real"; + case FLOAT8: + return "double precision"; + case VARCHAR: + return "character varying"; + case TIMESTAMP: + return "timestamp without time zone"; + case TIMESTAMPTZ: + return "timestamp with time zone"; + default: + return type.typeName(); + } + } + + /** Converts a row value to the column's table-model type ({@code null} stays null). */ + static Object convert(Object value, TSDataType type) { + if (value == null) { + return null; + } + switch (type) { + case INT64: + return ((Number) value).longValue(); + case INT32: + return ((Number) value).intValue(); + case FLOAT: + return ((Number) value).floatValue(); + case DOUBLE: + return ((Number) value).doubleValue(); + case BOOLEAN: + return value; + default: + return String.valueOf(value); + } + } +} diff --git a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/catalog/CatalogSnapshot.java b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/catalog/CatalogSnapshot.java new file mode 100644 index 0000000000000..0b9fd7e84c259 --- /dev/null +++ b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/catalog/CatalogSnapshot.java @@ -0,0 +1,292 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.catalog; + +import org.apache.iotdb.commons.schema.table.TreeViewSchema; +import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; +import org.apache.iotdb.postgresql.types.PgType; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.TreeMap; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.BiPredicate; +import java.util.function.Predicate; +import java.util.zip.CRC32; + +/** + * The databases (PostgreSQL schemas) and tables one user may see, with stable OIDs. OIDs are + * computed over all databases and tables, before the user's filter, so every user sees the same OID + * for the same table: a CRC-32 of the name mapped to [16384, 2^31), on a collision the next free + * value in name order. + */ +public final class CatalogSnapshot { + + private static final Logger LOGGER = LoggerFactory.getLogger(CatalogSnapshot.class); + + /** The first OID PostgreSQL gives to user objects. */ + static final long FIRST_USER_OID = 16384; + + private static final long OID_RANGE = Integer.MAX_VALUE - FIRST_USER_OID; + + private static final Set RESERVED = new HashSet<>(); + + static { + RESERVED.add("pg_catalog"); + RESERVED.add("information_schema"); + } + + private static final AtomicBoolean WARNED = new AtomicBoolean(); + + /** A schema. */ + public static final class Namespace { + final long oid; + final String name; + + Namespace(long oid, String name) { + this.oid = oid; + this.name = name; + } + + public long oid() { + return oid; + } + + public String name() { + return name; + } + } + + /** A column. */ + public static final class Attribute { + final String name; + final int number; + final PgType type; + final boolean notNull; + final String description; + + Attribute(String name, int number, PgType type, boolean notNull, String description) { + this.name = name; + this.number = number; + this.type = type; + this.notNull = notNull; + this.description = description; + } + + public String name() { + return name; + } + + public int number() { + return number; + } + + public PgType type() { + return type; + } + + public boolean notNull() { + return notNull; + } + + public String description() { + return description; + } + } + + /** A table or view. */ + public static final class Relation { + final long oid; + final Namespace namespace; + final String name; + final char kind; + final String description; + final List attributes; + + Relation( + long oid, + Namespace namespace, + String name, + char kind, + String description, + List attributes) { + this.oid = oid; + this.namespace = namespace; + this.name = name; + this.kind = kind; + this.description = description; + this.attributes = attributes; + } + + public long oid() { + return oid; + } + + public Namespace namespace() { + return namespace; + } + + public String name() { + return name; + } + + /** {@code r} for a table, {@code v} for a view. */ + public char kind() { + return kind; + } + + /** The table comment, or {@code null}. */ + public String description() { + return description; + } + + public List attributes() { + return attributes; + } + } + + private final List namespaces; + private final List relations; + + private CatalogSnapshot(List namespaces, List relations) { + this.namespaces = namespaces; + this.relations = relations; + } + + /** The user's databases, in name order. */ + public List namespaces() { + return namespaces; + } + + /** The user's tables, in database and table order. */ + public List relations() { + return relations; + } + + /** + * Builds the snapshot. + * + * @param tables database → table name → table, as {@code DataNodeTableCache} returns them + * @param canShowDatabase whether the user may see a database + * @param canShowTable whether the user may see a table of a database + */ + public static CatalogSnapshot of( + Map> tables, + Predicate canShowDatabase, + BiPredicate canShowTable) { + Map> sorted = new TreeMap<>(); + for (Map.Entry> database : tables.entrySet()) { + if (RESERVED.contains(database.getKey())) { + if (WARNED.compareAndSet(false, true)) { + LOGGER.warn( + "The database {} is hidden from PostgreSQL clients: its name is a reserved" + + " PostgreSQL schema", + database.getKey()); + } + continue; + } + sorted.put(database.getKey(), new TreeMap<>(database.getValue())); + } + + // OIDs over everything, then the filter + Set used = new HashSet<>(); + Map databaseOids = new HashMap<>(); + Map tableOids = new HashMap<>(); + for (Map.Entry> database : sorted.entrySet()) { + databaseOids.put(database.getKey(), allocate("n:" + database.getKey(), used)); + for (String table : database.getValue().keySet()) { + tableOids.put( + database.getKey() + "." + table, + allocate("r:" + database.getKey() + "." + table, used)); + } + } + + List namespaces = new ArrayList<>(); + List relations = new ArrayList<>(); + for (Map.Entry> database : sorted.entrySet()) { + String db = database.getKey(); + if (!canShowDatabase.test(db)) { + continue; + } + Namespace namespace = new Namespace(databaseOids.get(db), db); + namespaces.add(namespace); + for (Map.Entry entry : database.getValue().entrySet()) { + if (!canShowTable.test(db, entry.getKey())) { + continue; + } + relations.add(relation(tableOids.get(db + "." + entry.getKey()), namespace, entry)); + } + } + return new CatalogSnapshot( + Collections.unmodifiableList(namespaces), Collections.unmodifiableList(relations)); + } + + private static Relation relation( + long oid, Namespace namespace, Map.Entry entry) { + TsTable table = entry.getValue(); + List attributes = new ArrayList<>(); + for (TsTableColumnSchema column : table.getColumnList()) { + Optional type = PgType.of(column.getDataType()); + if (!type.isPresent()) { + // OBJECT and other types without a PostgreSQL equivalent stay hidden + continue; + } + TsTableColumnCategory category = column.getColumnCategory(); + String comment = column.getProps().get(TsTable.COMMENT_KEY); + attributes.add( + new Attribute( + column.getColumnName(), + attributes.size() + 1, + type.get(), + category == TsTableColumnCategory.TIME || category == TsTableColumnCategory.TAG, + comment == null ? category.name() : category.name() + ": " + comment)); + } + return new Relation( + oid, + namespace, + entry.getKey(), + TreeViewSchema.isTreeViewTable(table) ? 'v' : 'r', + table.getPropValue(TsTable.COMMENT_KEY).orElse(null), + Collections.unmodifiableList(attributes)); + } + + /** The OID for {@code key}: its hash, or the next free value after it. */ + static long allocate(String key, Set used) { + CRC32 crc = new CRC32(); + crc.update(key.getBytes(StandardCharsets.UTF_8)); + long oid = FIRST_USER_OID + crc.getValue() % OID_RANGE; + while (!used.add(oid)) { + oid = oid + 1 >= FIRST_USER_OID + OID_RANGE ? FIRST_USER_OID : oid + 1; + } + return oid; + } +} diff --git a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/catalog/PgCatalogExtension.java b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/catalog/PgCatalogExtension.java new file mode 100644 index 0000000000000..e3b7bf1c3d761 --- /dev/null +++ b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/catalog/PgCatalogExtension.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.catalog; + +import org.apache.iotdb.commons.schema.table.InformationSchema; +import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.commons.schema.table.column.AttributeColumnSchema; +import org.apache.iotdb.commons.schema.table.column.TagColumnSchema; + +import java.util.List; +import java.util.Map; +import java.util.function.BiConsumer; + +/** + * Registers the PostgreSQL catalog tables in IoTDB's {@code information_schema}. The first column + * of each table is its key (a tag column, as in IoTDB's own information-schema tables), the others + * are attribute columns; there is no time column. The rows come from {@link PgCatalogProvider}. + */ +public final class PgCatalogExtension implements InformationSchema.InformationSchemaExtension { + + @Override + public void registerAdditionalInformationSchemaTables(BiConsumer register) { + for (Map.Entry> table : + PgCatalogTables.tables().entrySet()) { + TsTable tsTable = new TsTable(table.getKey()); + boolean key = true; + for (PgCatalogTables.Column column : table.getValue()) { + tsTable.addColumnSchema( + key + ? new TagColumnSchema(column.name(), column.type()) + : new AttributeColumnSchema(column.name(), column.type())); + key = false; + } + register.accept(table.getKey(), tsTable); + } + } +} diff --git a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/catalog/PgCatalogProvider.java b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/catalog/PgCatalogProvider.java new file mode 100644 index 0000000000000..2c11b1a2f8277 --- /dev/null +++ b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/catalog/PgCatalogProvider.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.catalog; + +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; +import org.apache.iotdb.commons.audit.UserEntity; +import org.apache.iotdb.commons.queryengine.plan.relational.planner.Symbol; +import org.apache.iotdb.db.auth.AuthorityChecker; +import org.apache.iotdb.db.queryengine.common.DataNodeEndPoints; +import org.apache.iotdb.db.queryengine.execution.operator.OperatorContext; +import org.apache.iotdb.db.queryengine.execution.operator.source.relational.InformationSchemaContentSupplierFactory.IInformationSchemaContentSupplier; +import org.apache.iotdb.db.queryengine.execution.operator.source.relational.InformationSchemaContentSupplierFactory.TsBlockSupplier; +import org.apache.iotdb.db.queryengine.plan.execution.config.TableConfigTaskVisitor; +import org.apache.iotdb.db.queryengine.plan.relational.information.AdditionalInformationSchemaProvider; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.InformationSchemaTableScanNode; +import org.apache.iotdb.db.queryengine.plan.relational.security.AccessControl; +import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.BytesUtils; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * Supplies the rows of the catalog tables per query, for the querying user. The tables are + * evaluated on the local DataNode: the content comes from its table cache. + */ +public final class PgCatalogProvider implements AdditionalInformationSchemaProvider { + + @Override + public IInformationSchemaContentSupplier getContentSupplier( + String tableName, + OperatorContext context, + List dataTypes, + UserEntity userEntity, + InformationSchemaTableScanNode node) { + if (!PgCatalogTables.contains(tableName)) { + return null; + } + List columns = new ArrayList<>(); + for (Symbol symbol : node.getOutputSymbols()) { + columns.add(node.getAssignments().get(symbol).getName()); + } + CatalogRows catalog = new CatalogRows(userEntity.getUsername(), () -> snapshot(userEntity)); + return new RowSupplier(dataTypes, columns, catalog.rows(tableName)); + } + + @Override + public List getTableLocation(String tableName) { + return PgCatalogTables.contains(tableName) + ? Collections.singletonList(DataNodeEndPoints.getLocalDataNodeLocation()) + : null; + } + + private static CatalogSnapshot snapshot(UserEntity user) { + AccessControl access = AuthorityChecker.getAccessControl(); + String name = user.getUsername(); + return CatalogSnapshot.of( + DataNodeTableCache.getInstance().getTableSnapshot(), + database -> TableConfigTaskVisitor.canShowDB(access, name, database, user), + (database, table) -> + TableConfigTaskVisitor.canShowTable(access, name, database, table, user)); + } + + /** Writes rows into blocks, one column per output symbol of the scan. */ + private static final class RowSupplier extends TsBlockSupplier { + private final List types; + private final List columns; + private final List> rows; + private int next; + + RowSupplier(List types, List columns, List> rows) { + super(types); + this.types = types; + this.columns = columns; + this.rows = rows; + } + + @Override + public boolean hasNext() { + return next < rows.size(); + } + + @Override + protected void constructLine() { + Map row = rows.get(next++); + for (int i = 0; i < columns.size(); i++) { + TSDataType type = types.get(i); + Object value = CatalogRows.convert(row.get(columns.get(i)), type); + if (value == null) { + columnBuilders[i].appendNull(); + continue; + } + switch (type) { + case INT64: + columnBuilders[i].writeLong((Long) value); + break; + case INT32: + columnBuilders[i].writeInt((Integer) value); + break; + case FLOAT: + columnBuilders[i].writeFloat((Float) value); + break; + case DOUBLE: + columnBuilders[i].writeDouble((Double) value); + break; + case BOOLEAN: + columnBuilders[i].writeBoolean((Boolean) value); + break; + default: + columnBuilders[i].writeBinary(BytesUtils.valueOf((String) value)); + } + } + resultBuilder.declarePosition(); + } + } +} diff --git a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/catalog/PgCatalogTables.java b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/catalog/PgCatalogTables.java new file mode 100644 index 0000000000000..9cc993637aacb --- /dev/null +++ b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/catalog/PgCatalogTables.java @@ -0,0 +1,576 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.catalog; + +import org.apache.tsfile.enums.TSDataType; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * The catalog tables registered in IoTDB's {@code information_schema}: PostgreSQL 14 column names, + * types mapped to the table model ({@code oid} and {@code regproc} → INT64, {@code name}/{@code + * char}/{@code text} and arrays → STRING, {@code bool} → BOOLEAN, {@code int2}/{@code int4} → + * INT32, {@code float4} → FLOAT). + * + *

Only definitions, no other work: the core loads this class while it initialises its + * information schema, and a failure there would stop the database from starting. + */ +public final class PgCatalogTables { + + /** A column: name and table-model type. */ + public static final class Column { + private final String name; + private final TSDataType type; + private final boolean array; + + Column(String name, TSDataType type, boolean array) { + this.name = name; + this.type = type; + this.array = array; + } + + /** + * True for PostgreSQL array columns ({@code aclitem[]}, {@code oid[]}, {@code int2vector}, + * ...): STRING here and always NULL, because the table model has no arrays. + */ + public boolean array() { + return array; + } + + public String name() { + return name; + } + + public TSDataType type() { + return type; + } + } + + private static final TSDataType OID = TSDataType.INT64; + private static final TSDataType REGPROC = TSDataType.INT64; + private static final TSDataType NAME = TSDataType.STRING; + private static final TSDataType TEXT = TSDataType.STRING; + private static final TSDataType CHAR = TSDataType.STRING; + private static final TSDataType BOOL = TSDataType.BOOLEAN; + private static final TSDataType INT2 = TSDataType.INT32; + private static final TSDataType INT4 = TSDataType.INT32; + private static final TSDataType FLOAT4 = TSDataType.FLOAT; + + /** Prefix of PostgreSQL's own {@code information_schema} views. */ + public static final String INFORMATION_SCHEMA_PREFIX = "pg_is_"; + + private static final Map> TABLES = new LinkedHashMap<>(); + + static { + table("pg_namespace").col("oid", OID).col("nspname", NAME).col("nspowner", OID).array("nspacl"); + table("pg_class") + .col("oid", OID) + .col("relname", NAME) + .col("relnamespace", OID) + .col("reltype", OID) + .col("reloftype", OID) + .col("relowner", OID) + .col("relam", OID) + .col("relfilenode", OID) + .col("reltablespace", OID) + .col("relpages", INT4) + .col("reltuples", FLOAT4) + .col("relallvisible", INT4) + .col("reltoastrelid", OID) + .col("relhasindex", BOOL) + .col("relisshared", BOOL) + .col("relpersistence", CHAR) + .col("relkind", CHAR) + .col("relnatts", INT2) + .col("relchecks", INT2) + .col("relhasrules", BOOL) + .col("relhastriggers", BOOL) + .col("relhassubclass", BOOL) + .col("relrowsecurity", BOOL) + .col("relforcerowsecurity", BOOL) + .col("relispopulated", BOOL) + .col("relreplident", CHAR) + .col("relispartition", BOOL) + .col("relrewrite", OID) + .array("relacl") + .array("reloptions") + .col("relpartbound", TEXT); + table("pg_attribute") + .col("attrelid", OID) + .col("attname", NAME) + .col("atttypid", OID) + .col("attstattarget", INT4) + .col("attlen", INT2) + .col("attnum", INT2) + .col("attndims", INT4) + .col("attcacheoff", INT4) + .col("atttypmod", INT4) + .col("attbyval", BOOL) + .col("attalign", CHAR) + .col("attstorage", CHAR) + .col("attcompression", CHAR) + .col("attnotnull", BOOL) + .col("atthasdef", BOOL) + .col("atthasmissing", BOOL) + .col("attidentity", CHAR) + .col("attgenerated", CHAR) + .col("attisdropped", BOOL) + .col("attislocal", BOOL) + .col("attinhcount", INT4) + .col("attcollation", OID) + .array("attacl") + .array("attoptions") + .array("attfdwoptions"); + table("pg_type") + .col("oid", OID) + .col("typname", NAME) + .col("typnamespace", OID) + .col("typowner", OID) + .col("typlen", INT2) + .col("typbyval", BOOL) + .col("typtype", CHAR) + .col("typcategory", CHAR) + .col("typispreferred", BOOL) + .col("typisdefined", BOOL) + .col("typdelim", CHAR) + .col("typrelid", OID) + .col("typsubscript", REGPROC) + .col("typelem", OID) + .col("typarray", OID) + .col("typinput", REGPROC) + .col("typoutput", REGPROC) + .col("typreceive", REGPROC) + .col("typsend", REGPROC) + .col("typmodin", REGPROC) + .col("typmodout", REGPROC) + .col("typanalyze", REGPROC) + .col("typalign", CHAR) + .col("typstorage", CHAR) + .col("typnotnull", BOOL) + .col("typbasetype", OID) + .col("typtypmod", INT4) + .col("typndims", INT4) + .col("typcollation", OID) + .col("typdefaultbin", TEXT) + .col("typdefault", TEXT) + .array("typacl"); + table("pg_proc") + .col("oid", OID) + .col("proname", NAME) + .col("pronamespace", OID) + .col("proowner", OID) + .col("prolang", OID) + .col("procost", FLOAT4) + .col("prorows", FLOAT4) + .col("provariadic", OID) + .col("prosupport", REGPROC) + .col("prokind", CHAR) + .col("prosecdef", BOOL) + .col("proleakproof", BOOL) + .col("proisstrict", BOOL) + .col("proretset", BOOL) + .col("provolatile", CHAR) + .col("proparallel", CHAR) + .col("pronargs", INT2) + .col("pronargdefaults", INT2) + .col("prorettype", OID) + .array("proargtypes") + .array("proallargtypes") + .array("proargmodes") + .array("proargnames") + .col("proargdefaults", TEXT) + .array("protrftypes") + .col("prosrc", TEXT) + .col("probin", TEXT) + .col("prosqlbody", TEXT) + .array("proconfig") + .array("proacl"); + table("pg_description") + .col("objoid", OID) + .col("classoid", OID) + .col("objsubid", INT4) + .col("description", TEXT); + table("pg_range") + .col("rngtypid", OID) + .col("rngsubtype", OID) + .col("rngmultitypid", OID) + .col("rngcollation", OID) + .col("rngsubopc", OID) + .col("rngcanonical", REGPROC) + .col("rngsubdiff", REGPROC); + table("pg_enum") + .col("oid", OID) + .col("enumtypid", OID) + .col("enumsortorder", FLOAT4) + .col("enumlabel", NAME); + table("pg_index") + .col("indexrelid", OID) + .col("indrelid", OID) + .col("indnatts", INT2) + .col("indnkeyatts", INT2) + .col("indisunique", BOOL) + .col("indisprimary", BOOL) + .col("indisexclusion", BOOL) + .col("indimmediate", BOOL) + .col("indisclustered", BOOL) + .col("indisvalid", BOOL) + .col("indcheckxmin", BOOL) + .col("indisready", BOOL) + .col("indislive", BOOL) + .col("indisreplident", BOOL) + .array("indkey") + .array("indcollation") + .array("indclass") + .array("indoption") + .col("indexprs", TEXT) + .col("indpred", TEXT); + table("pg_constraint") + .col("oid", OID) + .col("conname", NAME) + .col("connamespace", OID) + .col("contype", CHAR) + .col("condeferrable", BOOL) + .col("condeferred", BOOL) + .col("convalidated", BOOL) + .col("conrelid", OID) + .col("contypid", OID) + .col("conindid", OID) + .col("conparentid", OID) + .col("confrelid", OID) + .col("confupdtype", CHAR) + .col("confdeltype", CHAR) + .col("confmatchtype", CHAR) + .col("conislocal", BOOL) + .col("coninhcount", INT4) + .col("connoinherit", BOOL) + .array("conkey") + .array("confkey") + .array("conpfeqop") + .array("conppeqop") + .array("conffeqop") + .array("conexclop") + .col("conbin", TEXT); + table("pg_inherits") + .col("inhrelid", OID) + .col("inhparent", OID) + .col("inhseqno", INT4) + .col("inhdetachpending", BOOL); + table("pg_attrdef").col("oid", OID).col("adrelid", OID).col("adnum", INT2).col("adbin", TEXT); + table("pg_am") + .col("oid", OID) + .col("amname", NAME) + .col("amhandler", REGPROC) + .col("amtype", CHAR); + table("pg_policy") + .col("oid", OID) + .col("polname", NAME) + .col("polrelid", OID) + .col("polcmd", CHAR) + .col("polpermissive", BOOL) + .array("polroles") + .col("polqual", TEXT) + .col("polwithcheck", TEXT); + table("pg_statistic_ext") + .col("oid", OID) + .col("stxrelid", OID) + .col("stxname", NAME) + .col("stxnamespace", OID) + .col("stxowner", OID) + .col("stxstattarget", INT4) + .array("stxkeys") + .array("stxkind") + .col("stxexprs", TEXT); + table("pg_publication") + .col("oid", OID) + .col("pubname", NAME) + .col("pubowner", OID) + .col("puballtables", BOOL) + .col("pubinsert", BOOL) + .col("pubupdate", BOOL) + .col("pubdelete", BOOL) + .col("pubtruncate", BOOL) + .col("pubviaroot", BOOL); + table("pg_publication_rel").col("oid", OID).col("prpubid", OID).col("prrelid", OID); + table("pg_collation") + .col("oid", OID) + .col("collname", NAME) + .col("collnamespace", OID) + .col("collowner", OID) + .col("collprovider", CHAR) + .col("collisdeterministic", BOOL) + .col("collencoding", INT4) + .col("collcollate", NAME) + .col("collctype", NAME) + .col("collversion", TEXT); + table("pg_database") + .col("oid", OID) + .col("datname", NAME) + .col("datdba", OID) + .col("encoding", INT4) + .col("datcollate", NAME) + .col("datctype", NAME) + .col("datistemplate", BOOL) + .col("datallowconn", BOOL) + .col("datconnlimit", INT4) + .col("datlastsysoid", OID) + .col("dattablespace", OID) + .array("datacl"); + table("pg_roles") + .col("oid", OID) + .col("rolname", NAME) + .col("rolsuper", BOOL) + .col("rolinherit", BOOL) + .col("rolcreaterole", BOOL) + .col("rolcreatedb", BOOL) + .col("rolcanlogin", BOOL) + .col("rolreplication", BOOL) + .col("rolconnlimit", INT4) + .col("rolpassword", TEXT) + .col("rolvaliduntil", TEXT) + .col("rolbypassrls", BOOL) + .array("rolconfig"); + table("pg_user") + .col("usename", NAME) + .col("usesysid", OID) + .col("usecreatedb", BOOL) + .col("usesuper", BOOL) + .col("userepl", BOOL) + .col("usebypassrls", BOOL) + .col("passwd", TEXT) + .col("valuntil", TEXT) + .array("useconfig"); + table("pg_settings") + .col("name", TEXT) + .col("setting", TEXT) + .col("unit", TEXT) + .col("category", TEXT) + .col("short_desc", TEXT) + .col("extra_desc", TEXT) + .col("context", TEXT) + .col("vartype", TEXT) + .col("source", TEXT) + .col("min_val", TEXT) + .col("max_val", TEXT) + .array("enumvals") + .col("boot_val", TEXT) + .col("reset_val", TEXT) + .col("sourcefile", TEXT) + .col("sourceline", INT4) + .col("pending_restart", BOOL); + + // PostgreSQL's information_schema views; sql_identifier, character_data and yes_or_no are + // text, cardinal_number is int4 + table("pg_is_schemata") + .col("catalog_name", TEXT) + .col("schema_name", TEXT) + .col("schema_owner", TEXT) + .col("default_character_set_catalog", TEXT) + .col("default_character_set_schema", TEXT) + .col("default_character_set_name", TEXT) + .col("sql_path", TEXT); + table("pg_is_tables") + .col("table_catalog", TEXT) + .col("table_schema", TEXT) + .col("table_name", TEXT) + .col("table_type", TEXT) + .col("self_referencing_column_name", TEXT) + .col("reference_generation", TEXT) + .col("user_defined_type_catalog", TEXT) + .col("user_defined_type_schema", TEXT) + .col("user_defined_type_name", TEXT) + .col("is_insertable_into", TEXT) + .col("is_typed", TEXT) + .col("commit_action", TEXT); + table("pg_is_columns") + .col("table_catalog", TEXT) + .col("table_schema", TEXT) + .col("table_name", TEXT) + .col("column_name", TEXT) + .col("ordinal_position", INT4) + .col("column_default", TEXT) + .col("is_nullable", TEXT) + .col("data_type", TEXT) + .col("character_maximum_length", INT4) + .col("character_octet_length", INT4) + .col("numeric_precision", INT4) + .col("numeric_precision_radix", INT4) + .col("numeric_scale", INT4) + .col("datetime_precision", INT4) + .col("interval_type", TEXT) + .col("interval_precision", INT4) + .col("character_set_catalog", TEXT) + .col("character_set_schema", TEXT) + .col("character_set_name", TEXT) + .col("collation_catalog", TEXT) + .col("collation_schema", TEXT) + .col("collation_name", TEXT) + .col("domain_catalog", TEXT) + .col("domain_schema", TEXT) + .col("domain_name", TEXT) + .col("udt_catalog", TEXT) + .col("udt_schema", TEXT) + .col("udt_name", TEXT) + .col("scope_catalog", TEXT) + .col("scope_schema", TEXT) + .col("scope_name", TEXT) + .col("maximum_cardinality", INT4) + .col("dtd_identifier", TEXT) + .col("is_self_referencing", TEXT) + .col("is_identity", TEXT) + .col("identity_generation", TEXT) + .col("identity_start", TEXT) + .col("identity_increment", TEXT) + .col("identity_maximum", TEXT) + .col("identity_minimum", TEXT) + .col("identity_cycle", TEXT) + .col("is_generated", TEXT) + .col("generation_expression", TEXT) + .col("is_updatable", TEXT); + table("pg_is_views") + .col("table_catalog", TEXT) + .col("table_schema", TEXT) + .col("table_name", TEXT) + .col("view_definition", TEXT) + .col("check_option", TEXT) + .col("is_updatable", TEXT) + .col("is_insertable_into", TEXT) + .col("is_trigger_updatable", TEXT) + .col("is_trigger_deletable", TEXT) + .col("is_trigger_insertable_into", TEXT); + table("pg_is_table_constraints") + .col("constraint_catalog", TEXT) + .col("constraint_schema", TEXT) + .col("constraint_name", TEXT) + .col("table_catalog", TEXT) + .col("table_schema", TEXT) + .col("table_name", TEXT) + .col("constraint_type", TEXT) + .col("is_deferrable", TEXT) + .col("initially_deferred", TEXT) + .col("enforced", TEXT); + table("pg_is_key_column_usage") + .col("constraint_catalog", TEXT) + .col("constraint_schema", TEXT) + .col("constraint_name", TEXT) + .col("table_catalog", TEXT) + .col("table_schema", TEXT) + .col("table_name", TEXT) + .col("column_name", TEXT) + .col("ordinal_position", INT4) + .col("position_in_unique_constraint", INT4); + table("pg_is_referential_constraints") + .col("constraint_catalog", TEXT) + .col("constraint_schema", TEXT) + .col("constraint_name", TEXT) + .col("unique_constraint_catalog", TEXT) + .col("unique_constraint_schema", TEXT) + .col("unique_constraint_name", TEXT) + .col("match_option", TEXT) + .col("update_rule", TEXT) + .col("delete_rule", TEXT); + } + + /** + * Tables that never have rows: the table model has no ranges, enums, indexes, constraints, + * defaults, inheritance, policies, extended statistics or publications. A query over one of them + * returns no rows whatever it selects (the translator relies on this). + */ + private static final java.util.Set ALWAYS_EMPTY = + new java.util.HashSet<>( + java.util.Arrays.asList( + "pg_range", + "pg_enum", + "pg_index", + "pg_constraint", + "pg_inherits", + "pg_attrdef", + "pg_policy", + "pg_statistic_ext", + "pg_publication", + "pg_publication_rel", + "pg_is_table_constraints", + "pg_is_key_column_usage", + "pg_is_referential_constraints")); + + private PgCatalogTables() {} + + /** True if {@code table} never has rows. */ + public static boolean alwaysEmpty(String table) { + return ALWAYS_EMPTY.contains(table); + } + + /** A column of a catalog table, or {@code null}. */ + public static Column column(String table, String column) { + List columns = TABLES.get(table); + if (columns != null) { + for (Column c : columns) { + if (c.name().equals(column)) { + return c; + } + } + } + return null; + } + + /** Table name → columns in order. */ + public static Map> tables() { + return Collections.unmodifiableMap(TABLES); + } + + /** True if {@code table} is one of the catalog tables. */ + public static boolean contains(String table) { + return TABLES.containsKey(table); + } + + /** The PostgreSQL name of a catalog table ({@code pg_is_tables} → {@code tables}). */ + public static String postgresName(String table) { + return table.startsWith(INFORMATION_SCHEMA_PREFIX) + ? table.substring(INFORMATION_SCHEMA_PREFIX.length()) + : table; + } + + private static Builder table(String name) { + List columns = new ArrayList<>(); + TABLES.put(name, columns); + return new Builder(columns); + } + + private static final class Builder { + private final List columns; + + Builder(List columns) { + this.columns = columns; + } + + Builder col(String name, TSDataType type) { + columns.add(new Column(name, type, false)); + return this; + } + + Builder array(String name) { + columns.add(new Column(name, TSDataType.STRING, true)); + return this; + } + } +} diff --git a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/catalog/StaticCatalog.java b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/catalog/StaticCatalog.java new file mode 100644 index 0000000000000..2022e04541492 --- /dev/null +++ b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/catalog/StaticCatalog.java @@ -0,0 +1,920 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.catalog; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * Constant catalog content: the types of the type mapping and the catalog types, their array types + * and their I/O functions as in PostgreSQL 14 (OIDs, names, sizes, alignment), access methods and + * settings. Npgsql 4.0 loads the types at connect time and classifies a type as an array by its + * receive function {@code array_recv}. + */ +public final class StaticCatalog { + + /** OIDs of the two system namespaces. */ + public static final long PG_CATALOG = 11; + + public static final long INFORMATION_SCHEMA = 13000; + + /** OID of the bootstrap superuser, the owner of every catalog object. */ + public static final long OWNER = 10; + + /** OID of {@code pg_class}, the {@code classoid} of table and column descriptions. */ + public static final long PG_CLASS = 1259; + + /** OID of the default collation, set for text columns. */ + public static final long DEFAULT_COLLATION = 100; + + private static final long ARRAY_IN = 750; + private static final long ARRAY_OUT = 751; + private static final long ARRAY_RECV = 2400; + private static final long ARRAY_SEND = 2401; + private static final long ARRAY_SUBSCRIPT = 6179; + private static final long RAW_ARRAY_SUBSCRIPT = 6180; + private static final long CSTRING = 2275; + private static final long BYTEA = 17; + + /** A {@code pg_type} row. */ + public static final class TypeDef { + final long oid; + final String name; + final int len; + final boolean byval; + final char typtype; + final char category; + final boolean preferred; + final char align; + final char storage; + final long elem; + final long array; + final long subscript; + final long[] io; + final long collation; + + TypeDef( + long oid, + String name, + int len, + boolean byval, + char typtype, + char category, + boolean preferred, + char align, + char storage, + long elem, + long array, + long subscript, + long[] io, + long collation) { + this.oid = oid; + this.name = name; + this.len = len; + this.byval = byval; + this.typtype = typtype; + this.category = category; + this.preferred = preferred; + this.align = align; + this.storage = storage; + this.elem = elem; + this.array = array; + this.subscript = subscript; + this.io = io; + this.collation = collation; + } + + public long oid() { + return oid; + } + + public String name() { + return name; + } + + public int len() { + return len; + } + + public boolean byval() { + return byval; + } + + public char align() { + return align; + } + + public char storage() { + return storage; + } + + public long collation() { + return collation; + } + } + + /** A system relation: a catalog table or view. */ + static final class SystemRelation { + final long oid; + final String table; + final long namespace; + final String name; + final char kind; + + SystemRelation(long oid, String table, long namespace, String name, char kind) { + this.oid = oid; + this.table = table; + this.namespace = namespace; + this.name = name; + this.kind = kind; + } + } + + private static final Map PROCS = new LinkedHashMap<>(); + private static final Map TYPES = new LinkedHashMap<>(); + private static final Map PROC_RESULT = new HashMap<>(); + private static final List RELATIONS = new ArrayList<>(); + + static { + // name, oid, len, byval, category, preferred, align, storage, array, collation, + // in, out, recv, send, typmodin, typmodout (0 = none) with the function names + base( + 16, + "bool", + 1, + true, + 'B', + true, + 'c', + 'p', + 1000, + 0, + f(1242, "boolin"), + f(1243, "boolout"), + f(2436, "boolrecv"), + f(2437, "boolsend")); + base( + 17, + "bytea", + -1, + false, + 'U', + false, + 'i', + 'x', + 1001, + 0, + f(1244, "byteain"), + f(31, "byteaout"), + f(2412, "bytearecv"), + f(2413, "byteasend")); + base( + 18, + "char", + 1, + true, + 'Z', + false, + 'c', + 'p', + 1002, + 0, + f(1245, "charin"), + f(33, "charout"), + f(2434, "charrecv"), + f(2435, "charsend")); + base( + 19, + "name", + 64, + false, + 'S', + false, + 'c', + 'p', + 1003, + 950, + f(34, "namein"), + f(35, "nameout"), + f(2422, "namerecv"), + f(2423, "namesend")); + base( + 20, + "int8", + 8, + true, + 'N', + false, + 'd', + 'p', + 1016, + 0, + f(460, "int8in"), + f(461, "int8out"), + f(2408, "int8recv"), + f(2409, "int8send")); + base( + 21, + "int2", + 2, + true, + 'N', + false, + 's', + 'p', + 1005, + 0, + f(38, "int2in"), + f(39, "int2out"), + f(2404, "int2recv"), + f(2405, "int2send")); + base( + 23, + "int4", + 4, + true, + 'N', + false, + 'i', + 'p', + 1007, + 0, + f(42, "int4in"), + f(43, "int4out"), + f(2406, "int4recv"), + f(2407, "int4send")); + base( + 24, + "regproc", + 4, + true, + 'N', + false, + 'i', + 'p', + 1008, + 0, + f(44, "regprocin"), + f(45, "regprocout"), + f(2444, "regprocrecv"), + f(2445, "regprocsend")); + base( + 25, + "text", + -1, + false, + 'S', + true, + 'i', + 'x', + 1009, + DEFAULT_COLLATION, + f(46, "textin"), + f(47, "textout"), + f(2414, "textrecv"), + f(2415, "textsend")); + base( + 26, + "oid", + 4, + true, + 'N', + true, + 'i', + 'p', + 1028, + 0, + f(1798, "oidin"), + f(1799, "oidout"), + f(2418, "oidrecv"), + f(2419, "oidsend")); + base( + 700, + "float4", + 4, + true, + 'N', + false, + 'i', + 'p', + 1021, + 0, + f(200, "float4in"), + f(201, "float4out"), + f(2424, "float4recv"), + f(2425, "float4send")); + base( + 701, + "float8", + 8, + true, + 'N', + true, + 'd', + 'p', + 1022, + 0, + f(214, "float8in"), + f(215, "float8out"), + f(2426, "float8recv"), + f(2427, "float8send")); + base( + 1043, + "varchar", + -1, + false, + 'S', + false, + 'i', + 'x', + 1015, + DEFAULT_COLLATION, + f(1046, "varcharin"), + f(1047, "varcharout"), + f(2432, "varcharrecv"), + f(2433, "varcharsend"), + f(2915, "varchartypmodin"), + f(2916, "varchartypmodout")); + base( + 1082, + "date", + 4, + true, + 'D', + false, + 'i', + 'p', + 1182, + 0, + f(1084, "date_in"), + f(1085, "date_out"), + f(2468, "date_recv"), + f(2469, "date_send")); + base( + 1114, + "timestamp", + 8, + true, + 'D', + false, + 'd', + 'p', + 1115, + 0, + f(1312, "timestamp_in"), + f(1313, "timestamp_out"), + f(2474, "timestamp_recv"), + f(2475, "timestamp_send"), + f(2905, "timestamptypmodin"), + f(2906, "timestamptypmodout")); + base( + 1184, + "timestamptz", + 8, + true, + 'D', + true, + 'd', + 'p', + 1185, + 0, + f(1150, "timestamptz_in"), + f(1151, "timestamptz_out"), + f(2476, "timestamptz_recv"), + f(2477, "timestamptz_send"), + f(2907, "timestamptztypmodin"), + f(2908, "timestamptztypmodout")); + base( + 1700, + "numeric", + -1, + false, + 'N', + false, + 'i', + 'm', + 1231, + 0, + f(1701, "numeric_in"), + f(1702, "numeric_out"), + f(2460, "numeric_recv"), + f(2461, "numeric_send"), + f(2917, "numerictypmodin"), + f(2918, "numerictypmodout")); + base( + 2206, + "regtype", + 4, + true, + 'N', + false, + 'i', + 'p', + 2211, + 0, + f(2220, "regtypein"), + f(2221, "regtypeout"), + f(2454, "regtyperecv"), + f(2455, "regtypesend")); + pseudo( + 705, + "unknown", + -2, + false, + 'X', + 'c', + 'p', + 0, + f(109, "unknownin"), + f(110, "unknownout"), + f(2416, "unknownrecv"), + f(2417, "unknownsend")); + pseudo( + 2249, + "record", + -1, + false, + 'P', + 'd', + 'x', + 2287, + f(2290, "record_in"), + f(2291, "record_out"), + f(2402, "record_recv"), + f(2403, "record_send")); + pseudo( + 2278, + "void", + 4, + true, + 'P', + 'i', + 'p', + 0, + f(2298, "void_in"), + f(2299, "void_out"), + f(3120, "void_recv"), + f(3121, "void_send")); + PROCS.put(ARRAY_IN, "array_in"); + PROCS.put(ARRAY_OUT, "array_out"); + PROCS.put(ARRAY_RECV, "array_recv"); + PROCS.put(ARRAY_SEND, "array_send"); + PROCS.put(ARRAY_SUBSCRIPT, "array_subscript_handler"); + PROCS.put(RAW_ARRAY_SUBSCRIPT, "raw_array_subscript_handler"); + PROC_RESULT.put(ARRAY_OUT, CSTRING); + PROC_RESULT.put(ARRAY_SEND, BYTEA); + // the array types, after their element types + for (TypeDef element : new ArrayList<>(TYPES.values())) { + if (element.array != 0) { + TYPES.put( + element.array, + new TypeDef( + element.array, + "_" + element.name, + -1, + false, + 'b', + 'A', + false, + element.align == 'd' ? 'd' : 'i', + 'x', + element.oid, + 0, + ARRAY_SUBSCRIPT, + new long[] {ARRAY_IN, ARRAY_OUT, ARRAY_RECV, ARRAY_SEND, 0, 0}, + element.collation)); + } + } + + // catalog relations with the OIDs PostgreSQL 14 gives them; the others get fixed OIDs + // below the first user OID, in the (append-only) order of PgCatalogTables + Map known = new HashMap<>(); + known.put("pg_type", 1247L); + known.put("pg_attribute", 1249L); + known.put("pg_proc", 1255L); + known.put("pg_class", PG_CLASS); + known.put("pg_database", 1262L); + known.put("pg_am", 2601L); + known.put("pg_attrdef", 2604L); + known.put("pg_constraint", 2606L); + known.put("pg_description", 2609L); + known.put("pg_index", 2610L); + known.put("pg_inherits", 2611L); + known.put("pg_namespace", 2615L); + known.put("pg_enum", 3501L); + known.put("pg_collation", 3456L); + known.put("pg_range", 3541L); + long next = 12100; + for (String table : PgCatalogTables.tables().keySet()) { + Long oid = known.get(table); + if (oid == null) { + oid = next++; + } + boolean view = + table.startsWith(PgCatalogTables.INFORMATION_SCHEMA_PREFIX) + || table.equals("pg_roles") + || table.equals("pg_user") + || table.equals("pg_settings"); + RELATIONS.add( + new SystemRelation( + oid, + table, + table.startsWith(PgCatalogTables.INFORMATION_SCHEMA_PREFIX) + ? INFORMATION_SCHEMA + : PG_CATALOG, + PgCatalogTables.postgresName(table), + view ? 'v' : 'r')); + } + } + + private StaticCatalog() {} + + /** The type with this OID, if it is in the catalog. */ + public static Optional type(long oid) { + return Optional.ofNullable(TYPES.get(oid)); + } + + /** SQL names of types as {@code format_type} prints them, where they differ from typname. */ + private static final Map SQL_NAMES = new HashMap<>(); + + /** Names a type literal may use for a type ({@code 'integer'::regtype}). */ + private static final Map ALIASES = new HashMap<>(); + + static { + SQL_NAMES.put("bool", "boolean"); + SQL_NAMES.put("char", "\"char\""); + SQL_NAMES.put("int8", "bigint"); + SQL_NAMES.put("int2", "smallint"); + SQL_NAMES.put("int4", "integer"); + SQL_NAMES.put("float4", "real"); + SQL_NAMES.put("float8", "double precision"); + SQL_NAMES.put("varchar", "character varying"); + SQL_NAMES.put("timestamp", "timestamp without time zone"); + SQL_NAMES.put("timestamptz", "timestamp with time zone"); + for (Map.Entry name : SQL_NAMES.entrySet()) { + ALIASES.put(name.getValue(), name.getKey()); + } + ALIASES.put("\"char\"", "char"); + ALIASES.put("int", "int4"); + ALIASES.put("float", "float8"); + ALIASES.put("character varying", "varchar"); + ALIASES.put("timestamp with time zone", "timestamptz"); + } + + /** + * {@code format_type(oid, -1)} for every type of the catalog: the SQL name, arrays as the element + * name with {@code []}. + */ + public static Map formatTypeNames() { + Map names = new LinkedHashMap<>(); + for (TypeDef type : TYPES.values()) { + if (type.category == 'A') { + TypeDef element = TYPES.get(type.elem); + names.put(type.oid, SQL_NAMES.getOrDefault(element.name, element.name) + "[]"); + } else { + names.put(type.oid, SQL_NAMES.getOrDefault(type.name, type.name)); + } + } + return names; + } + + /** + * The OID of a type given by name ({@code int4}, {@code integer}, {@code _int4}, {@code int4[]}). + */ + public static Optional typeOid(String name) { + String n = name.trim().replaceAll("\\s+", " "); + if (n.startsWith("pg_catalog.")) { + n = n.substring("pg_catalog.".length()); + } + boolean array = n.endsWith("[]"); + if (array) { + n = n.substring(0, n.length() - 2).trim(); + } + n = ALIASES.getOrDefault(n, n); + for (TypeDef type : TYPES.values()) { + if (type.name.equals(n)) { + if (!array) { + return Optional.of(type.oid); + } + return type.array == 0 ? Optional.empty() : Optional.of(type.array); + } + } + return Optional.empty(); + } + + /** The OID of a function of the catalog given by name ({@code array_in}). */ + public static Optional procOid(String name) { + String n = name.startsWith("pg_catalog.") ? name.substring("pg_catalog.".length()) : name; + for (Map.Entry proc : PROCS.entrySet()) { + if (proc.getValue().equals(n)) { + return Optional.of(proc.getKey()); + } + } + return Optional.empty(); + } + + /** + * The OID of a catalog table or view, by schema ({@code null} = search {@code pg_catalog}) and + * PostgreSQL name. + */ + public static Optional systemRelationOid(String schema, String name) { + for (SystemRelation relation : RELATIONS) { + boolean schemaMatches = + schema == null + ? relation.namespace == PG_CATALOG + : (schema.equals("pg_catalog") && relation.namespace == PG_CATALOG) + || (schema.equals("information_schema") + && relation.namespace == INFORMATION_SCHEMA); + if (schemaMatches && relation.name.equals(name)) { + return Optional.of(relation.oid); + } + } + return Optional.empty(); + } + + static List systemRelations() { + return Collections.unmodifiableList(RELATIONS); + } + + /** Rows of {@code pg_type}. */ + static List> types() { + List> rows = new ArrayList<>(); + for (TypeDef t : TYPES.values()) { + Map row = new HashMap<>(); + row.put("oid", t.oid); + row.put("typname", t.name); + row.put("typnamespace", PG_CATALOG); + row.put("typowner", OWNER); + row.put("typlen", t.len); + row.put("typbyval", t.byval); + row.put("typtype", String.valueOf(t.typtype)); + row.put("typcategory", String.valueOf(t.category)); + row.put("typispreferred", t.preferred); + row.put("typisdefined", true); + row.put("typdelim", ","); + row.put("typrelid", 0L); + row.put("typsubscript", t.subscript); + row.put("typelem", t.elem); + row.put("typarray", t.array); + row.put("typinput", t.io[0]); + row.put("typoutput", t.io[1]); + row.put("typreceive", t.io[2]); + row.put("typsend", t.io[3]); + row.put("typmodin", t.io[4]); + row.put("typmodout", t.io[5]); + row.put("typanalyze", 0L); + row.put("typalign", String.valueOf(t.align)); + row.put("typstorage", String.valueOf(t.storage)); + row.put("typnotnull", false); + row.put("typbasetype", 0L); + row.put("typtypmod", -1); + row.put("typndims", 0); + row.put("typcollation", t.collation); + rows.add(row); + } + return rows; + } + + /** Rows of {@code pg_proc}: the functions {@code pg_type} refers to. */ + static List> procs() { + List> rows = new ArrayList<>(); + for (Map.Entry proc : PROCS.entrySet()) { + Map row = new HashMap<>(); + row.put("oid", proc.getKey()); + row.put("proname", proc.getValue()); + row.put("pronamespace", PG_CATALOG); + row.put("proowner", OWNER); + row.put("prolang", 12L); + row.put("procost", 1.0f); + row.put("prorows", 0.0f); + row.put("provariadic", 0L); + row.put("prosupport", 0L); + row.put("prokind", "f"); + row.put("prosecdef", false); + row.put("proleakproof", false); + row.put("proisstrict", true); + row.put("proretset", false); + row.put("provolatile", "i"); + row.put("proparallel", "s"); + row.put("pronargs", 1); + row.put("pronargdefaults", 0); + row.put("prorettype", PROC_RESULT.getOrDefault(proc.getKey(), 0L)); + row.put("prosrc", proc.getValue()); + rows.add(row); + } + return rows; + } + + /** Rows of {@code pg_am}. */ + static List> accessMethods() { + List> rows = new ArrayList<>(); + rows.add(row("oid", 2L, "amname", "heap", "amhandler", 3L, "amtype", "t")); + rows.add(row("oid", 403L, "amname", "btree", "amhandler", 330L, "amtype", "i")); + return rows; + } + + /** Rows of {@code pg_collation}: the built-in collations PostgreSQL always has. */ + static List> collations() { + List> rows = new ArrayList<>(); + collation(rows, DEFAULT_COLLATION, "default", "d", -1, null); + collation(rows, 950, "C", "c", -1, "C"); + collation(rows, 951, "POSIX", "c", -1, "POSIX"); + return rows; + } + + private static void collation( + List> rows, + long oid, + String name, + String provider, + int encoding, + String locale) { + Map row = + row( + "oid", + oid, + "collname", + name, + "collnamespace", + PG_CATALOG, + "collowner", + OWNER, + "collprovider", + provider, + "collisdeterministic", + true, + "collencoding", + encoding); + row.put("collcollate", locale); + row.put("collctype", locale); + rows.add(row); + } + + /** Rows of {@code pg_settings}: the fixed parameters clients read. */ + static List> settings() { + List> rows = new ArrayList<>(); + setting(rows, "block_size", "8192", "integer"); + setting(rows, "client_encoding", "UTF8", "string"); + setting(rows, "DateStyle", "ISO, MDY", "string"); + setting(rows, "default_transaction_read_only", "on", "bool"); + setting(rows, "integer_datetimes", "on", "bool"); + setting(rows, "IntervalStyle", "postgres", "enum"); + setting(rows, "max_identifier_length", "63", "integer"); + setting(rows, "max_index_keys", "32", "integer"); + setting(rows, "server_encoding", "UTF8", "string"); + setting(rows, "server_version", "14.0", "string"); + setting(rows, "server_version_num", "140000", "integer"); + setting(rows, "standard_conforming_strings", "on", "bool"); + setting(rows, "transaction_isolation", "read committed", "string"); + setting(rows, "transaction_read_only", "on", "bool"); + return rows; + } + + private static void setting( + List> rows, String name, String value, String vartype) { + Map row = + row( + "name", + name, + "setting", + value, + "category", + "Preset Options", + "context", + "internal", + "vartype", + vartype, + "source", + "default"); + row.put("boot_val", value); + row.put("reset_val", value); + row.put("pending_restart", false); + rows.add(row); + } + + static Map row(Object... keysAndValues) { + Map row = new HashMap<>(); + for (int i = 0; i < keysAndValues.length; i += 2) { + row.put((String) keysAndValues[i], keysAndValues[i + 1]); + } + return row; + } + + // ---------------------------------------------------------------------------- definitions + + private static final class Fn { + final long oid; + final String name; + + Fn(long oid, String name) { + this.oid = oid; + this.name = name; + } + } + + private static Fn f(long oid, String name) { + return new Fn(oid, name); + } + + private static void base( + long oid, + String name, + int len, + boolean byval, + char category, + boolean preferred, + char align, + char storage, + long array, + long collation, + Fn... io) { + // name is a fixed-length array of char (subscriptable), as in PostgreSQL + long elem = name.equals("name") ? 18 : 0; + long subscript = name.equals("name") ? RAW_ARRAY_SUBSCRIPT : 0; + add( + new TypeDef( + oid, + name, + len, + byval, + 'b', + category, + preferred, + align, + storage, + elem, + array, + subscript, + register(oid, io), + collation)); + } + + private static void pseudo( + long oid, + String name, + int len, + boolean byval, + char category, + char align, + char storage, + long array, + Fn... io) { + add( + new TypeDef( + oid, + name, + len, + byval, + 'p', + category, + false, + align, + storage, + 0, + array, + 0, + register(oid, io), + 0)); + } + + private static void add(TypeDef type) { + TYPES.put(type.oid, type); + } + + /** Registers the functions and returns in, out, recv, send, typmodin, typmodout. */ + private static long[] register(long type, Fn... io) { + long[] oids = new long[6]; + for (int i = 0; i < io.length; i++) { + oids[i] = io[i].oid; + PROCS.put(io[i].oid, io[i].name); + } + // in and recv return the type, out returns cstring, send returns bytea + PROC_RESULT.put(oids[0], type); + PROC_RESULT.put(oids[1], CSTRING); + PROC_RESULT.put(oids[2], type); + PROC_RESULT.put(oids[3], BYTEA); + return oids; + } +} diff --git a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/config/BindAddresses.java b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/config/BindAddresses.java new file mode 100644 index 0000000000000..ae25054655bfb --- /dev/null +++ b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/config/BindAddresses.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.config; + +import java.net.InetAddress; +import java.net.NetworkInterface; +import java.net.SocketException; +import java.net.UnknownHostException; +import java.util.Optional; +import java.util.regex.Pattern; + +/** + * Checks the address the PostgreSQL endpoint is bound to: an IPv4 or IPv6 literal that is a + * loopback address, a wildcard or an address of this machine. Host names are refused (a name can + * resolve to several or changing addresses), and no DNS lookup is ever made. + */ +final class BindAddresses { + + private static final Pattern IPV4 = + Pattern.compile( + "^(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}$"); + + /** Hex digits, colons and an optional embedded IPv4 tail or zone id. */ + private static final Pattern IPV6 = + Pattern.compile("^[0-9A-Fa-f:]*:[0-9A-Fa-f:.]*(%[\\w.\\-]+)?$"); + + private BindAddresses() {} + + /** + * Why an address cannot be used for binding, or empty if it can. + * + * @param value the configured address + * @return the reason, or empty for a usable address + */ + static Optional problem(String value) { + if (value == null || value.trim().isEmpty()) { + return Optional.of("no address given; use an IP address such as 127.0.0.1"); + } + String trimmed = value.trim(); + InetAddress address = parseLiteral(trimmed); + if (address == null) { + return Optional.of( + "'" + trimmed + "' is not an IP address; use an IP address such as 127.0.0.1"); + } + if (address.isAnyLocalAddress() || address.isLoopbackAddress()) { + return Optional.empty(); + } + try { + if (NetworkInterface.getByInetAddress(address) != null) { + return Optional.empty(); + } + } catch (SocketException e) { + return Optional.of( + "the network interfaces of this machine cannot be read: " + e.getMessage()); + } + return Optional.of("'" + trimmed + "' is not an address of this machine"); + } + + /** Whether the value is a wildcard address (all interfaces). */ + static boolean isWildcard(String value) { + InetAddress address = value == null ? null : parseLiteral(value.trim()); + return address != null && address.isAnyLocalAddress(); + } + + /** Parses an IP literal without any DNS lookup; returns null for anything else. */ + static InetAddress parseLiteral(String value) { + String candidate = value; + if (candidate.startsWith("[") && candidate.endsWith("]")) { + candidate = candidate.substring(1, candidate.length() - 1); + } + if (!IPV4.matcher(candidate).matches() && !IPV6.matcher(candidate).matches()) { + return null; + } + try { + // a literal never triggers a lookup + return InetAddress.getByName(candidate); + } catch (UnknownHostException | SecurityException e) { + return null; + } + } +} diff --git a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/config/PostgreSqlServiceConfig.java b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/config/PostgreSqlServiceConfig.java new file mode 100644 index 0000000000000..13fbaef7e97e8 --- /dev/null +++ b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/config/PostgreSqlServiceConfig.java @@ -0,0 +1,208 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.config; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Function; + +/** + * The service's settings. Values that cannot be read are reported as problems and replaced by their + * defaults, so a configuration is always complete. + */ +public final class PostgreSqlServiceConfig { + + public static final String ENABLED = "enable_postgresql_service"; + public static final String PREFIX = "postgresql_service_"; + public static final String ADDRESS = PREFIX + "address"; + public static final String PORT = PREFIX + "port"; + public static final String MAX_CONNECTIONS = PREFIX + "max_connections"; + public static final String IDLE_TIMEOUT = PREFIX + "idle_timeout_in_ms"; + public static final String STARTUP_TIMEOUT = PREFIX + "startup_timeout_in_ms"; + public static final String WRITE_TIMEOUT = PREFIX + "write_timeout_in_ms"; + + public static final boolean DEFAULT_ENABLED = false; + public static final String DEFAULT_ADDRESS = "127.0.0.1"; + public static final int DEFAULT_PORT = 5432; + public static final int DEFAULT_MAX_CONNECTIONS = 32; + public static final Duration DEFAULT_IDLE_TIMEOUT = Duration.ofMinutes(30); + public static final Duration DEFAULT_STARTUP_TIMEOUT = Duration.ofSeconds(10); + public static final Duration DEFAULT_WRITE_TIMEOUT = Duration.ofMinutes(5); + + private final boolean enabled; + private final String address; + private final int port; + private final int maxConnections; + private final Duration idleTimeout; + private final Duration startupTimeout; + private final Duration writeTimeout; + private final List problems; + + private PostgreSqlServiceConfig(Function values) { + List found = new ArrayList<>(); + this.enabled = parseBoolean(values, ENABLED, DEFAULT_ENABLED, found); + this.address = parseAddress(values, found); + this.port = parseInt(values, PORT, DEFAULT_PORT, 1, 65535, found); + this.maxConnections = + parseInt(values, MAX_CONNECTIONS, DEFAULT_MAX_CONNECTIONS, 1, Integer.MAX_VALUE, found); + this.idleTimeout = parseDuration(values, IDLE_TIMEOUT, DEFAULT_IDLE_TIMEOUT, found); + this.startupTimeout = parseDuration(values, STARTUP_TIMEOUT, DEFAULT_STARTUP_TIMEOUT, found); + this.writeTimeout = parseDuration(values, WRITE_TIMEOUT, DEFAULT_WRITE_TIMEOUT, found); + this.problems = Collections.unmodifiableList(found); + } + + /** Reads the settings; keys that are missing or blank take their defaults. */ + public static PostgreSqlServiceConfig from(Map values) { + return new PostgreSqlServiceConfig(values::get); + } + + public boolean enabled() { + return enabled; + } + + public String address() { + return address; + } + + public int port() { + return port; + } + + public int maxConnections() { + return maxConnections; + } + + public Duration idleTimeout() { + return idleTimeout; + } + + public Duration startupTimeout() { + return startupTimeout; + } + + public Duration writeTimeout() { + return writeTimeout; + } + + /** Problems found while reading the values; empty when every value was valid. */ + public List problems() { + return problems; + } + + /** + * Whether the bind address is a loopback address. Only loopback addresses may accept unencrypted + * connections. The wildcard is not loopback. + */ + /** Whether the service listens on all interfaces. */ + public boolean isWildcard() { + return BindAddresses.isWildcard(address); + } + + public boolean isLoopback() { + if (BindAddresses.isWildcard(address)) { + return false; + } + try { + // the address is an IP literal (checked by BindAddresses), so no name lookup happens + return InetAddress.getByName(address).isLoopbackAddress(); + } catch (UnknownHostException e) { + return false; + } + } + + private static String text(Function values, String key) { + String value = values.apply(key); + return value == null || value.trim().isEmpty() ? null : value.trim(); + } + + private static boolean parseBoolean( + Function values, String key, boolean defaultValue, List problems) { + String value = text(values, key); + if (value == null) { + return defaultValue; + } + if (value.equalsIgnoreCase("true") || value.equalsIgnoreCase("false")) { + return Boolean.parseBoolean(value); + } + problems.add(key + ": must be true or false, was '" + value + "'"); + return defaultValue; + } + + private static String parseAddress(Function values, List problems) { + String value = text(values, ADDRESS); + if (value == null) { + return DEFAULT_ADDRESS; + } + Optional problem = BindAddresses.problem(value); + if (problem.isPresent()) { + problems.add(ADDRESS + ": " + problem.get()); + return DEFAULT_ADDRESS; + } + return value; + } + + private static int parseInt( + Function values, + String key, + int defaultValue, + int min, + int max, + List problems) { + String value = text(values, key); + if (value == null) { + return defaultValue; + } + try { + int parsed = Integer.parseInt(value); + if (parsed >= min && parsed <= max) { + return parsed; + } + } catch (NumberFormatException e) { + // reported below + } + problems.add( + key + ": must be a whole number from " + min + " to " + max + ", was '" + value + "'"); + return defaultValue; + } + + private static Duration parseDuration( + Function values, String key, Duration defaultValue, List problems) { + String value = text(values, key); + if (value == null) { + return defaultValue; + } + try { + long millis = Long.parseLong(value); + if (millis > 0) { + return Duration.ofMillis(millis); + } + } catch (NumberFormatException e) { + // reported below + } + problems.add(key + ": must be a positive number of milliseconds, was '" + value + "'"); + return defaultValue; + } +} diff --git a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/execution/ResultCursor.java b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/execution/ResultCursor.java new file mode 100644 index 0000000000000..82c6d3a1512a3 --- /dev/null +++ b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/execution/ResultCursor.java @@ -0,0 +1,250 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.execution; + +import org.apache.iotdb.postgresql.protocol.BackendMessages; +import org.apache.iotdb.postgresql.protocol.PgException; +import org.apache.iotdb.postgresql.protocol.SqlState; +import org.apache.iotdb.postgresql.types.PgType; +import org.apache.iotdb.postgresql.types.PgValueEncoder; + +import io.netty.buffer.ByteBuf; +import io.netty.channel.Channel; +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.block.TsBlock; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +/** + * Streams the rows of a {@link RunningQuery} as DataRow messages: block by block, writing only + * while the channel is writable, so memory stays bounded however large the result is. Keeps its + * position, so a portal can continue after PortalSuspended. + */ +public final class ResultCursor implements AutoCloseable { + + /** A client that stopped reading for longer than the write timeout. */ + public static final class WriteTimeoutException extends Exception { + private static final long serialVersionUID = 1L; + + WriteTimeoutException() { + super("client stopped reading", null, false, false); + } + } + + /** Bytes of DataRows collected before they are handed to the channel. */ + private static final int BATCH_BYTES = 64 * 1024; + + private final RunningQuery query; + private final PgType[] types; + private short[] formats; + private final PgValueEncoder encoder; + private TsBlock block; + private int position; + private boolean exhausted; + + /** + * @param formats result formats as sent in Bind: none (all text), one (for all columns) or one + * per column; 0 text, 1 binary + */ + public ResultCursor(RunningQuery query, short[] formats, PgValueEncoder encoder) + throws PgException { + this.query = query; + this.encoder = encoder; + List dataTypes = query.types(); + this.types = new PgType[dataTypes.size()]; + for (int i = 0; i < types.length; i++) { + TSDataType dataType = dataTypes.get(i); + Optional type = PgType.of(dataType); + if (!type.isPresent()) { + throw new PgException( + SqlState.FEATURE_NOT_SUPPORTED, + "column \"" + + query.names().get(i) + + "\" has type " + + dataType + + ", which is not" + + " available through PostgreSQL"); + } + types[i] = type.get(); + } + this.formats = expand(formats, types.length); + } + + private static short[] expand(short[] formats, int columns) throws PgException { + short[] result = new short[columns]; + if (formats == null || formats.length == 0) { + return result; + } + if (formats.length == 1) { + java.util.Arrays.fill(result, formats[0]); + return result; + } + if (formats.length != columns) { + throw new PgException( + SqlState.PROTOCOL_VIOLATION, + "bind message has " + + formats.length + + " result formats but query has " + + columns + + " columns"); + } + return formats.clone(); + } + + /** + * Changes the result formats for the rows sent from now on: a SQL cursor's FETCH chooses them per + * fetch (Bind of the FETCH, or BINARY of the DECLARE). + */ + public void setFormats(short[] formats) throws PgException { + this.formats = expand(formats, types.length); + } + + /** The RowDescription fields. */ + public List fields() { + List fields = new ArrayList<>(types.length); + for (int i = 0; i < types.length; i++) { + fields.add( + new BackendMessages.Field( + query.names().get(i), types[i].oid(), types[i].size(), formats[i])); + } + return fields; + } + + /** True once every row was sent. */ + public boolean exhausted() { + return exhausted; + } + + /** + * Sends up to {@code maxRows} rows (0 = all) as DataRow messages, appending to {@code out} and + * handing full batches to the channel. + * + * @return the number of rows sent + */ + public long send(Channel channel, ByteBuf out, long maxRows, Duration writeTimeout) + throws PgException, WriteTimeoutException { + return advance(channel, out, maxRows, writeTimeout); + } + + /** + * Skips up to {@code maxRows} rows (0 = all) without sending them (SQL {@code MOVE}). + * + * @return the number of rows skipped + */ + public long skip(long maxRows) throws PgException { + try { + return advance(null, null, maxRows, null); + } catch (WriteTimeoutException e) { + throw new IllegalStateException("nothing is written while skipping", e); + } + } + + /** Sends ({@code out != null}) or skips rows. */ + private long advance(Channel channel, ByteBuf out, long maxRows, Duration writeTimeout) + throws PgException, WriteTimeoutException { + long sent = 0; + while (maxRows == 0 || sent < maxRows) { + if (block == null || position >= block.getPositionCount()) { + Optional next = query.next(); + if (!next.isPresent()) { + exhausted = true; + break; + } + block = next.get(); + position = 0; + } + int end = block.getPositionCount(); + if (maxRows > 0) { + end = (int) Math.min(end, position + (maxRows - sent)); + } + if (out == null) { + sent += end - position; + position = end; + continue; + } + Column[] columns = new Column[types.length]; + for (int c = 0; c < columns.length; c++) { + columns[c] = block.getColumn(query.blockIndex(c)); + } + for (; position < end; position++) { + int row = BackendMessages.beginDataRow(out, columns.length); + for (int c = 0; c < columns.length; c++) { + if (columns[c].isNull(position)) { + BackendMessages.nullValue(out); + } else if (formats[c] == 0) { + BackendMessages.value(out, encoder.text(types[c], columns[c], position)); + } else { + BackendMessages.value(out, encoder.binary(types[c], columns[c], position)); + } + } + BackendMessages.end(out, row); + sent++; + if (out.readableBytes() >= BATCH_BYTES) { + flush(channel, out, writeTimeout); + } + } + } + return sent; + } + + /** Hands the collected bytes to the channel and waits until it can take more. */ + public static void flush(Channel channel, ByteBuf out, Duration writeTimeout) + throws WriteTimeoutException { + if (out.readableBytes() == 0) { + return; + } + // a copy: the caller keeps using its buffer while Netty writes this one + channel.writeAndFlush(out.copy()); + out.clear(); + awaitWritable(channel, writeTimeout); + } + + private static void awaitWritable(Channel channel, Duration writeTimeout) + throws WriteTimeoutException { + long deadline = System.nanoTime() + writeTimeout.toNanos(); + long pause = 1; + while (!channel.isWritable() && channel.isActive()) { + if (System.nanoTime() > deadline) { + throw new WriteTimeoutException(); + } + try { + Thread.sleep(pause); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } + pause = Math.min(pause * 2, 50); + } + } + + /** Cancels the query (CancelRequest, write timeout). */ + public void cancel() { + query.cancel(); + } + + @Override + public void close() { + query.close(); + } +} diff --git a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/execution/RunningQuery.java b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/execution/RunningQuery.java new file mode 100644 index 0000000000000..12b1b070808cc --- /dev/null +++ b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/execution/RunningQuery.java @@ -0,0 +1,213 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.execution; + +import org.apache.iotdb.commons.exception.IoTDBException; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Literal; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Statement; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.StringLiteral; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.protocol.session.IClientSession; +import org.apache.iotdb.db.protocol.session.SessionManager; +import org.apache.iotdb.db.queryengine.plan.Coordinator; +import org.apache.iotdb.db.queryengine.plan.execution.ExecutionResult; +import org.apache.iotdb.db.queryengine.plan.execution.IQueryExecution; +import org.apache.iotdb.db.queryengine.plan.planner.LocalExecutionPlanner; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExecuteImmediate; +import org.apache.iotdb.db.queryengine.plan.relational.sql.parser.SqlParser; +import org.apache.iotdb.postgresql.protocol.PgErrors; +import org.apache.iotdb.postgresql.protocol.PgException; +import org.apache.iotdb.postgresql.protocol.SqlState; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.block.TsBlock; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Optional; + +/** + * One table-model query started through the engine: parsed with the table dialect, run with the + * engine's query timeout as a read-only user query, and always cleaned up. + */ +public final class RunningQuery implements AutoCloseable { + + private final long queryId; + private final IQueryExecution execution; + private final List names; + private final List types; + private final int[] blockIndex; + private volatile boolean closed; + + private RunningQuery(long queryId, IQueryExecution execution) { + this.queryId = queryId; + this.execution = execution; + this.names = new ArrayList<>(execution.getDatasetHeader().getRespColumns()); + this.types = new ArrayList<>(execution.getDatasetHeader().getRespDataTypes()); + List mapping = execution.getDatasetHeader().getColumnIndex2TsBlockColumnIndexList(); + this.blockIndex = new int[names.size()]; + for (int i = 0; i < blockIndex.length; i++) { + blockIndex[i] = mapping == null ? i : mapping.get(i); + } + } + + /** + * Starts a query. + * + * @param sql table-model SQL (from the translator) + * @param parameters the values of its {@code ?} in order; empty for none + */ + public static RunningQuery start(IClientSession client, String sql, List parameters) + throws PgException { + return start(client, sql, parameters, java.time.Duration.ZERO); + } + + /** + * Starts a query with the session's {@code statement_timeout}. + * + * @param statementTimeout limits the engine's query timeout; zero for none + */ + public static RunningQuery start( + IClientSession client, + String sql, + List parameters, + java.time.Duration statementTimeout) + throws PgException { + SessionManager sessions = SessionManager.getInstance(); + SqlParser parser = new SqlParser(); + IClientSession previous = sessions.getCurrSession(); + sessions.setCurrSession(client); + Long queryId = null; + boolean started = false; + try { + Statement statement = + parameters.isEmpty() + ? parser.createStatement(sql, client.getZoneId(), client) + : new ExecuteImmediate(new StringLiteral(sql), parameters); + queryId = sessions.requestQueryId(); + ExecutionResult result = + Coordinator.getInstance() + .executeForTableModel( + statement, + parser, + client, + queryId, + sessions.getSessionInfo(client), + sql, + LocalExecutionPlanner.getInstance().metadata, + org.apache.iotdb.postgresql.session.StatementTimeout.engineTimeoutMillis( + IoTDBDescriptor.getInstance().getConfig().getQueryTimeoutThreshold(), + statementTimeout), + true, + false, + true); + if (result.status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode() + && result.status.getCode() != TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode()) { + throw PgErrors.of(result.status); + } + IQueryExecution execution = Coordinator.getInstance().getQueryExecution(queryId); + if (execution == null) { + throw new PgException(SqlState.INTERNAL_ERROR, "the statement did not start a query"); + } + started = true; + return new RunningQuery(queryId, execution); + } catch (PgException e) { + throw e; + } catch (RuntimeException e) { + throw PgErrors.of(errorCode(e), e); + } finally { + if (!started && queryId != null) { + Coordinator.getInstance().cleanupQueryExecution(queryId); + } + // the session stays registered (SHOW SESSIONS); only this thread forgets it + sessions.restoreSession(previous, null); + } + } + + private static int errorCode(Throwable error) { + for (Throwable t = error; t != null; t = t.getCause()) { + if (t instanceof IoTDBException) { + return ((IoTDBException) t).getErrorCode(); + } + if (t instanceof org.apache.iotdb.commons.exception.IoTDBRuntimeException) { + return ((org.apache.iotdb.commons.exception.IoTDBRuntimeException) t).getErrorCode(); + } + } + if (error.getClass().getSimpleName().equals("SemanticException")) { + return TSStatusCode.SEMANTIC_ERROR.getStatusCode(); + } + if (error.getClass().getSimpleName().equals("ParsingException")) { + return TSStatusCode.SQL_PARSE_ERROR.getStatusCode(); + } + return TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode(); + } + + /** Column names in result order. */ + public List names() { + return Collections.unmodifiableList(names); + } + + /** Column types in result order. */ + public List types() { + return Collections.unmodifiableList(types); + } + + /** The block column of result column {@code i}. */ + public int blockIndex(int i) { + return blockIndex[i]; + } + + /** The next non-empty block, or empty at the end. */ + public Optional next() throws PgException { + try { + while (execution.hasNextResult()) { + Optional block = execution.getBatchResult(); + if (!block.isPresent()) { + return Optional.empty(); + } + if (!block.get().isEmpty()) { + return block; + } + } + return Optional.empty(); + } catch (IoTDBException e) { + throw PgErrors.of(e.getErrorCode(), e); + } catch (RuntimeException e) { + if (closed) { + throw new PgException(SqlState.QUERY_CANCELED, "canceling statement due to user request"); + } + throw PgErrors.of(errorCode(e), e); + } + } + + /** Stops the query (CancelRequest, write timeout); the reader then gets 57014. */ + public void cancel() { + closed = true; + execution.cancel(); + } + + @Override + public void close() { + closed = true; + Coordinator.getInstance().cleanupQueryExecution(queryId); + } +} diff --git a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/protocol/BackendMessages.java b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/protocol/BackendMessages.java new file mode 100644 index 0000000000000..677117af765c4 --- /dev/null +++ b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/protocol/BackendMessages.java @@ -0,0 +1,242 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.protocol; + +import io.netty.buffer.ByteBuf; + +import java.nio.charset.StandardCharsets; +import java.util.List; + +/** + * Writes the server's messages of protocol 3.0 into a buffer. Each method appends one complete + * message; streaming rows use {@link #beginDataRow}, {@link #value}, {@link #nullValue} and {@link + * #end}. + */ +public final class BackendMessages { + + /** One column of a RowDescription. */ + public static final class Field { + final String name; + final int tableOid; + final short columnNumber; + final int typeOid; + final short typeSize; + final int typeModifier; + final short format; + + public Field(String name, int typeOid, short typeSize, short format) { + this(name, 0, (short) 0, typeOid, typeSize, -1, format); + } + + public Field( + String name, + int tableOid, + short columnNumber, + int typeOid, + short typeSize, + int typeModifier, + short format) { + this.name = name; + this.tableOid = tableOid; + this.columnNumber = columnNumber; + this.typeOid = typeOid; + this.typeSize = typeSize; + this.typeModifier = typeModifier; + this.format = format; + } + + public short format() { + return format; + } + } + + private BackendMessages() {} + + /** The single byte answering an SSLRequest or GSSENCRequest (not a framed message). */ + public static void sslResponse(ByteBuf out, boolean accept) { + out.writeByte(accept ? 'S' : 'N'); + } + + public static void authenticationOk(ByteBuf out) { + int start = begin(out, 'R'); + out.writeInt(0); + end(out, start); + } + + public static void authenticationCleartextPassword(ByteBuf out) { + int start = begin(out, 'R'); + out.writeInt(3); + end(out, start); + } + + public static void parameterStatus(ByteBuf out, String name, String value) { + int start = begin(out, 'S'); + cstring(out, name); + cstring(out, value); + end(out, start); + } + + public static void backendKeyData(ByteBuf out, int processId, int secretKey) { + int start = begin(out, 'K'); + out.writeInt(processId); + out.writeInt(secretKey); + end(out, start); + } + + /** {@code status}: 'I' idle, 'T' in a transaction block, 'E' in a failed transaction block. */ + public static void readyForQuery(ByteBuf out, char status) { + int start = begin(out, 'Z'); + out.writeByte(status); + end(out, start); + } + + public static void negotiateProtocolVersion( + ByteBuf out, int newestMinorVersion, List unsupportedOptions) { + int start = begin(out, 'v'); + out.writeInt(newestMinorVersion); + out.writeInt(unsupportedOptions.size()); + for (String option : unsupportedOptions) { + cstring(out, option); + } + end(out, start); + } + + public static void rowDescription(ByteBuf out, List fields) { + int start = begin(out, 'T'); + out.writeShort(fields.size()); + for (Field field : fields) { + cstring(out, field.name); + out.writeInt(field.tableOid); + out.writeShort(field.columnNumber); + out.writeInt(field.typeOid); + out.writeShort(field.typeSize); + out.writeInt(field.typeModifier); + out.writeShort(field.format); + } + end(out, start); + } + + /** Starts a DataRow; returns the position to pass to {@link #end}. */ + public static int beginDataRow(ByteBuf out, int columns) { + int start = begin(out, 'D'); + out.writeShort(columns); + return start; + } + + public static void value(ByteBuf out, byte[] value) { + out.writeInt(value.length); + out.writeBytes(value); + } + + public static void nullValue(ByteBuf out) { + out.writeInt(-1); + } + + public static void commandComplete(ByteBuf out, String tag) { + int start = begin(out, 'C'); + cstring(out, tag); + end(out, start); + } + + public static void emptyQueryResponse(ByteBuf out) { + end(out, begin(out, 'I')); + } + + public static void parseComplete(ByteBuf out) { + end(out, begin(out, '1')); + } + + public static void bindComplete(ByteBuf out) { + end(out, begin(out, '2')); + } + + public static void closeComplete(ByteBuf out) { + end(out, begin(out, '3')); + } + + public static void noData(ByteBuf out) { + end(out, begin(out, 'n')); + } + + public static void portalSuspended(ByteBuf out) { + end(out, begin(out, 's')); + } + + public static void parameterDescription(ByteBuf out, int[] typeOids) { + int start = begin(out, 't'); + out.writeShort(typeOids.length); + for (int oid : typeOids) { + out.writeInt(oid); + } + end(out, start); + } + + /** An ErrorResponse with severity ERROR or FATAL. */ + public static void errorResponse(ByteBuf out, PgException error) { + response( + out, + 'E', + error.isFatal() ? "FATAL" : "ERROR", + error.sqlState(), + error.getMessage(), + error.hint()); + } + + public static void noticeResponse(ByteBuf out, String severity, String sqlState, String message) { + response(out, 'N', severity, sqlState, message, null); + } + + private static void response( + ByteBuf out, char type, String severity, String sqlState, String message, String hint) { + int start = begin(out, type); + field(out, 'S', severity); + field(out, 'V', severity); + field(out, 'C', sqlState); + field(out, 'M', message == null ? "" : message); + if (hint != null) { + field(out, 'H', hint); + } + out.writeByte(0); + end(out, start); + } + + private static void field(ByteBuf out, char code, String value) { + out.writeByte(code); + cstring(out, value); + } + + /** Writes the type byte and a placeholder length; returns the length's position. */ + public static int begin(ByteBuf out, char type) { + out.writeByte(type); + int lengthIndex = out.writerIndex(); + out.writeInt(0); + return lengthIndex; + } + + /** Fills in the length of the message started at {@code lengthIndex}. */ + public static void end(ByteBuf out, int lengthIndex) { + out.setInt(lengthIndex, out.writerIndex() - lengthIndex); + } + + public static void cstring(ByteBuf out, String value) { + out.writeBytes(value.getBytes(StandardCharsets.UTF_8)); + out.writeByte(0); + } +} diff --git a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/protocol/FrontendMessage.java b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/protocol/FrontendMessage.java new file mode 100644 index 0000000000000..663612d2c733e --- /dev/null +++ b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/protocol/FrontendMessage.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.protocol; + +import java.nio.BufferUnderflowException; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; + +/** + * A message from the client: either a packet of the startup phase (no type byte; {@link #type()} is + * 0) or a typed message. The payload excludes the type byte and the length. + */ +public final class FrontendMessage { + + /** Request codes of startup packets. */ + public static final int PROTOCOL_3_0 = 196_608; + + public static final int SSL_REQUEST = 80_877_103; + public static final int GSSENC_REQUEST = 80_877_104; + public static final int CANCEL_REQUEST = 80_877_102; + + private final char type; + private final ByteBuffer payload; + + public FrontendMessage(char type, byte[] payload) { + this.type = type; + this.payload = ByteBuffer.wrap(payload); + } + + /** The type byte, or 0 for a startup packet. */ + public char type() { + return type; + } + + public boolean isStartupPacket() { + return type == 0; + } + + public int remaining() { + return payload.remaining(); + } + + public byte readByte() throws PgException { + try { + return payload.get(); + } catch (BufferUnderflowException e) { + throw truncated(); + } + } + + public short readInt16() throws PgException { + try { + return payload.getShort(); + } catch (BufferUnderflowException e) { + throw truncated(); + } + } + + public int readInt32() throws PgException { + try { + return payload.getInt(); + } catch (BufferUnderflowException e) { + throw truncated(); + } + } + + /** {@code n} bytes, or {@code null} for the length -1 used for NULL values. */ + public byte[] readBytes(int n) throws PgException { + if (n == -1) { + return null; + } + if (n < 0 || n > payload.remaining()) { + throw truncated(); + } + byte[] bytes = new byte[n]; + payload.get(bytes); + return bytes; + } + + /** A NUL-terminated UTF-8 string. */ + public String readCString() throws PgException { + int start = payload.position(); + for (int i = start; i < payload.limit(); i++) { + if (payload.get(i) == 0) { + String value = + new String( + payload.array(), payload.arrayOffset() + start, i - start, StandardCharsets.UTF_8); + payload.position(i + 1); + return value; + } + } + throw truncated(); + } + + private PgException truncated() { + return PgException.fatal( + SqlState.PROTOCOL_VIOLATION, + "invalid message format" + (type == 0 ? "" : " of message type '" + type + "'")); + } +} diff --git a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/protocol/PgErrors.java b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/protocol/PgErrors.java new file mode 100644 index 0000000000000..240c214744e50 --- /dev/null +++ b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/protocol/PgErrors.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.protocol; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.rpc.TSStatusCode; + +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.regex.Pattern; + +/** + * Maps the engine's status codes to SQLSTATEs. Unknown codes become XX000 with the engine's + * message. + */ +public final class PgErrors { + + private static final Map BY_CODE = new HashMap<>(); + private static final Pattern MISSING_TABLE = + Pattern.compile("table\\b.*\\b(does not exist|not exists?|cannot be found)"); + private static final Pattern MISSING_DATABASE = + Pattern.compile("database\\b.*\\b(does not exist|not exists?|cannot be found)"); + private static final Pattern MISSING_COLUMN = + Pattern.compile("column\\b.*\\b(does not exist|not exists?|cannot be resolved)"); + + static { + BY_CODE.put(TSStatusCode.SQL_PARSE_ERROR.getStatusCode(), SqlState.SYNTAX_ERROR); + BY_CODE.put(TSStatusCode.TABLE_NOT_EXISTS.getStatusCode(), SqlState.UNDEFINED_TABLE); + BY_CODE.put(TSStatusCode.COLUMN_NOT_EXISTS.getStatusCode(), SqlState.UNDEFINED_COLUMN); + BY_CODE.put(TSStatusCode.DATABASE_NOT_EXIST.getStatusCode(), SqlState.INVALID_SCHEMA_NAME); + BY_CODE.put(TSStatusCode.NO_PERMISSION.getStatusCode(), SqlState.INSUFFICIENT_PRIVILEGE); + BY_CODE.put(TSStatusCode.QUERY_WAS_KILLED.getStatusCode(), SqlState.QUERY_CANCELED); + BY_CODE.put(TSStatusCode.QUERY_TIMEOUT.getStatusCode(), SqlState.QUERY_CANCELED); + BY_CODE.put(TSStatusCode.MPP_MEMORY_NOT_ENOUGH.getStatusCode(), SqlState.OUT_OF_MEMORY); + BY_CODE.put( + TSStatusCode.QUERY_EXECUTION_MEMORY_NOT_ENOUGH.getStatusCode(), SqlState.OUT_OF_MEMORY); + BY_CODE.put( + TSStatusCode.TOO_MANY_CONCURRENT_QUERIES_ERROR.getStatusCode(), + SqlState.INSUFFICIENT_RESOURCES); + BY_CODE.put( + TSStatusCode.NUMERIC_VALUE_OUT_OF_RANGE.getStatusCode(), + SqlState.NUMERIC_VALUE_OUT_OF_RANGE); + BY_CODE.put(TSStatusCode.DIVISION_BY_ZERO.getStatusCode(), SqlState.DIVISION_BY_ZERO); + BY_CODE.put(TSStatusCode.DATE_OUT_OF_RANGE.getStatusCode(), SqlState.DATETIME_FIELD_OVERFLOW); + BY_CODE.put(TSStatusCode.UNSUPPORTED_OPERATION.getStatusCode(), SqlState.FEATURE_NOT_SUPPORTED); + } + + private PgErrors() {} + + /** The SQLSTATE for an engine status code and message. */ + public static String sqlState(int code, String message) { + String mapped = BY_CODE.get(code); + if (mapped != null) { + return mapped; + } + if (code == TSStatusCode.SEMANTIC_ERROR.getStatusCode()) { + // the analyzer reports missing objects as semantic errors with a message + String text = message == null ? "" : message.toLowerCase(Locale.ROOT); + if (MISSING_COLUMN.matcher(text).find()) { + return SqlState.UNDEFINED_COLUMN; + } + if (MISSING_TABLE.matcher(text).find()) { + return SqlState.UNDEFINED_TABLE; + } + if (MISSING_DATABASE.matcher(text).find()) { + return SqlState.INVALID_SCHEMA_NAME; + } + return SqlState.SYNTAX_OR_ACCESS_RULE; + } + return SqlState.INTERNAL_ERROR; + } + + /** An ERROR (not FATAL) for a failed engine status. */ + public static PgException of(TSStatus status) { + String message = + status.getMessage() == null ? "error " + status.getCode() : status.getMessage(); + return new PgException(sqlState(status.getCode(), message), message); + } + + /** An ERROR for an exception from the engine, with the status code it carries if any. */ + public static PgException of(int code, Throwable error) { + String message = error.getMessage() == null ? error.toString() : error.getMessage(); + return new PgException(sqlState(code, message), message); + } +} diff --git a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/protocol/PgException.java b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/protocol/PgException.java new file mode 100644 index 0000000000000..eb3d0e514fbae --- /dev/null +++ b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/protocol/PgException.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.protocol; + +/** An error that is sent to the client as an ErrorResponse with this SQLSTATE. */ +public class PgException extends Exception { + + private static final long serialVersionUID = 1L; + + private final String sqlState; + private final boolean fatal; + private final String hint; + + public PgException(String sqlState, String message) { + this(sqlState, message, null, false); + } + + public PgException(String sqlState, String message, String hint, boolean fatal) { + super(message); + this.sqlState = sqlState; + this.hint = hint; + this.fatal = fatal; + } + + /** A FATAL error, after which the server closes the connection. */ + public static PgException fatal(String sqlState, String message) { + return new PgException(sqlState, message, null, true); + } + + public String sqlState() { + return sqlState; + } + + public boolean isFatal() { + return fatal; + } + + /** A hint for the user, or {@code null}. */ + public String hint() { + return hint; + } +} diff --git a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/protocol/PgFrameDecoder.java b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/protocol/PgFrameDecoder.java new file mode 100644 index 0000000000000..879de1aa843bc --- /dev/null +++ b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/protocol/PgFrameDecoder.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.protocol; + +import io.netty.buffer.ByteBuf; +import io.netty.channel.ChannelHandlerContext; +import io.netty.handler.codec.ByteToMessageDecoder; +import io.netty.handler.codec.DecoderException; + +import java.util.List; + +/** + * Splits the byte stream into {@link FrontendMessage}s: untyped startup packets until {@link + * #startupComplete()}, typed messages afterwards. Lengths are checked before anything is buffered, + * so a client cannot make the server allocate more than {@code maxMessageSize}. + */ +public final class PgFrameDecoder extends ByteToMessageDecoder { + + /** Startup packets are small; PostgreSQL itself allows 10,000 bytes. */ + static final int MAX_STARTUP_PACKET = 10_000; + + private final int maxMessageSize; + private boolean startup = true; + + public PgFrameDecoder(int maxMessageSize) { + this.maxMessageSize = maxMessageSize; + } + + /** Switches to typed messages once the StartupMessage has been read. */ + public void startupComplete() { + startup = false; + } + + @Override + protected void decode(ChannelHandlerContext ctx, ByteBuf in, List out) { + if (startup) { + if (in.readableBytes() < 4) { + return; + } + int length = in.getInt(in.readerIndex()); + if (length < 8 || length > MAX_STARTUP_PACKET) { + throw new DecoderException( + PgException.fatal(SqlState.PROTOCOL_VIOLATION, "invalid length of startup packet")); + } + if (in.readableBytes() < length) { + return; + } + in.skipBytes(4); + byte[] payload = new byte[length - 4]; + in.readBytes(payload); + out.add(new FrontendMessage((char) 0, payload)); + return; + } + if (in.readableBytes() < 5) { + return; + } + char type = (char) (in.getByte(in.readerIndex()) & 0xFF); + int length = in.getInt(in.readerIndex() + 1); + if (length < 4 || length > maxMessageSize) { + throw new DecoderException( + PgException.fatal(SqlState.PROTOCOL_VIOLATION, "invalid message length")); + } + if (in.readableBytes() < 1 + length) { + return; + } + in.skipBytes(5); + byte[] payload = new byte[length - 4]; + in.readBytes(payload); + out.add(new FrontendMessage(type, payload)); + } +} diff --git a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/protocol/SqlState.java b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/protocol/SqlState.java new file mode 100644 index 0000000000000..c1538f6b64ac9 --- /dev/null +++ b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/protocol/SqlState.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.protocol; + +/** SQLSTATE codes the adapter sends. */ +public final class SqlState { + + public static final String READ_ONLY_SQL_TRANSACTION = "25006"; + public static final String FEATURE_NOT_SUPPORTED = "0A000"; + public static final String SYNTAX_ERROR = "42601"; + public static final String UNDEFINED_TABLE = "42P01"; + public static final String UNDEFINED_COLUMN = "42703"; + public static final String INVALID_SCHEMA_NAME = "3F000"; + public static final String INSUFFICIENT_PRIVILEGE = "42501"; + public static final String QUERY_CANCELED = "57014"; + public static final String TOO_MANY_CONNECTIONS = "53300"; + public static final String OUT_OF_MEMORY = "53200"; + public static final String INTERNAL_ERROR = "XX000"; + public static final String INVALID_PASSWORD = "28P01"; + public static final String INVALID_AUTHORIZATION = "28000"; + public static final String PROTOCOL_VIOLATION = "08P01"; + public static final String CONNECTION_FAILURE = "08006"; + public static final String ADMIN_SHUTDOWN = "57P01"; + public static final String INVALID_PARAMETER_VALUE = "22023"; + public static final String INVALID_CURSOR_NAME = "34000"; + public static final String DUPLICATE_CURSOR = "42P03"; + public static final String DUPLICATE_PREPARED_STATEMENT = "42P05"; + public static final String INVALID_SQL_STATEMENT_NAME = "26000"; + public static final String INVALID_TEXT_REPRESENTATION = "22P02"; + public static final String SYNTAX_OR_ACCESS_RULE = "42000"; + public static final String NUMERIC_VALUE_OUT_OF_RANGE = "22003"; + public static final String DIVISION_BY_ZERO = "22012"; + public static final String DATETIME_FIELD_OVERFLOW = "22008"; + public static final String INSUFFICIENT_RESOURCES = "53000"; + + private SqlState() {} +} diff --git a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/server/PgConnectionHandler.java b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/server/PgConnectionHandler.java new file mode 100644 index 0000000000000..ccc807db1d3fe --- /dev/null +++ b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/server/PgConnectionHandler.java @@ -0,0 +1,405 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.server; + +import org.apache.iotdb.commons.conf.IoTDBConstant; +import org.apache.iotdb.commons.queryengine.common.SqlDialect; +import org.apache.iotdb.db.protocol.basic.BasicOpenSessionResp; +import org.apache.iotdb.db.protocol.session.ExternalServiceClientSession; +import org.apache.iotdb.db.protocol.session.SessionManager; +import org.apache.iotdb.db.queryengine.plan.Coordinator; +import org.apache.iotdb.postgresql.protocol.BackendMessages; +import org.apache.iotdb.postgresql.protocol.FrontendMessage; +import org.apache.iotdb.postgresql.protocol.PgException; +import org.apache.iotdb.postgresql.protocol.PgFrameDecoder; +import org.apache.iotdb.postgresql.protocol.SqlState; +import org.apache.iotdb.postgresql.session.CancelRegistry; +import org.apache.iotdb.postgresql.session.PgSession; +import org.apache.iotdb.postgresql.session.QueryProtocol; +import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.iotdb.service.rpc.thrift.TSProtocolVersion; + +import io.netty.buffer.ByteBuf; +import io.netty.channel.ChannelFutureListener; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.SimpleChannelInboundHandler; +import io.netty.handler.codec.DecoderException; +import io.netty.handler.ssl.SslHandler; +import io.netty.util.concurrent.ScheduledFuture; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.net.ssl.SSLEngine; + +import java.net.InetSocketAddress; +import java.time.DateTimeException; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +/** One client connection: the startup phase, the login and then the query layer. */ +final class PgConnectionHandler extends SimpleChannelInboundHandler { + + private static final Logger LOGGER = LoggerFactory.getLogger(PgConnectionHandler.class); + + private enum State { + AWAIT_STARTUP, + PASSWORD, + READY, + CLOSED + } + + private final PgServerContext context; + private final PgFrameDecoder decoder; + private final SerialExecutor serial; + + private volatile State state = State.AWAIT_STARTUP; + private boolean tls; + private String user; + private final Map startupParameters = new LinkedHashMap<>(); + private ExternalServiceClientSession dbSession; + private CancelRegistry.Key key; + private QueryProtocol protocol; + private ScheduledFuture startupTimeout; + + PgConnectionHandler(PgServerContext context, PgFrameDecoder decoder) { + this.context = context; + this.decoder = decoder; + this.serial = new SerialExecutor(context.workers); + } + + @Override + public void channelActive(ChannelHandlerContext ctx) { + if (context.active.incrementAndGet() > context.config.maxConnections()) { + context.refusedConnections.incrementAndGet(); + fail( + ctx, PgException.fatal(SqlState.TOO_MANY_CONNECTIONS, "sorry, too many clients already")); + return; + } + startupTimeout = + ctx.executor() + .schedule( + () -> { + if (state != State.READY && state != State.CLOSED) { + ctx.close(); + } + }, + context.config.startupTimeout().toMillis(), + TimeUnit.MILLISECONDS); + ctx.fireChannelActive(); + } + + @Override + public void channelInactive(ChannelHandlerContext ctx) { + context.active.decrementAndGet(); + state = State.CLOSED; + if (startupTimeout != null) { + startupTimeout.cancel(false); + } + serial.execute(this::release); + ctx.fireChannelInactive(); + } + + private void release() { + if (key != null) { + context.cancels.unregister(key); + } + if (protocol != null) { + protocol.close(); + } + if (dbSession != null) { + SessionManager.getInstance() + .closeSession(dbSession, Coordinator.getInstance()::cleanupQueryExecution); + } + } + + @Override + protected void channelRead0(ChannelHandlerContext ctx, FrontendMessage message) { + try { + switch (state) { + case AWAIT_STARTUP: + startup(ctx, message); + return; + case PASSWORD: + authentication(ctx, message); + return; + case READY: + if (message.type() == 'X') { + ctx.close(); + return; + } + serial.execute(() -> protocol.handle(message, ctx.channel())); + return; + default: + // closed: ignore late messages + } + } catch (PgException e) { + fail(ctx, e); + } + } + + // ---------------------------------------------------------------------------- startup phase + + private void startup(ChannelHandlerContext ctx, FrontendMessage message) throws PgException { + if (!message.isStartupPacket()) { + throw PgException.fatal(SqlState.PROTOCOL_VIOLATION, "expected a startup message"); + } + int code = message.readInt32(); + switch (code) { + case FrontendMessage.SSL_REQUEST: + sslRequest(ctx); + return; + case FrontendMessage.GSSENC_REQUEST: + ByteBuf no = ctx.alloc().buffer(1); + BackendMessages.sslResponse(no, false); + ctx.writeAndFlush(no); + return; + case FrontendMessage.CANCEL_REQUEST: + int processId = message.readInt32(); + int secretKey = message.readInt32(); + context.cancels.cancel(processId, secretKey); + ctx.close(); + return; + default: + startupMessage(ctx, message, code); + } + } + + private void sslRequest(ChannelHandlerContext ctx) throws PgException { + if (tls) { + throw PgException.fatal(SqlState.PROTOCOL_VIOLATION, "SSL was already negotiated"); + } + ByteBuf answer = ctx.alloc().buffer(1); + if (context.sslContext == null) { + BackendMessages.sslResponse(answer, false); + ctx.writeAndFlush(answer); + return; + } + SSLEngine engine = context.sslContext.createSSLEngine(); + engine.setUseClientMode(false); + TlsPolicy.restrict(engine); + // startTls: the 'S' still goes out unencrypted, everything after it is TLS + ctx.pipeline().addFirst("tls", new SslHandler(engine, true)); + tls = true; + BackendMessages.sslResponse(answer, true); + ctx.writeAndFlush(answer); + } + + private void startupMessage(ChannelHandlerContext ctx, FrontendMessage message, int code) + throws PgException { + int major = code >>> 16; + int minor = code & 0xFFFF; + if (major != 3) { + throw PgException.fatal( + SqlState.PROTOCOL_VIOLATION, "unsupported frontend protocol " + major + "." + minor); + } + if (!tls && !context.loopback) { + // off loopback every connection negotiates TLS first + context.refusedUnencrypted.incrementAndGet(); + throw PgException.fatal( + SqlState.INVALID_AUTHORIZATION, + "encryption required: connect with SSL (sslmode=require)"); + } + List unsupportedOptions = new ArrayList<>(); + while (message.remaining() > 1) { + String name = message.readCString(); + String value = message.readCString(); + if (name.startsWith("_pq_.")) { + unsupportedOptions.add(name); + } else { + startupParameters.put(name.toLowerCase(Locale.ROOT), value); + } + } + user = startupParameters.get("user"); + if (user == null || user.isEmpty()) { + throw PgException.fatal(SqlState.INVALID_AUTHORIZATION, "no PostgreSQL user name specified"); + } + String encoding = startupParameters.get("client_encoding"); + if (encoding != null && !isUtf8(encoding)) { + throw PgException.fatal( + SqlState.FEATURE_NOT_SUPPORTED, "only client_encoding UTF8 is supported"); + } + decoder.startupComplete(); + ByteBuf out = ctx.alloc().buffer(); + if (minor > 0 || !unsupportedOptions.isEmpty()) { + BackendMessages.negotiateProtocolVersion(out, 0, unsupportedOptions); + } + InetSocketAddress remote = + ctx.channel().remoteAddress() instanceof InetSocketAddress + ? (InetSocketAddress) ctx.channel().remoteAddress() + : new InetSocketAddress(0); // not a TCP channel (tests) + dbSession = new ExternalServiceClientSession("postgresql", remote); + // The password inside TLS (off loopback every connection is encrypted, see above). The + // database stores no SCRAM verifiers, so SCRAM-SHA-256 cannot be offered. + state = State.PASSWORD; + BackendMessages.authenticationCleartextPassword(out); + ctx.writeAndFlush(out); + } + + private static boolean isUtf8(String encoding) { + String normalized = encoding.replace("'", "").replace("-", "").trim().toUpperCase(Locale.ROOT); + return normalized.equals("UTF8") || normalized.equals("UNICODE"); + } + + // ---------------------------------------------------------------------------- authentication + + private void authentication(ChannelHandlerContext ctx, FrontendMessage message) + throws PgException { + if (message.type() != 'p') { + throw PgException.fatal(SqlState.PROTOCOL_VIOLATION, "expected an authentication message"); + } + if (state != State.PASSWORD) { + throw PgException.fatal(SqlState.PROTOCOL_VIOLATION, "unexpected authentication message"); + } + String password = message.readCString(); + state = State.CLOSED; // until the login is done; no further message is valid + serial.execute(() -> passwordLogin(ctx, password)); + } + + private void passwordLogin(ChannelHandlerContext ctx, String password) { + try { + BasicOpenSessionResp response = + SessionManager.getInstance() + .login( + dbSession, + user, + password, + zone().getId(), + TSProtocolVersion.IOTDB_SERVICE_PROTOCOL_V3, + IoTDBConstant.ClientVersion.V_1_0, + SqlDialect.TABLE); + if (response.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + loginFailed(ctx, response); + return; + } + loggedIn(ctx, ctx.alloc().buffer()); + } catch (RuntimeException e) { + LOGGER.warn("PostgreSQL login failed", e); + fail(ctx, PgException.fatal(SqlState.CONNECTION_FAILURE, "login failed")); + } + } + + private void loginFailed(ChannelHandlerContext ctx, BasicOpenSessionResp response) { + context.failedLogins.incrementAndGet(); + int code = response.getCode(); + String sqlState = + code == TSStatusCode.USER_LOGIN_LOCKED.getStatusCode() + || code == TSStatusCode.NO_PERMISSION.getStatusCode() + ? SqlState.INVALID_AUTHORIZATION + : code == TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode() + ? SqlState.CONNECTION_FAILURE + : SqlState.INVALID_PASSWORD; + fail(ctx, PgException.fatal(sqlState, response.getMessage())); + } + + /** AuthenticationOk, the parameters, BackendKeyData and ReadyForQuery. */ + private void loggedIn(ChannelHandlerContext ctx, ByteBuf out) { + PgSession session = new PgSession(dbSession, user, zone()); + startupParameters.forEach(session::setParameter); + BackendMessages.authenticationOk(out); + Map reported = new LinkedHashMap<>(); + reported.put("server_version", context.serverVersion); + reported.put("server_encoding", "UTF8"); + reported.put("client_encoding", "UTF8"); + reported.put("DateStyle", "ISO, MDY"); + reported.put("IntervalStyle", "postgres"); + reported.put("TimeZone", session.zone().getId()); + reported.put("integer_datetimes", "on"); + reported.put("standard_conforming_strings", "on"); + reported.put("is_superuser", "off"); + reported.put("session_authorization", user); + reported.put("application_name", startupParameters.getOrDefault("application_name", "")); + reported.forEach( + (name, value) -> { + BackendMessages.parameterStatus(out, name, value); + session.setParameter(name.toLowerCase(Locale.ROOT), value); + }); + session.markStartup(); + protocol = context.protocols.apply(session); + key = context.cancels.register(protocol::cancel); + session.setProcessId(key.processId()); + BackendMessages.backendKeyData(out, key.processId(), key.secretKey()); + BackendMessages.readyForQuery(out, 'I'); + state = State.READY; + if (startupTimeout != null) { + startupTimeout.cancel(false); + } + ctx.writeAndFlush(out); + } + + private ZoneId zone() { + String zone = startupParameters.get("timezone"); + if (zone != null) { + try { + return ZoneId.of(zone.replace("'", "").trim()); + } catch (DateTimeException e) { + // fall back to the server's zone + } + } + return ZoneId.systemDefault(); + } + + // ---------------------------------------------------------------------------- errors + + @Override + public void userEventTriggered(ChannelHandlerContext ctx, Object event) { + if (event instanceof io.netty.handler.timeout.IdleStateEvent) { + // only connections without a running query are idle + if (state != State.READY || protocol == null || !protocol.busy()) { + fail( + ctx, + PgException.fatal( + SqlState.ADMIN_SHUTDOWN, "terminating connection due to idle timeout")); + } + return; + } + ctx.fireUserEventTriggered(event); + } + + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { + Throwable root = + cause instanceof DecoderException && cause.getCause() != null ? cause.getCause() : cause; + if (root instanceof PgException) { + fail(ctx, (PgException) root); + } else { + LOGGER.debug("PostgreSQL connection {} failed", ctx.channel().remoteAddress(), cause); + ctx.close(); + } + } + + /** Sends the error; a FATAL one closes the connection afterwards. */ + private void fail(ChannelHandlerContext ctx, PgException error) { + if (error.isFatal()) { + state = State.CLOSED; + } + ByteBuf out = ctx.alloc().buffer(); + BackendMessages.errorResponse(out, error); + if (error.isFatal()) { + ctx.writeAndFlush(out).addListener(ChannelFutureListener.CLOSE); + } else { + ctx.writeAndFlush(out); + } + } +} diff --git a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/server/PgServer.java b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/server/PgServer.java new file mode 100644 index 0000000000000..c59a189a75399 --- /dev/null +++ b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/server/PgServer.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.server; + +import org.apache.iotdb.postgresql.protocol.PgFrameDecoder; + +import io.netty.bootstrap.ServerBootstrap; +import io.netty.channel.Channel; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.ChannelOption; +import io.netty.channel.EventLoopGroup; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.SocketChannel; +import io.netty.channel.socket.nio.NioServerSocketChannel; +import io.netty.handler.timeout.IdleStateHandler; +import io.netty.util.concurrent.DefaultThreadFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.InetSocketAddress; +import java.util.concurrent.TimeUnit; + +/** The listening endpoint: Netty event loops, TLS on demand, timeouts, back pressure. */ +public final class PgServer { + + private static final Logger LOGGER = LoggerFactory.getLogger(PgServer.class); + + /** Largest client message; Bind messages with large values stay well below. */ + static final int MAX_MESSAGE_SIZE = 64 * 1024 * 1024; + + private final PgServerContext context; + private EventLoopGroup boss; + private EventLoopGroup workers; + private Channel channel; + + public PgServer(PgServerContext context) { + this.context = context; + } + + /** Binds the endpoint; fails (with the address named) when the port cannot be opened. */ + public void start() throws InterruptedException { + boss = new NioEventLoopGroup(1, new DefaultThreadFactory("pg-service-accept", true)); + workers = + new NioEventLoopGroup( + Math.min(4, Runtime.getRuntime().availableProcessors()), + new DefaultThreadFactory("pg-service-io", true)); + long idleMillis = context.config.idleTimeout().toMillis(); + ServerBootstrap bootstrap = + new ServerBootstrap() + .group(boss, workers) + .channel(NioServerSocketChannel.class) + .childOption(ChannelOption.TCP_NODELAY, true) + .childHandler( + new ChannelInitializer() { + @Override + protected void initChannel(SocketChannel ch) { + PgFrameDecoder decoder = new PgFrameDecoder(MAX_MESSAGE_SIZE); + ch.pipeline() + // the connection handler decides whether an idle connection is closed + .addLast( + "idle", new IdleStateHandler(idleMillis, 0, 0, TimeUnit.MILLISECONDS)) + .addLast("frames", decoder) + .addLast("connection", new PgConnectionHandler(context, decoder)); + } + }); + InetSocketAddress address = + context.config.isWildcard() + ? new InetSocketAddress(context.config.port()) + : new InetSocketAddress(context.config.address(), context.config.port()); + try { + channel = bootstrap.bind(address).sync().channel(); + } catch (RuntimeException | InterruptedException e) { + stop(); + throw e; + } + LOGGER.info("PostgreSQL endpoint listening on {}", channel.localAddress()); + } + + /** The bound address (with the actual port when port 0 was configured). */ + public InetSocketAddress localAddress() { + return channel == null ? null : (InetSocketAddress) channel.localAddress(); + } + + public void stop() { + if (channel != null) { + channel.close().syncUninterruptibly(); + channel = null; + } + if (boss != null) { + boss.shutdownGracefully(0, 2, TimeUnit.SECONDS); + boss = null; + } + if (workers != null) { + workers.shutdownGracefully(0, 2, TimeUnit.SECONDS); + workers = null; + } + } +} diff --git a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/server/PgServerContext.java b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/server/PgServerContext.java new file mode 100644 index 0000000000000..8e476d9db7161 --- /dev/null +++ b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/server/PgServerContext.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.server; + +import org.apache.iotdb.postgresql.config.PostgreSqlServiceConfig; +import org.apache.iotdb.postgresql.session.CancelRegistry; +import org.apache.iotdb.postgresql.session.PgSession; +import org.apache.iotdb.postgresql.session.QueryProtocol; + +import javax.net.ssl.SSLContext; + +import java.util.concurrent.Executor; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Function; + +/** What every connection of one server shares. */ +public final class PgServerContext { + + final PostgreSqlServiceConfig config; + final SSLContext sslContext; + final boolean loopback; + final CancelRegistry cancels = new CancelRegistry(); + final Executor workers; + final Function protocols; + final String serverVersion; + + final java.util.concurrent.atomic.AtomicInteger active = + new java.util.concurrent.atomic.AtomicInteger(); + final AtomicLong refusedConnections = new AtomicLong(); + final AtomicLong refusedUnencrypted = new AtomicLong(); + final AtomicLong failedLogins = new AtomicLong(); + + /** + * @param sslContext the TLS context, or {@code null} when no key store is configured (only + * allowed on loopback) + * @param protocols creates the query layer of a logged-in connection + * @param serverVersion the reported {@code server_version} + */ + public PgServerContext( + PostgreSqlServiceConfig config, + SSLContext sslContext, + Executor workers, + Function protocols, + String serverVersion) { + this.config = config; + this.sslContext = sslContext; + this.loopback = config.isLoopback(); + this.workers = workers; + this.protocols = protocols; + this.serverVersion = serverVersion; + } + + public int openConnections() { + return active.get(); + } + + public long refusedConnections() { + return refusedConnections.get(); + } + + public long refusedUnencrypted() { + return refusedUnencrypted.get(); + } + + public long failedLogins() { + return failedLogins.get(); + } +} diff --git a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/server/SerialExecutor.java b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/server/SerialExecutor.java new file mode 100644 index 0000000000000..3e649f409af28 --- /dev/null +++ b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/server/SerialExecutor.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.server; + +import java.util.ArrayDeque; +import java.util.Queue; +import java.util.concurrent.Executor; + +/** + * Runs one connection's tasks one after another on a shared pool, so blocking work (logins, + * queries) leaves Netty's event loop free while the connection's messages keep their order. + */ +final class SerialExecutor implements Executor { + + private final Executor pool; + private final Queue tasks = new ArrayDeque<>(); + private boolean running; + + SerialExecutor(Executor pool) { + this.pool = pool; + } + + @Override + public void execute(Runnable task) { + synchronized (tasks) { + tasks.add(task); + if (running) { + return; + } + running = true; + } + pool.execute(this::drain); + } + + private void drain() { + while (true) { + Runnable next; + synchronized (tasks) { + next = tasks.poll(); + if (next == null) { + running = false; + return; + } + } + try { + next.run(); + } catch (RuntimeException e) { + // a task reports its own errors to the client; never stop the queue + } + } + } +} diff --git a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/server/TlsPolicy.java b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/server/TlsPolicy.java new file mode 100644 index 0000000000000..f04cef26bf508 --- /dev/null +++ b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/server/TlsPolicy.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.server; + +import javax.net.ssl.SSLEngine; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * The TLS settings of the PostgreSQL endpoint: TLS 1.2 and 1.3 with AEAD cipher suites only. Older + * protocols and CBC suites are never enabled, even if the JVM still offers them. + */ +final class TlsPolicy { + + static final List PROTOCOLS = + Collections.unmodifiableList(Arrays.asList("TLSv1.3", "TLSv1.2")); + + static final List CIPHER_SUITES = + Collections.unmodifiableList( + Arrays.asList( + "TLS_AES_256_GCM_SHA384", + "TLS_AES_128_GCM_SHA256", + "TLS_CHACHA20_POLY1305_SHA256", + "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384", + "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256", + "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256", + "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", + "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", + "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256")); + + private TlsPolicy() {} + + /** Restricts the engine to the allowed protocols and suites; never falls back to weaker ones. */ + static void restrict(SSLEngine engine) { + String[] protocols = supported(PROTOCOLS, engine.getSupportedProtocols()); + String[] suites = supported(CIPHER_SUITES, engine.getSupportedCipherSuites()); + if (protocols.length == 0 || suites.length == 0) { + throw new IllegalStateException( + "The JVM supports none of the allowed TLS protocols " + PROTOCOLS + " or cipher suites"); + } + engine.setEnabledProtocols(protocols); + engine.setEnabledCipherSuites(suites); + } + + private static String[] supported(List allowed, String[] supportedByJvm) { + Set supported = new HashSet<>(Arrays.asList(supportedByJvm)); + List result = new ArrayList<>(); + for (String value : allowed) { + if (supported.contains(value)) { + result.add(value); + } + } + return result.toArray(new String[0]); + } +} diff --git a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/session/CancelRegistry.java b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/session/CancelRegistry.java new file mode 100644 index 0000000000000..fde7eb44e4718 --- /dev/null +++ b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/session/CancelRegistry.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.session; + +import java.security.SecureRandom; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Process ids and secret keys of the open connections (BackendKeyData), so that a CancelRequest on + * a new connection can find the connection whose query it cancels. + */ +public final class CancelRegistry { + + private static final SecureRandom RANDOM = new SecureRandom(); + + private static final class Entry { + final int secretKey; + final Runnable cancel; + + Entry(int secretKey, Runnable cancel) { + this.secretKey = secretKey; + this.cancel = cancel; + } + } + + private final AtomicInteger nextProcessId = new AtomicInteger(1000 + RANDOM.nextInt(50_000)); + private final Map entries = new ConcurrentHashMap<>(); + + /** A registered connection: its process id and secret key. */ + public static final class Key { + private final int processId; + private final int secretKey; + + Key(int processId, int secretKey) { + this.processId = processId; + this.secretKey = secretKey; + } + + public int processId() { + return processId; + } + + public int secretKey() { + return secretKey; + } + } + + /** Registers a connection; {@code cancel} stops its running query, if any. */ + public Key register(Runnable cancel) { + int processId = nextProcessId.getAndIncrement(); + int secretKey = RANDOM.nextInt(); + entries.put(processId, new Entry(secretKey, cancel)); + return new Key(processId, secretKey); + } + + public void unregister(Key key) { + entries.remove(key.processId); + } + + /** + * Cancels the query of the connection with this process id if the secret key matches; an unknown + * id or a wrong key does nothing (as in PostgreSQL, nothing is reported). + */ + public void cancel(int processId, int secretKey) { + Entry entry = entries.get(processId); + if (entry != null && entry.secretKey == secretKey) { + entry.cancel.run(); + } + } + + public int size() { + return entries.size(); + } +} diff --git a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/session/CursorCommand.java b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/session/CursorCommand.java new file mode 100644 index 0000000000000..99a658919fd14 --- /dev/null +++ b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/session/CursorCommand.java @@ -0,0 +1,183 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.session; + +import org.apache.iotdb.postgresql.protocol.PgException; +import org.apache.iotdb.postgresql.protocol.SqlState; + +import java.util.Locale; +import java.util.Optional; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * A SQL cursor statement: {@code DECLARE}, {@code FETCH}, {@code MOVE}, {@code CLOSE}. psycopg's + * server-side cursors use them instead of protocol portals. Only forward movement exists: the + * engine streams a query once. + */ +public final class CursorCommand { + + /** The kind of statement. */ + public enum Kind { + DECLARE, + FETCH, + MOVE, + CLOSE + } + + private static final int FLAGS = Pattern.CASE_INSENSITIVE | Pattern.DOTALL; + private static final String NAME = "(\"(?:[^\"]|\"\")+\"|[A-Za-z_][\\w$]*)"; + + private static final Pattern DECLARE = + Pattern.compile( + "DECLARE\\s+" + + NAME + + "\\s+((?:(?:BINARY|INSENSITIVE|ASENSITIVE|NO\\s+SCROLL|SCROLL)\\s+)*)" + + "CURSOR\\s+(?:(WITH|WITHOUT)\\s+HOLD\\s+)?FOR\\s+(.+)", + FLAGS); + private static final Pattern FETCH_OR_MOVE = + Pattern.compile( + "(FETCH|MOVE)" + + "(?:\\s+(FORWARD|NEXT|BACKWARD|PRIOR|FIRST|LAST|ABSOLUTE|RELATIVE))?" + + "(?:\\s+(ALL|[+-]?\\d+))?" + + "(?:\\s+(?:FROM|IN))?" + + "\\s+" + + NAME, + FLAGS); + private static final Pattern CLOSE = Pattern.compile("CLOSE\\s+(ALL|" + NAME + ")", FLAGS); + private static final Pattern START = Pattern.compile("(DECLARE|FETCH|MOVE|CLOSE)\\b.*", FLAGS); + + private final Kind kind; + private final String name; + private final long count; + private final boolean hold; + private final boolean binary; + private final String query; + + private CursorCommand( + Kind kind, String name, long count, boolean hold, boolean binary, String query) { + this.kind = kind; + this.name = name; + this.count = count; + this.hold = hold; + this.binary = binary; + this.query = query; + } + + /** The kind of statement. */ + public Kind kind() { + return kind; + } + + /** The cursor's name; {@code null} for {@code CLOSE ALL}. */ + public String name() { + return name; + } + + /** FETCH/MOVE: the number of rows, 0 for all. */ + public long count() { + return count; + } + + /** DECLARE: {@code WITH HOLD} (the cursor survives the end of the transaction). */ + public boolean hold() { + return hold; + } + + /** DECLARE: {@code BINARY} (rows in binary format by default). */ + public boolean binary() { + return binary; + } + + /** DECLARE: the query. */ + public String query() { + return query; + } + + /** + * Parses a cursor statement. + * + * @return empty if {@code statement} is not a cursor statement + * @throws PgException 0A000 for backward or absolute movement and scrollable cursors, 42601 for + * malformed cursor statements + */ + public static Optional parse(String statement) throws PgException { + String sql = statement.trim(); + if (!START.matcher(sql).matches()) { + return Optional.empty(); + } + Matcher m = DECLARE.matcher(sql); + if (m.matches()) { + String options = m.group(2).toUpperCase(Locale.ROOT); + if (options.matches("(?s).*(^|\\s)SCROLL\\b.*") && !options.contains("NO SCROLL")) { + throw new PgException(SqlState.FEATURE_NOT_SUPPORTED, "SCROLL cursors are not supported"); + } + return Optional.of( + new CursorCommand( + Kind.DECLARE, + fold(m.group(1)), + 0, + "WITH".equalsIgnoreCase(m.group(3)), + options.contains("BINARY"), + m.group(4).trim())); + } + m = FETCH_OR_MOVE.matcher(sql); + if (m.matches()) { + Kind kind = m.group(1).equalsIgnoreCase("FETCH") ? Kind.FETCH : Kind.MOVE; + String direction = m.group(2) == null ? "FORWARD" : m.group(2).toUpperCase(Locale.ROOT); + if (!direction.equals("FORWARD") && !direction.equals("NEXT")) { + throw new PgException( + SqlState.FEATURE_NOT_SUPPORTED, + kind + " " + direction + " is not supported: cursors only move forward"); + } + long count = 1; + if (m.group(3) != null) { + if (direction.equals("NEXT")) { + throw new PgException( + SqlState.SYNTAX_ERROR, "syntax error at or near \"" + m.group(3) + "\""); + } + if (m.group(3).equalsIgnoreCase("ALL")) { + count = 0; + } else { + count = Long.parseLong(m.group(3)); + if (count <= 0) { + throw new PgException( + SqlState.FEATURE_NOT_SUPPORTED, + kind + " " + m.group(3) + " is not supported: cursors only move forward"); + } + } + } + return Optional.of(new CursorCommand(kind, fold(m.group(4)), count, false, false, null)); + } + m = CLOSE.matcher(sql); + if (m.matches()) { + String name = m.group(1).equalsIgnoreCase("ALL") ? null : fold(m.group(2)); + return Optional.of(new CursorCommand(Kind.CLOSE, name, 0, false, false, null)); + } + throw new PgException(SqlState.SYNTAX_ERROR, "syntax error in cursor statement"); + } + + private static String fold(String identifier) { + if (identifier.startsWith("\"")) { + return identifier.substring(1, identifier.length() - 1).replace("\"\"", "\""); + } + return identifier.toLowerCase(Locale.ROOT); + } +} diff --git a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/session/PgSession.java b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/session/PgSession.java new file mode 100644 index 0000000000000..effded653f2b6 --- /dev/null +++ b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/session/PgSession.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.session; + +import org.apache.iotdb.db.protocol.session.ExternalServiceClientSession; + +import java.time.ZoneId; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * The state of one logged-in connection that the query layer needs: the database session, the + * session parameters and the transaction status. + */ +public final class PgSession { + + private final ExternalServiceClientSession session; + private final String user; + private final Map parameters = new LinkedHashMap<>(); + private final Map startup = new LinkedHashMap<>(); + private volatile ZoneId startupZone; + private volatile ZoneId zone; + private volatile char transactionStatus = 'I'; + private volatile int processId; + + public PgSession(ExternalServiceClientSession session, String user, ZoneId zone) { + this.session = session; + this.user = user; + this.zone = zone; + } + + public ExternalServiceClientSession session() { + return session; + } + + public String user() { + return user; + } + + public ZoneId zone() { + return zone; + } + + public void setZone(ZoneId zone) { + this.zone = zone; + // the engine reads the zone from the database session (time literals, date_bin origins) + session.setZoneId(zone); + } + + /** The process id of BackendKeyData ({@code pg_backend_pid()}). */ + public int processId() { + return processId; + } + + public void setProcessId(int processId) { + this.processId = processId; + } + + /** 'I' idle, 'T' in a transaction block, 'E' in a failed transaction block. */ + public char transactionStatus() { + return transactionStatus; + } + + public void setTransactionStatus(char status) { + this.transactionStatus = status; + } + + /** Session parameters by lower-case name (set at startup or with SET). */ + public synchronized String parameter(String name) { + return parameters.get(name); + } + + public synchronized void setParameter(String name, String value) { + parameters.put(name, value); + } + + /** Records the current parameters as the values {@code RESET} returns to (end of startup). */ + public synchronized void markStartup() { + startup.clear(); + startup.putAll(parameters); + startupZone = zone; + } + + /** + * {@code RESET name}: the value at the end of startup, or none. + * + * @return the value now in effect, {@code null} if the parameter has none + */ + public synchronized String reset(String name) { + String value = startup.get(name); + if (value == null) { + parameters.remove(name); + } else { + parameters.put(name, value); + } + if (name.equals("timezone") && startupZone != null) { + setZone(startupZone); + } + return value; + } + + /** {@code RESET ALL}: every parameter back to its value at the end of startup. */ + public synchronized void resetAll() { + parameters.clear(); + parameters.putAll(startup); + if (startupZone != null) { + setZone(startupZone); + } + } + + public synchronized Map parameters() { + return Collections.unmodifiableMap(new LinkedHashMap<>(parameters)); + } +} diff --git a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/session/QueryProtocol.java b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/session/QueryProtocol.java new file mode 100644 index 0000000000000..a3c5224a8a7eb --- /dev/null +++ b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/session/QueryProtocol.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.session; + +import org.apache.iotdb.postgresql.protocol.FrontendMessage; + +import io.netty.channel.Channel; + +/** + * Handles the messages of a logged-in connection (simple and extended query protocol). Called on + * the connection's own ordered executor, never on Netty's event loop, so it may block. + */ +public interface QueryProtocol { + + /** Handles one message of the ready state ('Q', 'P', 'B', 'D', 'E', 'S', 'C', 'H'). */ + void handle(FrontendMessage message, Channel channel); + + /** True while a query runs or a result is being sent; idle timeouts do not apply then. */ + boolean busy(); + + /** Cancels the running query, if any (CancelRequest); may be called from any thread. */ + void cancel(); + + /** Releases everything when the connection closes. */ + void close(); +} diff --git a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/session/SearchPath.java b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/session/SearchPath.java new file mode 100644 index 0000000000000..349cc46546c98 --- /dev/null +++ b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/session/SearchPath.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.session; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Locale; + +/** The {@code search_path} setting as a list of schema (database) names. */ +public final class SearchPath { + + private SearchPath() {} + + /** + * The folded names of a {@code search_path} value such as {@code "$user", public, mydb}; {@code + * $user} is left out (users have no schema of their own). Empty for no value. + */ + public static List parse(String value) { + if (value == null || value.trim().isEmpty()) { + return Collections.emptyList(); + } + List names = new ArrayList<>(); + for (String part : value.split(",")) { + String name = part.trim(); + if (name.length() >= 2 && name.startsWith("\"") && name.endsWith("\"")) { + name = name.substring(1, name.length() - 1).replace("\"\"", "\""); + } else if (name.length() >= 2 && name.startsWith("'") && name.endsWith("'")) { + name = name.substring(1, name.length() - 1); + } else { + name = name.toLowerCase(Locale.ROOT); + } + if (!name.isEmpty() && !name.equals("$user")) { + names.add(name); + } + } + return names; + } +} diff --git a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/session/SessionCommands.java b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/session/SessionCommands.java new file mode 100644 index 0000000000000..9f0e0f6698621 --- /dev/null +++ b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/session/SessionCommands.java @@ -0,0 +1,374 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.session; + +import org.apache.iotdb.postgresql.protocol.PgException; +import org.apache.iotdb.postgresql.protocol.SqlState; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.DateTimeException; +import java.time.ZoneId; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Statements handled by the adapter without the engine: transaction commands, SET/RESET/SHOW, + * DISCARD, UNLISTEN, DEALLOCATE and the pool-reset function call of Npgsql. SQL cursors ({@code + * DECLARE}, {@code FETCH}, {@code MOVE}, {@code CLOSE}) are {@link CursorCommand}s. + */ +public final class SessionCommands { + + private static final Logger LOGGER = LoggerFactory.getLogger(SessionCommands.class); + + /** What a session command produces. */ + public static final class Result { + private final String tag; + private final String showName; + private final String showValue; + private final Map parameterStatus; + + Result(String tag, String showName, String showValue, Map parameterStatus) { + this.tag = tag; + this.showName = showName; + this.showValue = showValue; + this.parameterStatus = parameterStatus; + } + + /** The CommandComplete tag. */ + public String tag() { + return tag; + } + + /** For SHOW and the function call: the column name, else {@code null}. */ + public String showName() { + return showName; + } + + public String showValue() { + return showValue; + } + + /** Reported parameters that changed (ParameterStatus to send). */ + public Map parameterStatus() { + return parameterStatus; + } + } + + /** Parameters reported in ParameterStatus, with their names as PostgreSQL writes them. */ + private static final Map REPORTED = new LinkedHashMap<>(); + + static { + for (String name : + Arrays.asList( + "application_name", + "client_encoding", + "DateStyle", + "IntervalStyle", + "TimeZone", + "standard_conforming_strings", + "integer_datetimes", + "is_superuser", + "server_encoding", + "server_version", + "session_authorization")) { + REPORTED.put(name.toLowerCase(Locale.ROOT), name); + } + } + + private static final Set READ_ONLY = + new HashSet<>( + Arrays.asList( + "server_version", + "server_version_num", + "server_encoding", + "integer_datetimes", + "is_superuser", + "session_authorization")); + + private static final int FLAGS = Pattern.CASE_INSENSITIVE | Pattern.DOTALL; + private static final Pattern BEGIN = Pattern.compile("(BEGIN|START\\s+TRANSACTION)\\b.*", FLAGS); + private static final Pattern COMMIT = + Pattern.compile("(COMMIT|END)(\\s+(WORK|TRANSACTION))?", FLAGS); + private static final Pattern ROLLBACK = + Pattern.compile("(ROLLBACK|ABORT)(\\s+(WORK|TRANSACTION))?", FLAGS); + private static final Pattern SET_TIME_ZONE = + Pattern.compile("SET\\s+(?:SESSION\\s+|LOCAL\\s+)?TIME\\s+ZONE\\s+(.+)", FLAGS); + private static final Pattern SET_AUTHORIZATION_DEFAULT = + Pattern.compile("SET\\s+SESSION\\s+AUTHORIZATION\\s+DEFAULT", FLAGS); + private static final Pattern SET = + Pattern.compile( + "SET\\s+(?:SESSION\\s+|LOCAL\\s+)?([A-Za-z_][\\w.]*)\\s*(?:=|\\s+TO\\s+)\\s*(.+)", FLAGS); + private static final Pattern RESET = Pattern.compile("RESET\\s+(ALL|[A-Za-z_][\\w.]*)", FLAGS); + private static final Pattern SHOW = Pattern.compile("SHOW\\s+([A-Za-z_][\\w.]*|ALL)", FLAGS); + private static final Pattern DISCARD = + Pattern.compile("DISCARD\\s+(ALL|TEMP|TEMPORARY|SEQUENCES|PLANS)", FLAGS); + private static final Pattern UNLISTEN = Pattern.compile("UNLISTEN\\s+\\*", FLAGS); + private static final Pattern DEALLOCATE = + Pattern.compile("DEALLOCATE\\s+(?:PREPARE\\s+)?(ALL|[A-Za-z_\"][\\w\"]*)", FLAGS); + private static final Pattern ADVISORY_UNLOCK_ALL = + Pattern.compile("SELECT\\s+(?:pg_catalog\\.)?pg_advisory_unlock_all\\s*\\(\\s*\\)", FLAGS); + + private SessionCommands() {} + + private static final Pattern[] ALL_COMMANDS = { + BEGIN, + COMMIT, + ROLLBACK, + SET_TIME_ZONE, + SET_AUTHORIZATION_DEFAULT, + SET, + RESET, + SHOW, + DISCARD, + UNLISTEN, + DEALLOCATE, + ADVISORY_UNLOCK_ALL + }; + + /** True if {@code statement} is a session command (nothing is executed). */ + public static boolean isCommand(String statement) { + String sql = statement.trim(); + for (Pattern pattern : ALL_COMMANDS) { + if (pattern.matcher(sql).matches()) { + return true; + } + } + return false; + } + + /** True if {@code statement} is a SHOW command (it returns one row). */ + public static boolean isShow(String statement) { + return SHOW.matcher(statement.trim()).matches(); + } + + /** The column name of a SHOW command's result. */ + public static String showName(String statement) { + Matcher m = SHOW.matcher(statement.trim()); + return m.matches() ? m.group(1) : "?column?"; + } + + /** The command's result if {@code statement} is a session command, else empty. */ + public static Optional execute(PgSession session, String statement) throws PgException { + String sql = statement.trim(); + if (BEGIN.matcher(sql).matches()) { + session.setTransactionStatus('T'); + return result("BEGIN"); + } + if (COMMIT.matcher(sql).matches()) { + session.setTransactionStatus('I'); + return result("COMMIT"); + } + if (ROLLBACK.matcher(sql).matches()) { + session.setTransactionStatus('I'); + return result("ROLLBACK"); + } + if (SET_AUTHORIZATION_DEFAULT.matcher(sql).matches()) { + return result("SET"); + } + Matcher m = SET_TIME_ZONE.matcher(sql); + if (m.matches()) { + return Optional.of(set(session, "timezone", m.group(1))); + } + m = SET.matcher(sql); + if (m.matches()) { + return Optional.of(set(session, m.group(1), m.group(2))); + } + m = RESET.matcher(sql); + if (m.matches()) { + String name = m.group(1).toLowerCase(Locale.ROOT); + if (name.equals("all")) { + session.resetAll(); + return result("RESET"); + } + if (READ_ONLY.contains(name)) { + throw new PgException( + SqlState.INVALID_PARAMETER_VALUE, "parameter \"" + m.group(1) + "\" cannot be changed"); + } + String value = session.reset(name); + String reported = REPORTED.get(name); + if (reported != null && value != null) { + return Optional.of( + new Result("RESET", null, null, Collections.singletonMap(reported, value))); + } + return result("RESET"); + } + m = SHOW.matcher(sql); + if (m.matches()) { + String name = m.group(1).toLowerCase(Locale.ROOT); + return Optional.of( + new Result("SHOW", m.group(1), show(session, name), Collections.emptyMap())); + } + m = DISCARD.matcher(sql); + if (m.matches()) { + return result("DISCARD " + m.group(1).toUpperCase(Locale.ROOT)); + } + if (UNLISTEN.matcher(sql).matches()) { + return result("UNLISTEN"); + } + m = DEALLOCATE.matcher(sql); + if (m.matches()) { + return result(m.group(1).equalsIgnoreCase("ALL") ? "DEALLOCATE ALL" : "DEALLOCATE"); + } + if (ADVISORY_UNLOCK_ALL.matcher(sql).matches()) { + return Optional.of( + new Result("SELECT 1", "pg_advisory_unlock_all", "", Collections.emptyMap())); + } + return Optional.empty(); + } + + private static Optional result(String tag) { + return Optional.of(new Result(tag, null, null, Collections.emptyMap())); + } + + private static Result set(PgSession session, String rawName, String rawValue) throws PgException { + String name = rawName.toLowerCase(Locale.ROOT); + String value = unquote(rawValue.trim()); + if (READ_ONLY.contains(name)) { + throw new PgException( + SqlState.INVALID_PARAMETER_VALUE, "parameter \"" + rawName + "\" cannot be changed"); + } + switch (name) { + case "client_encoding": + if (!value.replace("-", "").equalsIgnoreCase("UTF8") + && !value.equalsIgnoreCase("UNICODE")) { + throw new PgException( + SqlState.FEATURE_NOT_SUPPORTED, "only client_encoding UTF8 is supported"); + } + value = "UTF8"; + break; + case "bytea_output": + if (!value.equalsIgnoreCase("hex")) { + throw new PgException( + SqlState.FEATURE_NOT_SUPPORTED, "only bytea_output hex is supported"); + } + break; + case "intervalstyle": + if (!value.equalsIgnoreCase("postgres")) { + throw new PgException( + SqlState.FEATURE_NOT_SUPPORTED, "only IntervalStyle postgres is supported"); + } + break; + case "datestyle": + if (!value.toUpperCase(Locale.ROOT).startsWith("ISO")) { + throw new PgException(SqlState.FEATURE_NOT_SUPPORTED, "only DateStyle ISO is supported"); + } + value = "ISO, MDY"; + break; + case "timezone": + try { + ZoneId zone = + value.equalsIgnoreCase("LOCAL") || value.equalsIgnoreCase("DEFAULT") + ? ZoneId.systemDefault() + : ZoneId.of(value); + session.setZone(zone); + value = zone.getId(); + } catch (DateTimeException e) { + throw new PgException( + SqlState.INVALID_PARAMETER_VALUE, + "invalid value for parameter \"TimeZone\": \"" + value + "\""); + } + break; + case "statement_timeout": + try { + StatementTimeout.parse(value); + } catch (IllegalArgumentException e) { + throw new PgException(SqlState.INVALID_PARAMETER_VALUE, e.getMessage()); + } + break; + case "search_path": + case "application_name": + case "extra_float_digits": + case "lock_timeout": + case "client_min_messages": + case "standard_conforming_strings": + case "ssl_renegotiation_limit": + // kept; the query layer reads the ones it honours + break; + default: + LOGGER.debug("Unknown PostgreSQL parameter {} accepted and ignored", name); + } + session.setParameter(name, value); + Map status = Collections.emptyMap(); + String reported = REPORTED.get(name); + if (reported != null) { + status = Collections.singletonMap(reported, value); + } + return new Result("SET", null, null, status); + } + + private static String show(PgSession session, String name) throws PgException { + switch (name) { + case "server_version_num": + return "140000"; + case "timezone": + return session.zone().getId(); + case "transaction_isolation": + case "default_transaction_isolation": + return "read committed"; + case "transaction_read_only": + case "default_transaction_read_only": + return "on"; + case "standard_conforming_strings": + case "integer_datetimes": + return "on"; + case "client_encoding": + case "server_encoding": + return "UTF8"; + case "datestyle": + return "ISO, MDY"; + case "intervalstyle": + return "postgres"; + case "max_identifier_length": + return "63"; + case "statement_timeout": + String timeout = session.parameter(name); + return timeout == null ? "0" : timeout; + case "application_name": + String application = session.parameter(name); + return application == null ? "" : application; + default: + String value = session.parameter(name); + if (value == null) { + throw new PgException("42704", "unrecognized configuration parameter \"" + name + "\""); + } + return value; + } + } + + private static String unquote(String value) { + if (value.length() >= 2 + && (value.startsWith("'") && value.endsWith("'") + || value.startsWith("\"") && value.endsWith("\""))) { + char q = value.charAt(0); + return value.substring(1, value.length() - 1).replace("" + q + q, "" + q); + } + return value; + } +} diff --git a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/session/SqlProtocol.java b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/session/SqlProtocol.java new file mode 100644 index 0000000000000..4342a50410cef --- /dev/null +++ b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/session/SqlProtocol.java @@ -0,0 +1,930 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.session; + +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Literal; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.NullLiteral; +import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; +import org.apache.iotdb.postgresql.execution.ResultCursor; +import org.apache.iotdb.postgresql.execution.RunningQuery; +import org.apache.iotdb.postgresql.protocol.BackendMessages; +import org.apache.iotdb.postgresql.protocol.FrontendMessage; +import org.apache.iotdb.postgresql.protocol.PgException; +import org.apache.iotdb.postgresql.protocol.SqlState; +import org.apache.iotdb.postgresql.sql.PgToTableSql; +import org.apache.iotdb.postgresql.sql.PgToTableSql.Translation; +import org.apache.iotdb.postgresql.sql.TranslationContext; +import org.apache.iotdb.postgresql.types.ParameterDecoder; +import org.apache.iotdb.postgresql.types.PgType; +import org.apache.iotdb.postgresql.types.PgValueEncoder; +import org.apache.iotdb.postgresql.types.TimestampPrecision; + +import io.netty.buffer.ByteBuf; +import io.netty.channel.Channel; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; + +/** + * The query layer: session commands, and queries translated to table-model SQL and run by the + * engine, through the simple and the extended query protocol . Runs on the connection's ordered + * executor. + */ +public final class SqlProtocol implements QueryProtocol { + + private static final Logger LOGGER = LoggerFactory.getLogger(SqlProtocol.class); + + /** A parsed statement (Parse). */ + private static final class Prepared { + final String sql; + final boolean empty; + final boolean command; + final Translation translation; + final int[] parameterTypes; + final CursorCommand cursor; + + Prepared(String sql, boolean empty, boolean command, Translation translation, int[] types) { + this(sql, empty, command, translation, types, null); + } + + Prepared( + String sql, + boolean empty, + boolean command, + Translation translation, + int[] types, + CursorCommand cursor) { + this.sql = sql; + this.empty = empty; + this.command = command; + this.translation = translation; + this.parameterTypes = types; + this.cursor = cursor; + } + } + + /** A SQL cursor (DECLARE): a running query fetched by name. */ + private static final class SqlCursor { + final ResultCursor rows; + final boolean hold; + final boolean binary; + final boolean inTransaction; + + SqlCursor(ResultCursor rows, boolean hold, boolean binary, boolean inTransaction) { + this.rows = rows; + this.hold = hold; + this.binary = binary; + this.inTransaction = inTransaction; + } + } + + /** A bound statement (Bind), possibly with a running query. */ + private static final class Portal { + final Prepared statement; + final List parameters; + final short[] resultFormats; + ResultCursor cursor; + boolean finished; + + Portal(Prepared statement, List parameters, short[] resultFormats) { + this.statement = statement; + this.parameters = parameters; + this.resultFormats = resultFormats; + } + } + + private final PgSession session; + private final Duration writeTimeout; + private final TimestampPrecision precision; + private final Map statements = new HashMap<>(); + private final Map portals = new HashMap<>(); + private final Map cursors = new HashMap<>(); + private boolean skipUntilSync; + private volatile ResultCursor running; + private volatile boolean cancelRequested; + private volatile boolean busy; + + public SqlProtocol(PgSession session, Duration writeTimeout, TimestampPrecision precision) { + this.session = session; + this.writeTimeout = writeTimeout; + this.precision = precision; + } + + // ---------------------------------------------------------------------------- dispatch + + @Override + public void handle(FrontendMessage message, Channel channel) { + ByteBuf out = channel.alloc().buffer(); + if (!session.session().isLogin()) { + // the session was ended from outside + LOGGER.info("PostgreSQL session of {} was ended; closing the connection", session.user()); + closePortals(); + closeCursors(); + BackendMessages.errorResponse( + out, + PgException.fatal( + SqlState.ADMIN_SHUTDOWN, "terminating connection due to administrator command")); + channel.writeAndFlush(out).addListener(f -> channel.close()); + return; + } + busy = true; + try { + char type = message.type(); + if (skipUntilSync && type != 'S' && type != 'Q') { + return; + } + switch (type) { + case 'Q': + simpleQuery(message.readCString(), channel, out); + break; + case 'P': + parse(message, out); + break; + case 'B': + bind(message, out); + break; + case 'D': + describe(message, out); + break; + case 'E': + execute(message, channel, out); + break; + case 'C': + close(message, out); + break; + case 'S': + sync(out); + break; + case 'H': + break; + case 'F': + throw new PgException(SqlState.FEATURE_NOT_SUPPORTED, "the function call protocol"); + case 'd': + case 'c': + case 'f': + throw new PgException(SqlState.FEATURE_NOT_SUPPORTED, "COPY is not supported"); + default: + throw PgException.fatal( + SqlState.PROTOCOL_VIOLATION, "invalid frontend message type '" + type + "'"); + } + } catch (PgException e) { + fail(e, out); + if (e.isFatal()) { + busy = false; + channel.writeAndFlush(out).addListener(f -> channel.close()); + return; + } + if (message.type() != 'Q') { + // extended protocol: discard messages until Sync + skipUntilSync = true; + } + } catch (ResultCursor.WriteTimeoutException e) { + LOGGER.info( + "PostgreSQL client {} stopped reading results for {}; closing the connection", + session.user(), + writeTimeout); + closePortals(); + out.clear(); + BackendMessages.errorResponse( + out, + PgException.fatal( + SqlState.ADMIN_SHUTDOWN, + "terminating connection because the client stopped reading")); + busy = false; + channel.writeAndFlush(out).addListener(f -> channel.close()); + return; + } catch (RuntimeException | LinkageError e) { + // never leave the client waiting: an unexpected failure is reported like any other error + LOGGER.warn("PostgreSQL message '{}' of {} failed", message.type(), session.user(), e); + closePortals(); + out.clear(); + fail(new PgException(SqlState.INTERNAL_ERROR, "internal error: " + e), out); + if (message.type() != 'Q') { + skipUntilSync = true; + } else { + BackendMessages.readyForQuery(out, session.transactionStatus()); + } + } finally { + busy = false; + } + if (out.readableBytes() > 0) { + channel.writeAndFlush(out); + } else { + out.release(); + } + } + + private void fail(PgException e, ByteBuf out) { + BackendMessages.errorResponse(out, e); + if (session.transactionStatus() == 'T') { + session.setTransactionStatus('E'); + } + } + + // ---------------------------------------------------------------------------- simple query + + private void simpleQuery(String sql, Channel channel, ByteBuf out) + throws ResultCursor.WriteTimeoutException { + skipUntilSync = false; + // a simple query closes the unnamed statement and portal + statements.remove(""); + closePortal(""); + List parts = StatementSplitter.split(sql); + if (parts.isEmpty()) { + BackendMessages.emptyQueryResponse(out); + } + for (String statement : parts) { + if (statement.isEmpty()) { + BackendMessages.emptyQueryResponse(out); + continue; + } + try { + if (session.transactionStatus() == 'E' && !isTransactionEnd(statement)) { + throw new PgException( + "25P02", + "current transaction is aborted, commands ignored until end of transaction block"); + } + Prepared prepared = prepare(statement, new int[0]); + Portal portal = new Portal(prepared, Collections.emptyList(), new short[0]); + try { + run(portal, channel, out, 0, true); + } finally { + if (portal.cursor != null) { + portal.cursor.close(); + } + } + } catch (PgException e) { + fail(e, out); + break; + } + } + BackendMessages.readyForQuery(out, session.transactionStatus()); + } + + private static boolean isTransactionEnd(String statement) { + String s = statement.trim().toUpperCase(Locale.ROOT); + return s.startsWith("COMMIT") + || s.startsWith("ROLLBACK") + || s.startsWith("END") + || s.startsWith("ABORT"); + } + + // ---------------------------------------------------------------------------- extended query + + private void parse(FrontendMessage message, ByteBuf out) throws PgException { + String name = message.readCString(); + String sql = message.readCString(); + int count = message.readInt16(); + int[] types = new int[count]; + for (int i = 0; i < count; i++) { + types[i] = message.readInt32(); + } + if (!name.isEmpty() && statements.containsKey(name)) { + throw new PgException( + SqlState.DUPLICATE_PREPARED_STATEMENT, + "prepared statement \"" + name + "\" already exists"); + } + if (session.transactionStatus() == 'E' && !isTransactionEnd(sql)) { + throw new PgException( + "25P02", + "current transaction is aborted, commands ignored until end of transaction block"); + } + statements.put(name, prepare(sql, types)); + BackendMessages.parseComplete(out); + } + + private Prepared prepare(String sql, int[] types) throws PgException { + List parts = StatementSplitter.split(sql); + if (parts.isEmpty() || (parts.size() == 1 && parts.get(0).isEmpty())) { + return new Prepared(sql, true, false, null, types); + } + if (parts.size() > 1) { + throw new PgException( + SqlState.SYNTAX_ERROR, "cannot insert multiple commands into a prepared statement"); + } + String statement = parts.get(0); + Optional cursor = CursorCommand.parse(statement); + if (cursor.isPresent()) { + Translation query = null; + int[] allTypes = types; + if (cursor.get().kind() == CursorCommand.Kind.DECLARE) { + query = PgToTableSql.translate(cursor.get().query(), translationContext()); + if (query.parameterCount() > types.length) { + allTypes = java.util.Arrays.copyOf(types, query.parameterCount()); + } + } + return new Prepared(statement, false, false, query, allTypes, cursor.get()); + } + if (SessionCommands.isCommand(statement)) { + return new Prepared(statement, false, true, null, types); + } + Translation translation = + PgToTableSql.translate(statement, translationContext()).withTypedParameters(types); + int[] allTypes = types; + if (translation.parameterCount() > types.length) { + allTypes = java.util.Arrays.copyOf(types, translation.parameterCount()); + } + return new Prepared(statement, false, false, translation, allTypes); + } + + private void bind(FrontendMessage message, ByteBuf out) throws PgException { + String portalName = message.readCString(); + String statementName = message.readCString(); + Prepared statement = statements.get(statementName); + if (statement == null) { + throw new PgException( + SqlState.INVALID_SQL_STATEMENT_NAME, + "prepared statement \"" + statementName + "\" does not exist"); + } + short[] parameterFormats = readFormats(message); + int count = message.readInt16(); + if (count != statement.parameterTypes.length) { + throw new PgException( + SqlState.PROTOCOL_VIOLATION, + "bind message supplies " + + count + + " parameters, but prepared statement \"" + + statementName + + "\" requires " + + statement.parameterTypes.length); + } + ParameterDecoder decoder = new ParameterDecoder(session.zone(), precision); + Literal[] values = new Literal[count]; + for (int i = 0; i < count; i++) { + int length = message.readInt32(); + byte[] value = message.readBytes(length); + short format = + parameterFormats.length == 0 + ? 0 + : parameterFormats.length == 1 ? parameterFormats[0] : parameterFormats[i]; + values[i] = decoder.decode(statement.parameterTypes[i], format == 1, value); + } + short[] resultFormats = readFormats(message); + List ordered = new ArrayList<>(); + if (statement.translation != null) { + for (int n : statement.translation.parameterOrder()) { + ordered.add(values[n - 1]); + } + } + if (!portalName.isEmpty() && portals.containsKey(portalName)) { + throw new PgException( + SqlState.DUPLICATE_CURSOR, "portal \"" + portalName + "\" already exists"); + } + closePortal(portalName); + portals.put(portalName, new Portal(statement, ordered, resultFormats)); + BackendMessages.bindComplete(out); + } + + private static short[] readFormats(FrontendMessage message) throws PgException { + int count = message.readInt16(); + short[] formats = new short[count]; + for (int i = 0; i < count; i++) { + formats[i] = message.readInt16(); + if (formats[i] != 0 && formats[i] != 1) { + throw new PgException( + SqlState.PROTOCOL_VIOLATION, "unsupported format code: " + formats[i]); + } + } + return formats; + } + + private void describe(FrontendMessage message, ByteBuf out) throws PgException { + char kind = (char) message.readByte(); + String name = message.readCString(); + if (kind == 'S') { + Prepared statement = statements.get(name); + if (statement == null) { + throw new PgException( + SqlState.INVALID_SQL_STATEMENT_NAME, + "prepared statement \"" + name + "\" does not exist"); + } + int[] reported = statement.parameterTypes.clone(); + for (int i = 0; i < reported.length; i++) { + if (reported[i] == 0) { + reported[i] = PgType.TEXT.oid(); + } + } + BackendMessages.parameterDescription(out, reported); + describeResult(statement, null, out); + return; + } + if (kind != 'P') { + throw new PgException(SqlState.PROTOCOL_VIOLATION, "invalid DESCRIBE message subtype"); + } + Portal portal = portals.get(name); + if (portal == null) { + // SQL cursors are portals too (psycopg describes its DECLAREd cursor this way) + SqlCursor cursor = cursors.get(name); + if (cursor == null) { + throw new PgException( + SqlState.INVALID_CURSOR_NAME, "portal \"" + name + "\" does not exist"); + } + cursor.rows.setFormats(fetchFormats(cursor, null)); + BackendMessages.rowDescription(out, cursor.rows.fields()); + return; + } + describeResult(portal.statement, portal, out); + } + + /** + * RowDescription or NoData. For a portal the query starts here (Execute continues it); for a + * statement the query is planned with NULL parameters and stopped at once. + */ + private void describeResult(Prepared statement, Portal portal, ByteBuf out) throws PgException { + if (statement.empty) { + BackendMessages.noData(out); + return; + } + if (statement.cursor != null) { + if (statement.cursor.kind() == CursorCommand.Kind.FETCH) { + SqlCursor cursor = cursor(statement.cursor.name()); + cursor.rows.setFormats(fetchFormats(cursor, portal)); + BackendMessages.rowDescription(out, cursor.rows.fields()); + } else { + BackendMessages.noData(out); + } + return; + } + if (statement.command) { + if (SessionCommands.isShow(statement.sql)) { + BackendMessages.rowDescription( + out, + Collections.singletonList( + new BackendMessages.Field( + SessionCommands.showName(statement.sql), + PgType.TEXT.oid(), + PgType.TEXT.size(), + (short) 0))); + } else { + BackendMessages.noData(out); + } + return; + } + if (portal != null) { + start(portal); + BackendMessages.rowDescription(out, portal.cursor.fields()); + return; + } + // planned with a value of each declared type: the engine refuses comparisons with untyped + // NULLs; untyped parameters are tried as a number, then as text, then as NULL + PgException failure = null; + boolean untyped = false; + for (int n : statement.translation.parameterOrder()) { + untyped |= n - 1 >= statement.parameterTypes.length || statement.parameterTypes[n - 1] == 0; + } + for (int attempt = 0; attempt < (untyped ? 3 : 1); attempt++) { + List values = new ArrayList<>(); + for (int n : statement.translation.parameterOrder()) { + int oid = n - 1 < statement.parameterTypes.length ? statement.parameterTypes[n - 1] : 0; + values.add(placeholder(oid == 0 ? new int[] {20, 25, 0}[attempt] : oid)); + } + try (RunningQuery query = startQuery(statement.translation, values)) { + ResultCursor cursor = new ResultCursor(query, new short[0], encoder()); + BackendMessages.rowDescription(out, cursor.fields()); + return; + } catch (PgException e) { + failure = failure == null ? e : failure; + } + } + throw failure; + } + + /** A value of the given parameter type, for planning a statement without its parameters. */ + private static Literal placeholder(int oid) { + switch (oid) { + case 16: + return new org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.BooleanLiteral( + "false"); + case 20: + case 21: + case 23: + case 26: + case 1114: + case 1184: + return new org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.LongLiteral("0"); + case 700: + case 701: + case 1700: + return new org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.DoubleLiteral("0"); + case 25: + case 1043: + case 19: + case 1042: + return new org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.StringLiteral(""); + case 1082: + return new org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.StringLiteral( + "1970-01-01"); + case 17: + return new org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.BinaryLiteral( + new byte[0]); + default: + return new NullLiteral(); + } + } + + private void execute(FrontendMessage message, Channel channel, ByteBuf out) + throws PgException, ResultCursor.WriteTimeoutException { + String name = message.readCString(); + int maxRows = message.readInt32(); + Portal portal = portals.get(name); + if (portal == null) { + throw new PgException(SqlState.INVALID_CURSOR_NAME, "portal \"" + name + "\" does not exist"); + } + if (session.transactionStatus() == 'E' + && !(portal.statement.command && isTransactionEnd(portal.statement.sql))) { + throw new PgException( + "25P02", + "current transaction is aborted, commands ignored until end of transaction block"); + } + run(portal, channel, out, Math.max(0, maxRows), false); + } + + private void close(FrontendMessage message, ByteBuf out) throws PgException { + char kind = (char) message.readByte(); + String name = message.readCString(); + if (kind == 'S') { + statements.remove(name); + } else if (kind == 'P') { + closePortal(name); + SqlCursor cursor = cursors.remove(name); + if (cursor != null) { + cursor.rows.close(); + } + } else { + throw new PgException(SqlState.PROTOCOL_VIOLATION, "invalid CLOSE message subtype"); + } + BackendMessages.closeComplete(out); + } + + private void sync(ByteBuf out) { + skipUntilSync = false; + if (session.transactionStatus() == 'I') { + // end of the implicit transaction: its portals and cursors without HOLD end too + closePortals(); + endTransactionCursors(false); + } + BackendMessages.readyForQuery(out, session.transactionStatus()); + } + + // ---------------------------------------------------------------------------- running + + /** + * Runs a portal: a session command, or the next rows of its query. + * + * @param maxRows 0 for all rows + * @param describe whether to send RowDescription first (simple query) + */ + private void run(Portal portal, Channel channel, ByteBuf out, long maxRows, boolean describe) + throws PgException, ResultCursor.WriteTimeoutException { + Prepared statement = portal.statement; + if (statement.empty) { + BackendMessages.emptyQueryResponse(out); + return; + } + if (statement.cursor != null) { + runCursorCommand(portal, channel, out, describe); + return; + } + if (statement.command) { + Optional result = SessionCommands.execute(session, statement.sql); + if (!result.isPresent()) { + throw new PgException(SqlState.INTERNAL_ERROR, "not a session command"); + } + SessionCommands.Result r = result.get(); + r.parameterStatus().forEach((n, v) -> BackendMessages.parameterStatus(out, n, v)); + if (r.showName() != null) { + if (describe) { + BackendMessages.rowDescription( + out, + Collections.singletonList( + new BackendMessages.Field( + r.showName(), PgType.TEXT.oid(), PgType.TEXT.size(), (short) 0))); + } + int row = BackendMessages.beginDataRow(out, 1); + BackendMessages.value(out, r.showValue().getBytes(StandardCharsets.UTF_8)); + BackendMessages.end(out, row); + } + BackendMessages.commandComplete(out, r.tag()); + if (r.tag().equals("COMMIT") || r.tag().equals("ROLLBACK")) { + closePortalsExcept(portal); + endTransactionCursors(r.tag().equals("ROLLBACK")); + } else if (r.tag().equals("DISCARD ALL")) { + closeCursors(); + } + return; + } + if (portal.finished) { + BackendMessages.commandComplete(out, "SELECT 0"); + return; + } + start(portal); + if (describe) { + BackendMessages.rowDescription(out, portal.cursor.fields()); + } + running = portal.cursor; + long sent; + try { + sent = portal.cursor.send(channel, out, maxRows, writeTimeout); + } catch (PgException e) { + closeCursor(portal); + if (cancelRequested) { + throw new PgException(SqlState.QUERY_CANCELED, "canceling statement due to user request"); + } + throw e; + } finally { + running = null; + } + if (portal.cursor.exhausted()) { + closeCursor(portal); + portal.finished = true; + BackendMessages.commandComplete(out, "SELECT " + sent); + } else { + BackendMessages.portalSuspended(out); + } + } + + private void start(Portal portal) throws PgException { + if (portal.cursor != null) { + return; + } + cancelRequested = false; + RunningQuery query = startQuery(portal.statement.translation, portal.parameters); + try { + portal.cursor = new ResultCursor(query, portal.resultFormats, encoder()); + } catch (PgException e) { + query.close(); + throw e; + } + } + + private RunningQuery startQuery(Translation translation, List parameters) + throws PgException { + return RunningQuery.start( + session.session(), + translation.sqlFor(parameters), + translation.remaining(parameters), + StatementTimeout.parse(session.parameter("statement_timeout"))); + } + + private PgValueEncoder encoder() { + return new PgValueEncoder(session.zone(), precision); + } + + private static void closeCursor(Portal portal) { + if (portal.cursor != null) { + portal.cursor.close(); + portal.cursor = null; + } + } + + private void closePortal(String name) { + Portal portal = portals.remove(name); + if (portal != null) { + closeCursor(portal); + } + } + + private void closePortals() { + for (Portal portal : portals.values()) { + closeCursor(portal); + } + portals.clear(); + } + + private void closePortalsExcept(Portal keep) { + portals + .entrySet() + .removeIf( + e -> { + if (e.getValue() == keep) { + return false; + } + closeCursor(e.getValue()); + return true; + }); + } + + // ---------------------------------------------------------------------------- SQL cursors + + private void runCursorCommand(Portal portal, Channel channel, ByteBuf out, boolean describe) + throws PgException, ResultCursor.WriteTimeoutException { + CursorCommand command = portal.statement.cursor; + switch (command.kind()) { + case DECLARE: + declare(portal, command, out); + return; + case FETCH: + { + SqlCursor cursor = cursor(command.name()); + cursor.rows.setFormats(fetchFormats(cursor, describe ? null : portal)); + if (describe) { + BackendMessages.rowDescription(out, cursor.rows.fields()); + } + long sent = moveOrFetch(command.name(), cursor, channel, out, command.count()); + BackendMessages.commandComplete(out, "FETCH " + sent); + return; + } + case MOVE: + { + SqlCursor cursor = cursor(command.name()); + long skipped = moveOrFetch(command.name(), cursor, null, null, command.count()); + BackendMessages.commandComplete(out, "MOVE " + skipped); + return; + } + case CLOSE: + default: + if (command.name() == null) { + closeCursors(); + } else { + cursor(command.name()); + cursors.remove(command.name()).rows.close(); + } + BackendMessages.commandComplete(out, "CLOSE CURSOR"); + } + } + + private void declare(Portal portal, CursorCommand command, ByteBuf out) throws PgException { + if (!command.hold() && session.transactionStatus() == 'I') { + throw new PgException("25P01", "DECLARE CURSOR can only be used in transaction blocks"); + } + if (cursors.containsKey(command.name()) || portals.containsKey(command.name())) { + throw new PgException( + SqlState.DUPLICATE_CURSOR, "cursor \"" + command.name() + "\" already exists"); + } + cancelRequested = false; + RunningQuery query = startQuery(portal.statement.translation, portal.parameters); + ResultCursor rows; + try { + rows = new ResultCursor(query, new short[0], encoder()); + } catch (PgException e) { + query.close(); + throw e; + } + cursors.put( + command.name(), + new SqlCursor(rows, command.hold(), command.binary(), session.transactionStatus() == 'T')); + BackendMessages.commandComplete(out, "DECLARE CURSOR"); + } + + /** Sends ({@code out != null}) or skips up to {@code count} rows (0 = all). */ + private long moveOrFetch(String name, SqlCursor cursor, Channel channel, ByteBuf out, long count) + throws PgException, ResultCursor.WriteTimeoutException { + running = cursor.rows; + try { + return out == null + ? cursor.rows.skip(count) + : cursor.rows.send(channel, out, count, writeTimeout); + } catch (PgException e) { + cursors.remove(name); + cursor.rows.close(); + if (cancelRequested) { + throw new PgException(SqlState.QUERY_CANCELED, "canceling statement due to user request"); + } + throw e; + } finally { + running = null; + } + } + + private SqlCursor cursor(String name) throws PgException { + SqlCursor cursor = cursors.get(name); + if (cursor == null) { + throw new PgException(SqlState.INVALID_CURSOR_NAME, "cursor \"" + name + "\" does not exist"); + } + return cursor; + } + + /** + * The result formats of a FETCH: those of its Bind (extended protocol), else binary for a BINARY + * cursor and text otherwise. + */ + private static short[] fetchFormats(SqlCursor cursor, Portal portal) { + if (portal != null && portal.resultFormats.length > 0) { + return portal.resultFormats; + } + return cursor.binary ? new short[] {1} : new short[0]; + } + + /** At COMMIT, ROLLBACK or the end of an implicit transaction. */ + private void endTransactionCursors(boolean rollback) { + cursors + .entrySet() + .removeIf( + e -> { + SqlCursor cursor = e.getValue(); + // ROLLBACK also ends the HOLD cursors declared in the rolled-back transaction + if (!cursor.hold || (rollback && cursor.inTransaction)) { + cursor.rows.close(); + return true; + } + return false; + }); + } + + private void closeCursors() { + for (SqlCursor cursor : cursors.values()) { + cursor.rows.close(); + } + cursors.clear(); + } + + // ---------------------------------------------------------------------------- translation + + private TranslationContext translationContext() { + return new TranslationContext() { + @Override + public List searchPath() { + return SearchPath.parse(session.parameter("search_path")); + } + + @Override + public boolean tableExists(String database, String table) { + try { + return DataNodeTableCache.getInstance().getTable(database, table) != null; + } catch (RuntimeException e) { + return false; + } + } + + @Override + public String user() { + return session.user(); + } + + @Override + public String version() { + return "PostgreSQL " + session.parameter("server_version") + ", read-only adapter"; + } + + @Override + public String currentDatabase() { + String database = session.parameter("database"); + return database == null || database.isEmpty() ? session.user() : database; + } + + @Override + public int backendPid() { + return session.processId(); + } + + @Override + public java.time.ZoneId zone() { + return session.zone(); + } + + @Override + public TimestampPrecision precision() { + return precision; + } + }; + } + + // ---------------------------------------------------------------------------- QueryProtocol + + @Override + public boolean busy() { + return busy; + } + + @Override + public void cancel() { + ResultCursor cursor = running; + if (cursor != null) { + cancelRequested = true; + cursor.cancel(); + } + } + + @Override + public void close() { + closePortals(); + closeCursors(); + statements.clear(); + } +} diff --git a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/session/StatementSplitter.java b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/session/StatementSplitter.java new file mode 100644 index 0000000000000..779d20fff84c4 --- /dev/null +++ b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/session/StatementSplitter.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.session; + +import java.util.ArrayList; +import java.util.List; + +/** + * Splits a simple-query string into statements at semicolons outside quotes, dollar quotes and + * comments, like PostgreSQL. Empty statements are kept as empty strings (EmptyQueryResponse). + */ +public final class StatementSplitter { + + private StatementSplitter() {} + + public static List split(String sql) { + List statements = new ArrayList<>(); + StringBuilder current = new StringBuilder(); + int i = 0; + int n = sql.length(); + while (i < n) { + char c = sql.charAt(i); + if (c == '\'' || c == '"') { + int end = closing(sql, i + 1, c); + current.append(sql, i, end); + i = end; + } else if (c == '-' && i + 1 < n && sql.charAt(i + 1) == '-') { + int end = sql.indexOf('\n', i); + end = end < 0 ? n : end; + current.append(sql, i, end); + i = end; + } else if (c == '/' && i + 1 < n && sql.charAt(i + 1) == '*') { + int end = sql.indexOf("*/", i + 2); + end = end < 0 ? n : end + 2; + current.append(sql, i, end); + i = end; + } else if (c == '$' && dollarTag(sql, i) != null) { + String tag = dollarTag(sql, i); + int end = sql.indexOf(tag, i + tag.length()); + end = end < 0 ? n : end + tag.length(); + current.append(sql, i, end); + i = end; + } else if (c == ';') { + statements.add(current.toString().trim()); + current.setLength(0); + i++; + } else { + current.append(c); + i++; + } + } + String last = current.toString().trim(); + if (!last.isEmpty() || statements.isEmpty()) { + statements.add(last); + } + return statements; + } + + /** The end (exclusive) of a quoted part starting after the opening quote; '' and "" escape. */ + private static int closing(String sql, int from, char quote) { + int i = from; + while (i < sql.length()) { + if (sql.charAt(i) == quote) { + if (i + 1 < sql.length() && sql.charAt(i + 1) == quote) { + i += 2; + continue; + } + return i + 1; + } + i++; + } + return sql.length(); + } + + /** {@code $tag$} starting at {@code i}, or null. */ + private static String dollarTag(String sql, int i) { + int end = i + 1; + while (end < sql.length() + && (Character.isLetterOrDigit(sql.charAt(end)) || sql.charAt(end) == '_')) { + end++; + } + if (end < sql.length() && sql.charAt(end) == '$') { + return sql.substring(i, end + 1); + } + return null; + } +} diff --git a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/session/StatementTimeout.java b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/session/StatementTimeout.java new file mode 100644 index 0000000000000..f6d79ca7feccb --- /dev/null +++ b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/session/StatementTimeout.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.session; + +import java.time.Duration; +import java.util.Locale; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * PostgreSQL's {@code statement_timeout}: a number of milliseconds or a number with a unit ({@code + * us}, {@code ms}, {@code s}, {@code min}, {@code h}, {@code d}); 0 switches it off. + */ +public final class StatementTimeout { + + private static final Pattern VALUE = + Pattern.compile("(\\d+(?:\\.\\d+)?)\\s*(us|ms|s|min|h|d)?", Pattern.CASE_INSENSITIVE); + + private StatementTimeout() {} + + /** + * Parses a value. + * + * @return {@link Duration#ZERO} for no timeout (0 or no value) + * @throws IllegalArgumentException if the value is not a valid time + */ + public static Duration parse(String value) { + if (value == null || value.trim().isEmpty()) { + return Duration.ZERO; + } + Matcher m = VALUE.matcher(value.trim()); + if (!m.matches()) { + throw new IllegalArgumentException( + "invalid value for parameter \"statement_timeout\": \"" + value + "\""); + } + double amount = Double.parseDouble(m.group(1)); + String unit = m.group(2) == null ? "ms" : m.group(2).toLowerCase(Locale.ROOT); + double millis; + switch (unit) { + case "us": + millis = amount / 1000; + break; + case "s": + millis = amount * 1000; + break; + case "min": + millis = amount * 60_000; + break; + case "h": + millis = amount * 3_600_000; + break; + case "d": + millis = amount * 86_400_000; + break; + default: + millis = amount; + } + // PostgreSQL rounds to whole milliseconds; a positive value below 1 ms is 1 ms + long rounded = Math.round(millis); + return Duration.ofMillis(amount > 0 && rounded == 0 ? 1 : rounded); + } + + /** The timeout the engine gets: the smaller of its own and the session's (if set). */ + public static long engineTimeoutMillis(long engineTimeout, Duration statementTimeout) { + if (statementTimeout.isZero()) { + return engineTimeout; + } + long session = statementTimeout.toMillis(); + return engineTimeout <= 0 ? session : Math.min(engineTimeout, session); + } +} diff --git a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/sql/ConstructCheck.java b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/sql/ConstructCheck.java new file mode 100644 index 0000000000000..ae3492d4e326c --- /dev/null +++ b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/sql/ConstructCheck.java @@ -0,0 +1,440 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.sql; + +import net.sf.jsqlparser.expression.Function; +import net.sf.jsqlparser.statement.select.Distinct; +import net.sf.jsqlparser.statement.select.Join; +import net.sf.jsqlparser.statement.select.OrderByElement; +import net.sf.jsqlparser.statement.select.PlainSelect; +import net.sf.jsqlparser.statement.select.WithItem; + +import java.lang.reflect.Array; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.IdentityHashMap; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Checks a parsed statement against the constructs with an exact translation and prepares it: every + * node class must be on the allow list, and flags that change the meaning of an allowed node + * (DISTINCT ON, NATURAL JOIN, ORDER BY inside an aggregate, ...) are refused. Walks the syntax tree + * through its fields, so a construct the list does not know can never pass unnoticed. + * + *

Preparation: ORDER BY items get PostgreSQL's default null ordering (NULLS LAST ascending, + * NULLS FIRST descending) written out, and the names of common table expressions are collected. + */ +final class ConstructCheck { + + private static final String PACKAGE = "net.sf.jsqlparser."; + + private static final Set ALLOWED = + new HashSet<>( + Arrays.asList( + "statement.select.PlainSelect", + "statement.select.SetOperationList", + "statement.select.UnionOp", + "statement.select.ParenthesedSelect", + "statement.select.SelectItem", + "statement.select.AllColumns", + "statement.select.AllTableColumns", + "statement.select.Join", + "statement.select.OrderByElement", + "statement.select.GroupByElement", + "statement.select.Limit", + "statement.select.Offset", + "statement.select.Fetch", + "statement.select.Distinct", + "statement.select.WithItem", + "statement.select.ParenthesedFromItem", + "statement.create.table.ColDataType", + "schema.Table", + "schema.Column", + "schema.Database", + "schema.Server", + "expression.Alias", + "expression.BooleanValue", + "expression.LongValue", + "expression.DoubleValue", + "expression.StringValue", + "expression.NullValue", + "expression.JdbcParameter", + "expression.Function", + "expression.CastExpression", + "expression.CaseExpression", + "expression.WhenClause", + "expression.SignedExpression", + "expression.NotExpression", + "expression.TimeKeyExpression", + "expression.operators.conditional.AndExpression", + "expression.operators.conditional.OrExpression", + "expression.operators.relational.EqualsTo", + "expression.operators.relational.NotEqualsTo", + "expression.operators.relational.GreaterThan", + "expression.operators.relational.GreaterThanEquals", + "expression.operators.relational.MinorThan", + "expression.operators.relational.MinorThanEquals", + "expression.operators.relational.InExpression", + "expression.operators.relational.Between", + "expression.operators.relational.IsNullExpression", + "expression.operators.relational.IsBooleanExpression", + "expression.operators.relational.LikeExpression", + "expression.operators.relational.RegExpMatchOperator", + "expression.ExtractExpression", + "expression.IntervalExpression", + "expression.IntervalQualifier", + "expression.DateTimeLiteralExpression", + "expression.DateUnitExpression", + "expression.ArrayExpression", + "expression.AnalyticExpression", + "expression.WindowDefinition", + "expression.PartitionByClause", + "expression.OrderByClause", + "expression.operators.relational.ExistsExpression", + "expression.operators.relational.ExpressionList", + "expression.operators.relational.ParenthesedExpressionList", + "expression.operators.arithmetic.Addition", + "expression.operators.arithmetic.Subtraction", + "expression.operators.arithmetic.Multiplication", + "expression.operators.arithmetic.Division", + "expression.operators.arithmetic.Modulo", + "expression.operators.arithmetic.Concat")); + + private static final Map NAMES = new HashMap<>(); + + static { + NAMES.put("ArrayConstructor", "ARRAY"); + NAMES.put("AnyComparisonExpression", "ANY/ALL/SOME"); + NAMES.put("IntervalExpression", "INTERVAL"); + NAMES.put("Values", "VALUES"); + NAMES.put("ParenthesedValues", "VALUES"); + NAMES.put("ExtractExpression", "EXTRACT"); + NAMES.put("IntersectOp", "INTERSECT"); + NAMES.put("ExceptOp", "EXCEPT"); + NAMES.put("MinusOp", "MINUS"); + NAMES.put("SimilarToExpression", "SIMILAR TO"); + NAMES.put("RegExpMatchOperator", "regular expression operators"); + NAMES.put("JsonExpression", "JSON operators"); + NAMES.put("JsonOperator", "JSON operators"); + NAMES.put("ForClause", "FOR UPDATE/SHARE"); + NAMES.put("LateralSubSelect", "LATERAL"); + NAMES.put("TableFunction", "table functions in FROM"); + NAMES.put("JdbcNamedParameter", "named parameters"); + NAMES.put("UserVariable", "variables"); + NAMES.put("HexValue", "hexadecimal literals"); + NAMES.put("DateValue", "{d ...} literals"); + NAMES.put("TimestampValue", "{ts ...} literals"); + NAMES.put("TimeValue", "{t ...} literals"); + NAMES.put("CollateExpression", "COLLATE"); + NAMES.put("TrimFunction", "TRIM"); + NAMES.put("OverlapsCondition", "OVERLAPS"); + NAMES.put("RowConstructor", "row constructors"); + NAMES.put("XorExpression", "XOR"); + NAMES.put("BitwiseAnd", "bitwise operators"); + NAMES.put("BitwiseOr", "bitwise operators"); + NAMES.put("BitwiseXor", "bitwise operators"); + NAMES.put("BitwiseLeftShift", "bitwise operators"); + NAMES.put("BitwiseRightShift", "bitwise operators"); + NAMES.put("IntegerDivision", "DIV"); + NAMES.put("GroupingSets", "GROUPING SETS"); + } + + private static final Map, Field[]> FIELDS = new ConcurrentHashMap<>(); + + private final IdentityHashMap visited = new IdentityHashMap<>(); + private final Set cteNames = new HashSet<>(); + + /** Table qualifier (alias or name) → catalog table it refers to; {@code null} if ambiguous. */ + private final Map catalogTables = new HashMap<>(); + + private boolean otherTables; + + /** What the check found out about the statement. */ + static final class Result { + final Set cteNames; + final Map catalogTables; + final boolean otherTables; + + Result(Set cteNames, Map catalogTables, boolean otherTables) { + this.cteNames = cteNames; + this.catalogTables = catalogTables; + this.otherTables = otherTables; + } + } + + private ConstructCheck() {} + + /** + * Checks and prepares the statement. + * + * @return the folded names of its common table expressions + * @throws UnsupportedConstructException naming the first construct without exact translation + */ + static Result check(Object statement) { + ConstructCheck check = new ConstructCheck(); + check.walk(statement); + // a CTE named like a catalog table hides it + check.catalogTables.keySet().removeIf(check.cteNames::contains); + return new Result(check.cteNames, check.catalogTables, check.otherTables); + } + + /** Records which catalog table a table reference names, under its alias or name. */ + private void recordTable(net.sf.jsqlparser.schema.Table table) { + String schema = Identifiers.fold(table.getSchemaName()); + String name = Identifiers.fold(table.getName()); + if (name == null) { + return; + } + String catalog = null; + if (schema == null && name.startsWith("pg_")) { + catalog = name; + } else if ("pg_catalog".equals(schema)) { + catalog = name; + } else if ("information_schema".equals(schema)) { + catalog = "pg_is_" + name; + } + String key = table.getAlias() != null ? Identifiers.fold(table.getAlias().getName()) : name; + if (catalog == null || !org.apache.iotdb.postgresql.catalog.PgCatalogTables.contains(catalog)) { + // a user table (or a CTE) under the same qualifier makes the qualifier ambiguous + otherTables = true; + catalogTables.put(key, null); + return; + } + if (catalogTables.containsKey(key) && !catalog.equals(catalogTables.get(key))) { + catalogTables.put(key, null); + } else { + catalogTables.put(key, catalog); + } + } + + private void walk(Object node) { + if (node == null || visited.containsKey(node)) { + return; + } + if (node instanceof Collection) { + for (Object element : (Collection) node) { + walk(element); + } + return; + } + Class type = node.getClass(); + if (type.isArray()) { + if (!type.getComponentType().isPrimitive()) { + for (int i = 0; i < Array.getLength(node); i++) { + walk(Array.get(node, i)); + } + } + return; + } + String name = type.getName(); + if (!name.startsWith(PACKAGE) || type.isEnum() || name.startsWith(PACKAGE + "parser.")) { + return; + } + visited.put(node, Boolean.TRUE); + String relative = name.substring(PACKAGE.length()); + if (!ALLOWED.contains(relative)) { + throw new UnsupportedConstructException(describe(type)); + } + checkFlags(node); + for (Field field : fields(type)) { + try { + walk(field.get(node)); + } catch (IllegalAccessException e) { + throw new IllegalStateException("cannot inspect " + field, e); + } + } + } + + private void checkFlags(Object node) { + if (node instanceof net.sf.jsqlparser.schema.Column) { + // the qualifier of a column is a Table node too, but not a table reference + net.sf.jsqlparser.schema.Table qualifier = + ((net.sf.jsqlparser.schema.Column) node).getTable(); + if (qualifier != null) { + visited.put(qualifier, Boolean.TRUE); + } + } else if (node instanceof net.sf.jsqlparser.schema.Table) { + recordTable((net.sf.jsqlparser.schema.Table) node); + } else if (node instanceof net.sf.jsqlparser.expression.AnalyticExpression) { + checkWindow((net.sf.jsqlparser.expression.AnalyticExpression) node); + } else if (node instanceof PlainSelect) { + PlainSelect select = (PlainSelect) node; + if (select.getIntoTables() != null || select.getIntoTempTable() != null) { + throw new UnsupportedConstructException("SELECT INTO"); + } + if (select.getWindowDefinitions() != null && !select.getWindowDefinitions().isEmpty()) { + throw new UnsupportedConstructException("WINDOW clauses"); + } + if (select.getForMode() != null) { + throw new UnsupportedConstructException("FOR UPDATE/SHARE"); + } + } else if (node instanceof Distinct) { + Distinct distinct = (Distinct) node; + if (distinct.getOnSelectItems() != null && !distinct.getOnSelectItems().isEmpty()) { + throw new UnsupportedConstructException("DISTINCT ON"); + } + if (distinct.isUseUnique() || distinct.isUseDistinctRow()) { + throw new UnsupportedConstructException("SELECT UNIQUE"); + } + } else if (node instanceof Join) { + Join join = (Join) node; + if (join.isNatural()) { + throw new UnsupportedConstructException("NATURAL JOIN"); + } + if (join.isApply() || join.isSemi() || join.isStraight() || join.isWindowJoin()) { + throw new UnsupportedConstructException("this kind of JOIN"); + } + } else if (node instanceof Function) { + Function function = (Function) node; + if (function.getOrderByElements() != null && !function.getOrderByElements().isEmpty()) { + throw new UnsupportedConstructException("ORDER BY inside " + function.getName() + "()"); + } + // Function.getAttribute()/getAttributeName() fail on functions without one (JSqlParser 5.4) + if (field(function, "attributeExpression") != null + || field(function, "attributeColumn") != null) { + throw new UnsupportedConstructException("field access on " + function.getName() + "()"); + } + if (function.isUnique() + || function.isIgnoreNulls() + || function.getNullHandling() != null + || function.getExtraKeyword() != null + || (function.getKeywordArguments() != null && !function.getKeywordArguments().isEmpty()) + || function.getKeep() != null + || function.getLimit() != null + || function.getHavingClause() != null + || function.getNamedParameters() != null + || function.getChainedParameters() != null) { + throw new UnsupportedConstructException("this form of " + function.getName() + "()"); + } + } else if (node instanceof OrderByElement) { + OrderByElement order = (OrderByElement) node; + if (order.getNullOrdering() == null) { + // PostgreSQL's defaults, written out so the engine's defaults do not matter + order.setNullOrdering( + order.isAsc() + ? OrderByElement.NullOrdering.NULLS_LAST + : OrderByElement.NullOrdering.NULLS_FIRST); + } + } else if (node instanceof WithItem) { + WithItem with = (WithItem) node; + if (with.isRecursive()) { + throw new UnsupportedConstructException("WITH RECURSIVE"); + } + if (with.getWithItemList() != null && !with.getWithItemList().isEmpty()) { + throw new UnsupportedConstructException("column lists of WITH queries"); + } + if (with.getAlias() != null) { + cteNames.add(Identifiers.fold(with.getAlias().getName())); + } + } + } + + /** Window functions the engine has with PostgreSQL's meaning, and aggregates over windows. */ + private static final Set WINDOW_FUNCTIONS = + new HashSet<>( + Arrays.asList( + "row_number", + "rank", + "dense_rank", + "percent_rank", + "cume_dist", + "ntile", + "first_value", + "last_value", + "nth_value", + "lead", + "lag", + "count", + "sum", + "avg", + "min", + "max")); + + private static void checkWindow(net.sf.jsqlparser.expression.AnalyticExpression window) { + String name = window.getName().toLowerCase(java.util.Locale.ROOT); + if (!WINDOW_FUNCTIONS.contains(name)) { + throw new UnsupportedConstructException("the window function " + name + "()"); + } + if (window.getType() != net.sf.jsqlparser.expression.AnalyticType.OVER + || window.getWindowElement() != null + || window.getWindowName() != null + || window.isDistinct() + || window.isUnique() + || window.isIgnoreNulls() + || window.isIgnoreNullsOutside() + || window.getFilterExpression() != null + || window.getKeep() != null + || window.getFuncOrderBy() != null) { + // explicit frames, FILTER, named windows, DISTINCT and IGNORE NULLS stay out + throw new UnsupportedConstructException("this form of the window function " + name + "()"); + } + } + + private static Object field(Object node, String name) { + for (Class c = node.getClass(); c != null && c != Object.class; c = c.getSuperclass()) { + try { + Field field = c.getDeclaredField(name); + field.setAccessible(true); + return field.get(node); + } catch (NoSuchFieldException e) { + // look in the superclass + } catch (IllegalAccessException e) { + throw new IllegalStateException("cannot inspect " + name, e); + } + } + return null; + } + + private static Field[] fields(Class type) { + return FIELDS.computeIfAbsent( + type, + t -> { + java.util.List result = new java.util.ArrayList<>(); + for (Class c = t; c != null && c != Object.class; c = c.getSuperclass()) { + for (Field field : c.getDeclaredFields()) { + if (Modifier.isStatic(field.getModifiers()) + || field.getType().isPrimitive() + || field.getType() == String.class) { + continue; + } + field.setAccessible(true); + result.add(field); + } + } + return result.toArray(new Field[0]); + }); + } + + private static String describe(Class type) { + String simple = type.getSimpleName(); + String known = NAMES.get(simple); + if (known != null) { + return known; + } + // CamelCase class name to words: "SimilarToExpression" -> "similar to expression" + return simple.replaceAll("([a-z])([A-Z])", "$1 $2").toLowerCase(java.util.Locale.ROOT); + } +} diff --git a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/sql/DateTimeRules.java b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/sql/DateTimeRules.java new file mode 100644 index 0000000000000..57339f9882a46 --- /dev/null +++ b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/sql/DateTimeRules.java @@ -0,0 +1,398 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.sql; + +import org.apache.iotdb.postgresql.types.TimestampPrecision; + +import java.time.Instant; +import java.time.LocalDate; +import java.time.ZoneId; +import java.time.zone.ZoneOffsetTransition; +import java.time.zone.ZoneRules; +import java.util.Locale; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Date and time rules: {@code date_trunc}, {@code date_part}/{@code EXTRACT} and interval + * arithmetic, each only where the table-model form gives PostgreSQL's result in the session time + * zone. The engine's {@code date_bin} is calendar-aware for months (in the session time zone) and + * fixed-length for everything else; its casts between TIMESTAMP and DATE use the session time zone. + * + *

Templates contain {@code %s} where the argument goes, already cast to TIMESTAMP. + */ +final class DateTimeRules { + + /** Transitions after this instant are checked (data before 1970 is not expected). */ + private static final Instant SINCE = Instant.parse("1970-01-01T00:00:00Z"); + + private static final Instant UNTIL = Instant.parse("2100-01-01T00:00:00Z"); + + private static final long MICROS_PER_SECOND = 1_000_000L; + + private DateTimeRules() {} + + /** The template for {@code date_trunc(unit, %s)}. */ + static String truncate(String rawUnit, ZoneId zone, TimestampPrecision precision) { + String unit = rawUnit.toLowerCase(Locale.ROOT); + switch (unit) { + case "microseconds": + return fixedBin(1, zone, precision, 1, unit); + case "milliseconds": + return fixedBin(1_000, zone, precision, 1, unit); + case "second": + return fixedBin(MICROS_PER_SECOND, zone, precision, 1, unit); + case "minute": + return fixedBin(60 * MICROS_PER_SECOND, zone, precision, 60, unit); + case "hour": + return fixedBin(3600 * MICROS_PER_SECOND, zone, precision, 3600, unit); + case "day": + // the engine's casts between TIMESTAMP and DATE use the session time zone: local midnight + return "CAST(CAST(%s AS DATE) AS TIMESTAMP)"; + case "week": + if (!withoutTransitions(zone)) { + throw new UnsupportedConstructException( + "date_trunc('week') in a time zone with daylight saving time (" + zone + ")"); + } + // weeks start on Monday; 1970-01-05 was one + return "date_bin(1w, %s, " + localMidnight(LocalDate.of(1970, 1, 5), zone, precision) + ")"; + case "month": + return monthBin(1, zone, precision); + case "quarter": + return monthBin(3, zone, precision); + case "year": + return monthBin(12, zone, precision); + case "decade": + return monthBin(120, zone, precision); + default: + throw new UnsupportedConstructException("date_trunc('" + rawUnit + "')"); + } + } + + private static String fixedBin( + long micros, ZoneId zone, TimestampPrecision precision, int offsetMultiple, String unit) { + if (offsetMultiple > 1 && !offsetsAreMultiplesOf(zone, offsetMultiple)) { + throw new UnsupportedConstructException( + "date_trunc('" + unit + "') in the time zone " + zone + " (its UTC offset is not whole)"); + } + return "date_bin(" + duration(micros, precision, "date_trunc('" + unit + "')") + ", %s, 0)"; + } + + private static String monthBin(int months, ZoneId zone, TimestampPrecision precision) { + // month bins are calendar-aware; the origin is local 1970-01-01 00:00 + return "date_bin(" + + months + + "mo, %s, " + + localMidnight(LocalDate.of(1970, 1, 1), zone, precision) + + ")"; + } + + /** The table-model duration literal of a whole number of microseconds. */ + private static String duration(long micros, TimestampPrecision precision, String what) { + long units = precision.fromEpochMicros(micros); + if (precision.toEpochMicros(units) != micros) { + throw new UnsupportedConstructException(what + " below the timestamp precision"); + } + if (micros % (3600 * MICROS_PER_SECOND) == 0) { + return micros / (3600 * MICROS_PER_SECOND) + "h"; + } + if (micros % (60 * MICROS_PER_SECOND) == 0) { + return micros / (60 * MICROS_PER_SECOND) + "m"; + } + if (micros % MICROS_PER_SECOND == 0) { + return micros / MICROS_PER_SECOND + "s"; + } + if (micros % 1000 == 0) { + return micros / 1000 + "ms"; + } + return micros + "us"; + } + + private static long localMidnight(LocalDate date, ZoneId zone, TimestampPrecision precision) { + Instant instant = date.atStartOfDay(zone).toInstant(); + return precision.fromEpochMicros( + instant.getEpochSecond() * MICROS_PER_SECOND + instant.getNano() / 1000); + } + + /** True if every UTC offset of {@code zone} since 1970 is a multiple of {@code seconds}. */ + static boolean offsetsAreMultiplesOf(ZoneId zone, int seconds) { + ZoneRules rules = zone.getRules(); + if (rules.getOffset(SINCE).getTotalSeconds() % seconds != 0) { + return false; + } + ZoneOffsetTransition transition = rules.nextTransition(SINCE); + while (transition != null && transition.getInstant().isBefore(UNTIL)) { + if (transition.getOffsetAfter().getTotalSeconds() % seconds != 0) { + return false; + } + transition = rules.nextTransition(transition.getInstant()); + } + return true; + } + + /** True if {@code zone} has had one UTC offset since 1970 (no daylight saving time). */ + static boolean withoutTransitions(ZoneId zone) { + ZoneRules rules = zone.getRules(); + if (rules.isFixedOffset()) { + return true; + } + ZoneOffsetTransition transition = rules.nextTransition(SINCE); + return transition == null || !transition.getInstant().isBefore(UNTIL); + } + + // ---------------------------------------------------------------------------- literals + + private static final Pattern TIMESTAMP = + Pattern.compile( + "(\\d{4}-\\d{1,2}-\\d{1,2})(?:[ T](\\d{1,2}):(\\d{2})(?::(\\d{2})(?:\\.(\\d{1,9}))?)?)?" + + "\\s*(Z|[+-]\\d{1,2}(?::?\\d{2})?)?"); + + /** + * A timestamp literal in timestamp units, as PostgreSQL reads it: {@code timestamp} literals + * ignore an offset and are local times in the session time zone; {@code timestamptz} literals use + * their offset, or the session time zone without one. + */ + static long timestampLiteral( + String text, boolean withTimeZone, ZoneId zone, TimestampPrecision precision) { + Matcher m = TIMESTAMP.matcher(text.trim()); + if (!m.matches()) { + throw new UnsupportedConstructException("the timestamp literal '" + text + "'"); + } + java.time.LocalDateTime local; + try { + local = + java.time.LocalDate.parse(normalizeDate(m.group(1))) + .atTime( + m.group(2) == null ? 0 : Integer.parseInt(m.group(2)), + m.group(3) == null ? 0 : Integer.parseInt(m.group(3)), + m.group(4) == null ? 0 : Integer.parseInt(m.group(4)), + m.group(5) == null + ? 0 + : Integer.parseInt((m.group(5) + "000000000").substring(0, 9))); + } catch (java.time.DateTimeException e) { + throw new UnsupportedConstructException("the timestamp literal '" + text + "'"); + } + Instant instant; + if (withTimeZone && m.group(6) != null) { + instant = local.toInstant(offset(m.group(6))); + } else { + instant = local.atZone(zone).toInstant(); + } + long micros = instant.getEpochSecond() * MICROS_PER_SECOND + instant.getNano() / 1000; + long units = precision.fromEpochMicros(micros); + if (precision.toEpochMicros(units) != micros) { + throw new UnsupportedConstructException( + "the timestamp literal '" + text + "' (below the timestamp precision)"); + } + return units; + } + + /** A DATE literal as ISO text ({@code 2026-3-1} → {@code 2026-03-01}). */ + static String dateLiteral(String text) { + try { + return java.time.LocalDate.parse(normalizeDate(text.trim())).toString(); + } catch (java.time.DateTimeException e) { + throw new UnsupportedConstructException("the date literal '" + text + "'"); + } + } + + private static String normalizeDate(String date) { + String[] parts = date.split("-"); + if (parts.length != 3) { + return date; + } + return parts[0] + + "-" + + (parts[1].length() == 1 ? "0" : "") + + parts[1] + + "-" + + (parts[2].length() == 1 ? "0" : "") + + parts[2]; + } + + private static java.time.ZoneOffset offset(String text) { + if (text.equals("Z")) { + return java.time.ZoneOffset.UTC; + } + String digits = text.substring(1).replace(":", ""); + int hours = + Integer.parseInt(digits.length() <= 2 ? digits : digits.substring(0, digits.length() - 2)); + int minutes = + digits.length() <= 2 ? 0 : Integer.parseInt(digits.substring(digits.length() - 2)); + int sign = text.startsWith("-") ? -1 : 1; + return java.time.ZoneOffset.ofHoursMinutes(sign * hours, sign * minutes); + } + + // ---------------------------------------------------------------------------- EXTRACT + + /** The expression for {@code EXTRACT(field FROM %s)} / {@code date_part('field', %s)}. */ + static String extract(String rawField, ZoneId zone, TimestampPrecision precision) { + String field = rawField.toLowerCase(Locale.ROOT); + switch (field) { + case "year": + case "quarter": + case "month": + case "day": + case "doy": + case "hour": + case "minute": + return "EXTRACT(" + field.toUpperCase(Locale.ROOT) + " FROM %s)"; + case "isodow": + // the engine counts Monday = 1 ... Sunday = 7, as isodow + return "EXTRACT(DOW FROM %s)"; + case "dow": + // PostgreSQL: Sunday = 0 ... Saturday = 6 + return "(EXTRACT(DOW FROM %s) % 7)"; + case "second": + { + if (!offsetsAreMultiplesOf(zone, 60)) { + throw new UnsupportedConstructException( + "the date part 'second' in the time zone " + + zone + + " (its UTC offset is not whole)"); + } + // with the fraction, as PostgreSQL (microsecond resolution) + long units = precision.fromEpochMicros(MICROS_PER_SECOND); + long minute = 60 * units; + return "(CAST(((CAST(%s AS INT64) % " + + minute + + ") + " + + minute + + ") % " + + minute + + " AS DOUBLE) / " + + units + + ")"; + } + case "epoch": + return "(CAST(CAST(%s AS INT64) AS DOUBLE) / " + + precision.fromEpochMicros(MICROS_PER_SECOND) + + ")"; + default: + throw new UnsupportedConstructException("the date part '" + rawField + "'"); + } + } + + // ---------------------------------------------------------------------------- intervals + + private static final Pattern PART = + Pattern.compile("([+-]?\\d+(?:\\.\\d+)?)\\s*([a-z]+)", Pattern.CASE_INSENSITIVE); + private static final Pattern CLOCK = + Pattern.compile("([+-]?)(\\d+):(\\d{2})(?::(\\d{2}(?:\\.\\d+)?))?"); + + /** + * The length of an interval literal in timestamp units, for arithmetic with timestamps. Only + * units of fixed length: days and weeks only in time zones without daylight saving time (in + * PostgreSQL, adding a day keeps the local time across a DST change); months and years never. + */ + static long intervalUnits(String text, ZoneId zone, TimestampPrecision precision) { + String rest = text.trim().toLowerCase(Locale.ROOT); + double micros = 0; + boolean any = false; + Matcher clock = CLOCK.matcher(rest); + if (clock.find() && clock.end() == rest.length()) { + double sign = clock.group(1).equals("-") ? -1 : 1; + double seconds = + Long.parseLong(clock.group(2)) * 3600 + + Long.parseLong(clock.group(3)) * 60 + + (clock.group(4) == null ? 0 : Double.parseDouble(clock.group(4))); + micros += sign * seconds * MICROS_PER_SECOND; + rest = rest.substring(0, clock.start()).trim(); + any = true; + } + Matcher m = PART.matcher(rest); + int position = 0; + while (m.find()) { + if (!rest.substring(position, m.start()).trim().isEmpty()) { + throw new UnsupportedConstructException("the interval '" + text + "'"); + } + position = m.end(); + double amount = Double.parseDouble(m.group(1)); + micros += amount * unitMicros(m.group(2), zone, text); + any = true; + } + if (!any || !rest.substring(position).trim().isEmpty()) { + throw new UnsupportedConstructException("the interval '" + text + "'"); + } + long whole = Math.round(micros); + if (Math.abs(micros - whole) > 1e-6) { + throw new UnsupportedConstructException("the interval '" + text + "' (below a microsecond)"); + } + long units = precision.fromEpochMicros(whole); + if (precision.toEpochMicros(units) != whole) { + throw new UnsupportedConstructException( + "the interval '" + text + "' (below the timestamp precision)"); + } + return units; + } + + private static double unitMicros(String unit, ZoneId zone, String text) { + switch (unit) { + case "us": + case "usec": + case "usecs": + case "microsecond": + case "microseconds": + return 1; + case "ms": + case "msec": + case "msecs": + case "millisecond": + case "milliseconds": + return 1_000; + case "s": + case "sec": + case "secs": + case "second": + case "seconds": + return MICROS_PER_SECOND; + case "m": + case "min": + case "mins": + case "minute": + case "minutes": + return 60.0 * MICROS_PER_SECOND; + case "h": + case "hr": + case "hrs": + case "hour": + case "hours": + return 3600.0 * MICROS_PER_SECOND; + case "d": + case "day": + case "days": + case "w": + case "week": + case "weeks": + if (!withoutTransitions(zone)) { + throw new UnsupportedConstructException( + "interval days in a time zone with daylight saving time (" + zone + ")"); + } + return (unit.startsWith("w") ? 7 : 1) * 86_400.0 * MICROS_PER_SECOND; + default: + throw new UnsupportedConstructException( + "the interval unit '" + + unit + + "' in '" + + text + + "' (months and years have no fixed length)"); + } + } +} diff --git a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/sql/EmptyRelations.java b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/sql/EmptyRelations.java new file mode 100644 index 0000000000000..d90ecb6fd6195 --- /dev/null +++ b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/sql/EmptyRelations.java @@ -0,0 +1,388 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.sql; + +import org.apache.iotdb.postgresql.catalog.PgCatalogTables; + +import net.sf.jsqlparser.expression.Alias; +import net.sf.jsqlparser.expression.BooleanValue; +import net.sf.jsqlparser.expression.CastExpression; +import net.sf.jsqlparser.expression.Expression; +import net.sf.jsqlparser.expression.Function; +import net.sf.jsqlparser.expression.NullValue; +import net.sf.jsqlparser.schema.Column; +import net.sf.jsqlparser.schema.Table; +import net.sf.jsqlparser.statement.create.table.ColDataType; +import net.sf.jsqlparser.statement.select.FromItem; +import net.sf.jsqlparser.statement.select.Join; +import net.sf.jsqlparser.statement.select.PlainSelect; +import net.sf.jsqlparser.statement.select.SelectItem; + +import java.lang.reflect.Array; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashSet; +import java.util.IdentityHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Set; + +/** + * Queries over catalog tables that never have rows: a SELECT whose FROM reads such a table through + * inner, cross or left joins (as the left side), without aggregates outside a GROUP BY, returns no + * rows whatever it selects. Its select list becomes typed NULLs with the same output names, its + * WHERE {@code FALSE}, so that constructs without a translation there (psql's {@code ANY(array)}, + * {@code ARRAY(...)}, statistics functions) do not refuse the statement. Runs before {@link + * ConstructCheck}. + */ +final class EmptyRelations { + + private static final String PACKAGE = "net.sf.jsqlparser."; + + private static final Set AGGREGATES = + new HashSet<>( + Arrays.asList( + "count", + "sum", + "avg", + "min", + "max", + "bool_and", + "bool_or", + "every", + "string_agg", + "array_agg", + "stddev", + "stddev_pop", + "stddev_samp", + "variance", + "var_pop", + "var_samp")); + + private final IdentityHashMap visited = new IdentityHashMap<>(); + private final List selects = new ArrayList<>(); + + private EmptyRelations() {} + + /** Rewrites every qualifying SELECT of {@code statement} in place. */ + static void simplify(Object statement) { + EmptyRelations finder = new EmptyRelations(); + finder.walk(statement); + // innermost first: a SELECT over a derived table that is empty is empty as well + java.util.Set emptied = + java.util.Collections.newSetFromMap(new IdentityHashMap<>()); + for (int i = finder.selects.size() - 1; i >= 0; i--) { + PlainSelect select = finder.selects.get(i); + String table = emptyTable(select); + if (table != null) { + empty(select, table); + emptied.add(select); + } else if (emptyDerived(select, emptied)) { + emptyOverDerived(select); + emptied.add(select); + } + } + } + + /** The always-empty catalog table that makes {@code select} empty, or {@code null}. */ + private static String emptyTable(PlainSelect select) { + if (select.getGroupBy() == null && hasAggregate(select.getSelectItems())) { + // an aggregate over no rows still returns one row + return null; + } + List joins = select.getJoins() == null ? new ArrayList<>() : select.getJoins(); + String first = catalogTable(select.getFromItem()); + if (first != null && PgCatalogTables.alwaysEmpty(first)) { + for (Join join : joins) { + if (join.isRight() || join.isFull()) { + return null; + } + } + return first; + } + for (Join join : joins) { + String table = catalogTable(join.getFromItem()); + boolean inner = join.isInner() || join.isCross() || join.isSimple() || isPlain(join); + if (table != null && PgCatalogTables.alwaysEmpty(table) && inner) { + // an inner or cross join with an empty table is empty, wherever it stands + return table; + } + } + return null; + } + + private static boolean isPlain(Join join) { + return !join.isLeft() + && !join.isRight() + && !join.isFull() + && !join.isOuter() + && !join.isNatural() + && !join.isSemi() + && !join.isApply(); + } + + static boolean hasAggregate(List> items) { + for (SelectItem item : items) { + if (containsAggregate(item.getExpression(), new IdentityHashMap<>())) { + return true; + } + } + return false; + } + + private static boolean containsAggregate(Object node, IdentityHashMap seen) { + if (node == null || seen.containsKey(node)) { + return false; + } + if (node instanceof Function + && AGGREGATES.contains(((Function) node).getName().toLowerCase(Locale.ROOT))) { + return true; + } + if (node instanceof Collection) { + for (Object element : (Collection) node) { + if (containsAggregate(element, seen)) { + return true; + } + } + return false; + } + Class type = node.getClass(); + if (!type.getName().startsWith(PACKAGE) + || type.isEnum() + || node instanceof net.sf.jsqlparser.statement.select.Select) { + // a sub-query's aggregates are its own + return false; + } + seen.put(node, Boolean.TRUE); + for (Field field : fields(type)) { + try { + if (containsAggregate(field.get(node), seen)) { + return true; + } + } catch (IllegalAccessException e) { + throw new IllegalStateException("cannot inspect " + field, e); + } + } + return false; + } + + /** True if {@code select} reads an emptied derived table (FROM or inner/cross join). */ + private static boolean emptyDerived(PlainSelect select, java.util.Set emptied) { + if (select.getGroupBy() == null && hasAggregate(select.getSelectItems())) { + return false; + } + if (isEmptied(select.getFromItem(), emptied)) { + if (select.getJoins() != null) { + for (Join join : select.getJoins()) { + if (join.isRight() || join.isFull()) { + return false; + } + } + } + return true; + } + return false; + } + + private static boolean isEmptied(FromItem item, java.util.Set emptied) { + return item instanceof net.sf.jsqlparser.statement.select.ParenthesedSelect + && ((net.sf.jsqlparser.statement.select.ParenthesedSelect) item).getSelect() + instanceof PlainSelect + && emptied.contains( + (PlainSelect) + ((net.sf.jsqlparser.statement.select.ParenthesedSelect) item).getSelect()); + } + + /** Like {@link #empty}, keeping the (empty) derived table as the FROM item. */ + private static void emptyOverDerived(PlainSelect select) { + List> items = new ArrayList<>(); + for (SelectItem item : select.getSelectItems()) { + String name = + item.getAlias() != null ? Identifiers.fold(item.getAlias().getName()) : outputName(item); + items.add(typedNull(org.apache.tsfile.enums.TSDataType.STRING, name)); + } + select.setSelectItems(items); + select.setJoins(null); + select.setWhere(new BooleanValue(false)); + select.setGroupByElement(null); + select.setHaving(null); + select.setDistinct(null); + select.setOrderByElements(null); + select.setLimit(null); + select.setOffset(null); + select.setFetch(null); + } + + /** The catalog table a FROM item names (PgCatalogTables name), or {@code null}. */ + private static String catalogTable(FromItem item) { + if (!(item instanceof Table)) { + return null; + } + Table table = (Table) item; + String schema = Identifiers.fold(table.getSchemaName()); + String name = Identifiers.fold(table.getName()); + String catalog = null; + if (schema == null && name != null && name.startsWith("pg_")) { + catalog = name; + } else if ("pg_catalog".equals(schema)) { + catalog = name; + } else if ("information_schema".equals(schema)) { + catalog = "pg_is_" + name; + } + return catalog != null && PgCatalogTables.contains(catalog) ? catalog : null; + } + + private static void empty(PlainSelect select, String table) { + List> items = new ArrayList<>(); + for (SelectItem item : select.getSelectItems()) { + if (item.getExpression() instanceof net.sf.jsqlparser.statement.select.AllColumns) { + for (PgCatalogTables.Column column : PgCatalogTables.tables().get(table)) { + items.add(typedNull(column.type(), column.name())); + } + continue; + } + PgCatalogTables.Column column = column(item.getExpression(), table); + String name = + item.getAlias() != null ? Identifiers.fold(item.getAlias().getName()) : outputName(item); + items.add( + typedNull( + column == null ? org.apache.tsfile.enums.TSDataType.STRING : column.type(), name)); + } + select.setSelectItems(items); + Table from = new Table("pg_catalog", PgCatalogTables.postgresName(table)); + if (table.startsWith(PgCatalogTables.INFORMATION_SCHEMA_PREFIX)) { + from = new Table("information_schema", PgCatalogTables.postgresName(table)); + } + select.setFromItem(from); + select.setJoins(null); + select.setWhere(new BooleanValue(false)); + select.setGroupByElement(null); + select.setHaving(null); + select.setDistinct(null); + select.setOrderByElements(null); + select.setLimit(null); + select.setOffset(null); + select.setFetch(null); + } + + private static PgCatalogTables.Column column(Expression expression, String table) { + if (expression instanceof Column) { + return PgCatalogTables.column(table, Identifiers.fold(((Column) expression).getColumnName())); + } + return null; + } + + /** The name PostgreSQL gives an output column without alias. */ + private static String outputName(SelectItem item) { + Expression expression = item.getExpression(); + while (expression instanceof CastExpression) { + expression = ((CastExpression) expression).getLeftExpression(); + } + if (expression instanceof Column) { + return Identifiers.fold(((Column) expression).getColumnName()); + } + if (expression instanceof Function) { + List parts = ((Function) expression).getMultipartName(); + return Identifiers.fold(parts.get(parts.size() - 1)); + } + return "?column?"; + } + + private static SelectItem typedNull(org.apache.tsfile.enums.TSDataType type, String name) { + CastExpression cast = new CastExpression(); + cast.setLeftExpression(new NullValue()); + cast.setColDataType(new ColDataType(pgType(type))); + SelectItem item = new SelectItem<>(cast); + item.setAlias(new Alias(Identifiers.quote(name), true)); + return item; + } + + private static String pgType(org.apache.tsfile.enums.TSDataType type) { + switch (type) { + case INT32: + return "int4"; + case INT64: + return "int8"; + case FLOAT: + return "float4"; + case DOUBLE: + return "float8"; + case BOOLEAN: + return "bool"; + default: + return "text"; + } + } + + // ---------------------------------------------------------------------------- walking + + private void walk(Object node) { + if (node == null || visited.containsKey(node)) { + return; + } + if (node instanceof Collection) { + for (Object element : (Collection) node) { + walk(element); + } + return; + } + Class type = node.getClass(); + if (type.isArray()) { + if (!type.getComponentType().isPrimitive()) { + for (int i = 0; i < Array.getLength(node); i++) { + walk(Array.get(node, i)); + } + } + return; + } + if (!type.getName().startsWith(PACKAGE) + || type.isEnum() + || type.getName().startsWith(PACKAGE + "parser.")) { + return; + } + visited.put(node, Boolean.TRUE); + if (node instanceof PlainSelect) { + selects.add((PlainSelect) node); + } + for (Field field : fields(type)) { + try { + walk(field.get(node)); + } catch (IllegalAccessException e) { + throw new IllegalStateException("cannot inspect " + field, e); + } + } + } + + private static List fields(Class type) { + List fields = new ArrayList<>(); + for (Class c = type; c != null && c != Object.class; c = c.getSuperclass()) { + for (Field field : c.getDeclaredFields()) { + if (!Modifier.isStatic(field.getModifiers()) && !field.getType().isPrimitive()) { + field.setAccessible(true); + fields.add(field); + } + } + } + return fields; + } +} diff --git a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/sql/Identifiers.java b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/sql/Identifiers.java new file mode 100644 index 0000000000000..6d97c7bab6a26 --- /dev/null +++ b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/sql/Identifiers.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.sql; + +import java.util.Locale; + +/** PostgreSQL identifier rules: unquoted names fold to lower case, quoted names keep theirs. */ +final class Identifiers { + + private Identifiers() {} + + /** The name an identifier as written in SQL stands for. */ + static String fold(String identifier) { + if (identifier == null) { + return null; + } + if (identifier.length() >= 2 && identifier.startsWith("\"") && identifier.endsWith("\"")) { + return identifier.substring(1, identifier.length() - 1).replace("\"\"", "\""); + } + return identifier.toLowerCase(Locale.ROOT); + } + + /** The name as a double-quoted table-model identifier. */ + static String quote(String name) { + return '"' + name.replace("\"", "\"\"") + '"'; + } + + /** A string literal of the table model. */ + static String literal(String value) { + return '\'' + value.replace("'", "''") + '\''; + } +} diff --git a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/sql/OuterConditions.java b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/sql/OuterConditions.java new file mode 100644 index 0000000000000..4a8e6f9d29aed --- /dev/null +++ b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/sql/OuterConditions.java @@ -0,0 +1,185 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.sql; + +import net.sf.jsqlparser.expression.Expression; +import net.sf.jsqlparser.expression.operators.conditional.AndExpression; +import net.sf.jsqlparser.schema.Column; +import net.sf.jsqlparser.schema.Table; +import net.sf.jsqlparser.statement.select.FromItem; +import net.sf.jsqlparser.statement.select.Join; +import net.sf.jsqlparser.statement.select.PlainSelect; +import net.sf.jsqlparser.statement.select.Select; + +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.IdentityHashMap; +import java.util.List; +import java.util.Set; + +/** + * The conditions of a sub-query's WHERE that only read the outer row: every column they use is + * qualified with a name the sub-query does not define. Used to move them in front of the sub-query, + * where the engine can evaluate them. + */ +final class OuterConditions { + + private OuterConditions() {} + + /** + * The outer-only conjuncts of {@code select}'s WHERE; empty if there are none or the sub-query + * aggregates or groups (then no rows does not mean NULL). + */ + static List split(PlainSelect select) { + if (select.getWhere() == null + || select.getGroupBy() != null + || select.getHaving() != null + || EmptyRelations.hasAggregate(select.getSelectItems())) { + return Collections.emptyList(); + } + Set local = new HashSet<>(); + addName(local, select.getFromItem()); + if (select.getJoins() != null) { + for (Join join : select.getJoins()) { + addName(local, join.getFromItem()); + } + } + List outer = new ArrayList<>(); + for (Expression conjunct : conjuncts(select.getWhere())) { + List columns = new ArrayList<>(); + if (!columns(conjunct, columns, new IdentityHashMap<>()) || columns.isEmpty()) { + continue; + } + boolean onlyOuter = true; + for (Column column : columns) { + Table table = column.getTable(); + if (table == null + || table.getName() == null + || table.getSchemaName() != null + || local.contains(Identifiers.fold(table.getName()))) { + onlyOuter = false; + break; + } + } + if (onlyOuter) { + outer.add(conjunct); + } + } + if (outer.size() == conjuncts(select.getWhere()).size()) { + // nothing would stay inside: keep the statement as it is + return Collections.emptyList(); + } + return outer; + } + + /** The WHERE without {@code removed} (by identity); {@code null} if nothing remains. */ + static Expression remaining(Expression where, List removed) { + List kept = new ArrayList<>(); + for (Expression conjunct : conjuncts(where)) { + boolean drop = false; + for (Expression r : removed) { + drop |= r == conjunct; + } + if (!drop) { + kept.add(conjunct); + } + } + return kept.isEmpty() ? null : and(kept); + } + + static Expression and(List conjuncts) { + Expression result = conjuncts.get(0); + for (int i = 1; i < conjuncts.size(); i++) { + result = new AndExpression(result, conjuncts.get(i)); + } + return result; + } + + private static List conjuncts(Expression where) { + List result = new ArrayList<>(); + if (where instanceof AndExpression) { + result.addAll(conjuncts(((AndExpression) where).getLeftExpression())); + result.addAll(conjuncts(((AndExpression) where).getRightExpression())); + } else { + result.add(where); + } + return result; + } + + private static void addName(Set names, FromItem item) { + if (item == null) { + return; + } + if (item.getAlias() != null) { + names.add(Identifiers.fold(item.getAlias().getName())); + } + if (item instanceof Table) { + names.add(Identifiers.fold(((Table) item).getName())); + } + } + + /** Collects the columns of {@code node}; false if it contains a sub-query. */ + private static boolean columns( + Object node, List columns, IdentityHashMap seen) { + if (node == null || seen.containsKey(node)) { + return true; + } + if (node instanceof Select) { + return false; + } + if (node instanceof Column) { + columns.add((Column) node); + return true; + } + if (node instanceof Collection) { + for (Object element : (Collection) node) { + if (!columns(element, columns, seen)) { + return false; + } + } + return true; + } + Class type = node.getClass(); + if (!type.getName().startsWith("net.sf.jsqlparser.") || type.isEnum()) { + return true; + } + seen.put(node, Boolean.TRUE); + for (Class c = type; c != null && c != Object.class; c = c.getSuperclass()) { + for (Field field : c.getDeclaredFields()) { + if (Modifier.isStatic(field.getModifiers()) || field.getType().isPrimitive()) { + continue; + } + field.setAccessible(true); + try { + if (!columns(field.get(node), columns, seen)) { + return false; + } + } catch (IllegalAccessException e) { + throw new IllegalStateException("cannot inspect " + field, e); + } + } + } + return true; + } +} diff --git a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/sql/OutputNames.java b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/sql/OutputNames.java new file mode 100644 index 0000000000000..3ae73108f584e --- /dev/null +++ b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/sql/OutputNames.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.sql; + +import net.sf.jsqlparser.expression.CaseExpression; +import net.sf.jsqlparser.expression.CastExpression; +import net.sf.jsqlparser.expression.Expression; +import net.sf.jsqlparser.expression.ExtractExpression; +import net.sf.jsqlparser.expression.Function; +import net.sf.jsqlparser.expression.TimeKeyExpression; +import net.sf.jsqlparser.schema.Column; +import net.sf.jsqlparser.statement.select.AllColumns; +import net.sf.jsqlparser.statement.select.Select; + +import java.util.List; +import java.util.Locale; + +/** + * The name PostgreSQL gives an output column without alias ({@code FigureColname}): a function's + * name, a cast's operand name or else its type, {@code case}, {@code ?column?} for anything else. + */ +final class OutputNames { + + private OutputNames() {} + + /** The name, or {@code null} where the engine's name is already PostgreSQL's (columns, *). */ + static String of(Expression expression) { + if (expression instanceof Column) { + String name = Identifiers.fold(((Column) expression).getColumnName()); + // keywords that are values, not columns + switch (name) { + case "current_user": + case "session_user": + case "current_catalog": + case "current_schema": + case "user": + return name; + default: + return null; + } + } + if (expression instanceof AllColumns || expression instanceof Select) { + return null; + } + String name = figure(expression); + return name == null ? "?column?" : name; + } + + private static String figure(Expression expression) { + if (expression instanceof Column) { + return Identifiers.fold(((Column) expression).getColumnName()); + } + if (expression instanceof Function) { + List parts = ((Function) expression).getMultipartName(); + return Identifiers.fold(parts.get(parts.size() - 1)); + } + if (expression instanceof CastExpression) { + CastExpression cast = (CastExpression) expression; + String inner = figure(cast.getLeftExpression()); + if (inner != null) { + return inner; + } + String type = PgTypeNames.baseName(cast.getColDataType()); + return CANONICAL.getOrDefault(type, type); + } + if (expression instanceof CaseExpression) { + return "case"; + } + if (expression instanceof ExtractExpression) { + return "extract"; + } + if (expression instanceof TimeKeyExpression) { + String key = ((TimeKeyExpression) expression).getStringValue().toLowerCase(Locale.ROOT); + int paren = key.indexOf('('); + return (paren < 0 ? key : key.substring(0, paren)).trim(); + } + return null; + } + + /** PostgreSQL's internal names of SQL type names, as a cast's column name shows them. */ + private static final java.util.Map CANONICAL = new java.util.HashMap<>(); + + static { + CANONICAL.put("integer", "int4"); + CANONICAL.put("int", "int4"); + CANONICAL.put("smallint", "int2"); + CANONICAL.put("bigint", "int8"); + CANONICAL.put("real", "float4"); + CANONICAL.put("double precision", "float8"); + CANONICAL.put("float", "float8"); + CANONICAL.put("boolean", "bool"); + CANONICAL.put("character varying", "varchar"); + CANONICAL.put("timestamp with time zone", "timestamptz"); + CANONICAL.put("timestamp without time zone", "timestamp"); + } +} diff --git a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/sql/PgToTableSql.java b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/sql/PgToTableSql.java new file mode 100644 index 0000000000000..92f2e68cbb875 --- /dev/null +++ b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/sql/PgToTableSql.java @@ -0,0 +1,1682 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.sql; + +import org.apache.iotdb.postgresql.catalog.StaticCatalog; +import org.apache.iotdb.postgresql.protocol.PgException; +import org.apache.iotdb.postgresql.protocol.SqlState; + +import net.sf.jsqlparser.JSQLParserException; +import net.sf.jsqlparser.expression.Alias; +import net.sf.jsqlparser.expression.BooleanValue; +import net.sf.jsqlparser.expression.CastExpression; +import net.sf.jsqlparser.expression.Expression; +import net.sf.jsqlparser.expression.Function; +import net.sf.jsqlparser.expression.JdbcParameter; +import net.sf.jsqlparser.expression.StringValue; +import net.sf.jsqlparser.expression.TimeKeyExpression; +import net.sf.jsqlparser.expression.operators.relational.LikeExpression; +import net.sf.jsqlparser.parser.CCJSqlParserUtil; +import net.sf.jsqlparser.schema.Column; +import net.sf.jsqlparser.schema.Table; +import net.sf.jsqlparser.statement.Statement; +import net.sf.jsqlparser.statement.select.Limit; +import net.sf.jsqlparser.statement.select.PlainSelect; +import net.sf.jsqlparser.statement.select.Select; +import net.sf.jsqlparser.util.deparser.ExpressionDeParser; +import net.sf.jsqlparser.util.deparser.SelectDeParser; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +/** + * Translates a PostgreSQL query into table-model SQL: names, casts, parameters, catalog functions + * and the constructs with identical meaning. Anything else is refused with 0A000 naming the + * construct; the translator never evaluates data itself. + */ +public final class PgToTableSql { + + private static final Logger LOGGER = LoggerFactory.getLogger(PgToTableSql.class); + + /** The translated statement. */ + public static final class Translation { + private final String sql; + private final List parameterOrder; + private final boolean[] dateParameters; + + Translation(String sql, List parameterOrder) { + this(sql, parameterOrder, null); + } + + private Translation(String sql, List parameterOrder, boolean[] dateParameters) { + this.sql = sql; + this.parameterOrder = Collections.unmodifiableList(parameterOrder); + this.dateParameters = dateParameters; + } + + /** The table-model SQL. */ + public String sql() { + return sql; + } + + /** + * For each {@code ?} of {@link #sql()} in order, the 1-based number of the PostgreSQL parameter + * ({@code $n}) it stands for; a parameter may appear several times. + */ + public List parameterOrder() { + return parameterOrder; + } + + /** + * The statement with the parameters the client declared as {@code date} marked: the engine has + * no date literal and cannot bind a parameter inside a CAST, so their values are written into + * the SQL as {@code CAST('yyyy-mm-dd' AS DATE)} when bound ({@link #sqlFor}). + * + * @param types the parameter type OIDs from Parse (index = PostgreSQL parameter number - 1) + */ + public Translation withTypedParameters(int[] types) { + boolean[] dates = new boolean[parameterOrder.size()]; + boolean any = false; + for (int k = 0; k < dates.length; k++) { + int number = parameterOrder.get(k); + dates[k] = number - 1 < types.length && types[number - 1] == 1082; + any |= dates[k]; + } + return any ? new Translation(sql, parameterOrder, dates) : this; + } + + /** The SQL to run with {@code values} (one per {@code ?}, in order). */ + public String sqlFor(List values) { + if (dateParameters == null) { + return sql; + } + StringBuilder out = new StringBuilder(sql.length()); + int k = 0; + char quote = 0; + for (int i = 0; i < sql.length(); i++) { + char c = sql.charAt(i); + if (quote != 0) { + out.append(c); + if (c == quote) { + quote = 0; + } + continue; + } + if (c == '\'' || c == '"') { + quote = c; + out.append(c); + continue; + } + if (c == '?' && k < dateParameters.length) { + if (dateParameters[k]) { + Object value = values.get(k); + String text = + value + instanceof + org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.StringLiteral + ? ((org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.StringLiteral) + value) + .getValue() + : null; + // the value was parsed as a date by the parameter decoder: digits and dashes only + out.append( + text == null + ? "CAST(NULL AS DATE)" + : "CAST(" + + Identifiers.literal(java.time.LocalDate.parse(text).toString()) + + " AS DATE)"); + } else { + out.append('?'); + } + k++; + continue; + } + out.append(c); + } + return out.toString(); + } + + /** The values still bound as parameters (those not written into {@link #sqlFor}). */ + public List remaining(List values) { + if (dateParameters == null) { + return values; + } + List kept = new ArrayList<>(); + for (int k = 0; k < values.size(); k++) { + if (!dateParameters[k]) { + kept.add(values.get(k)); + } + } + return kept; + } + + /** The highest PostgreSQL parameter number used, 0 if none. */ + public int parameterCount() { + int max = 0; + for (int n : parameterOrder) { + max = Math.max(max, n); + } + return max; + } + } + + /** Functions with the same name, arguments and meaning in both dialects. */ + private static final Set SAME = + new HashSet<>( + Arrays.asList( + "count", + "sum", + "avg", + "min", + "max", + "stddev", + "stddev_pop", + "stddev_samp", + "variance", + "var_pop", + "var_samp", + "corr", + "covar_pop", + "covar_samp", + "regr_slope", + "regr_intercept", + "lower", + "upper", + "length", + "trim", + "ltrim", + "rtrim", + "replace", + "substring", + "strpos", + "starts_with", + "abs", + "sign", + "ceil", + "floor", + "round", + "sqrt", + "exp", + "ln", + "log10", + "sin", + "cos", + "tan", + "asin", + "acos", + "atan", + "degrees", + "radians", + "pi", + "greatest", + "least", + "coalesce")); + + /** Functions with another name in the table model. */ + private static final Map RENAMED = new HashMap<>(); + + static { + RENAMED.put("char_length", "length"); + RENAMED.put("character_length", "length"); + RENAMED.put("ceiling", "ceil"); + RENAMED.put("substr", "substring"); + } + + /** Functions that are always true because the catalog only shows what the user may read. */ + private static final Set ALWAYS_TRUE = + new HashSet<>( + Arrays.asList( + "has_table_privilege", + "has_schema_privilege", + "has_database_privilege", + "has_column_privilege", + "pg_table_is_visible", + "pg_type_is_visible", + "pg_function_is_visible")); + + /** Cast targets that print a value as text. */ + private static final Set TEXT_TYPES = + new HashSet<>(Arrays.asList("text", "varchar", "character varying", "name", "bpchar")); + + /** PostgreSQL's encoding names by number ({@code pg_enc}). */ + private static final String[] ENCODINGS = { + "SQL_ASCII", + "EUC_JP", + "EUC_CN", + "EUC_KR", + "EUC_TW", + "EUC_JIS_2004", + "UTF8", + "MULE_INTERNAL", + "LATIN1", + "LATIN2", + "LATIN3", + "LATIN4", + "LATIN5", + "LATIN6", + "LATIN7", + "LATIN8", + "LATIN9", + "LATIN10", + "WIN1256", + "WIN1258", + "WIN866", + "WIN874", + "KOI8R", + "WIN1251", + "WIN1252", + "ISO_8859_5", + "ISO_8859_6", + "ISO_8859_7", + "ISO_8859_8", + "WIN1250", + "WIN1253", + "WIN1254", + "WIN1255", + "WIN1257", + "KOI8U", + "SJIS", + "BIG5", + "GBK", + "UHC", + "GB18030", + "JOHAB", + "SHIFT_JIS_2004" + }; + + private PgToTableSql() {} + + /** + * Translates one statement (already split, without the trailing semicolon). + * + * @throws PgException 42601 for syntax errors, 0A000 for constructs without exact translation, + * 25006 for statements that write, 42P01 for tables not found on the search path + */ + public static Translation translate(String sql, TranslationContext context) throws PgException { + return translate(sql, context, true); + } + + /** + * As {@link #translate(String, TranslationContext)}; {@code outputNames} false leaves output + * columns without alias unnamed (rule tests that look at expressions only). + */ + static Translation translate(String sql, TranslationContext context, boolean outputNames) + throws PgException { + Statement statement; + try { + statement = CCJSqlParserUtil.parse(PreParse.rewrite(sql)); + } catch (JSQLParserException e) { + throw new PgException(SqlState.SYNTAX_ERROR, "syntax error: " + firstLine(e)); + } + if (!(statement instanceof Select)) { + throw notAQuery(statement); + } + try { + EmptyRelations.simplify(statement); + ConstructCheck.Result checked = ConstructCheck.check(statement); + Writer writer = + new Writer(context, checked.cteNames, checked.otherTables ? null : checked.catalogTables); + writer.qualifiedCatalogTables = checked.catalogTables; + writer.outputNames = outputNames; + writer.write((Select) statement); + return new Translation(writer.builder.toString(), writer.parameters); + } catch (UnsupportedConstructException e) { + LOGGER.debug("Refused PostgreSQL statement ({}): {}", e.construct(), sql); + throw new PgException(SqlState.FEATURE_NOT_SUPPORTED, e.getMessage()); + } catch (Refusal e) { + throw new PgException(e.sqlState, e.getMessage()); + } catch (TableNotFound e) { + throw new PgException( + SqlState.UNDEFINED_TABLE, + "relation \"" + e.getMessage() + "\" does not exist", + "Qualify the table with its database, for example mydb." + e.getMessage(), + false); + } + } + + private static PgException notAQuery(Statement statement) { + String kind = statement.getClass().getSimpleName().toLowerCase(Locale.ROOT); + if (kind.matches( + "insert|update|delete|upsert|merge|truncate|create.*|alter.*|drop|grant|replace")) { + return new PgException( + SqlState.READ_ONLY_SQL_TRANSACTION, + "cannot execute " + kind.toUpperCase(Locale.ROOT) + " in a read-only transaction"); + } + return new PgException( + SqlState.FEATURE_NOT_SUPPORTED, + "statements of type " + statement.getClass().getSimpleName() + " are not supported"); + } + + private static String firstLine(Exception e) { + String message = e.getCause() != null ? e.getCause().getMessage() : e.getMessage(); + if (message == null) { + return "cannot parse the statement"; + } + int newline = message.indexOf('\n'); + String line = newline < 0 ? message : message.substring(0, newline); + // the parser's class name and lexer state mean nothing to a client + return line.replaceFirst("^[\\w.]+Exception: ", "") + .replaceFirst(",? in lexical state \\w+\\.?$", ""); + } + + /** A table not found on the search path (carries the table name). */ + private static final class TableNotFound extends RuntimeException { + private static final long serialVersionUID = 1L; + + TableNotFound(String table) { + super(table, null, false, false); + } + } + + /** A statement PostgreSQL itself would refuse with this SQLSTATE (unknown type, ...). */ + private static final class Refusal extends RuntimeException { + private static final long serialVersionUID = 1L; + private final String sqlState; + + Refusal(String sqlState, String message) { + super(message, null, false, false); + this.sqlState = sqlState; + } + } + + /** The deparsers with the translation rules. */ + private static final class Writer { + final StringBuilder builder = new StringBuilder(); + final List parameters = new ArrayList<>(); + final TranslationContext context; + final Set cteNames; + + /** Catalog tables by qualifier, {@code null} if the statement also reads other tables. */ + final Map catalogTables; + + /** Catalog tables by qualifier, for qualified column references. */ + Map qualifiedCatalogTables; + + final Expressions expressions; + final Selects selects; + + Writer(TranslationContext context, Set cteNames, Map catalogTables) { + this.context = context; + this.cteNames = cteNames; + this.catalogTables = catalogTables; + this.expressions = new Expressions(this); + this.selects = new Selects(this); + expressions.setSelectVisitor(selects); + expressions.setBuilder(builder); + selects.setExpressionVisitor(expressions); + selects.setBuilder(builder); + } + + /** The select list whose names the client sees (PostgreSQL: the first branch's). */ + List outputItems; + + boolean outputNames = true; + + void write(Select select) { + Select first = select; + while (first instanceof net.sf.jsqlparser.statement.select.SetOperationList) { + first = ((net.sf.jsqlparser.statement.select.SetOperationList) first).getSelects().get(0); + } + while (first instanceof net.sf.jsqlparser.statement.select.ParenthesedSelect) { + first = ((net.sf.jsqlparser.statement.select.ParenthesedSelect) first).getSelect(); + } + if (first instanceof PlainSelect && outputNames) { + outputItems = ((PlainSelect) first).getSelectItems(); + } + select.accept( + (net.sf.jsqlparser.statement.select.SelectVisitor) selects, null); + } + + /** The table-model name of a table reference. */ + String table(Table table) { + String schema = Identifiers.fold(table.getSchemaName()); + String name = Identifiers.fold(table.getName()); + if (schema == null) { + if (cteNames.contains(name)) { + return Identifiers.quote(name); + } + if (name.startsWith("pg_")) { + return "information_schema." + Identifiers.quote(name); + } + for (String candidate : context.searchPath()) { + if (candidate.equals("pg_catalog") || candidate.equals("information_schema")) { + continue; + } + if (context.tableExists(candidate, name)) { + return Identifiers.quote(candidate) + "." + Identifiers.quote(name); + } + } + throw new TableNotFound(name); + } + switch (schema) { + case "pg_catalog": + return "information_schema." + Identifiers.quote(name); + case "information_schema": + // IoTDB's own information_schema stays hidden from PostgreSQL clients + return "information_schema." + Identifiers.quote("pg_is_" + name); + default: + return Identifiers.quote(schema) + "." + Identifiers.quote(name); + } + } + } + + /** Statement-level rules: table names, FETCH FIRST. */ + private static final class Selects extends SelectDeParser { + private final Writer writer; + + Selects(Writer writer) { + this.writer = writer; + } + + @Override + public StringBuilder visit(Table table, S context) { + builder.append(writer.table(table)); + Alias alias = table.getAlias(); + if (alias != null) { + if (alias.getAliasColumns() != null && !alias.getAliasColumns().isEmpty()) { + throw new UnsupportedConstructException("column aliases of a table"); + } + builder.append(" AS ").append(Identifiers.quote(Identifiers.fold(alias.getName()))); + } + return builder; + } + + @Override + protected void deparseSelectItemsClause( + List> items) { + for (int i = 0; i < items.size(); i++) { + if (i > 0) { + builder.append(", "); + } + net.sf.jsqlparser.statement.select.SelectItem item = items.get(i); + if (item.getExpression() instanceof net.sf.jsqlparser.expression.NullValue) { + // PostgreSQL returns an untyped NULL output column as text + builder.append("CAST(NULL AS STRING)"); + } else { + item.getExpression().accept(writer.expressions, null); + } + Alias alias = item.getAlias(); + if (alias != null) { + // output names follow the identifier rules: unquoted aliases fold to lower case + builder.append(" AS ").append(Identifiers.quote(Identifiers.fold(alias.getName()))); + } else if (items == writer.outputItems) { + // the name PostgreSQL gives an output column without alias (the engine's are _colN) + String name = OutputNames.of(item.getExpression()); + if (name != null) { + builder.append(" AS ").append(Identifiers.quote(name)); + } + } + } + } + + @Override + public StringBuilder visit(PlainSelect select, S context) { + if (select.getFetch() != null) { + // FETCH FIRST n ROWS ONLY = LIMIT n (WITH TIES has no table-model form) + List options = select.getFetch().getFetchParameters(); + if (options != null + && options.stream().anyMatch(p -> p.toUpperCase(Locale.ROOT).contains("TIES"))) { + throw new UnsupportedConstructException("FETCH ... WITH TIES"); + } + select.setLimit( + new Limit() + .withRowCount( + select.getFetch().getExpression() == null + ? new net.sf.jsqlparser.expression.LongValue(1) + : select.getFetch().getExpression())); + select.setFetch(null); + } + return super.visit(select, context); + } + } + + /** Expression-level rules. */ + private static final class Expressions extends ExpressionDeParser { + private final Writer writer; + + Expressions(Writer writer) { + this.writer = writer; + } + + @Override + public StringBuilder visit(Column column, S context) { + Table table = column.getTable(); + String name = Identifiers.fold(column.getColumnName()); + if ((table == null || table.getName() == null) && !isQuoted(column.getColumnName())) { + // keywords PostgreSQL parses as values, not columns + switch (name) { + case "current_user": + case "session_user": + case "user": + case "current_role": + builder.append(Identifiers.literal(writer.context.user())); + return builder; + case "current_catalog": + builder.append(Identifiers.literal(writer.context.currentDatabase())); + return builder; + case "current_schema": + appendCurrentSchema(); + return builder; + case "true": + case "false": + builder.append(name.toUpperCase(Locale.ROOT)); + return builder; + default: + break; + } + } + if (table != null && table.getName() != null) { + if (table.getSchemaName() != null) { + builder.append(writer.table(table)); + } else { + builder.append(Identifiers.quote(Identifiers.fold(table.getName()))); + } + builder.append('.'); + } + builder.append(Identifiers.quote(name)); + return builder; + } + + @Override + public StringBuilder visit(Function function, S context) { + List parts = function.getMultipartName(); + if (parts.size() > 2 + || (parts.size() == 2 && !"pg_catalog".equals(Identifiers.fold(parts.get(0))))) { + throw new UnsupportedConstructException("function " + function.getName() + "()"); + } + String name = Identifiers.fold(parts.get(parts.size() - 1)); + int arguments = + function.getParameters() == null ? 0 : function.getParameters().getExpressions().size(); + switch (name) { + case "version": + builder.append(Identifiers.literal(writer.context.version())); + return builder; + case "current_database": + builder.append(Identifiers.literal(writer.context.currentDatabase())); + return builder; + case "current_schema": + appendCurrentSchema(); + return builder; + case "pg_backend_pid": + builder.append(writer.context.backendPid()); + return builder; + case "format_type": + if (arguments == 2) { + appendTypeName( + (Expression) function.getParameters().getExpressions().get(0), context, false); + return builder; + } + break; + case "pg_get_expr": + // only pg_attrdef and pg_index hold expressions, and both are empty + if (arguments == 2 || arguments == 3) { + builder.append("CAST(NULL AS STRING)"); + return builder; + } + break; + case "obj_description": + if (arguments == 1 || arguments == 2) { + List args = function.getParameters().getExpressions(); + appendDescription( + (Expression) args.get(0), + null, + arguments == 2 ? (Expression) args.get(1) : null, + context); + return builder; + } + break; + case "col_description": + if (arguments == 2) { + List args = function.getParameters().getExpressions(); + appendDescription((Expression) args.get(0), (Expression) args.get(1), null, context); + return builder; + } + break; + case "pg_encoding_to_char": + if (arguments == 1) { + appendEncodingName( + (Expression) function.getParameters().getExpressions().get(0), context); + return builder; + } + break; + case "array_to_string": + if (arguments == 2 || arguments == 3) { + Object array = function.getParameters().getExpressions().get(0); + org.apache.iotdb.postgresql.catalog.PgCatalogTables.Column column = + array instanceof Column ? catalogColumn((Column) array) : null; + if (column != null && column.array()) { + // the catalog's array columns are always NULL, and so is the result + builder.append("CAST(NULL AS STRING)"); + return builder; + } + } + break; + case "date_trunc": + if (arguments == 2) { + List args = function.getParameters().getExpressions(); + appendTemplate( + DateTimeRules.truncate( + literalText(args.get(0), "date_trunc"), + writer.context.zone(), + writer.context.precision()), + (Expression) args.get(1), + context); + return builder; + } + break; + case "date_part": + if (arguments == 2) { + List args = function.getParameters().getExpressions(); + // date_part returns double precision + builder.append("CAST("); + appendTemplate( + DateTimeRules.extract( + literalText(args.get(0), "date_part"), + writer.context.zone(), + writer.context.precision()), + (Expression) args.get(1), + context); + builder.append(" AS DOUBLE)"); + return builder; + } + break; + case "nullif": + if (arguments == 2) { + // the engine cannot evaluate NULLIF; this is its definition, without an untyped NULL + List args = function.getParameters().getExpressions(); + Expression a = (Expression) args.get(0); + Expression b = (Expression) args.get(1); + builder.append("CASE WHEN "); + a.accept(this, context); + builder.append(" <> "); + b.accept(this, context); + builder.append(" OR "); + a.accept(this, context); + builder.append(" IS NULL OR "); + b.accept(this, context); + builder.append(" IS NULL THEN "); + a.accept(this, context); + builder.append(" END"); + return builder; + } + break; + case "concat": + // PostgreSQL's concat skips NULLs and returns '' when all are NULL + builder.append("COALESCE("); + function.setName("concat"); + super.visit(function, context); + builder.append(", '')"); + return builder; + case "pg_get_userbyid": + if (arguments == 1) { + // every catalog object is shown as owned by the querying user (contract) + builder.append(Identifiers.literal(writer.context.user())); + return builder; + } + break; + case "now": + if (arguments == 0) { + builder.append("now()"); + return builder; + } + break; + default: + break; + } + if (ALWAYS_TRUE.contains(name)) { + builder.append("TRUE"); + return builder; + } + String mapped = SAME.contains(name) ? name : RENAMED.get(name); + if (mapped == null) { + throw new UnsupportedConstructException("function " + name + "()"); + } + function.setName(mapped); + return super.visit(function, context); + } + + @Override + public StringBuilder visit(BooleanValue value, S context) { + builder.append(value.getValue() ? "TRUE" : "FALSE"); + return builder; + } + + @Override + public StringBuilder visit(CastExpression cast, S context) { + String target = PgTypeNames.baseName(cast.getColDataType()); + if (TEXT_TYPES.contains(target) && cast.getLeftExpression() instanceof CastExpression) { + CastExpression inner = (CastExpression) cast.getLeftExpression(); + if (PgTypeNames.baseName(inner.getColDataType()).equals("regtype") + && !(inner.getLeftExpression() instanceof StringValue)) { + // regtype prints the type name, or the number for an unknown OID + appendTypeName(inner.getLeftExpression(), context, true); + return builder; + } + } + if (target.startsWith("reg")) { + appendRegCast(target, cast.getLeftExpression()); + return builder; + } + if (cast.getLeftExpression() instanceof StringValue + && ((StringValue) cast.getLeftExpression()).getPrefix() == null + && !target.equals("interval")) { + // literals are read as PostgreSQL reads them (time zone, formats) + String text = ((StringValue) cast.getLeftExpression()).getValue().replace("''", "'"); + switch (target) { + case "date": + appendDate(text); + return builder; + case "timestamp": + case "timestamp without time zone": + appendTimestamp(text, false); + return builder; + case "timestamptz": + case "timestamp with time zone": + appendTimestamp(text, true); + return builder; + default: + break; + } + } + String type = PgTypeNames.tableType(cast.getColDataType()); + builder.append("CAST("); + cast.getLeftExpression().accept(this, context); + builder.append(" AS ").append(type).append(')'); + return builder; + } + + @Override + public StringBuilder visit(JdbcParameter parameter, S context) { + Integer index = parameter.getIndex(); + writer.parameters.add(index == null ? writer.parameters.size() + 1 : index); + builder.append('?'); + return builder; + } + + @Override + public StringBuilder visit(StringValue value, S context) { + String prefix = value.getPrefix(); + if (prefix == null) { + if (value.getQuoteStr() == null || "'".equals(value.getQuoteStr())) { + // standard_conforming_strings: the body is already a table-model literal body + builder.append('\'').append(value.getValue()).append('\''); + } else { + builder.append(Identifiers.literal(value.getValue())); + } + return builder; + } + if (prefix.equalsIgnoreCase("E")) { + builder.append(Identifiers.literal(StringLiterals.unescape(value.getValue()))); + return builder; + } + throw new UnsupportedConstructException(prefix + "'...' literals"); + } + + @Override + public StringBuilder visit(TimeKeyExpression key, S context) { + String value = key.getStringValue().toUpperCase(Locale.ROOT).replaceAll("\\s+", " "); + if (value.startsWith("CURRENT_TIMESTAMP") || value.startsWith("LOCALTIMESTAMP")) { + if (value.contains("(")) { + throw new UnsupportedConstructException(value + " with a precision"); + } + builder.append("now()"); + } else if (value.equals("CURRENT_DATE")) { + builder.append("CAST(now() AS DATE)"); + } else { + throw new UnsupportedConstructException(value); + } + return builder; + } + + @Override + public StringBuilder visit(LikeExpression like, S context) { + switch (like.getLikeKeyWord()) { + case LIKE: + return super.visit(like, context); + case ILIKE: + builder.append("lower("); + like.getLeftExpression().accept(this, context); + builder.append(like.isNot() ? ") NOT LIKE lower(" : ") LIKE lower("); + like.getRightExpression().accept(this, context); + builder.append(')'); + if (like.getEscape() != null) { + builder.append(" ESCAPE "); + like.getEscape().accept(this, context); + } + return builder; + default: + throw new UnsupportedConstructException(like.getLikeKeyWord().name()); + } + } + + /** + * {@code 'name'::regclass} and the other object-identifier casts of a literal: the OID as a + * constant for catalog objects, as a lookup in the catalog tables for user tables. Casts of + * other expressions are refused: PostgreSQL prints such values as names, the engine would print + * numbers. + */ + private void appendRegCast(String type, Expression operand) { + if (!(operand instanceof StringValue)) { + throw new UnsupportedConstructException("casts of expressions to " + type); + } + StringValue literal = (StringValue) operand; + String text = + "E".equalsIgnoreCase(literal.getPrefix()) + ? StringLiterals.unescape(literal.getValue()) + : literal.getValue().replace("''", "'"); + switch (type) { + case "regclass": + appendRegclass(text); + return; + case "regtype": + builder.append( + StaticCatalog.typeOid(text.toLowerCase(Locale.ROOT)) + .orElseThrow(() -> new Refusal("42704", "type \"" + text + "\" does not exist"))); + return; + case "regproc": + case "regprocedure": + String proc = text.contains("(") ? text.substring(0, text.indexOf('(')) : text; + builder.append( + StaticCatalog.procOid(Identifiers.fold(proc.trim())) + .orElseThrow( + () -> new Refusal("42883", "function \"" + text + "\" does not exist"))); + return; + case "regnamespace": + String schema = Identifiers.fold(text.trim()); + if (schema.equals("pg_catalog")) { + builder.append(StaticCatalog.PG_CATALOG); + } else if (schema.equals("information_schema")) { + builder.append(StaticCatalog.INFORMATION_SCHEMA); + } else { + builder + .append("(SELECT max(pg_rn.\"oid\") FROM information_schema.\"pg_namespace\" pg_rn") + .append(" WHERE pg_rn.\"nspname\" = ") + .append(Identifiers.literal(schema)) + .append(')'); + } + return; + default: + throw new UnsupportedConstructException("casts to " + type); + } + } + + private void appendRegclass(String text) { + List parts = qualifiedName(text); + if (parts.size() > 2) { + throw new UnsupportedConstructException("cross-database references in regclass"); + } + String schema = parts.size() == 2 ? parts.get(0) : null; + String name = parts.get(parts.size() - 1); + if (schema == null || schema.equals("pg_catalog") || schema.equals("information_schema")) { + java.util.Optional system = StaticCatalog.systemRelationOid(schema, name); + if (system.isPresent()) { + builder.append(system.get()); + return; + } + if (schema != null) { + throw new Refusal("42P01", "relation \"" + text + "\" does not exist"); + } + schema = searchPathDatabase(name); + } + // a table without read permission gives NULL, as an absent one: no existence leak + builder + .append("(SELECT max(pg_rc.\"oid\") FROM information_schema.\"pg_class\" pg_rc") + .append(" JOIN information_schema.\"pg_namespace\" pg_rn") + .append(" ON pg_rc.\"relnamespace\" = pg_rn.\"oid\"") + .append(" WHERE pg_rn.\"nspname\" = ") + .append(Identifiers.literal(schema)) + .append(" AND pg_rc.\"relname\" = ") + .append(Identifiers.literal(name)) + .append(')'); + } + + private String searchPathDatabase(String table) { + for (String candidate : writer.context.searchPath()) { + if (!candidate.equals("pg_catalog") + && !candidate.equals("information_schema") + && writer.context.tableExists(candidate, table)) { + return candidate; + } + } + throw new Refusal("42P01", "relation \"" + table + "\" does not exist"); + } + + /** Splits {@code a.b} or {@code "A"."b"} into folded parts. */ + private static List qualifiedName(String text) { + List parts = new ArrayList<>(); + StringBuilder part = new StringBuilder(); + boolean quoted = false; + boolean wasQuoted = false; + for (int i = 0; i < text.length(); i++) { + char c = text.charAt(i); + if (c == '"') { + if (quoted && i + 1 < text.length() && text.charAt(i + 1) == '"') { + part.append('"'); + i++; + } else { + quoted = !quoted; + wasQuoted = true; + } + } else if (c == '.' && !quoted) { + parts.add(wasQuoted ? part.toString() : part.toString().trim().toLowerCase(Locale.ROOT)); + part.setLength(0); + wasQuoted = false; + } else { + part.append(c); + } + } + parts.add(wasQuoted ? part.toString() : part.toString().trim().toLowerCase(Locale.ROOT)); + return parts; + } + + /** {@code format_type(oid, typmod)}: the type names of the catalog (typmods are always -1). */ + private void appendTypeName(Expression oid, S context, boolean numberForUnknown) { + // CASE needs equal types; every catalog OID is below 2^31, so INT32 holds it + builder.append("CASE CAST("); + oid.accept(this, context); + builder.append(" AS INT32)"); + for (Map.Entry type : StaticCatalog.formatTypeNames().entrySet()) { + builder + .append(" WHEN ") + .append(type.getKey()) + .append(" THEN ") + .append(Identifiers.literal(type.getValue())); + } + if (numberForUnknown) { + builder.append(" ELSE CAST("); + oid.accept(this, context); + builder.append(" AS STRING) END"); + return; + } + builder.append(" ELSE CASE WHEN "); + oid.accept(this, context); + builder.append(" IS NULL THEN CAST(NULL AS STRING) ELSE '???' END END"); + } + + /** + * {@code obj_description(oid [, catalog])} and {@code col_description(table, column)}: a lookup + * in {@code pg_description}. + */ + private void appendDescription( + Expression object, Expression column, Expression catalog, S context) { + builder.append( + "(SELECT max(pg_d.\"description\") FROM information_schema.\"pg_description\" pg_d" + + " WHERE pg_d.\"objoid\" = "); + object.accept(this, context); + builder.append(" AND pg_d.\"objsubid\" = "); + if (column == null) { + builder.append('0'); + } else { + column.accept(this, context); + } + if (column != null) { + builder.append(" AND pg_d.\"classoid\" = ").append(StaticCatalog.PG_CLASS); + } else if (catalog != null) { + if (!(catalog instanceof StringValue)) { + throw new UnsupportedConstructException("obj_description with a computed catalog name"); + } + String name = Identifiers.fold(((StringValue) catalog).getValue()); + builder + .append(" AND pg_d.\"classoid\" = ") + .append( + StaticCatalog.systemRelationOid(null, name) + .orElseThrow( + () -> new Refusal("42P01", "relation \"" + name + "\" does not exist"))); + } + builder.append(')'); + } + + /** + * Comparisons: a quoted literal compared with a typed catalog column takes the column's type, + * as an untyped literal does in PostgreSQL ({@code a.attrelid = '16386'}). + */ + @Override + public StringBuilder deparse( + net.sf.jsqlparser.expression.operators.relational.OldOracleJoinBinaryExpression expression, + String operator, + S context) { + Expression left = expression.getLeftExpression(); + Expression right = expression.getRightExpression(); + String leftLiteral = typedLiteral(right, left); + String rightLiteral = typedLiteral(left, right); + if (leftLiteral == null && rightLiteral == null) { + return super.deparse(expression, operator, context); + } + if (leftLiteral != null) { + builder.append(leftLiteral); + } else { + left.accept(this, context); + } + builder.append(operator.startsWith(" ") ? operator : " " + operator + " "); + if (rightLiteral != null) { + builder.append(rightLiteral); + } else { + right.accept(this, context); + } + return builder; + } + + /** + * The literal {@code value} in the type of the catalog column {@code other}, or {@code null} if + * nothing is to change. + */ + private String typedLiteral(Expression other, Expression value) { + if (!(other instanceof Column) || !(value instanceof StringValue)) { + return null; + } + StringValue literal = (StringValue) value; + if (literal.getPrefix() != null) { + return null; + } + org.apache.tsfile.enums.TSDataType type = catalogColumnType((Column) other); + if (type == null) { + return null; + } + String text = literal.getValue().replace("''", "'").trim(); + try { + switch (type) { + case INT64: + case INT32: + return String.valueOf(Long.parseLong(text)); + case FLOAT: + case DOUBLE: + return String.valueOf(Double.parseDouble(text)); + case BOOLEAN: + String b = text.toLowerCase(Locale.ROOT); + if (Arrays.asList("t", "true", "y", "yes", "on", "1").contains(b)) { + return "TRUE"; + } + if (Arrays.asList("f", "false", "n", "no", "off", "0").contains(b)) { + return "FALSE"; + } + throw new NumberFormatException(text); + default: + return null; + } + } catch (NumberFormatException e) { + throw new Refusal( + "22P02", + "invalid input syntax for type " + + type.name().toLowerCase(Locale.ROOT) + + ": \"" + + text + + "\""); + } + } + + /** The type of a column of a catalog table, or {@code null} if it is not one. */ + private org.apache.tsfile.enums.TSDataType catalogColumnType(Column column) { + org.apache.iotdb.postgresql.catalog.PgCatalogTables.Column found = catalogColumn(column); + return found == null ? null : found.type(); + } + + /** The catalog column a column reference names, or {@code null} if unknown or ambiguous. */ + private org.apache.iotdb.postgresql.catalog.PgCatalogTables.Column catalogColumn( + Column column) { + String name = Identifiers.fold(column.getColumnName()); + Table table = column.getTable(); + List candidates = new ArrayList<>(); + if (table != null && table.getName() != null) { + String catalog = writer.qualifiedCatalogTables.get(Identifiers.fold(table.getName())); + if (catalog == null) { + return null; + } + candidates.add(catalog); + } else { + if (writer.catalogTables == null) { + // the column may belong to a user table + return null; + } + for (String catalog : writer.catalogTables.values()) { + if (catalog != null) { + candidates.add(catalog); + } + } + } + org.apache.iotdb.postgresql.catalog.PgCatalogTables.Column found = null; + for (String catalog : candidates) { + for (org.apache.iotdb.postgresql.catalog.PgCatalogTables.Column c : + org.apache.iotdb.postgresql.catalog.PgCatalogTables.tables().get(catalog)) { + if (c.name().equals(name)) { + if (found != null && (found.type() != c.type() || found.array() != c.array())) { + return null; + } + found = c; + } + } + } + return found; + } + + /** + * PostgreSQL gives a bare {@code NULL} result of a CASE the type of the other results; the + * engine wants equal types, so the NULL is written as a cast when another result's type is + * known. + */ + @Override + public StringBuilder visit( + net.sf.jsqlparser.expression.CaseExpression expression, S context) { + List whens = expression.getWhenClauses(); + String type = null; + for (net.sf.jsqlparser.expression.WhenClause when : whens) { + type = type != null ? type : pgTypeOf(when.getThenExpression()); + } + type = type != null ? type : pgTypeOf(expression.getElseExpression()); + if (type != null) { + for (net.sf.jsqlparser.expression.WhenClause when : whens) { + if (when.getThenExpression() instanceof net.sf.jsqlparser.expression.NullValue) { + when.setThenExpression(typedNull(type)); + } + } + if (expression.getElseExpression() instanceof net.sf.jsqlparser.expression.NullValue) { + expression.setElseExpression(typedNull(type)); + } + } + return super.visit(expression, context); + } + + private static Expression typedNull(String pgType) { + net.sf.jsqlparser.expression.CastExpression cast = + new net.sf.jsqlparser.expression.CastExpression(); + cast.setLeftExpression(new net.sf.jsqlparser.expression.NullValue()); + cast.setColDataType(new net.sf.jsqlparser.statement.create.table.ColDataType(pgType)); + return cast; + } + + /** The PostgreSQL type name of an expression whose type is known here, else {@code null}. */ + private String pgTypeOf(Expression expression) { + if (expression instanceof StringValue) { + return "text"; + } + if (expression instanceof net.sf.jsqlparser.expression.LongValue) { + long value = ((net.sf.jsqlparser.expression.LongValue) expression).getValue(); + return value == (int) value ? "int4" : "int8"; + } + if (expression instanceof net.sf.jsqlparser.expression.DoubleValue) { + return "float8"; + } + if (expression instanceof BooleanValue) { + return "bool"; + } + if (expression instanceof CastExpression) { + return ((CastExpression) expression).getColDataType().getDataType(); + } + if (expression instanceof net.sf.jsqlparser.expression.CaseExpression) { + net.sf.jsqlparser.expression.CaseExpression nested = + (net.sf.jsqlparser.expression.CaseExpression) expression; + for (net.sf.jsqlparser.expression.WhenClause when : nested.getWhenClauses()) { + String type = pgTypeOf(when.getThenExpression()); + if (type != null) { + return type; + } + } + return pgTypeOf(nested.getElseExpression()); + } + if (expression instanceof Column) { + org.apache.tsfile.enums.TSDataType type = catalogColumnType((Column) expression); + if (type == null) { + return null; + } + switch (type) { + case INT32: + return "int4"; + case INT64: + return "int8"; + case FLOAT: + return "float4"; + case DOUBLE: + return "float8"; + case BOOLEAN: + return "bool"; + default: + return "text"; + } + } + return null; + } + + /** + * {@code a ~ 'pattern'} and its variants, for patterns with an exact LIKE form (psql's {@code + * ^(name)$}, {@code ^pg_}); others are refused. + */ + @Override + public StringBuilder visit( + net.sf.jsqlparser.expression.operators.relational.RegExpMatchOperator match, S context) { + Expression pattern = match.getRightExpression(); + if (!(pattern instanceof StringValue) || ((StringValue) pattern).getPrefix() != null) { + throw new UnsupportedConstructException("regular expressions that are not literals"); + } + String regex = ((StringValue) pattern).getValue().replace("''", "'"); + RegexPatterns.LikeForm like = RegexPatterns.toLike(regex); + if (like == null) { + throw new UnsupportedConstructException("the regular expression '" + regex + "'"); + } + String type = match.getOperatorType().name(); + boolean negated = type.startsWith("NOT_"); + boolean insensitive = type.endsWith("CASEINSENSITIVE"); + builder.append(negated ? "NOT (" : "("); + if (like.exact()) { + appendOperand(match.getLeftExpression(), insensitive, context); + if (like.literals.size() == 1) { + builder.append(" = "); + appendPattern(like.literals.get(0), insensitive); + } else { + builder.append(" IN ("); + for (int i = 0; i < like.literals.size(); i++) { + builder.append(i > 0 ? ", " : ""); + appendPattern(like.literals.get(i), insensitive); + } + builder.append(')'); + } + } else { + for (int i = 0; i < like.patterns.size(); i++) { + builder.append(i > 0 ? " OR " : ""); + appendOperand(match.getLeftExpression(), insensitive, context); + builder.append(" LIKE "); + appendPattern(like.patterns.get(i), insensitive); + if (like.escaped) { + builder.append(" ESCAPE '\\'"); + } + } + } + builder.append(')'); + return builder; + } + + private void appendOperand(Expression operand, boolean lower, S context) { + if (lower) { + builder.append("lower("); + } + operand.accept(this, context); + if (lower) { + builder.append(')'); + } + } + + private void appendPattern(String text, boolean lower) { + builder.append(Identifiers.literal(lower ? text.toLowerCase(Locale.ROOT) : text)); + } + + /** {@code pg_encoding_to_char(n)}: PostgreSQL's encoding names, '' for unknown numbers. */ + private void appendEncodingName(Expression encoding, S context) { + builder.append("CASE CAST("); + encoding.accept(this, context); + builder.append(" AS INT32)"); + for (int i = 0; i < ENCODINGS.length; i++) { + builder.append(" WHEN ").append(i).append(" THEN '").append(ENCODINGS[i]).append('\''); + } + builder.append(" ELSE CASE WHEN "); + encoding.accept(this, context); + builder.append(" IS NULL THEN CAST(NULL AS STRING) ELSE '' END END"); + } + + /** True while writing the sub-query of IN or EXISTS (not a scalar sub-query). */ + private boolean nonScalar; + + @Override + public StringBuilder visit( + net.sf.jsqlparser.expression.operators.relational.ExistsExpression exists, S context) { + boolean saved = nonScalar; + nonScalar = true; + try { + return super.visit(exists, context); + } finally { + nonScalar = saved; + } + } + + @Override + public StringBuilder visit( + net.sf.jsqlparser.expression.operators.relational.InExpression in, S context) { + boolean saved = nonScalar; + nonScalar = true; + try { + return super.visit(in, context); + } finally { + nonScalar = saved; + } + } + + /** + * A scalar sub-query whose WHERE has conditions on the outer row only ({@code AND a.atthasdef} + * in psql's {@code \d}): the engine cannot plan those as join criteria. {@code (SELECT x ... + * WHERE c AND p)} equals {@code CASE WHEN p THEN (SELECT x ... WHERE c) END} when the sub-query + * has no aggregates (no rows means NULL either way). + */ + @Override + public StringBuilder visit(net.sf.jsqlparser.statement.select.Select any, S context) { + if (!(any instanceof net.sf.jsqlparser.statement.select.ParenthesedSelect)) { + return super.visit(any, context); + } + net.sf.jsqlparser.statement.select.ParenthesedSelect select = + (net.sf.jsqlparser.statement.select.ParenthesedSelect) any; + boolean scalar = !nonScalar; + nonScalar = false; + try { + PlainSelect inner = + select.getSelect() instanceof PlainSelect ? (PlainSelect) select.getSelect() : null; + if (scalar + && inner != null + && inner.getWhere() instanceof BooleanValue + && !((BooleanValue) inner.getWhere()).getValue() + && inner.getSelectItems().size() == 1 + && inner.getGroupBy() == null + && !EmptyRelations.hasAggregate(inner.getSelectItems())) { + // a scalar sub-query without rows is NULL (EmptyRelations wrote a typed NULL) + inner.getSelectItems().get(0).getExpression().accept(this, context); + return builder; + } + List outer = + scalar && inner != null ? OuterConditions.split(inner) : Collections.emptyList(); + if (outer.isEmpty()) { + return super.visit(any, context); + } + Expression where = inner.getWhere(); + inner.setWhere(OuterConditions.remaining(where, outer)); + try { + builder.append("CASE WHEN "); + OuterConditions.and(outer).accept(this, context); + builder.append(" THEN "); + super.visit(any, context); + builder.append(" END"); + } finally { + inner.setWhere(where); + } + return builder; + } finally { + nonScalar = !scalar; + } + } + + /** The text of a string literal argument (unit and field names). */ + private String literalText(Object argument, String function) { + if (!(argument instanceof StringValue)) { + throw new UnsupportedConstructException(function + "() with a computed unit"); + } + return ((StringValue) argument).getValue(); + } + + /** + * Writes {@code template} with its {@code %s} replaced by {@code argument}: a column as it is + * (so the engine still sees {@code date_bin(..., time)}), anything else cast to TIMESTAMP. + */ + private void appendTemplate(String template, Expression argument, S context) { + String[] parts = template.split("%s", -1); + for (int i = 0; i < parts.length; i++) { + builder.append(parts[i]); + if (i + 1 < parts.length) { + if (argument instanceof Column || isNow(argument)) { + argument.accept(this, context); + } else { + builder.append("CAST("); + argument.accept(this, context); + builder.append(" AS TIMESTAMP)"); + } + } + } + } + + private static boolean isNow(Expression expression) { + if (expression instanceof TimeKeyExpression) { + String key = ((TimeKeyExpression) expression).getStringValue().toUpperCase(Locale.ROOT); + return key.startsWith("CURRENT_TIMESTAMP") || key.startsWith("LOCALTIMESTAMP"); + } + return expression instanceof Function + && Identifiers.fold(((Function) expression).getName()).equals("now") + && (((Function) expression).getParameters() == null + || ((Function) expression).getParameters().getExpressions().isEmpty()); + } + + @Override + public StringBuilder visit( + net.sf.jsqlparser.expression.ExtractExpression extract, S context) { + appendTemplate( + DateTimeRules.extract( + extract.getName(), writer.context.zone(), writer.context.precision()), + extract.getExpression(), + context); + return builder; + } + + /** An interval literal ({@code interval '2 hours'}, {@code '2 hours'::interval}), or null. */ + private static String intervalText(Expression expression) { + if (expression instanceof net.sf.jsqlparser.expression.IntervalExpression) { + net.sf.jsqlparser.expression.IntervalExpression interval = + (net.sf.jsqlparser.expression.IntervalExpression) expression; + String parameter = interval.getParameter(); + if (parameter == null) { + throw new UnsupportedConstructException("INTERVAL with a computed value"); + } + String text = parameter.trim(); + if (text.length() >= 2 && text.startsWith("'") && text.endsWith("'")) { + text = text.substring(1, text.length() - 1).replace("''", "'"); + } + if (interval.getIntervalQualifier() != null) { + String qualifier = interval.getIntervalQualifier().toString().trim(); + if (qualifier.toUpperCase(Locale.ROOT).contains(" TO ")) { + throw new UnsupportedConstructException("INTERVAL ... TO ..."); + } + return text + " " + qualifier; + } + return interval.getIntervalType() == null ? text : text + " " + interval.getIntervalType(); + } + if (expression instanceof CastExpression + && PgTypeNames.baseName(((CastExpression) expression).getColDataType()).equals("interval") + && ((CastExpression) expression).getLeftExpression() instanceof StringValue) { + return ((StringValue) ((CastExpression) expression).getLeftExpression()) + .getValue() + .replace("''", "'"); + } + return null; + } + + @Override + public StringBuilder visit( + net.sf.jsqlparser.expression.operators.arithmetic.Addition addition, S context) { + String right = intervalText(addition.getRightExpression()); + String left = intervalText(addition.getLeftExpression()); + if (right != null) { + appendShifted(addition.getLeftExpression(), "+", right, context); + return builder; + } + if (left != null) { + appendShifted(addition.getRightExpression(), "+", left, context); + return builder; + } + return super.visit(addition, context); + } + + @Override + public StringBuilder visit( + net.sf.jsqlparser.expression.operators.arithmetic.Subtraction subtraction, S context) { + String right = intervalText(subtraction.getRightExpression()); + if (right != null) { + appendShifted(subtraction.getLeftExpression(), "-", right, context); + return builder; + } + if (intervalText(subtraction.getLeftExpression()) != null) { + throw new UnsupportedConstructException("an interval minus a timestamp"); + } + return super.visit(subtraction, context); + } + + /** {@code timestamp ± interval}: the engine adds whole timestamp units to a TIMESTAMP. */ + private void appendShifted(Expression time, String operator, String interval, S context) { + long units = + DateTimeRules.intervalUnits(interval, writer.context.zone(), writer.context.precision()); + builder.append('('); + if (isNow(time)) { + time.accept(this, context); + } else { + // a DATE operand becomes a timestamp first, as in PostgreSQL + builder.append("CAST("); + time.accept(this, context); + builder.append(" AS TIMESTAMP)"); + } + builder.append(' ').append(operator).append(' ').append(units).append(')'); + } + + @Override + public StringBuilder visit( + net.sf.jsqlparser.expression.IntervalExpression interval, S context) { + throw new UnsupportedConstructException( + "INTERVAL values outside timestamp + interval and timestamp - interval"); + } + + @Override + public StringBuilder visit( + net.sf.jsqlparser.expression.DateTimeLiteralExpression literal, S context) { + String value = literal.getValue().trim(); + if (value.length() >= 2 && value.startsWith("'") && value.endsWith("'")) { + value = value.substring(1, value.length() - 1); + } + switch (literal.getType()) { + case DATE: + appendDate(value); + break; + case TIMESTAMP: + case DATETIME: + appendTimestamp(value, false); + break; + case TIMESTAMPTZ: + appendTimestamp(value, true); + break; + default: + throw new UnsupportedConstructException(literal.getType() + " literals"); + } + return builder; + } + + private void appendDate(String text) { + builder + .append("CAST(") + .append(Identifiers.literal(DateTimeRules.dateLiteral(text))) + .append(" AS DATE)"); + } + + private void appendTimestamp(String text, boolean withTimeZone) { + builder + .append("CAST(") + .append( + DateTimeRules.timestampLiteral( + text, withTimeZone, writer.context.zone(), writer.context.precision())) + .append(" AS TIMESTAMP)"); + } + + /** + * {@code (current_schemas(b))[n]} (pgJDBC's {@code getSchemas}): the n-th schema of the + * effective search path, with {@code pg_catalog} first if implicit schemas are included. Other + * array subscripts are refused. + */ + @Override + public StringBuilder visit(net.sf.jsqlparser.expression.ArrayExpression array, S context) { + Expression object = array.getObjExpression(); + while (object + instanceof net.sf.jsqlparser.expression.operators.relational.ParenthesedExpressionList + && ((net.sf.jsqlparser.expression.operators.relational.ParenthesedExpressionList) + object) + .size() + == 1) { + object = + (Expression) + ((net.sf.jsqlparser.expression.operators.relational.ParenthesedExpressionList) + object) + .get(0); + } + if (object instanceof Function + && array.getStartIndexExpression() == null + && array.getIndexExpression() instanceof net.sf.jsqlparser.expression.LongValue) { + Function function = (Function) object; + List parts = function.getMultipartName(); + String name = Identifiers.fold(parts.get(parts.size() - 1)); + if (name.equals("current_schemas") + && function.getParameters() != null + && function.getParameters().size() == 1 + && function.getParameters().get(0) instanceof BooleanValue) { + List schemas = new ArrayList<>(); + if (((BooleanValue) function.getParameters().get(0)).getValue()) { + schemas.add("pg_catalog"); + } + for (String schema : writer.context.searchPath()) { + if (!schemas.contains(schema)) { + schemas.add(schema); + } + } + long index = + ((net.sf.jsqlparser.expression.LongValue) array.getIndexExpression()).getValue(); + builder.append( + index >= 1 && index <= schemas.size() + ? Identifiers.literal(schemas.get((int) index - 1)) + : "CAST(NULL AS STRING)"); + return builder; + } + } + throw new UnsupportedConstructException("array subscripts"); + } + + /** + * JSqlParser reads {@code day}, {@code month}, ... as date units where PostgreSQL sees column + * names; outside EXTRACT and INTERVAL (handled before) they are columns. + */ + @Override + public StringBuilder visit( + net.sf.jsqlparser.expression.DateUnitExpression unit, S context) { + builder.append(Identifiers.quote(unit.toString().toLowerCase(Locale.ROOT))); + return builder; + } + + /** {@code a || b}: NULL if either is NULL (the engine's concat would skip it). */ + @Override + public StringBuilder visit( + net.sf.jsqlparser.expression.operators.arithmetic.Concat concat, S context) { + builder.append("CASE WHEN "); + concat.getLeftExpression().accept(this, context); + builder.append(" IS NULL OR "); + concat.getRightExpression().accept(this, context); + builder.append(" IS NULL THEN CAST(NULL AS STRING) ELSE concat("); + concat.getLeftExpression().accept(this, context); + builder.append(", "); + concat.getRightExpression().accept(this, context); + builder.append(") END"); + return builder; + } + + private void appendCurrentSchema() { + for (String schema : writer.context.searchPath()) { + builder.append(Identifiers.literal(schema)); + return; + } + builder.append("NULL"); + } + + private static boolean isQuoted(String identifier) { + return identifier != null && identifier.startsWith("\""); + } + } +} diff --git a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/sql/PgTypeNames.java b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/sql/PgTypeNames.java new file mode 100644 index 0000000000000..d420f81bcc048 --- /dev/null +++ b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/sql/PgTypeNames.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.sql; + +import net.sf.jsqlparser.statement.create.table.ColDataType; + +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +/** PostgreSQL type names in casts, mapped to table-model types. */ +final class PgTypeNames { + + private static final Map TYPES = new HashMap<>(); + + static { + for (String name : new String[] {"bool", "boolean"}) { + TYPES.put(name, "BOOLEAN"); + } + for (String name : new String[] {"int2", "smallint", "int4", "int", "integer"}) { + TYPES.put(name, "INT32"); + } + for (String name : new String[] {"int8", "bigint", "oid"}) { + TYPES.put(name, "INT64"); + } + for (String name : new String[] {"float4", "real"}) { + TYPES.put(name, "FLOAT"); + } + for (String name : new String[] {"float8", "double precision", "float"}) { + TYPES.put(name, "DOUBLE"); + } + for (String name : + new String[] { + "text", "varchar", "character varying", "char", "character", "bpchar", "name" + }) { + TYPES.put(name, "STRING"); + } + for (String name : + new String[] { + "timestamptz", "timestamp", "timestamp with time zone", "timestamp without time zone" + }) { + TYPES.put(name, "TIMESTAMP"); + } + TYPES.put("date", "DATE"); + TYPES.put("bytea", "BLOB"); + } + + private PgTypeNames() {} + + /** The name of a cast target: lower case, without {@code pg_catalog.} and modifiers. */ + static String baseName(ColDataType type) { + String name = type.getDataType().toLowerCase(Locale.ROOT).replaceAll("\\s+", " ").trim(); + if (name.startsWith("pg_catalog.")) { + name = name.substring("pg_catalog.".length()); + } + int open = name.indexOf('('); + if (open >= 0) { + name = name.substring(0, open).trim(); + } + return Identifiers.fold(name); + } + + /** The table-model type for a cast target. */ + static String tableType(ColDataType type) { + String name = type.getDataType().toLowerCase(Locale.ROOT).replaceAll("\\s+", " ").trim(); + if (name.startsWith("pg_catalog.")) { + name = name.substring("pg_catalog.".length()); + } + boolean modifiers = false; + int open = name.indexOf('('); + if (open >= 0) { + // JSqlParser keeps the type modifiers in the name: "varchar (10)", "numeric (10, 2)" + modifiers = true; + name = name.substring(0, open).trim(); + } + name = Identifiers.fold(name); + if (type.getArrayData() != null && !type.getArrayData().isEmpty()) { + throw new UnsupportedConstructException("casts to array types"); + } + String mapped = TYPES.get(name); + if (mapped == null) { + throw new UnsupportedConstructException("casts to " + name); + } + List arguments = type.getArgumentsStringList(); + boolean hasArguments = modifiers || (arguments != null && !arguments.isEmpty()); + if (hasArguments && !mapped.equals("STRING") && !name.startsWith("timestamp")) { + throw new UnsupportedConstructException("casts to " + name + " with modifiers"); + } + return mapped; + } +} diff --git a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/sql/PreParse.java b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/sql/PreParse.java new file mode 100644 index 0000000000000..07f639111f93a --- /dev/null +++ b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/sql/PreParse.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.sql; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Textual rewrites before parsing, for PostgreSQL syntax JSqlParser does not read and that has a + * plain equivalent: {@code OPERATOR(pg_catalog.op)} is the operator {@code op} (psql qualifies its + * operators), and {@code COLLATE pg_catalog.default} is the default collation, which changes + * nothing. Only text outside literals, quoted identifiers and comments is touched. + */ +final class PreParse { + + private static final Pattern OPERATOR = + Pattern.compile( + "OPERATOR\\s*\\(\\s*(?:pg_catalog\\s*\\.\\s*)?([~!<>=*+\\-/%^@#&|]+)\\s*\\)", + Pattern.CASE_INSENSITIVE); + private static final Pattern DEFAULT_COLLATION = + Pattern.compile( + "\\s+COLLATE\\s+(?:pg_catalog\\s*\\.\\s*)?(?:default|\"default\")\\b", + Pattern.CASE_INSENSITIVE); + + private PreParse() {} + + static String rewrite(String sql) { + StringBuilder result = new StringBuilder(sql.length()); + StringBuilder code = new StringBuilder(); + int i = 0; + while (i < sql.length()) { + char c = sql.charAt(i); + int end = -1; + if (c == '\'' || c == '"') { + boolean escapes = + c == '\'' + && i > 0 + && (sql.charAt(i - 1) == 'E' || sql.charAt(i - 1) == 'e') + && (i == 1 || !Character.isLetterOrDigit(sql.charAt(i - 2))); + end = closing(sql, i, c, escapes); + } else if (c == '-' && i + 1 < sql.length() && sql.charAt(i + 1) == '-') { + int newline = sql.indexOf('\n', i); + end = newline < 0 ? sql.length() : newline; + } else if (c == '/' && i + 1 < sql.length() && sql.charAt(i + 1) == '*') { + int close = sql.indexOf("*/", i + 2); + end = close < 0 ? sql.length() : close + 2; + } + if (end < 0) { + code.append(c); + i++; + continue; + } + result.append(rewriteCode(code.toString())); + code.setLength(0); + result.append(sql, i, end); + i = end; + } + result.append(rewriteCode(code.toString())); + return result.toString(); + } + + /** The index after the closing quote (a doubled quote, or in E'' a backslash, escapes it). */ + private static int closing(String sql, int start, char quote, boolean escapes) { + int i = start + 1; + while (i < sql.length()) { + if (escapes && sql.charAt(i) == '\\') { + i += 2; + continue; + } + if (sql.charAt(i) == quote) { + if (i + 1 < sql.length() && sql.charAt(i + 1) == quote) { + i += 2; + continue; + } + return i + 1; + } + i++; + } + return sql.length(); + } + + private static String rewriteCode(String code) { + Matcher m = OPERATOR.matcher(code); + String rewritten = m.replaceAll(" $1 "); + return DEFAULT_COLLATION.matcher(rewritten).replaceAll(""); + } +} diff --git a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/sql/RegexPatterns.java b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/sql/RegexPatterns.java new file mode 100644 index 0000000000000..5c3b10ccd0c05 --- /dev/null +++ b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/sql/RegexPatterns.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.sql; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * Regular expressions that have an exact LIKE form: the patterns psql and catalog queries build, + * such as {@code ^(sensors)$}, {@code ^pg_}, {@code ^(sens.*)$} or {@code ^(a|b)$}. Supported: + * literal characters (backslash escapes punctuation), {@code .} (one character, as {@code _}), + * {@code .*} (any characters, as {@code %}), the anchors {@code ^} and {@code $}, and one group of + * alternatives. Anything else has no exact LIKE form. + */ +final class RegexPatterns { + + private static final String META = ".^$|()[]{}*+?\\"; + + /** One LIKE pattern per alternative; {@link #exact} if none of them has a wildcard. */ + static final class LikeForm { + final List patterns; + final List literals; + final boolean escaped; + + LikeForm(List patterns, List literals, boolean escaped) { + this.patterns = patterns; + this.literals = literals; + this.escaped = escaped; + } + + /** True if every alternative is a fixed string ({@link #literals}). */ + boolean exact() { + return literals != null; + } + } + + private RegexPatterns() {} + + /** + * The LIKE form of {@code regex}, or {@code null} if it has none. + * + *

LIKE patterns use {@code \} as the escape character when {@link LikeForm#escaped}. + */ + static LikeForm toLike(String regex) { + String body = regex; + boolean start = body.startsWith("^"); + if (start) { + body = body.substring(1); + } + boolean end = body.endsWith("$") && !body.endsWith("\\$"); + if (end) { + body = body.substring(0, body.length() - 1); + } + List alternatives; + if (body.startsWith("(") && body.endsWith(")") && body.indexOf('(', 1) < 0) { + String inner = body.substring(1, body.length() - 1); + if (inner.indexOf(')') >= 0) { + return null; + } + alternatives = split(inner); + if (alternatives == null) { + return null; + } + } else { + alternatives = Collections.singletonList(body); + } + if (alternatives.size() > 1 && (!start || !end)) { + // a|b without both anchors binds the anchors to the first and last alternative only + return null; + } + List patterns = new ArrayList<>(); + List literals = new ArrayList<>(); + boolean escaped = false; + for (String alternative : alternatives) { + StringBuilder like = new StringBuilder(); + StringBuilder literal = new StringBuilder(); + boolean wildcard = !start || !end; + if (!start) { + like.append('%'); + } + for (int i = 0; i < alternative.length(); i++) { + char c = alternative.charAt(i); + if (c == '\\') { + if (i + 1 >= alternative.length()) { + return null; + } + char next = alternative.charAt(++i); + if (Character.isLetterOrDigit(next)) { + // \d, \w, \m, ... are classes or constraints, not literals + return null; + } + c = next; + } else if (c == '.') { + if (i + 1 < alternative.length() && alternative.charAt(i + 1) == '*') { + like.append('%'); + i++; + } else { + like.append('_'); + } + wildcard = true; + continue; + } else if (META.indexOf(c) >= 0) { + return null; + } + if (c == '%' || c == '_' || c == '\\') { + like.append('\\'); + escaped = true; + } + like.append(c); + literal.append(c); + } + if (!end) { + like.append('%'); + } + patterns.add(like.toString()); + literals.add(wildcard ? null : literal.toString()); + } + boolean exact = literals.stream().allMatch(l -> l != null); + return new LikeForm(patterns, exact ? literals : null, escaped); + } + + /** Splits alternatives at top-level {@code |}; {@code null} if a part is malformed. */ + private static List split(String inner) { + List parts = new ArrayList<>(); + StringBuilder part = new StringBuilder(); + for (int i = 0; i < inner.length(); i++) { + char c = inner.charAt(i); + if (c == '\\' && i + 1 < inner.length()) { + part.append(c).append(inner.charAt(++i)); + } else if (c == '|') { + parts.add(part.toString()); + part.setLength(0); + } else { + part.append(c); + } + } + parts.add(part.toString()); + for (String p : parts) { + if (p.isEmpty()) { + return null; + } + } + return parts; + } +} diff --git a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/sql/StringLiterals.java b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/sql/StringLiterals.java new file mode 100644 index 0000000000000..2fdef6769aa46 --- /dev/null +++ b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/sql/StringLiterals.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.sql; + +/** PostgreSQL's escape string literals ({@code E'...'}). */ +final class StringLiterals { + + private StringLiterals() {} + + /** + * The value of the body of an {@code E'...'} literal as written between the quotes: {@code ''} + * and C-style backslash escapes (backspace, form feed, newline, carriage return, tab, octal, + * hexadecimal, 16- and 32-bit unicode; any other character escaped by itself). + */ + static String unescape(String body) { + StringBuilder out = new StringBuilder(body.length()); + for (int i = 0; i < body.length(); i++) { + char c = body.charAt(i); + if (c == '\'' && i + 1 < body.length() && body.charAt(i + 1) == '\'') { + out.append('\''); + i++; + continue; + } + if (c != '\\' || i + 1 >= body.length()) { + out.append(c); + continue; + } + char e = body.charAt(++i); + switch (e) { + case 'b': + out.append('\b'); + break; + case 'f': + out.append('\f'); + break; + case 'n': + out.append('\n'); + break; + case 'r': + out.append('\r'); + break; + case 't': + out.append('\t'); + break; + case 'x': + { + int end = i + 1; + while (end < body.length() && end < i + 3 && isHex(body.charAt(end))) { + end++; + } + if (end == i + 1) { + out.append('x'); + } else { + out.append((char) Integer.parseInt(body.substring(i + 1, end), 16)); + i = end - 1; + } + break; + } + case 'u': + case 'U': + { + int digits = e == 'u' ? 4 : 8; + if (i + digits >= body.length()) { + throw new UnsupportedConstructException("incomplete unicode escape"); + } + String hex = body.substring(i + 1, i + 1 + digits); + for (char h : hex.toCharArray()) { + if (!isHex(h)) { + throw new UnsupportedConstructException("invalid unicode escape"); + } + } + out.appendCodePoint(Integer.parseInt(hex, 16)); + i += digits; + break; + } + default: + if (e >= '0' && e <= '7') { + int end = i; + while (end < body.length() + && end < i + 3 + && body.charAt(end) >= '0' + && body.charAt(end) <= '7') { + end++; + } + out.append((char) Integer.parseInt(body.substring(i, end), 8)); + i = end - 1; + } else { + out.append(e); + } + } + } + return out.toString(); + } + + private static boolean isHex(char c) { + return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); + } +} diff --git a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/sql/TranslationContext.java b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/sql/TranslationContext.java new file mode 100644 index 0000000000000..3c18571a7c5b3 --- /dev/null +++ b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/sql/TranslationContext.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.sql; + +import java.util.List; + +/** What the translator needs to know about the session and the schema. */ +public interface TranslationContext { + + /** The session's {@code search_path} schemas in order, folded, without {@code $user}. */ + List searchPath(); + + /** True if the database (PostgreSQL schema) has a table of this name (both folded). */ + boolean tableExists(String database, String table); + + /** The logged-in user. */ + String user(); + + /** The value of {@code version()}. */ + String version(); + + /** The value of {@code current_database()}. */ + String currentDatabase(); + + /** The value of {@code pg_backend_pid()}. */ + int backendPid(); + + /** The session time zone (date and time rules). */ + default java.time.ZoneId zone() { + return java.time.ZoneOffset.UTC; + } + + /** The database's timestamp precision (interval lengths, origins). */ + default org.apache.iotdb.postgresql.types.TimestampPrecision precision() { + return org.apache.iotdb.postgresql.types.TimestampPrecision.of("ms"); + } +} diff --git a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/sql/UnsupportedConstructException.java b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/sql/UnsupportedConstructException.java new file mode 100644 index 0000000000000..637fbb283f81c --- /dev/null +++ b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/sql/UnsupportedConstructException.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.sql; + +/** + * A PostgreSQL construct without an exact translation to table-model SQL: the statement is refused + * with 0A000 naming the construct, never run with a different meaning. + */ +public final class UnsupportedConstructException extends RuntimeException { + + private static final long serialVersionUID = 1L; + + private final String construct; + + public UnsupportedConstructException(String construct) { + super(construct + " is not supported"); + this.construct = construct; + } + + /** The construct, as named in the error message. */ + public String construct() { + return construct; + } +} diff --git a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/types/ParameterDecoder.java b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/types/ParameterDecoder.java new file mode 100644 index 0000000000000..31f4dce7d674e --- /dev/null +++ b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/types/ParameterDecoder.java @@ -0,0 +1,257 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.types; + +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.BinaryLiteral; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.BooleanLiteral; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.DoubleLiteral; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Literal; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.LongLiteral; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.NullLiteral; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.StringLiteral; +import org.apache.iotdb.postgresql.protocol.PgException; +import org.apache.iotdb.postgresql.protocol.SqlState; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.OffsetDateTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeFormatterBuilder; +import java.time.format.DateTimeParseException; +import java.time.temporal.ChronoField; +import java.util.Locale; +import java.util.regex.Pattern; + +/** + * Turns Bind parameter values into table-model literals: by the parameter's type OID in text or + * binary format. Timestamps become the epoch value in the database's precision; parameters without + * a type (OID 0) are recognised from their text (integer, decimal, timestamp, else text), as + * PostgreSQL would infer them from a comparison with a column of that type. + */ +public final class ParameterDecoder { + + private static final Pattern INTEGER = Pattern.compile("[+-]?\\d{1,18}"); + private static final Pattern DECIMAL = + Pattern.compile("[+-]?(\\d+\\.\\d*|\\.\\d+|\\d+)([eE][+-]?\\d+)?"); + private static final Pattern TIMESTAMP = + Pattern.compile("\\d{4}-\\d{2}-\\d{2}[ T]\\d{2}:\\d{2}(:\\d{2}(\\.\\d+)?)?.*"); + + /** PostgreSQL's text form: {@code 2026-11-19 10:15:30.123456+01}, offset optional. */ + private static final DateTimeFormatter PG_TIMESTAMP = + new DateTimeFormatterBuilder() + .parseCaseInsensitive() + .append(DateTimeFormatter.ISO_LOCAL_DATE) + .optionalStart() + .appendLiteral(' ') + .optionalEnd() + .optionalStart() + .appendLiteral('T') + .optionalEnd() + .appendValue(ChronoField.HOUR_OF_DAY, 2) + .appendLiteral(':') + .appendValue(ChronoField.MINUTE_OF_HOUR, 2) + .optionalStart() + .appendLiteral(':') + .appendValue(ChronoField.SECOND_OF_MINUTE, 2) + .optionalStart() + .appendFraction(ChronoField.NANO_OF_SECOND, 0, 9, true) + .optionalEnd() + .optionalEnd() + .optionalStart() + .appendOffset("+HH:mm", "Z") + .optionalEnd() + .optionalStart() + .appendOffset("+HH", "Z") + .optionalEnd() + .toFormatter(Locale.ROOT); + + private static final long PG_EPOCH_MICROS = 946_684_800_000_000L; + private static final int PG_EPOCH_DAYS = 10_957; + + private final ZoneId zone; + private final TimestampPrecision precision; + + public ParameterDecoder(ZoneId zone, TimestampPrecision precision) { + this.zone = zone; + this.precision = precision; + } + + /** + * The literal for one parameter value. + * + * @param typeOid the type from Parse, 0 if unspecified + * @param binary whether the value is in binary format + * @param value the value, {@code null} for SQL NULL + */ + public Literal decode(int typeOid, boolean binary, byte[] value) throws PgException { + if (value == null) { + return new NullLiteral(); + } + try { + return binary + ? binary(typeOid, value) + : text(typeOid, new String(value, StandardCharsets.UTF_8)); + } catch (RuntimeException e) { + throw new PgException( + SqlState.INVALID_TEXT_REPRESENTATION, + "invalid value for a parameter of type " + typeName(typeOid) + ": " + e.getMessage()); + } + } + + private Literal text(int oid, String text) { + switch (oid) { + case 21: + case 23: + case 20: + case 26: + return new LongLiteral(text.trim()); + case 700: + case 701: + case 1700: + return new DoubleLiteral(text.trim()); + case 16: + return new BooleanLiteral(parseBoolean(text) ? "true" : "false"); + case 1184: + case 1114: + return new LongLiteral(String.valueOf(timestamp(text, oid == 1114))); + case 1082: + return new StringLiteral(LocalDate.parse(text.trim()).toString()); + case 17: + return new BinaryLiteral(byteaText(text)); + case 0: + case 705: + return infer(text); + default: + return new StringLiteral(text); + } + } + + private Literal infer(String text) { + String trimmed = text.trim(); + if (INTEGER.matcher(trimmed).matches()) { + return new LongLiteral(trimmed); + } + if (DECIMAL.matcher(trimmed).matches()) { + return new DoubleLiteral(trimmed); + } + if (TIMESTAMP.matcher(trimmed).matches()) { + try { + return new LongLiteral(String.valueOf(timestamp(trimmed, false))); + } catch (DateTimeParseException e) { + // not a timestamp after all: text + } + } + return new StringLiteral(text); + } + + private Literal binary(int oid, byte[] value) { + ByteBuffer buffer = ByteBuffer.wrap(value); + switch (oid) { + case 21: + return new LongLiteral(String.valueOf(buffer.getShort())); + case 23: + case 26: + return new LongLiteral(String.valueOf(buffer.getInt())); + case 20: + return new LongLiteral(String.valueOf(buffer.getLong())); + case 700: + return new DoubleLiteral(String.valueOf(buffer.getFloat())); + case 701: + return new DoubleLiteral(String.valueOf(buffer.getDouble())); + case 16: + return new BooleanLiteral(buffer.get() != 0 ? "true" : "false"); + case 1184: + case 1114: + return new LongLiteral( + String.valueOf(precision.fromEpochMicros(buffer.getLong() + PG_EPOCH_MICROS))); + case 1082: + return new StringLiteral( + LocalDate.ofEpochDay(buffer.getInt() + (long) PG_EPOCH_DAYS).toString()); + case 17: + return new BinaryLiteral(value); + case 25: + case 1043: + case 19: + case 1042: + case 0: + case 705: + return text(oid == 0 || oid == 705 ? 0 : 25, new String(value, StandardCharsets.UTF_8)); + default: + throw new IllegalArgumentException("binary format of type " + oid + " is not supported"); + } + } + + /** A timestamp in the database's precision; without an offset it is in the session's zone. */ + long timestamp(String text, boolean withoutZone) { + String trimmed = text.trim(); + java.time.temporal.TemporalAccessor parsed = PG_TIMESTAMP.parse(trimmed); + java.time.Instant instant; + if (!withoutZone && parsed.isSupported(ChronoField.OFFSET_SECONDS)) { + instant = OffsetDateTime.from(parsed).toInstant(); + } else { + instant = LocalDateTime.from(parsed).atZone(zone).toInstant(); + } + long micros = + Math.addExact( + Math.multiplyExact(instant.getEpochSecond(), 1_000_000L), instant.getNano() / 1000); + return precision.fromEpochMicros(micros); + } + + private static boolean parseBoolean(String text) { + String t = text.trim().toLowerCase(Locale.ROOT); + switch (t) { + case "t": + case "true": + case "y": + case "yes": + case "on": + case "1": + return true; + case "f": + case "false": + case "n": + case "no": + case "off": + case "0": + return false; + default: + throw new IllegalArgumentException("not a boolean: " + text); + } + } + + private static byte[] byteaText(String text) { + if (!text.startsWith("\\x")) { + throw new IllegalArgumentException("only the hex format of bytea is supported"); + } + String hex = text.substring(2); + byte[] bytes = new byte[hex.length() / 2]; + for (int i = 0; i < bytes.length; i++) { + bytes[i] = (byte) Integer.parseInt(hex.substring(2 * i, 2 * i + 2), 16); + } + return bytes; + } + + private static String typeName(int oid) { + return PgType.ofOid(oid).map(PgType::typeName).orElse("oid " + oid); + } +} diff --git a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/types/PgType.java b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/types/PgType.java new file mode 100644 index 0000000000000..76c13cee00279 --- /dev/null +++ b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/types/PgType.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.types; + +import org.apache.tsfile.enums.TSDataType; + +import java.util.Optional; + +/** + * The PostgreSQL types of table-model values, with their built-in OIDs, the OIDs of their array + * types and their wire sizes. + */ +public enum PgType { + BOOL(16, 1000, 1, "bool"), + INT2(21, 1005, 2, "int2"), + INT4(23, 1007, 4, "int4"), + INT8(20, 1016, 8, "int8"), + FLOAT4(700, 1021, 4, "float4"), + FLOAT8(701, 1022, 8, "float8"), + TEXT(25, 1009, -1, "text"), + VARCHAR(1043, 1015, -1, "varchar"), + BYTEA(17, 1001, -1, "bytea"), + DATE(1082, 1182, 4, "date"), + TIMESTAMP(1114, 1115, 8, "timestamp"), + TIMESTAMPTZ(1184, 1185, 8, "timestamptz"), + /** Parameters whose type the client leaves to the server. */ + UNKNOWN(705, 0, -2, "unknown"); + + private final int oid; + private final int arrayOid; + private final short size; + private final String typeName; + + PgType(int oid, int arrayOid, int size, String typeName) { + this.oid = oid; + this.arrayOid = arrayOid; + this.size = (short) size; + this.typeName = typeName; + } + + public int oid() { + return oid; + } + + public int arrayOid() { + return arrayOid; + } + + /** {@code typlen}: bytes of a value, -1 for variable length, -2 for C strings. */ + public short size() { + return size; + } + + public String typeName() { + return typeName; + } + + /** + * The PostgreSQL type of a table-model column; empty for types that are not exposed (OBJECT, + * VECTOR, UNKNOWN). + */ + public static Optional of(TSDataType dataType) { + switch (dataType) { + case BOOLEAN: + return Optional.of(BOOL); + case INT32: + return Optional.of(INT4); + case INT64: + return Optional.of(INT8); + case FLOAT: + return Optional.of(FLOAT4); + case DOUBLE: + return Optional.of(FLOAT8); + case TEXT: + case STRING: + return Optional.of(TEXT); + case BLOB: + return Optional.of(BYTEA); + case DATE: + return Optional.of(DATE); + case TIMESTAMP: + return Optional.of(TIMESTAMPTZ); + default: + return Optional.empty(); + } + } + + /** The type with this OID, or empty if it is not one the adapter knows. */ + public static Optional ofOid(int oid) { + for (PgType type : values()) { + if (type.oid == oid) { + return Optional.of(type); + } + } + return Optional.empty(); + } +} diff --git a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/types/PgValueEncoder.java b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/types/PgValueEncoder.java new file mode 100644 index 0000000000000..1cd24b33b2d21 --- /dev/null +++ b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/types/PgValueEncoder.java @@ -0,0 +1,230 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.types; + +import org.apache.tsfile.block.column.Column; + +import java.math.BigDecimal; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.time.LocalDate; +import java.time.OffsetDateTime; +import java.time.ZoneId; +import java.time.ZoneOffset; + +/** + * Encodes table-model values in PostgreSQL's text and binary formats. Null values are not encoded + * here; the row writer sends them as length -1. + */ +public final class PgValueEncoder { + + /** Days from 1970-01-01 to PostgreSQL's epoch 2000-01-01. */ + static final long PG_EPOCH_DAYS = 10_957L; + + /** Microseconds from 1970-01-01T00:00Z to PostgreSQL's epoch 2000-01-01T00:00Z. */ + static final long PG_EPOCH_MICROS = PG_EPOCH_DAYS * 86_400_000_000L; + + private static final byte[] TRUE = {'t'}; + private static final byte[] FALSE = {'f'}; + private static final char[] HEX = "0123456789abcdef".toCharArray(); + + private final ZoneId zone; + private final TimestampPrecision precision; + + /** + * @param zone the session's time zone, for the text format of timestamps + * @param precision the table model's timestamp precision + */ + public PgValueEncoder(ZoneId zone, TimestampPrecision precision) { + this.zone = zone; + this.precision = precision; + } + + /** The value at {@code position} in PostgreSQL's text format (UTF-8). */ + public byte[] text(PgType type, Column column, int position) { + switch (type) { + case BOOL: + return column.getBoolean(position) ? TRUE : FALSE; + case INT4: + return ascii(Integer.toString(column.getInt(position))); + case INT8: + return ascii(Long.toString(column.getLong(position))); + case FLOAT4: + return ascii(float4Text(column.getFloat(position))); + case FLOAT8: + return ascii(float8Text(column.getDouble(position))); + case TEXT: + case VARCHAR: + return column.getBinary(position).getValues(); + case BYTEA: + return ascii(byteaText(column.getBinary(position).getValues())); + case DATE: + return ascii(dateOf(column.getInt(position)).toString()); + case TIMESTAMPTZ: + return ascii(timestamptzText(precision.toEpochMicros(column.getLong(position)), zone)); + default: + throw new IllegalArgumentException("No text format for " + type); + } + } + + /** The value at {@code position} in PostgreSQL's binary format. */ + public byte[] binary(PgType type, Column column, int position) { + switch (type) { + case BOOL: + return new byte[] {(byte) (column.getBoolean(position) ? 1 : 0)}; + case INT4: + return ByteBuffer.allocate(4).putInt(column.getInt(position)).array(); + case INT8: + return ByteBuffer.allocate(8).putLong(column.getLong(position)).array(); + case FLOAT4: + return ByteBuffer.allocate(4).putFloat(column.getFloat(position)).array(); + case FLOAT8: + return ByteBuffer.allocate(8).putDouble(column.getDouble(position)).array(); + case TEXT: + case VARCHAR: + case BYTEA: + return column.getBinary(position).getValues(); + case DATE: + return ByteBuffer.allocate(4) + .putInt((int) (dateOf(column.getInt(position)).toEpochDay() - PG_EPOCH_DAYS)) + .array(); + case TIMESTAMPTZ: + return ByteBuffer.allocate(8) + .putLong(precision.toEpochMicros(column.getLong(position)) - PG_EPOCH_MICROS) + .array(); + default: + throw new IllegalArgumentException("No binary format for " + type); + } + } + + /** A table-model DATE (an int {@code yyyyMMdd}) as a date. */ + static LocalDate dateOf(int yyyyMMdd) { + return LocalDate.of(yyyyMMdd / 10_000, yyyyMMdd / 100 % 100, yyyyMMdd % 100); + } + + /** PostgreSQL's ISO output of {@code timestamptz}, e.g. {@code 2026-10-25 01:30:00.5+02}. */ + static String timestamptzText(long epochMicros, ZoneId zone) { + Instant instant = + Instant.ofEpochSecond( + Math.floorDiv(epochMicros, 1_000_000L), Math.floorMod(epochMicros, 1_000_000L) * 1000L); + OffsetDateTime time = instant.atZone(zone).toOffsetDateTime(); + StringBuilder out = new StringBuilder(32); + out.append(time.toLocalDate()).append(' '); + two(out, time.getHour()).append(':'); + two(out, time.getMinute()).append(':'); + two(out, time.getSecond()); + int micros = time.getNano() / 1000; + if (micros != 0) { + String fraction = String.format("%06d", micros); + int end = fraction.length(); + while (fraction.charAt(end - 1) == '0') { + end--; + } + out.append('.').append(fraction, 0, end); + } + ZoneOffset offset = time.getOffset(); + int seconds = offset.getTotalSeconds(); + out.append(seconds < 0 ? '-' : '+'); + seconds = Math.abs(seconds); + two(out, seconds / 3600); + if (seconds % 3600 != 0) { + out.append(':'); + two(out, seconds % 3600 / 60); + if (seconds % 60 != 0) { + out.append(':'); + two(out, seconds % 60); + } + } + return out.toString(); + } + + /** PostgreSQL's float8 output with the default extra_float_digits (shortest exact). */ + static String float8Text(double value) { + if (Double.isNaN(value)) { + return "NaN"; + } + if (Double.isInfinite(value)) { + return value > 0 ? "Infinity" : "-Infinity"; + } + return shortest(Double.toString(value), value == 0.0 && 1 / value < 0, 15); + } + + /** PostgreSQL's float4 output with the default extra_float_digits (shortest exact). */ + static String float4Text(float value) { + if (Float.isNaN(value)) { + return "NaN"; + } + if (Float.isInfinite(value)) { + return value > 0 ? "Infinity" : "-Infinity"; + } + return shortest(Float.toString(value), value == 0.0f && 1 / value < 0, 6); + } + + /** + * Plain notation when the decimal exponent is within [-4, digits), else {@code d.ddde+XX}, like + * PostgreSQL's {@code float8out}/{@code float4out}. + */ + private static String shortest(String javaText, boolean negativeZero, int digits) { + if (negativeZero) { + return "-0"; + } + BigDecimal decimal = new BigDecimal(javaText).stripTrailingZeros(); + if (decimal.signum() == 0) { + return "0"; + } + int exponent = decimal.precision() - decimal.scale() - 1; + if (exponent >= -4 && exponent < digits) { + return decimal.toPlainString(); + } + String unscaled = decimal.unscaledValue().abs().toString(); + StringBuilder out = new StringBuilder(); + if (decimal.signum() < 0) { + out.append('-'); + } + out.append(unscaled.charAt(0)); + if (unscaled.length() > 1) { + out.append('.').append(unscaled, 1, unscaled.length()); + } + out.append('e').append(exponent < 0 ? '-' : '+'); + two(out, Math.abs(exponent)); + return out.toString(); + } + + /** PostgreSQL's hex output of {@code bytea}: {@code \x} followed by lower-case hex digits. */ + static String byteaText(byte[] value) { + StringBuilder out = new StringBuilder(2 + value.length * 2).append("\\x"); + for (byte b : value) { + out.append(HEX[(b >> 4) & 0xF]).append(HEX[b & 0xF]); + } + return out.toString(); + } + + private static StringBuilder two(StringBuilder out, int value) { + if (value < 10) { + out.append('0'); + } + return out.append(value); + } + + private static byte[] ascii(String value) { + return value.getBytes(StandardCharsets.US_ASCII); + } +} diff --git a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/types/TimestampPrecision.java b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/types/TimestampPrecision.java new file mode 100644 index 0000000000000..ec82e84ce4231 --- /dev/null +++ b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/types/TimestampPrecision.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.types; + +/** The table model's timestamp precision ({@code timestamp_precision}). */ +public enum TimestampPrecision { + MS, + US, + NS; + + /** The precision configured as {@code ms}, {@code us} or {@code ns}. */ + public static TimestampPrecision of(String configured) { + switch (configured == null ? "ms" : configured.trim().toLowerCase(java.util.Locale.ROOT)) { + case "us": + return US; + case "ns": + return NS; + default: + return MS; + } + } + + /** Microseconds since 1970 (PostgreSQL's resolution; nanoseconds are truncated toward -∞). */ + public long toEpochMicros(long value) { + switch (this) { + case MS: + return Math.multiplyExact(value, 1000L); + case NS: + return Math.floorDiv(value, 1000L); + default: + return value; + } + } + + /** A value in this precision from microseconds since 1970. */ + public long fromEpochMicros(long micros) { + switch (this) { + case MS: + return Math.floorDiv(micros, 1000L); + case NS: + return Math.multiplyExact(micros, 1000L); + default: + return micros; + } + } +} diff --git a/external-service-impl/postgresql/src/main/resources/META-INF/services/org.apache.iotdb.commons.schema.table.InformationSchema$InformationSchemaExtension b/external-service-impl/postgresql/src/main/resources/META-INF/services/org.apache.iotdb.commons.schema.table.InformationSchema$InformationSchemaExtension new file mode 100644 index 0000000000000..caae0516fd8c0 --- /dev/null +++ b/external-service-impl/postgresql/src/main/resources/META-INF/services/org.apache.iotdb.commons.schema.table.InformationSchema$InformationSchemaExtension @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +org.apache.iotdb.postgresql.catalog.PgCatalogExtension diff --git a/external-service-impl/postgresql/src/main/resources/META-INF/services/org.apache.iotdb.db.queryengine.plan.relational.information.AdditionalInformationSchemaProvider b/external-service-impl/postgresql/src/main/resources/META-INF/services/org.apache.iotdb.db.queryengine.plan.relational.information.AdditionalInformationSchemaProvider new file mode 100644 index 0000000000000..b9521ac5166bc --- /dev/null +++ b/external-service-impl/postgresql/src/main/resources/META-INF/services/org.apache.iotdb.db.queryengine.plan.relational.information.AdditionalInformationSchemaProvider @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +org.apache.iotdb.postgresql.catalog.PgCatalogProvider diff --git a/external-service-impl/postgresql/src/main/resources/META-INF/services/org.apache.iotdb.externalservice.api.IExternalService b/external-service-impl/postgresql/src/main/resources/META-INF/services/org.apache.iotdb.externalservice.api.IExternalService new file mode 100644 index 0000000000000..e938558e69f0a --- /dev/null +++ b/external-service-impl/postgresql/src/main/resources/META-INF/services/org.apache.iotdb.externalservice.api.IExternalService @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +org.apache.iotdb.postgresql.PostgreSqlService diff --git a/external-service-impl/postgresql/src/test/corpus-capture/PgJdbcCapture.java b/external-service-impl/postgresql/src/test/corpus-capture/PgJdbcCapture.java new file mode 100644 index 0000000000000..4bbdd21dc64e8 --- /dev/null +++ b/external-service-impl/postgresql/src/test/corpus-capture/PgJdbcCapture.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.Statement; +import java.sql.Timestamp; +import java.time.Instant; +import java.util.Properties; + +/** Records what pgJDBC (DBeaver's driver) sends: metadata, prepared statements, fetch size. */ +public class PgJdbcCapture { + public static void main(String[] args) throws Exception { + Properties props = new Properties(); + props.setProperty("user", "postgres"); + props.setProperty("password", Files.readString(Path.of(args[0])).trim()); + props.setProperty("ApplicationName", "pgjdbc"); + String url = "jdbc:postgresql://127.0.0.1:54314/iotdb"; + try (Connection c = DriverManager.getConnection(url, props)) { + DatabaseMetaData md = c.getMetaData(); + drain(md.getCatalogs()); + drain(md.getSchemas()); + drain(md.getTables(null, "refdb", "%", new String[] {"TABLE", "VIEW"})); + drain(md.getColumns(null, "refdb", "sensors", "%")); + drain(md.getPrimaryKeys(null, "refdb", "sensors")); + drain(md.getIndexInfo(null, "refdb", "sensors", false, true)); + drain(md.getImportedKeys(null, "refdb", "sensors")); + drain(md.getTypeInfo()); + try (Statement s = c.createStatement()) { + drain(s.executeQuery("SELECT * FROM refdb.sensors LIMIT 5")); + } + try (PreparedStatement p = + c.prepareStatement( + "SELECT \"time\", machine, temperature FROM refdb.sensors" + + " WHERE machine = ? AND \"time\" >= ? ORDER BY \"time\" LIMIT ?")) { + for (int i = 1; i <= 6; i++) { // the 5th execution switches to a server-side prepared statement + p.setString(1, "m" + (i % 5 + 1)); + p.setTimestamp(2, Timestamp.from(Instant.parse("2026-10-25T00:00:00Z"))); + p.setInt(3, 10); + drain(p.executeQuery()); + } + } + c.setAutoCommit(false); + try (Statement s = c.createStatement()) { + s.setFetchSize(1000); // portal with row limit + drain(s.executeQuery("SELECT * FROM refdb.events")); + } + c.commit(); + } + } + + private static void drain(ResultSet rs) throws Exception { + try (rs) { + int columns = rs.getMetaData().getColumnCount(); + while (rs.next()) { + for (int i = 1; i <= columns; i++) { + rs.getObject(i); + } + } + } + } +} diff --git a/external-service-impl/postgresql/src/test/corpus-capture/extract.py b/external-service-impl/postgresql/src/test/corpus-capture/extract.py new file mode 100644 index 0000000000000..0934416efa04a --- /dev/null +++ b/external-service-impl/postgresql/src/test/corpus-capture/extract.py @@ -0,0 +1,84 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +import re, sys, collections + +LICENSE = '''-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +''' + +# Usage: extract.py =... +log, outdir, pgv = sys.argv[1], sys.argv[2], sys.argv[3] +versions = dict(a.split('=',1) for a in sys.argv[4:]) +start = re.compile(r'^\d{4}-\d\d-\d\d \d\d:\d\d:\d\d\.\d+ \w+ \[\d+\] app=(\S*) user=\S* db=\S* (LOG|DETAIL|ERROR|STATEMENT|FATAL): (.*)$') +records = [] +for line in open(log, encoding='utf-8', errors='replace'): + line = line.rstrip('\n') + m = start.match(line) + if m: + records.append([m.group(1), m.group(2), m.group(3)]) + elif records and not re.match(r'^\d{4}-\d\d-\d\d ', line): + records[-1][2] += '\n' + line +# skip the setup (it ran as psql too): everything up to the final ANALYZE +cut = max(i for i, r in enumerate(records) if r[1] == 'LOG' and r[2].strip() == 'statement: ANALYZE;') +records = records[cut + 1:] +per = collections.OrderedDict() +for app, kind, text in records: + if app not in versions: + continue + if kind == 'LOG': + m = re.match(r'(?:statement|execute [^:]*): (.*)', text, re.S) + if m: + per.setdefault(app, []).append([m.group(1).strip(), None]) + elif kind == 'DETAIL' and text.startswith('parameters:') and per.get(app): + per[app][-1][1] = text +for app, stmts in per.items(): + with open(f'{outdir}/{app}.sql', 'w') as f: + f.write(LICENSE) + f.write(f'-- Client: {app} {versions[app]}\n-- Server: PostgreSQL {pgv} (docker postgres:14-alpine), log_statement=all\n') + f.write('-- Captured: 2026-11-19 (T001, spec 012); schema refdb/other from corpus/README.md\n') + collapsed = [] + for sql, params in stmts: + if collapsed and collapsed[-1][0] == sql and collapsed[-1][1] == params: + collapsed[-1][2] += 1 + else: + collapsed.append([sql, params, 1]) + for sql, params, times in collapsed: + f.write('-- next\n') + if times > 1: + f.write(f'-- executed {times} times in a row (portal fetches or repeated executions)\n') + if params: + f.write(f'-- {params}\n') + f.write(sql + '\n') + print(app, len(stmts), 'statements,', len(collapsed), 'entries') diff --git a/external-service-impl/postgresql/src/test/corpus-capture/psycopg_capture.py b/external-service-impl/postgresql/src/test/corpus-capture/psycopg_capture.py new file mode 100644 index 0000000000000..afbe81115e9c2 --- /dev/null +++ b/external-service-impl/postgresql/src/test/corpus-capture/psycopg_capture.py @@ -0,0 +1,36 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +import sys, datetime, psycopg +pw = open(sys.argv[1]).read().strip() +dsn = f"host=127.0.0.1 port=54314 dbname=iotdb user=postgres password={pw} application_name=psycopg" +with psycopg.connect(dsn) as conn: + with conn.cursor() as cur: + cur.execute("SELECT version()"); cur.fetchall() + cur.execute("SELECT * FROM refdb.sensors LIMIT 5"); cur.fetchall() + for m in ("m1", "m2", "m3", "m4", "m5", "m1"): # prepared after 5 executions + cur.execute('SELECT "time", temperature FROM refdb.sensors WHERE machine = %s AND "time" >= %s LIMIT %s', + (m, datetime.datetime(2026, 10, 25, tzinfo=datetime.timezone.utc), 10)) + cur.fetchall() + cur.execute("SELECT * FROM refdb.sensors WHERE running = %b AND level > %s LIMIT 3", (True, 50), binary=True); cur.fetchall() + with conn.cursor(name="big") as cur: # server-side cursor + cur.itersize = 5000 + cur.execute("SELECT * FROM refdb.events") + n = sum(1 for _ in cur) + conn.commit() +print("ok", n) diff --git a/external-service-impl/postgresql/src/test/corpus-capture/setup.sql b/external-service-impl/postgresql/src/test/corpus-capture/setup.sql new file mode 100644 index 0000000000000..804bd6ce6ed72 --- /dev/null +++ b/external-service-impl/postgresql/src/test/corpus-capture/setup.sql @@ -0,0 +1,45 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +-- Reference schema for recording client SQL (spec 012, T001). Run against PostgreSQL 14 as +-- postgres: creates database iotdb with schemas refdb and other. The time range covers the +-- end of summer time in Europe (2026-10-25). +CREATE DATABASE iotdb; +\c iotdb +CREATE SCHEMA refdb; +CREATE TABLE refdb.sensors ( + "time" timestamptz NOT NULL, + plant text, line text, machine text, model text, + temperature double precision, pressure real, count bigint, level integer, + running boolean, state text, note text, blob bytea, day date, ts timestamptz); +COMMENT ON COLUMN refdb.sensors.plant IS 'TAG'; +COMMENT ON COLUMN refdb.sensors.model IS 'ATTRIBUTE'; +INSERT INTO refdb.sensors +SELECT t, 'plant' || p, 'line' || l, 'm' || m, 'model' || (m % 3), + 20 + 5 * sin(extract(epoch FROM t) / 3600) + random(), (1 + random())::real, + (extract(epoch FROM t))::bigint, (random() * 100)::int, random() > 0.2, + (ARRAY['RUN','IDLE','STOP'])[1 + (random() * 2)::int], NULL, NULL, t::date, t +FROM generate_series(1, 3) p, generate_series(1, 4) l, generate_series(1, 5) m, + generate_series(timestamptz '2026-10-20 00:00+00', timestamptz '2026-11-03 00:00+00', interval '5 minutes') t; +CREATE TABLE refdb.events ("time" timestamptz NOT NULL, machine text, code integer, message text); +INSERT INTO refdb.events +SELECT timestamptz '2026-10-20 00:00+00' + (i * interval '37 seconds'), 'm' || (i % 5 + 1), i % 17, 'event ' || i +FROM generate_series(1, 100000) i; +CREATE SCHEMA other; +CREATE TABLE other.weather ("time" timestamptz NOT NULL, station text, temperature double precision); +ANALYZE; diff --git a/external-service-impl/postgresql/src/test/java/org/apache/iotdb/postgresql/catalog/CatalogRowsTest.java b/external-service-impl/postgresql/src/test/java/org/apache/iotdb/postgresql/catalog/CatalogRowsTest.java new file mode 100644 index 0000000000000..a1b797aad9aa6 --- /dev/null +++ b/external-service-impl/postgresql/src/test/java/org/apache/iotdb/postgresql/catalog/CatalogRowsTest.java @@ -0,0 +1,247 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.catalog; + +import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.commons.schema.table.column.AttributeColumnSchema; +import org.apache.iotdb.commons.schema.table.column.FieldColumnSchema; +import org.apache.iotdb.commons.schema.table.column.TagColumnSchema; +import org.apache.iotdb.commons.schema.table.column.TimeColumnSchema; +import org.apache.iotdb.postgresql.types.PgType; + +import org.apache.tsfile.enums.TSDataType; +import org.junit.Test; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; + +public class CatalogRowsTest { + + private static Map> tables() { + TsTable sensors = new TsTable("sensors"); + sensors.addColumnSchema(new TimeColumnSchema("time", TSDataType.TIMESTAMP)); + sensors.addColumnSchema(new TagColumnSchema("device", TSDataType.STRING)); + sensors.addColumnSchema(new AttributeColumnSchema("site", TSDataType.STRING)); + sensors.addColumnSchema(new FieldColumnSchema("blob", TSDataType.OBJECT)); + FieldColumnSchema temp = new FieldColumnSchema("temp", TSDataType.DOUBLE); + temp.getProps().put(TsTable.COMMENT_KEY, "degrees"); + sensors.addColumnSchema(temp); + TsTable secret = new TsTable("secret"); + secret.addColumnSchema(new TimeColumnSchema("time", TSDataType.TIMESTAMP)); + Map plant = new HashMap<>(); + plant.put("sensors", sensors); + plant.put("secret", secret); + Map hidden = new HashMap<>(); + hidden.put("t", new TsTable("t")); + Map> all = new HashMap<>(); + all.put("plant", plant); + all.put("other", hidden); + all.put("pg_catalog", new HashMap<>(hidden)); + return all; + } + + private static CatalogSnapshot userSnapshot() { + return CatalogSnapshot.of( + tables(), db -> db.equals("plant"), (db, table) -> !table.equals("secret")); + } + + @Test + public void everyMappedTypeHasABaseAndAnArrayRowWithItsFunctions() { + CatalogRows rows = new CatalogRows("u", CatalogRowsTest::userSnapshot); + Map> types = new HashMap<>(); + for (Map row : rows.rows("pg_type")) { + types.put((Long) row.get("oid"), row); + } + Set procs = + rows.rows("pg_proc").stream().map(r -> (Long) r.get("oid")).collect(Collectors.toSet()); + for (PgType type : PgType.values()) { + if (type == PgType.UNKNOWN) { + assertEquals("p", types.get((long) type.oid()).get("typtype")); + continue; + } + Map base = types.get((long) type.oid()); + assertEquals(type.typeName(), base.get("typname")); + assertEquals("b", base.get("typtype")); + assertEquals((long) type.arrayOid(), base.get("typarray")); + Map array = types.get((long) type.arrayOid()); + assertEquals("_" + type.typeName(), array.get("typname")); + assertEquals((long) type.oid(), array.get("typelem")); + assertEquals(2400L, array.get("typreceive")); + } + for (Map type : types.values()) { + for (String function : new String[] {"typinput", "typoutput", "typreceive", "typsend"}) { + assertTrue(type.get("typname") + "." + function, procs.contains((Long) type.get(function))); + } + } + assertTrue(types.containsKey(2249L)); // record + assertTrue(types.containsKey(2287L)); // _record + assertTrue(types.containsKey(2278L)); // void + } + + @Test + public void theUserSeesOnlyReadableDatabasesTablesAndExposedColumns() { + CatalogRows rows = new CatalogRows("u", CatalogRowsTest::userSnapshot); + List schemas = + rows.rows("pg_namespace").stream() + .map(r -> (String) r.get("nspname")) + .collect(Collectors.toList()); + assertEquals(3, schemas.size()); + assertTrue( + schemas.containsAll(java.util.Arrays.asList("pg_catalog", "information_schema", "plant"))); + + List> userTables = + rows.rows("pg_is_tables").stream() + .filter(r -> "plant".equals(r.get("table_schema"))) + .collect(Collectors.toList()); + assertEquals(1, userTables.size()); + assertEquals("sensors", userTables.get(0).get("table_name")); + assertEquals("BASE TABLE", userTables.get(0).get("table_type")); + + List> columns = + rows.rows("pg_is_columns").stream() + .filter(r -> "sensors".equals(r.get("table_name"))) + .collect(Collectors.toList()); + assertEquals("OBJECT columns are hidden", 4, columns.size()); + assertEquals("time", columns.get(0).get("column_name")); + assertEquals("timestamp with time zone", columns.get(0).get("data_type")); + assertEquals("NO", columns.get(0).get("is_nullable")); + assertEquals("NO", columns.get(1).get("is_nullable")); // tag + assertEquals("YES", columns.get(2).get("is_nullable")); // attribute + assertEquals("temp", columns.get(3).get("column_name")); + assertEquals(4, columns.get(3).get("ordinal_position")); + assertEquals("float8", columns.get(3).get("udt_name")); + } + + @Test + public void classesAttributesAndDescriptionsAgree() { + CatalogRows rows = new CatalogRows("u", CatalogRowsTest::userSnapshot); + Map sensors = + rows.rows("pg_class").stream() + .filter(r -> "sensors".equals(r.get("relname"))) + .findFirst() + .get(); + long oid = (Long) sensors.get("oid"); + assertTrue(oid >= 16384 && oid < Integer.MAX_VALUE); + assertEquals("r", sensors.get("relkind")); + assertEquals(4, sensors.get("relnatts")); + long plant = + rows.rows("pg_namespace").stream() + .filter(r -> "plant".equals(r.get("nspname"))) + .map(r -> (Long) r.get("oid")) + .findFirst() + .get(); + assertEquals(plant, sensors.get("relnamespace")); + + List> attributes = + rows.rows("pg_attribute").stream() + .filter(r -> r.get("attrelid").equals(oid)) + .collect(Collectors.toList()); + assertEquals(4, attributes.size()); + assertEquals(1184L, attributes.get(0).get("atttypid")); + assertEquals(true, attributes.get(0).get("attnotnull")); + assertEquals(701L, attributes.get(3).get("atttypid")); + + Map descriptions = new HashMap<>(); + for (Map row : rows.rows("pg_description")) { + if (row.get("objoid").equals(oid)) { + descriptions.put((Integer) row.get("objsubid"), row.get("description")); + } + } + assertEquals("TIME", descriptions.get(1)); + assertEquals("TAG", descriptions.get(2)); + assertEquals("FIELD: degrees", descriptions.get(4)); + + // catalog relations are listed too, with PostgreSQL's OIDs + assertTrue( + rows.rows("pg_class").stream() + .anyMatch(r -> "pg_type".equals(r.get("relname")) && r.get("oid").equals(1247L))); + } + + @Test + public void oidsDoNotDependOnTheUser() { + CatalogSnapshot all = CatalogSnapshot.of(tables(), db -> true, (db, table) -> true); + CatalogSnapshot some = userSnapshot(); + long sensorsForAll = + all.relations().stream().filter(r -> r.name().equals("sensors")).findFirst().get().oid(); + long sensorsForSome = + some.relations().stream().filter(r -> r.name().equals("sensors")).findFirst().get().oid(); + assertEquals(sensorsForAll, sensorsForSome); + assertFalse( + "reserved names are hidden", + all.namespaces().stream().anyMatch(n -> n.name().equals("pg_catalog"))); + } + + @Test + public void collisionsTakeTheNextFreeOid() { + Set used = new HashSet<>(); + long first = CatalogSnapshot.allocate("x", used); + long second = CatalogSnapshot.allocate("x", used); + assertNotEquals(first, second); + assertEquals(first + 1, second); + } + + @Test + public void everyRegisteredTableHasRowsOrIsEmpty() { + CatalogRows rows = new CatalogRows("u", CatalogRowsTest::userSnapshot); + for (Map.Entry> table : + PgCatalogTables.tables().entrySet()) { + Set names = + table.getValue().stream().map(PgCatalogTables.Column::name).collect(Collectors.toSet()); + for (Map row : rows.rows(table.getKey())) { + for (String column : row.keySet()) { + assertTrue(table.getKey() + " has no column " + column, names.contains(column)); + } + for (PgCatalogTables.Column column : table.getValue()) { + // every value converts to its column type + CatalogRows.convert(row.get(column.name()), column.type()); + } + } + } + assertEquals("u", rows.rows("pg_roles").get(0).get("rolname")); + } + + @Test + public void alwaysEmptyTablesAreEmptyAndArrayColumnsAreNeverFilled() { + CatalogRows rows = new CatalogRows("u", CatalogRowsTest::userSnapshot); + for (Map.Entry> table : + PgCatalogTables.tables().entrySet()) { + List> content = rows.rows(table.getKey()); + if (PgCatalogTables.alwaysEmpty(table.getKey())) { + assertTrue(table.getKey(), content.isEmpty()); + } + for (PgCatalogTables.Column column : table.getValue()) { + if (column.array()) { + for (Map row : content) { + assertEquals(table.getKey() + "." + column.name(), null, row.get(column.name())); + } + } + } + } + } +} diff --git a/external-service-impl/postgresql/src/test/java/org/apache/iotdb/postgresql/config/PostgreSqlServiceConfigTest.java b/external-service-impl/postgresql/src/test/java/org/apache/iotdb/postgresql/config/PostgreSqlServiceConfigTest.java new file mode 100644 index 0000000000000..a6f44e6f044af --- /dev/null +++ b/external-service-impl/postgresql/src/test/java/org/apache/iotdb/postgresql/config/PostgreSqlServiceConfigTest.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.config; + +import org.junit.Test; + +import java.time.Duration; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class PostgreSqlServiceConfigTest { + + @Test + public void defaultsMatchTheContract() { + PostgreSqlServiceConfig config = PostgreSqlServiceConfig.from(Collections.emptyMap()); + assertFalse(config.enabled()); + assertEquals("127.0.0.1", config.address()); + assertEquals(5432, config.port()); + assertEquals(32, config.maxConnections()); + assertEquals(Duration.ofMinutes(30), config.idleTimeout()); + assertEquals(Duration.ofSeconds(10), config.startupTimeout()); + assertEquals(Duration.ofMinutes(5), config.writeTimeout()); + assertTrue(config.isLoopback()); + assertTrue(config.problems().isEmpty()); + } + + @Test + public void valuesAreRead() { + Map values = new HashMap<>(); + values.put(PostgreSqlServiceConfig.ENABLED, "true"); + values.put(PostgreSqlServiceConfig.ADDRESS, "0.0.0.0"); + values.put(PostgreSqlServiceConfig.PORT, "15432"); + values.put(PostgreSqlServiceConfig.MAX_CONNECTIONS, "4"); + values.put(PostgreSqlServiceConfig.IDLE_TIMEOUT, "60000"); + values.put(PostgreSqlServiceConfig.STARTUP_TIMEOUT, " 2000 "); + values.put(PostgreSqlServiceConfig.WRITE_TIMEOUT, "3000"); + PostgreSqlServiceConfig config = PostgreSqlServiceConfig.from(values); + assertTrue(config.enabled()); + assertEquals("0.0.0.0", config.address()); + assertFalse("the wildcard is not loopback", config.isLoopback()); + assertEquals(15432, config.port()); + assertEquals(4, config.maxConnections()); + assertEquals(Duration.ofMinutes(1), config.idleTimeout()); + assertEquals(Duration.ofSeconds(2), config.startupTimeout()); + assertEquals(Duration.ofSeconds(3), config.writeTimeout()); + assertTrue(config.problems().isEmpty()); + } + + @Test + public void ipv6LoopbackIsLoopback() { + Map values = new HashMap<>(); + values.put(PostgreSqlServiceConfig.ADDRESS, "::1"); + assertTrue(PostgreSqlServiceConfig.from(values).isLoopback()); + } + + @Test + public void invalidValuesAreReportedAndFallBackToDefaults() { + Map values = new HashMap<>(); + values.put(PostgreSqlServiceConfig.ENABLED, "yes"); + values.put(PostgreSqlServiceConfig.ADDRESS, "db.example.com"); + values.put(PostgreSqlServiceConfig.PORT, "70000"); + values.put(PostgreSqlServiceConfig.MAX_CONNECTIONS, "0"); + values.put(PostgreSqlServiceConfig.IDLE_TIMEOUT, "30m"); + values.put(PostgreSqlServiceConfig.WRITE_TIMEOUT, "0"); + PostgreSqlServiceConfig config = PostgreSqlServiceConfig.from(values); + assertEquals( + values.keySet().stream().sorted().collect(Collectors.toList()), + config.problems().stream() + .map(p -> p.substring(0, p.indexOf(':'))) + .sorted() + .collect(Collectors.toList())); + assertFalse(config.enabled()); + assertEquals("127.0.0.1", config.address()); + assertEquals(5432, config.port()); + assertEquals(32, config.maxConnections()); + assertEquals(Duration.ofMinutes(30), config.idleTimeout()); + assertEquals(Duration.ofMinutes(5), config.writeTimeout()); + } +} diff --git a/external-service-impl/postgresql/src/test/java/org/apache/iotdb/postgresql/protocol/CodecTest.java b/external-service-impl/postgresql/src/test/java/org/apache/iotdb/postgresql/protocol/CodecTest.java new file mode 100644 index 0000000000000..3b0dd7c01da24 --- /dev/null +++ b/external-service-impl/postgresql/src/test/java/org/apache/iotdb/postgresql/protocol/CodecTest.java @@ -0,0 +1,171 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.protocol; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import io.netty.channel.embedded.EmbeddedChannel; +import io.netty.handler.codec.DecoderException; +import org.junit.Test; + +import java.nio.charset.StandardCharsets; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class CodecTest { + + private static byte[] bytes(ByteBuf buf) { + byte[] out = new byte[buf.readableBytes()]; + buf.getBytes(buf.readerIndex(), out); + buf.release(); + return out; + } + + private static ByteBuf startupMessage(String... keyValues) { + ByteBuf body = Unpooled.buffer(); + body.writeInt(FrontendMessage.PROTOCOL_3_0); + for (String s : keyValues) { + BackendMessages.cstring(body, s); + } + body.writeByte(0); + ByteBuf framed = Unpooled.buffer(); + framed.writeInt(body.readableBytes() + 4).writeBytes(body); + body.release(); + return framed; + } + + @Test + public void startupPacketIsSplitEvenWhenItArrivesInPieces() throws Exception { + EmbeddedChannel channel = new EmbeddedChannel(new PgFrameDecoder(1 << 20)); + ByteBuf packet = startupMessage("user", "alice", "database", "x"); + channel.writeInbound(packet.readRetainedSlice(3)); + assertNull(channel.readInbound()); + channel.writeInbound(packet); + FrontendMessage message = channel.readInbound(); + assertTrue(message.isStartupPacket()); + assertEquals(FrontendMessage.PROTOCOL_3_0, message.readInt32()); + assertEquals("user", message.readCString()); + assertEquals("alice", message.readCString()); + assertEquals("database", message.readCString()); + assertEquals("x", message.readCString()); + assertEquals("", message.readCString()); + channel.finishAndReleaseAll(); + } + + @Test + public void typedMessagesFollowTheStartup() throws Exception { + PgFrameDecoder decoder = new PgFrameDecoder(1 << 20); + EmbeddedChannel channel = new EmbeddedChannel(decoder); + decoder.startupComplete(); + ByteBuf query = Unpooled.buffer(); + int start = BackendMessages.begin(query, 'Q'); + BackendMessages.cstring(query, "SELECT 1"); + BackendMessages.end(query, start); + ByteBuf sync = Unpooled.buffer(); + BackendMessages.end(sync, BackendMessages.begin(sync, 'S')); + channel.writeInbound(Unpooled.wrappedBuffer(query, sync)); + FrontendMessage q = channel.readInbound(); + assertEquals('Q', q.type()); + assertEquals("SELECT 1", q.readCString()); + FrontendMessage s = channel.readInbound(); + assertEquals('S', s.type()); + assertEquals(0, s.remaining()); + channel.finishAndReleaseAll(); + } + + @Test + public void oversizedOrBrokenLengthsAreRefusedBeforeBuffering() { + PgFrameDecoder decoder = new PgFrameDecoder(1024); + EmbeddedChannel channel = new EmbeddedChannel(decoder); + decoder.startupComplete(); + ByteBuf huge = Unpooled.buffer().writeByte('Q').writeInt(Integer.MAX_VALUE); + try { + channel.writeInbound(huge); + fail("an oversized message must be refused"); + } catch (DecoderException e) { + assertEquals(SqlState.PROTOCOL_VIOLATION, ((PgException) e.getCause()).sqlState()); + } + EmbeddedChannel startup = new EmbeddedChannel(new PgFrameDecoder(1024)); + try { + startup.writeInbound(Unpooled.buffer().writeInt(3)); + fail("a startup packet shorter than its header must be refused"); + } catch (DecoderException e) { + assertTrue(((PgException) e.getCause()).isFatal()); + } + } + + @Test + public void truncatedPayloadIsAProtocolViolation() { + FrontendMessage message = new FrontendMessage('B', new byte[] {0, 1}); + try { + message.readInt32(); + fail(); + } catch (PgException e) { + assertEquals(SqlState.PROTOCOL_VIOLATION, e.sqlState()); + assertTrue(e.isFatal()); + } + try { + new FrontendMessage('Q', "no terminator".getBytes(StandardCharsets.UTF_8)).readCString(); + fail(); + } catch (PgException e) { + assertEquals(SqlState.PROTOCOL_VIOLATION, e.sqlState()); + } + } + + @Test + public void backendMessagesHaveTheDocumentedBytes() { + ByteBuf out = Unpooled.buffer(); + BackendMessages.readyForQuery(out, 'I'); + assertArrayEquals(new byte[] {'Z', 0, 0, 0, 5, 'I'}, bytes(out)); + + out = Unpooled.buffer(); + BackendMessages.authenticationOk(out); + assertArrayEquals(new byte[] {'R', 0, 0, 0, 8, 0, 0, 0, 0}, bytes(out)); + + out = Unpooled.buffer(); + BackendMessages.authenticationCleartextPassword(out); + assertArrayEquals(new byte[] {'R', 0, 0, 0, 8, 0, 0, 0, 3}, bytes(out)); + + out = Unpooled.buffer(); + int row = BackendMessages.beginDataRow(out, 2); + BackendMessages.value(out, new byte[] {'4', '2'}); + BackendMessages.nullValue(out); + BackendMessages.end(out, row); + assertArrayEquals( + new byte[] {'D', 0, 0, 0, 16, 0, 2, 0, 0, 0, 2, '4', '2', -1, -1, -1, -1}, bytes(out)); + } + + @Test + public void errorResponseCarriesSeverityCodeMessageAndHint() { + ByteBuf out = Unpooled.buffer(); + BackendMessages.errorResponse( + out, + new PgException( + SqlState.UNDEFINED_TABLE, "relation \"t\" does not exist", "qualify it", false)); + String text = new String(bytes(out), StandardCharsets.UTF_8); + assertTrue(text.startsWith("E")); + assertTrue( + text.contains("SERROR\0VERROR\0C42P01\0Mrelation \"t\" does not exist\0Hqualify it\0\0")); + } +} diff --git a/external-service-impl/postgresql/src/test/java/org/apache/iotdb/postgresql/protocol/PgErrorsTest.java b/external-service-impl/postgresql/src/test/java/org/apache/iotdb/postgresql/protocol/PgErrorsTest.java new file mode 100644 index 0000000000000..bccb5656aaf98 --- /dev/null +++ b/external-service-impl/postgresql/src/test/java/org/apache/iotdb/postgresql/protocol/PgErrorsTest.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.protocol; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +public class PgErrorsTest { + + private static String state(TSStatusCode code, String message) { + return PgErrors.sqlState(code.getStatusCode(), message); + } + + @Test + public void codesMapToTheContractTable() { + assertEquals("42601", state(TSStatusCode.SQL_PARSE_ERROR, "line 1:8: mismatched input")); + assertEquals("42P01", state(TSStatusCode.TABLE_NOT_EXISTS, "x")); + assertEquals("42703", state(TSStatusCode.COLUMN_NOT_EXISTS, "x")); + assertEquals("3F000", state(TSStatusCode.DATABASE_NOT_EXIST, "x")); + assertEquals("42501", state(TSStatusCode.NO_PERMISSION, "x")); + assertEquals("57014", state(TSStatusCode.QUERY_WAS_KILLED, "x")); + assertEquals("57014", state(TSStatusCode.QUERY_TIMEOUT, "x")); + assertEquals("53200", state(TSStatusCode.MPP_MEMORY_NOT_ENOUGH, "x")); + assertEquals("0A000", state(TSStatusCode.UNSUPPORTED_OPERATION, "x")); + assertEquals("22012", state(TSStatusCode.DIVISION_BY_ZERO, "x")); + assertEquals("XX000", state(TSStatusCode.INTERNAL_SERVER_ERROR, "x")); + } + + @Test + public void semanticErrorsAreReadFromTheMessage() { + assertEquals("42P01", state(TSStatusCode.SEMANTIC_ERROR, "Table 'db.t' does not exist.")); + assertEquals("42703", state(TSStatusCode.SEMANTIC_ERROR, "Column 'speed' cannot be resolved")); + assertEquals("3F000", state(TSStatusCode.SEMANTIC_ERROR, "Database db2 does not exist")); + assertEquals("42000", state(TSStatusCode.SEMANTIC_ERROR, "GROUP BY clause is missing")); + } + + @Test + public void engineErrorsAreNotFatalAndKeepTheMessage() { + PgException error = + PgErrors.of(new TSStatus(TSStatusCode.TABLE_NOT_EXISTS.getStatusCode()).setMessage("gone")); + assertFalse(error.isFatal()); + assertEquals("42P01", error.sqlState()); + assertEquals("gone", error.getMessage()); + } +} diff --git a/external-service-impl/postgresql/src/test/java/org/apache/iotdb/postgresql/server/StartupTest.java b/external-service-impl/postgresql/src/test/java/org/apache/iotdb/postgresql/server/StartupTest.java new file mode 100644 index 0000000000000..42e360e2c2c3f --- /dev/null +++ b/external-service-impl/postgresql/src/test/java/org/apache/iotdb/postgresql/server/StartupTest.java @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.server; + +import org.apache.iotdb.postgresql.config.PostgreSqlServiceConfig; +import org.apache.iotdb.postgresql.protocol.FrontendMessage; +import org.apache.iotdb.postgresql.protocol.PgFrameDecoder; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import io.netty.channel.embedded.EmbeddedChannel; +import io.netty.handler.ssl.SslHandler; +import org.junit.Test; + +import javax.net.ssl.SSLContext; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +/** The connection start, on an in-memory channel (no network). */ +public class StartupTest { + + private static EmbeddedChannel channel(String address, SSLContext ssl) { + Map values = new HashMap<>(); + values.put(PostgreSqlServiceConfig.ENABLED, "true"); + values.put(PostgreSqlServiceConfig.ADDRESS, address); + values.put(PostgreSqlServiceConfig.STARTUP_TIMEOUT, "1000"); + PgServerContext context = + new PgServerContext( + PostgreSqlServiceConfig.from(values), ssl, Runnable::run, session -> null, "14.0"); + PgFrameDecoder decoder = new PgFrameDecoder(1 << 20); + return new EmbeddedChannel(decoder, new PgConnectionHandler(context, decoder)); + } + + private static EmbeddedChannel loopback() { + return channel("127.0.0.1", null); + } + + private static ByteBuf packet(int code, String... parameters) { + ByteBuf body = Unpooled.buffer(); + body.writeInt(code); + if (parameters.length > 0) { + for (String p : parameters) { + body.writeBytes(p.getBytes(StandardCharsets.UTF_8)).writeByte(0); + } + body.writeByte(0); + } + ByteBuf out = Unpooled.buffer(); + out.writeInt(body.readableBytes() + 4).writeBytes(body); + return out; + } + + private static ByteBuf startup(int minor, String... extra) { + List parameters = new ArrayList<>(); + parameters.add("user"); + parameters.add("alice"); + parameters.add("database"); + parameters.add("any"); + for (String e : extra) { + parameters.add(e); + } + return packet((3 << 16) | minor, parameters.toArray(new String[0])); + } + + /** All bytes the server wrote so far. */ + private static ByteBuf written(EmbeddedChannel ch) { + ByteBuf all = Unpooled.buffer(); + ByteBuf next; + while ((next = ch.readOutbound()) != null) { + all.writeBytes(next); + next.release(); + } + return all; + } + + /** The type bytes of the typed messages in a buffer. */ + private static String types(ByteBuf buf) { + StringBuilder types = new StringBuilder(); + while (buf.readableBytes() >= 5) { + types.append((char) buf.readByte()); + buf.skipBytes(buf.readInt() - 4); + } + return types.toString(); + } + + @Test + public void sslRequestWithoutKeyStoreIsAnsweredWithN() { + EmbeddedChannel ch = loopback(); + ch.writeInbound(packet(FrontendMessage.SSL_REQUEST)); + ByteBuf answer = written(ch); + assertEquals(1, answer.readableBytes()); + assertEquals('N', answer.readByte()); + assertTrue(ch.isOpen()); + ch.writeInbound(startup(0)); + ByteBuf auth = written(ch); + assertEquals('R', auth.readByte()); + auth.readInt(); + assertEquals("AuthenticationCleartextPassword", 3, auth.readInt()); + } + + @Test + public void sslRequestWithKeyStoreSwitchesToTls() throws Exception { + EmbeddedChannel ch = channel("127.0.0.1", SSLContext.getDefault()); + ch.writeInbound(packet(FrontendMessage.SSL_REQUEST)); + ByteBuf answer = written(ch); + assertEquals(1, answer.readableBytes()); + assertEquals("S goes out unencrypted", 'S', answer.readByte()); + assertNotNull("everything after S is TLS", ch.pipeline().get(SslHandler.class)); + } + + @Test + public void gssEncryptionIsDeclined() { + EmbeddedChannel ch = loopback(); + ch.writeInbound(packet(FrontendMessage.GSSENC_REQUEST)); + ByteBuf answer = written(ch); + assertEquals('N', answer.readByte()); + assertTrue(ch.isOpen()); + } + + @Test + public void newerMinorVersionsAndOptionsAreNegotiatedDown() { + EmbeddedChannel ch = loopback(); + ch.writeInbound(startup(2, "_pq_.compression", "on")); + ByteBuf out = written(ch); + assertEquals('v', out.getByte(0)); + ByteBuf negotiate = out.slice(5, out.getInt(1) - 4); + assertEquals("highest minor version", 0, negotiate.readInt()); + assertEquals(1, negotiate.readInt()); + assertTrue(negotiate.toString(StandardCharsets.US_ASCII).startsWith("_pq_.compression")); + assertEquals("vR", types(out)); + } + + @Test + public void aTlsClientHelloAsFirstBytesClosesTheConnection() { + EmbeddedChannel ch = loopback(); + ByteBuf hello = Unpooled.buffer(); + hello.writeBytes(new byte[] {0x16, 0x03, 0x01, 0x02, 0x00, 0x01, 0x00, 0x01, (byte) 0xfc}); + ch.writeInbound(hello); + ch.runPendingTasks(); + assertFalse(ch.isOpen()); + assertEquals("E", types(written(ch))); + } + + @Test + public void offLoopbackPlainStartupIsRefused() { + EmbeddedChannel ch = channel("0.0.0.0", null); + ch.writeInbound(startup(0)); + ch.runPendingTasks(); + ByteBuf out = written(ch); + assertEquals('E', out.getByte(0)); + assertTrue(out.toString(StandardCharsets.UTF_8).contains("28000")); + assertTrue(out.toString(StandardCharsets.UTF_8).contains("encryption required")); + assertFalse(ch.isOpen()); + } + + @Test + public void theStartupTimeoutClosesTheConnection() { + EmbeddedChannel ch = loopback(); + ch.writeInbound(packet(FrontendMessage.SSL_REQUEST)); + assertTrue(ch.isOpen()); + ch.advanceTimeBy(2, TimeUnit.SECONDS); + ch.runScheduledPendingTasks(); + assertFalse(ch.isOpen()); + } +} diff --git a/external-service-impl/postgresql/src/test/java/org/apache/iotdb/postgresql/session/CursorCommandTest.java b/external-service-impl/postgresql/src/test/java/org/apache/iotdb/postgresql/session/CursorCommandTest.java new file mode 100644 index 0000000000000..7e346986ba401 --- /dev/null +++ b/external-service-impl/postgresql/src/test/java/org/apache/iotdb/postgresql/session/CursorCommandTest.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.session; + +import org.apache.iotdb.postgresql.protocol.PgException; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class CursorCommandTest { + + private static CursorCommand parse(String sql) throws PgException { + return CursorCommand.parse(sql).orElseThrow(() -> new AssertionError("not a cursor: " + sql)); + } + + private static void refused(String sql, String sqlState) { + try { + CursorCommand.parse(sql); + fail("refused: " + sql); + } catch (PgException e) { + assertEquals(e.getMessage(), sqlState, e.sqlState()); + } + } + + @Test + public void declare() throws Exception { + CursorCommand c = parse("DECLARE \"big\" CURSOR FOR SELECT * FROM refdb.events"); + assertEquals(CursorCommand.Kind.DECLARE, c.kind()); + assertEquals("big", c.name()); + assertEquals("SELECT * FROM refdb.events", c.query()); + assertFalse(c.hold()); + c = parse("declare Cur binary no scroll cursor with hold for\nselect 1"); + assertEquals("cur", c.name()); + assertTrue(c.hold()); + assertTrue(c.binary()); + assertEquals("select 1", c.query()); + refused("DECLARE c SCROLL CURSOR FOR SELECT 1", "0A000"); + } + + @Test + public void fetchAndMove() throws Exception { + CursorCommand c = parse("FETCH FORWARD 5000 FROM \"big\""); + assertEquals(CursorCommand.Kind.FETCH, c.kind()); + assertEquals(5000, c.count()); + assertEquals("big", c.name()); + assertEquals(1, parse("FETCH big").count()); + assertEquals(1, parse("FETCH NEXT FROM big").count()); + assertEquals(0, parse("FETCH ALL IN big").count()); + assertEquals(0, parse("fetch forward all from big").count()); + assertEquals(10, parse("FETCH 10 big").count()); + c = parse("MOVE FORWARD 3 IN c"); + assertEquals(CursorCommand.Kind.MOVE, c.kind()); + assertEquals(3, c.count()); + refused("FETCH BACKWARD 1 FROM c", "0A000"); + refused("FETCH PRIOR FROM c", "0A000"); + refused("FETCH ABSOLUTE 5 FROM c", "0A000"); + refused("FETCH -1 FROM c", "0A000"); + refused("FETCH 0 FROM c", "0A000"); + } + + @Test + public void close() throws Exception { + assertEquals("big", parse("CLOSE \"big\"").name()); + assertNull(parse("close all").name()); + } + + @Test + public void otherStatementsAreNotCursors() throws Exception { + assertFalse(CursorCommand.parse("SELECT 1").isPresent()); + assertFalse(CursorCommand.parse("SET x = 1").isPresent()); + } +} diff --git a/external-service-impl/postgresql/src/test/java/org/apache/iotdb/postgresql/session/SessionEndTest.java b/external-service-impl/postgresql/src/test/java/org/apache/iotdb/postgresql/session/SessionEndTest.java new file mode 100644 index 0000000000000..c7650467d1a6c --- /dev/null +++ b/external-service-impl/postgresql/src/test/java/org/apache/iotdb/postgresql/session/SessionEndTest.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.session; + +import org.apache.iotdb.db.protocol.session.ExternalServiceClientSession; +import org.apache.iotdb.postgresql.protocol.FrontendMessage; +import org.apache.iotdb.postgresql.types.TimestampPrecision; + +import io.netty.buffer.ByteBuf; +import io.netty.channel.embedded.EmbeddedChannel; +import org.junit.Test; + +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.time.ZoneOffset; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** A session ended from outside closes the connection with FATAL 57P01. */ +public class SessionEndTest { + + @Test + public void anEndedSessionClosesTheConnectionWith57P01() { + ExternalServiceClientSession dbSession = + new ExternalServiceClientSession("postgresql", new InetSocketAddress("127.0.0.1", 5432)); + dbSession.setLogin(true); + PgSession session = new PgSession(dbSession, "alice", ZoneOffset.UTC); + SqlProtocol protocol = + new SqlProtocol(session, Duration.ofSeconds(5), TimestampPrecision.of("ms")); + EmbeddedChannel channel = new EmbeddedChannel(); + + // ended from outside: the login flag is false + dbSession.setLogin(false); + protocol.handle( + new FrontendMessage('Q', "SHOW TimeZone\0".getBytes(StandardCharsets.UTF_8)), channel); + channel.runPendingTasks(); + + ByteBuf out = channel.readOutbound(); + assertEquals('E', (char) out.readByte()); + String body = out.toString(StandardCharsets.UTF_8); + assertTrue(body, body.contains("FATAL")); + assertTrue(body, body.contains("57P01")); + out.release(); + assertFalse(channel.isOpen()); + } +} diff --git a/external-service-impl/postgresql/src/test/java/org/apache/iotdb/postgresql/session/StatementTimeoutTest.java b/external-service-impl/postgresql/src/test/java/org/apache/iotdb/postgresql/session/StatementTimeoutTest.java new file mode 100644 index 0000000000000..626b714960b94 --- /dev/null +++ b/external-service-impl/postgresql/src/test/java/org/apache/iotdb/postgresql/session/StatementTimeoutTest.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.session; + +import org.junit.Test; + +import java.time.Duration; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +public class StatementTimeoutTest { + + @Test + public void valuesAndUnits() { + assertEquals(Duration.ZERO, StatementTimeout.parse(null)); + assertEquals(Duration.ZERO, StatementTimeout.parse("0")); + assertEquals(Duration.ofMillis(5000), StatementTimeout.parse("5000")); + assertEquals(Duration.ofMillis(500), StatementTimeout.parse("500ms")); + assertEquals(Duration.ofSeconds(5), StatementTimeout.parse("5s")); + assertEquals(Duration.ofMinutes(2), StatementTimeout.parse("2 min")); + assertEquals(Duration.ofHours(1), StatementTimeout.parse("1h")); + assertEquals(Duration.ofMillis(1), StatementTimeout.parse("10us")); + try { + StatementTimeout.parse("soon"); + fail("invalid"); + } catch (IllegalArgumentException e) { + // expected + } + } + + @Test + public void theSmallerTimeoutWins() { + assertEquals(60_000, StatementTimeout.engineTimeoutMillis(60_000, Duration.ZERO)); + assertEquals(5_000, StatementTimeout.engineTimeoutMillis(60_000, Duration.ofSeconds(5))); + assertEquals(60_000, StatementTimeout.engineTimeoutMillis(60_000, Duration.ofHours(1))); + assertEquals(5_000, StatementTimeout.engineTimeoutMillis(0, Duration.ofSeconds(5))); + } +} diff --git a/external-service-impl/postgresql/src/test/java/org/apache/iotdb/postgresql/sql/PgToTableSqlTest.java b/external-service-impl/postgresql/src/test/java/org/apache/iotdb/postgresql/sql/PgToTableSqlTest.java new file mode 100644 index 0000000000000..c1cef0ab3582a --- /dev/null +++ b/external-service-impl/postgresql/src/test/java/org/apache/iotdb/postgresql/sql/PgToTableSqlTest.java @@ -0,0 +1,548 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.sql; + +import org.apache.iotdb.postgresql.protocol.PgException; + +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class PgToTableSqlTest { + + private static final TranslationContext CONTEXT = + new TranslationContext() { + @Override + public List searchPath() { + return Arrays.asList("plant", "public"); + } + + @Override + public boolean tableExists(String database, String table) { + return database.equals("plant") && table.equals("sensors"); + } + + @Override + public String user() { + return "alice"; + } + + @Override + public String version() { + return "PostgreSQL 14.0 (IoTDB)"; + } + + @Override + public String currentDatabase() { + return "iotdb"; + } + + @Override + public int backendPid() { + return 42; + } + }; + + /** The translation without PostgreSQL output names (tested in {@link #outputColumnNames}). */ + private static String sql(String pg) throws PgException { + return PgToTableSql.translate(pg, CONTEXT, false).sql(); + } + + private static void refused(String pg, String sqlState, String messagePart) { + try { + fail("refused, but translated to: " + sql(pg)); + } catch (PgException e) { + assertEquals(e.getMessage(), sqlState, e.sqlState()); + assertTrue(e.getMessage(), e.getMessage().contains(messagePart)); + } + } + + @Test + public void namesAreFoldedAndQuoted() throws Exception { + assertEquals( + "SELECT \"time\", \"Device\", \"t\".\"temp\" FROM \"mydb\".\"sensors\" AS \"t\"", + sql("SELECT Time, \"Device\", t.TEMP FROM MyDb.sensors t")); + } + + @Test + public void unqualifiedTablesUseTheSearchPath() throws Exception { + assertEquals("SELECT * FROM \"plant\".\"sensors\"", sql("SELECT * FROM sensors")); + refused("SELECT * FROM missing", "42P01", "relation \"missing\" does not exist"); + } + + @Test + public void catalogNamesGoToInformationSchema() throws Exception { + assertEquals( + "SELECT \"nspname\" FROM information_schema.\"pg_namespace\"", + sql("SELECT nspname FROM pg_catalog.pg_namespace")); + assertEquals( + "SELECT \"oid\" FROM information_schema.\"pg_type\"", sql("SELECT oid FROM pg_type")); + assertEquals( + "SELECT * FROM information_schema.\"pg_is_tables\"", + sql("SELECT * FROM information_schema.tables")); + } + + @Test + public void castsParametersAndLiterals() throws Exception { + PgToTableSql.Translation t = + PgToTableSql.translate( + "SELECT temp::float8, CAST(x AS varchar(10)) FROM plant.sensors" + + " WHERE time > $2 AND device = $1 AND other = $2", + CONTEXT, + false); + assertEquals( + "SELECT CAST(\"temp\" AS DOUBLE), CAST(\"x\" AS STRING) FROM \"plant\".\"sensors\"" + + " WHERE \"time\" > ? AND \"device\" = ? AND \"other\" = ?", + t.sql()); + assertEquals(Arrays.asList(2, 1, 2), t.parameterOrder()); + assertEquals(2, t.parameterCount()); + assertEquals("SELECT 'it''s', 'a\nb', TRUE", sql("SELECT 'it''s', E'a\\nb', true")); + } + + @Test + public void sessionFunctionsBecomeConstants() throws Exception { + assertEquals( + "SELECT 'PostgreSQL 14.0 (IoTDB)', 'plant', 'iotdb', 42, 'alice', TRUE", + sql( + "SELECT version(), current_schema(), current_database(), pg_backend_pid()," + + " current_user, has_table_privilege('x', 'select')")); + } + + @Test + public void limitsOrderingAndIlike() throws Exception { + assertEquals( + "SELECT * FROM \"plant\".\"sensors\" WHERE lower(\"device\") LIKE lower('a%')" + + " ORDER BY \"time\" DESC NULLS FIRST, \"device\" NULLS LAST LIMIT 5", + sql( + "SELECT * FROM sensors WHERE device ILIKE 'a%' ORDER BY time DESC, device" + + " FETCH FIRST 5 ROWS ONLY")); + assertEquals( + "SELECT count(DISTINCT \"device\"), avg(\"temp\") FROM \"plant\".\"sensors\"" + + " GROUP BY \"device\" LIMIT 10 OFFSET 20", + sql( + "SELECT count(DISTINCT device), avg(temp) FROM sensors GROUP BY device" + + " LIMIT 10 OFFSET 20")); + } + + @Test + public void constructsWithoutExactTranslationAreRefusedByName() { + refused("SELECT DISTINCT ON (device) * FROM sensors", "0A000", "DISTINCT ON"); + refused("SELECT a = ANY(ARRAY[1,2]) FROM sensors", "0A000", "ARRAY"); + refused("SELECT now() - interval '1 month'", "0A000", "months and years"); + refused("SELECT interval '1 day'", "0A000", "INTERVAL values"); + refused( + "SELECT sum(temp) OVER (ORDER BY time ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) FROM sensors", + "0A000", + "this form of the window function sum()"); + refused("SELECT median(temp) OVER () FROM sensors", "0A000", "window function median()"); + refused("SELECT x::numeric(10,2) FROM sensors", "0A000", "numeric"); + refused("SELECT md5(device) FROM sensors", "0A000", "function md5()"); + refused("SELECT * FROM sensors FOR UPDATE", "0A000", "FOR UPDATE"); + } + + @Test + public void writesAreReadOnlyErrors() { + refused("INSERT INTO plant.sensors VALUES (1)", "25006", "read-only"); + refused("DELETE FROM plant.sensors", "25006", "read-only"); + refused("CREATE TABLE x (a int)", "25006", "read-only"); + } + + @Test + public void syntaxErrorsAre42601() { + refused("SELEC 1", "42601", "syntax error"); + } + + @Test + public void commonTableExpressionsStayUnqualified() throws Exception { + assertEquals( + "WITH w AS (SELECT \"device\" FROM \"plant\".\"sensors\") SELECT * FROM \"w\"", + sql("WITH w AS (SELECT device FROM sensors) SELECT * FROM w")); + } + + @Test + public void objectIdentifierCastsOfLiterals() throws Exception { + assertEquals("SELECT 1259", sql("SELECT 'pg_class'::regclass")); + assertEquals("SELECT 1259", sql("SELECT 'pg_catalog.pg_class'::pg_catalog.regclass")); + assertEquals( + "SELECT (SELECT max(pg_rc.\"oid\") FROM information_schema.\"pg_class\" pg_rc" + + " JOIN information_schema.\"pg_namespace\" pg_rn" + + " ON pg_rc.\"relnamespace\" = pg_rn.\"oid\"" + + " WHERE pg_rn.\"nspname\" = 'plant' AND pg_rc.\"relname\" = 'sensors')", + sql("SELECT 'sensors'::regclass")); + assertEquals("SELECT 23, 1185", sql("SELECT 'integer'::regtype, 'timestamptz[]'::regtype")); + assertEquals("SELECT 750", sql("SELECT 'pg_catalog.array_in'::regproc")); + assertEquals("SELECT 11", sql("SELECT 'pg_catalog'::regnamespace")); + refused("SELECT c.oid::regclass FROM pg_class c", "0A000", "casts of expressions to regclass"); + refused("SELECT 'nosuchtype'::regtype", "42704", "nosuchtype"); + refused("SELECT 'nosuchfunc'::regproc", "42883", "nosuchfunc"); + refused("SELECT 'missing'::regclass", "42P01", "missing"); + } + + @Test + public void catalogFunctions() throws Exception { + String formatType = sql("SELECT format_type(a.atttypid, a.atttypmod) FROM pg_attribute a"); + assertTrue(formatType, formatType.startsWith("SELECT CASE CAST(\"a\".\"atttypid\" AS INT32)")); + assertTrue(formatType, formatType.contains("WHEN 1184 THEN 'timestamp with time zone'")); + assertTrue(formatType, formatType.contains("WHEN 1007 THEN 'integer[]'")); + assertEquals( + "SELECT CAST(NULL AS STRING) AS \"pg_get_expr\" FROM information_schema.\"pg_attrdef\"" + + " WHERE FALSE", + sql("SELECT pg_get_expr(d.adbin, d.adrelid) FROM pg_attrdef d")); + assertEquals( + "SELECT (SELECT max(pg_d.\"description\") FROM information_schema.\"pg_description\" pg_d" + + " WHERE pg_d.\"objoid\" = \"c\".\"oid\" AND pg_d.\"objsubid\" = 0" + + " AND pg_d.\"classoid\" = 1259) FROM information_schema.\"pg_class\" AS \"c\"", + sql("SELECT obj_description(c.oid, 'pg_class') FROM pg_class c")); + assertEquals( + "SELECT (SELECT max(pg_d.\"description\") FROM information_schema.\"pg_description\" pg_d" + + " WHERE pg_d.\"objoid\" = \"a\".\"attrelid\" AND pg_d.\"objsubid\" =" + + " \"a\".\"attnum\" AND pg_d.\"classoid\" = 1259)" + + " FROM information_schema.\"pg_attribute\" AS \"a\"", + sql("SELECT col_description(a.attrelid, a.attnum) FROM pg_attribute a")); + assertEquals("SELECT 'alice'", sql("SELECT pg_get_userbyid(10)")); + } + + @Test + public void quotedLiteralsTakeTheTypeOfCatalogColumns() throws Exception { + assertEquals( + "SELECT \"a\".\"attname\" FROM information_schema.\"pg_attribute\" AS \"a\"" + + " WHERE \"a\".\"attrelid\" = 16386 AND \"a\".\"attisdropped\" = FALSE", + sql( + "SELECT a.attname FROM pg_attribute a WHERE a.attrelid = '16386'" + + " AND a.attisdropped = 'f'")); + // unqualified: only when every table is a catalog table + assertEquals( + "SELECT * FROM information_schema.\"pg_class\" WHERE \"oid\" = 5", + sql("SELECT * FROM pg_class WHERE oid = '5'")); + assertEquals( + "SELECT * FROM information_schema.\"pg_class\", \"plant\".\"sensors\"" + + " WHERE \"oid\" = '5'", + sql("SELECT * FROM pg_class, sensors WHERE oid = '5'")); + // text columns keep the literal + assertEquals( + "SELECT * FROM information_schema.\"pg_class\" AS \"c\" WHERE \"c\".\"relkind\" = 'r'", + sql("SELECT * FROM pg_class c WHERE c.relkind = 'r'")); + refused("SELECT * FROM pg_class c WHERE c.oid = 'abc'", "22P02", "abc"); + } + + @Test + public void nullResultsOfCaseTakeTheTypeOfTheOthers() throws Exception { + assertEquals( + "SELECT CASE WHEN \"a\".\"attstattarget\" = -1 THEN CAST(NULL AS INT32)" + + " ELSE \"a\".\"attstattarget\" END FROM information_schema.\"pg_attribute\" AS \"a\"", + sql( + "SELECT CASE WHEN a.attstattarget = -1 THEN NULL ELSE a.attstattarget END" + + " FROM pg_attribute a")); + assertEquals( + "SELECT CASE WHEN TRUE THEN 'x' ELSE CAST(NULL AS STRING) END", + sql("SELECT CASE WHEN true THEN 'x' ELSE NULL END")); + } + + @Test + public void outputNamesAreFolded() throws Exception { + assertEquals( + "SELECT \"temp\" AS \"table_cat\", \"temp\" AS \"Mixed\" FROM \"plant\".\"sensors\"", + sql("SELECT temp AS TABLE_CAT, temp AS \"Mixed\" FROM plant.sensors")); + } + + @Test + public void anchoredRegularExpressionsBecomeLikeOrEquality() throws Exception { + assertEquals( + "SELECT * FROM information_schema.\"pg_class\" AS \"c\" WHERE (\"c\".\"relname\" = 'sensors')", + sql( + "SELECT * FROM pg_class c WHERE c.relname OPERATOR(pg_catalog.~) '^(sensors)$'" + + " COLLATE pg_catalog.default")); + assertEquals( + "SELECT * FROM information_schema.\"pg_namespace\" AS \"n\"" + + " WHERE NOT (\"n\".\"nspname\" LIKE 'pg\\_%' ESCAPE '\\')", + sql("SELECT * FROM pg_namespace n WHERE n.nspname !~ '^pg_'")); + assertEquals( + "SELECT * FROM information_schema.\"pg_class\" AS \"c\"" + + " WHERE (\"c\".\"relname\" LIKE 'sens%')", + sql("SELECT * FROM pg_class c WHERE c.relname ~ '^(sens.*)$'")); + assertEquals( + "SELECT * FROM information_schema.\"pg_class\" AS \"c\"" + + " WHERE (\"c\".\"relname\" IN ('a', 'b'))", + sql("SELECT * FROM pg_class c WHERE c.relname ~ '^(a|b)$'")); + assertEquals( + "SELECT * FROM information_schema.\"pg_class\" AS \"c\"" + + " WHERE (lower(\"c\".\"relname\") LIKE '%ab_c%')", + sql("SELECT * FROM pg_class c WHERE c.relname ~* 'AB.c'")); + refused("SELECT * FROM pg_class c WHERE c.relname ~ '^[a-z]+$'", "0A000", "[a-z]+"); + refused("SELECT * FROM pg_class c WHERE c.relname ~ c.relname", "0A000", "not literals"); + // OPERATOR(...) inside a literal is text, not syntax + assertEquals("SELECT 'OPERATOR(pg_catalog.~)'", sql("SELECT 'OPERATOR(pg_catalog.~)'")); + } + + @Test + public void queriesOverAlwaysEmptyCatalogTablesSelectTypedNulls() throws Exception { + assertEquals( + "SELECT CAST(NULL AS STRING) AS \"polname\", CAST(NULL AS STRING) AS \"r\"" + + " FROM information_schema.\"pg_policy\" WHERE FALSE", + sql( + "SELECT pol.polname, array_to_string(array(select 1), ',') AS r" + + " FROM pg_catalog.pg_policy pol WHERE 'd' = any(pol.polroles) ORDER BY 1")); + // an aggregate over no rows is still one row: not rewritten + assertEquals( + "SELECT count(*) FROM information_schema.\"pg_inherits\"", + sql("SELECT count(*) FROM pg_inherits")); + // a scalar sub-query over an empty table is NULL + assertEquals( + "SELECT CAST(NULL AS STRING) AS \"adbin\" FROM information_schema.\"pg_class\"", + sql( + "SELECT (SELECT d.adbin FROM pg_attrdef d WHERE d.adrelid = pg_class.oid) AS adbin" + + " FROM pg_class")); + } + + @Test + public void outerConditionsMoveInFrontOfScalarSubQueries() throws Exception { + assertEquals( + "SELECT CASE WHEN \"a\".\"atthasdef\" THEN (SELECT \"t\".\"typname\"" + + " FROM information_schema.\"pg_type\" AS \"t\" WHERE \"t\".\"oid\" = \"a\".\"atttypid\")" + + " END FROM information_schema.\"pg_attribute\" AS \"a\"", + sql( + "SELECT (SELECT t.typname FROM pg_type t WHERE t.oid = a.atttypid AND a.atthasdef)" + + " FROM pg_attribute a")); + // not for EXISTS, and not for aggregating sub-queries + String exists = + sql( + "SELECT 1 FROM pg_attribute a WHERE EXISTS (SELECT 1 FROM pg_type t" + + " WHERE t.oid = a.atttypid AND a.atthasdef)"); + assertTrue(exists, !exists.contains("CASE")); + String count = + sql( + "SELECT (SELECT count(*) FROM pg_type t WHERE t.oid = a.atttypid AND a.atthasdef)" + + " FROM pg_attribute a"); + assertTrue(count, !count.contains("CASE")); + } + + @Test + public void morePsqlFunctions() throws Exception { + String encoding = sql("SELECT pg_encoding_to_char(d.encoding) FROM pg_database d"); + assertTrue(encoding, encoding.contains("WHEN 6 THEN 'UTF8'")); + assertTrue(encoding, encoding.contains("ELSE ''")); + assertEquals( + "SELECT CAST(NULL AS STRING) FROM information_schema.\"pg_database\" AS \"d\"", + sql("SELECT array_to_string(d.datacl, E'\\n') FROM pg_database d")); + refused( + "SELECT array_to_string(d.datname, ',') FROM pg_database d", "0A000", "array_to_string"); + String regtype = sql("SELECT c.reloftype::pg_catalog.regtype::pg_catalog.text FROM pg_class c"); + assertTrue(regtype, regtype.contains("WHEN 23 THEN 'integer'")); + assertTrue( + regtype, + regtype.endsWith( + "ELSE CAST(\"c\".\"reloftype\" AS STRING) END" + + " FROM information_schema.\"pg_class\" AS \"c\"")); + assertEquals("SELECT CAST(NULL AS STRING) AS \"x\"", sql("SELECT NULL AS x")); + } + + private static TranslationContext inZone(String zone) { + return new TranslationContext() { + @Override + public List searchPath() { + return CONTEXT.searchPath(); + } + + @Override + public boolean tableExists(String database, String table) { + return CONTEXT.tableExists(database, table); + } + + @Override + public String user() { + return "alice"; + } + + @Override + public String version() { + return "v"; + } + + @Override + public String currentDatabase() { + return "iotdb"; + } + + @Override + public int backendPid() { + return 42; + } + + @Override + public java.time.ZoneId zone() { + return java.time.ZoneId.of(zone); + } + }; + } + + private static String sqlIn(String zone, String pg) throws PgException { + return PgToTableSql.translate(pg, inZone(zone), false).sql(); + } + + @Test + public void dateTruncInTheSessionTimeZone() throws Exception { + // Europe/Berlin: local 1970-01-01 00:00 is -3600000 ms + assertEquals( + "SELECT date_bin(1mo, \"time\", -3600000) FROM \"plant\".\"sensors\"", + sqlIn("Europe/Berlin", "SELECT date_trunc('month', time) FROM sensors")); + assertEquals( + "SELECT date_bin(3mo, \"time\", -3600000), date_bin(12mo, \"time\", -3600000)" + + " FROM \"plant\".\"sensors\"", + sqlIn( + "Europe/Berlin", + "SELECT date_trunc('quarter', time), date_trunc('YEAR', time) FROM sensors")); + assertEquals( + "SELECT CAST(CAST(\"time\" AS DATE) AS TIMESTAMP) FROM \"plant\".\"sensors\"", + sqlIn("Europe/Berlin", "SELECT date_trunc('day', time) FROM sensors")); + assertEquals( + "SELECT date_bin(1h, \"time\", 0) FROM \"plant\".\"sensors\"", + sqlIn("Europe/Berlin", "SELECT date_trunc('hour', time) FROM sensors")); + assertEquals( + "SELECT date_bin(1w, \"time\", 345600000) FROM \"plant\".\"sensors\"", + sql("SELECT date_trunc('week', time) FROM sensors")); + // half-hour offsets: hours are not whole; DST: weeks have no fixed length + refused2("Asia/Kolkata", "SELECT date_trunc('hour', time) FROM sensors", "hour"); + refused2("Europe/Berlin", "SELECT date_trunc('week', time) FROM sensors", "daylight saving"); + refused2("UTC", "SELECT date_trunc('century', time) FROM sensors", "century"); + // other expressions are cast to TIMESTAMP (a DATE becomes local midnight) + assertEquals( + "SELECT CAST(CAST(CAST(CAST('2026-03-29' AS DATE) AS TIMESTAMP) AS DATE) AS TIMESTAMP)", + sql("SELECT date_trunc('day', CAST('2026-03-29' AS date))")); + } + + @Test + public void datePartsAsPostgreSql() throws Exception { + assertEquals( + "SELECT EXTRACT(YEAR FROM \"time\"), (EXTRACT(DOW FROM \"time\") % 7)" + + " FROM \"plant\".\"sensors\"", + sql("SELECT extract(year from time), extract(dow from time) FROM sensors")); + assertEquals( + "SELECT CAST(EXTRACT(MONTH FROM \"time\") AS DOUBLE) FROM \"plant\".\"sensors\"", + sql("SELECT date_part('month', time) FROM sensors")); + assertEquals( + "SELECT (CAST(CAST(\"time\" AS INT64) AS DOUBLE) / 1000) FROM \"plant\".\"sensors\"", + sql("SELECT extract(epoch from time) FROM sensors")); + refused("SELECT extract(week from time) FROM sensors", "0A000", "'week'"); + } + + @Test + public void intervalArithmetic() throws Exception { + assertEquals( + "SELECT * FROM \"plant\".\"sensors\" WHERE \"time\" >= (now() - 7200000)", + sqlIn("Europe/Berlin", "SELECT * FROM sensors WHERE time >= now() - interval '2 hours'")); + assertEquals( + "SELECT (CAST(\"time\" AS TIMESTAMP) + 5400000) FROM \"plant\".\"sensors\"", + sql("SELECT time + '1:30'::interval FROM sensors")); + assertEquals( + "SELECT (now() - 2592000000)", sql("SELECT CURRENT_TIMESTAMP - INTERVAL '30' DAY")); + assertEquals("SELECT (now() - 90000)", sql("SELECT now() - interval '1 min 30 sec'")); + refused2("Europe/Berlin", "SELECT now() - interval '7 days'", "daylight saving time"); + refused("SELECT now() - interval '1 microsecond'", "0A000", "precision"); + } + + @Test + public void stringRules() throws Exception { + assertEquals( + "SELECT CASE WHEN \"a\" IS NULL OR \"b\" IS NULL THEN CAST(NULL AS STRING)" + + " ELSE concat(\"a\", \"b\") END FROM \"plant\".\"sensors\"", + sql("SELECT a || b FROM sensors")); + assertEquals( + "SELECT COALESCE(concat(\"a\", 'x'), '') FROM \"plant\".\"sensors\"", + sql("SELECT concat(a, 'x') FROM sensors")); + refused("SELECT lpad(a, 5) FROM sensors", "0A000", "lpad"); + } + + private static void refused2(String zone, String pg, String messagePart) { + try { + fail("refused, but translated to: " + sqlIn(zone, pg)); + } catch (PgException e) { + assertEquals(e.getMessage(), "0A000", e.sqlState()); + assertTrue(e.getMessage(), e.getMessage().contains(messagePart)); + } + } + + @Test + public void dateAndTimestampLiteralsAsPostgreSqlReadsThem() throws Exception { + // timestamp: local time in the session zone, an offset is ignored + assertEquals( + "SELECT CAST(1792879200000 AS TIMESTAMP)", + sqlIn("Europe/Berlin", "SELECT TIMESTAMP '2026-10-25 00:00:00'")); + assertEquals( + "SELECT CAST(1792879200000 AS TIMESTAMP)", + sqlIn("Europe/Berlin", "SELECT TIMESTAMP '2026-10-25 00:00:00+05'")); + // timestamptz: its offset counts + assertEquals( + "SELECT CAST(1792886400000 AS TIMESTAMP)", + sqlIn("Europe/Berlin", "SELECT '2026-10-25 00:00:00+00'::timestamptz")); + assertEquals( + "SELECT CAST(1792886400250 AS TIMESTAMP)", + sql("SELECT timestamptz '2026-10-25T00:00:00.25Z'")); + assertEquals("SELECT CAST('2026-03-01' AS DATE)", sql("SELECT DATE '2026-3-1'")); + assertEquals("SELECT CAST('2026-03-01' AS DATE)", sql("SELECT '2026-03-01'::date")); + refused("SELECT TIMESTAMP 'yesterday'", "0A000", "yesterday"); + refused("SELECT TIME '10:00'", "0A000", "TIME literals"); + } + + @Test + public void dateParametersAreWrittenIntoTheStatement() throws Exception { + PgToTableSql.Translation t = + PgToTableSql.translate( + "SELECT * FROM sensors WHERE label = '?' AND day >= $2 AND n = $1", CONTEXT) + .withTypedParameters(new int[] {23, 1082}); + java.util.List values = + java.util.Arrays.asList( + new org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.StringLiteral( + "2026-01-02"), + new org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.LongLiteral("5")); + assertEquals( + "SELECT * FROM \"plant\".\"sensors\" WHERE \"label\" = '?'" + + " AND \"day\" >= CAST('2026-01-02' AS DATE) AND \"n\" = ?", + t.sqlFor(values)); + assertEquals(1, t.remaining(values).size()); + assertTrue( + t.remaining(values).get(0) + instanceof org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.LongLiteral); + } + + @Test + public void outputColumnNames() throws Exception { + assertEquals( + "SELECT count(*) AS \"count\", \"temp\", CAST(\"temp\" AS STRING) AS \"temp\"," + + " CAST('1' AS INT32) AS \"int4\", 1 AS \"?column?\", now() AS \"now\"," + + " \"temp\" AS \"t\" FROM \"plant\".\"sensors\" GROUP BY \"temp\"", + PgToTableSql.translate( + "SELECT count(*), temp, temp::text, '1'::integer, 1, now(), temp AS t" + + " FROM sensors GROUP BY temp", + CONTEXT) + .sql()); + // only the outer select list is named; a UNION takes the first branch's names + assertEquals( + "SELECT max(\"n\") AS \"max\" FROM (SELECT count(*) FROM \"plant\".\"sensors\") x" + + " UNION ALL SELECT 1", + PgToTableSql.translate( + "SELECT max(n) FROM (SELECT count(*) FROM sensors) x UNION ALL SELECT 1", CONTEXT) + .sql()); + } +} diff --git a/external-service-impl/postgresql/src/test/java/org/apache/iotdb/postgresql/sql/TranslatorGoldenTest.java b/external-service-impl/postgresql/src/test/java/org/apache/iotdb/postgresql/sql/TranslatorGoldenTest.java new file mode 100644 index 0000000000000..31a18e3b07b8c --- /dev/null +++ b/external-service-impl/postgresql/src/test/java/org/apache/iotdb/postgresql/sql/TranslatorGoldenTest.java @@ -0,0 +1,217 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.sql; + +import org.apache.iotdb.postgresql.protocol.PgException; +import org.apache.iotdb.postgresql.session.CursorCommand; +import org.apache.iotdb.postgresql.session.SessionCommands; + +import org.junit.Test; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * Golden tests: every statement of {@code corpus/.sql} either translates to the table-model + * SQL in {@code corpus/expected//.sql}, is a session command, or is refused with the + * SQLSTATE and message recorded there. + * + *

Run with {@code -Dgolden.update=true} to rewrite the expected files after a deliberate change; + * the maintainer reviews the diff. + */ +public class TranslatorGoldenTest { + + /** The source directory (not the copy in target/), so that updates land in the sources. */ + private static final Path CORPUS = corpusDirectory(); + + private static Path corpusDirectory() { + try { + // target/test-classes -> module directory + Path classes = + Paths.get( + TranslatorGoldenTest.class + .getProtectionDomain() + .getCodeSource() + .getLocation() + .toURI()); + return classes + .getParent() + .getParent() + .resolve(Paths.get("src", "test", "resources", "corpus")); + } catch (java.net.URISyntaxException e) { + throw new IllegalStateException(e); + } + } + + private static final String LICENSE = + "--\n" + + "-- Licensed to the Apache Software Foundation (ASF) under one\n" + + "-- or more contributor license agreements. See the NOTICE file\n" + + "-- distributed with this work for additional information\n" + + "-- regarding copyright ownership. The ASF licenses this file\n" + + "-- to you under the Apache License, Version 2.0 (the\n" + + "-- \"License\"); you may not use this file except in compliance\n" + + "-- with the License. You may obtain a copy of the License at\n" + + "--\n" + + "-- http://www.apache.org/licenses/LICENSE-2.0\n" + + "--\n" + + "-- Unless required by applicable law or agreed to in writing,\n" + + "-- software distributed under the License is distributed on an\n" + + "-- \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n" + + "-- KIND, either express or implied. See the License for the\n" + + "-- specific language governing permissions and limitations\n" + + "-- under the License.\n" + + "--\n"; + + /** The session of the recordings: database refdb (spec corpus/README.md), no search path. */ + static final TranslationContext CONTEXT = + new TranslationContext() { + @Override + public List searchPath() { + return Collections.emptyList(); + } + + @Override + public boolean tableExists(String database, String table) { + return database.equals("refdb") && Arrays.asList("sensors", "events").contains(table); + } + + @Override + public String user() { + return "admin"; + } + + @Override + public String version() { + return "PostgreSQL 14.0 (IoTDB), read-only adapter"; + } + + @Override + public String currentDatabase() { + return "refdb"; + } + + @Override + public int backendPid() { + return 4242; + } + }; + + /** The statements of a corpus file, comments removed. */ + static List statements(Path file) throws IOException { + List statements = new ArrayList<>(); + StringBuilder current = null; + for (String line : Files.readAllLines(file, StandardCharsets.UTF_8)) { + if (line.equals("-- next")) { + if (current != null) { + statements.add(current.toString().trim()); + } + current = new StringBuilder(); + } else if (current != null && !line.startsWith("--")) { + current.append(line).append('\n'); + } + } + if (current != null) { + statements.add(current.toString().trim()); + } + return statements; + } + + /** What the adapter does with a statement, as written to the expected file. */ + static String outcome(String statement) { + try { + java.util.Optional cursor = CursorCommand.parse(statement); + if (cursor.isPresent()) { + if (cursor.get().kind() == CursorCommand.Kind.DECLARE) { + return "-- cursor DECLARE " + + cursor.get().name() + + " FOR\n" + + PgToTableSql.translate(cursor.get().query(), CONTEXT).sql() + + "\n"; + } + return "-- cursor " + cursor.get().kind() + " " + cursor.get().name() + "\n"; + } + } catch (PgException e) { + return "-- refused " + e.sqlState() + ": " + e.getMessage() + "\n"; + } + if (SessionCommands.isCommand(statement)) { + return "-- session command\n"; + } + try { + return PgToTableSql.translate(statement, CONTEXT).sql() + "\n"; + } catch (PgException e) { + return "-- refused " + e.sqlState() + ": " + e.getMessage() + "\n"; + } + } + + @Test + public void theCorpusTranslatesAsRecorded() throws IOException { + boolean update = Boolean.getBoolean("golden.update"); + List files; + try (Stream list = Files.list(CORPUS)) { + files = list.filter(p -> p.toString().endsWith(".sql")).sorted().collect(Collectors.toList()); + } + assertFalse("no corpus files in " + CORPUS.toAbsolutePath(), files.isEmpty()); + List mismatches = new ArrayList<>(); + int checked = 0; + for (Path file : files) { + String client = file.getFileName().toString().replace(".sql", ""); + Path expectedDir = CORPUS.resolve("expected").resolve(client); + List statements = statements(file); + for (int i = 0; i < statements.size(); i++) { + String actual = outcome(statements.get(i)); + Path expectedFile = expectedDir.resolve((i + 1) + ".sql"); + if (update) { + Files.createDirectories(expectedDir); + Files.write(expectedFile, (LICENSE + actual).getBytes(StandardCharsets.UTF_8)); + continue; + } + assertTrue( + "missing " + expectedFile + "; run with -Dgolden.update=true", + Files.exists(expectedFile)); + String expected = + new String(Files.readAllBytes(expectedFile), StandardCharsets.UTF_8) + .substring(LICENSE.length()); + checked++; + if (!expected.equals(actual)) { + mismatches.add( + client + "/" + (i + 1) + "\n expected: " + expected + " actual: " + actual); + } + } + } + assertEquals(String.join("\n", mismatches), Collections.emptyList(), mismatches); + if (!update) { + assertTrue(checked > 0); + } + } +} diff --git a/external-service-impl/postgresql/src/test/java/org/apache/iotdb/postgresql/types/ParameterDecoderTest.java b/external-service-impl/postgresql/src/test/java/org/apache/iotdb/postgresql/types/ParameterDecoderTest.java new file mode 100644 index 0000000000000..5bf58494566be --- /dev/null +++ b/external-service-impl/postgresql/src/test/java/org/apache/iotdb/postgresql/types/ParameterDecoderTest.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.types; + +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.BinaryLiteral; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.BooleanLiteral; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.DoubleLiteral; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Literal; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.LongLiteral; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.NullLiteral; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.StringLiteral; +import org.apache.iotdb.postgresql.protocol.PgException; + +import org.junit.Test; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.time.ZoneId; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** Parameters of every mapped type, text and binary, all three precisions. */ +public class ParameterDecoderTest { + + private static final ZoneId BERLIN = ZoneId.of("Europe/Berlin"); + + /** 2026-10-25T00:00:00Z: microseconds since 2000-01-01 (PostgreSQL's binary epoch). */ + private static final long PG_MICROS = 1792886400_000_000L - 946684800_000_000L; + + private static Literal text(ParameterDecoder d, int oid, String value) throws PgException { + return d.decode(oid, false, value.getBytes(StandardCharsets.UTF_8)); + } + + private static String longValue(Literal literal) { + assertTrue(literal.getClass().getSimpleName(), literal instanceof LongLiteral); + return ((LongLiteral) literal).getValue(); + } + + @Test + public void textValues() throws Exception { + ParameterDecoder d = new ParameterDecoder(BERLIN, TimestampPrecision.MS); + assertEquals("42", longValue(text(d, 23, " 42 "))); + assertEquals("-7", longValue(text(d, 21, "-7"))); + assertEquals("9000000000", longValue(text(d, 20, "9000000000"))); + assertEquals(2.5, ((DoubleLiteral) text(d, 701, "2.5")).getValue(), 0); + assertEquals(0.25, ((DoubleLiteral) text(d, 700, "0.25")).getValue(), 0); + assertTrue(((BooleanLiteral) text(d, 16, "t")).getValue()); + assertTrue(!((BooleanLiteral) text(d, 16, "off")).getValue()); + assertEquals("2026-03-01", ((StringLiteral) text(d, 1082, "2026-03-01")).getValue()); + assertEquals("hello", ((StringLiteral) text(d, 25, "hello")).getValue()); + assertEquals("hello", ((StringLiteral) text(d, 1043, "hello")).getValue()); + assertTrue(text(d, 17, "\\x0102") instanceof BinaryLiteral); + assertTrue(d.decode(23, false, null) instanceof NullLiteral); + } + + @Test + public void timestampsInEveryPrecision() throws Exception { + long ms = 1792886400_000L; + String[] precisions = {"ms", "us", "ns"}; + long[] expected = {ms, ms * 1000, ms * 1_000_000}; + for (int i = 0; i < precisions.length; i++) { + ParameterDecoder d = new ParameterDecoder(BERLIN, TimestampPrecision.of(precisions[i])); + // timestamptz with an offset + assertEquals(String.valueOf(expected[i]), longValue(text(d, 1184, "2026-10-25 00:00:00+00"))); + // timestamp: local time in the session zone (CEST here, UTC+2), an offset is ignored + assertEquals( + String.valueOf(expected[i] - 7_200_000L * (expected[i] / ms)), + longValue(text(d, 1114, "2026-10-25 00:00:00+05"))); + // binary: microseconds since 2000-01-01 UTC + byte[] binary = ByteBuffer.allocate(8).putLong(PG_MICROS).array(); + assertEquals(String.valueOf(expected[i]), longValue(d.decode(1184, true, binary))); + } + } + + @Test + public void binaryValues() throws Exception { + ParameterDecoder d = new ParameterDecoder(BERLIN, TimestampPrecision.MS); + assertEquals( + "7", longValue(d.decode(21, true, ByteBuffer.allocate(2).putShort((short) 7).array()))); + assertEquals("42", longValue(d.decode(23, true, ByteBuffer.allocate(4).putInt(42).array()))); + assertEquals("-1", longValue(d.decode(20, true, ByteBuffer.allocate(8).putLong(-1).array()))); + assertEquals( + 1.5, + ((DoubleLiteral) d.decode(701, true, ByteBuffer.allocate(8).putDouble(1.5).array())) + .getValue(), + 0); + assertEquals( + 0.5, + ((DoubleLiteral) d.decode(700, true, ByteBuffer.allocate(4).putFloat(0.5f).array())) + .getValue(), + 0); + assertTrue(((BooleanLiteral) d.decode(16, true, new byte[] {1})).getValue()); + // dates: days since 2000-01-01 + assertEquals( + "2000-01-31", + ((StringLiteral) d.decode(1082, true, ByteBuffer.allocate(4).putInt(30).array())) + .getValue()); + assertEquals( + "abc", + ((StringLiteral) d.decode(1043, true, "abc".getBytes(StandardCharsets.UTF_8))).getValue()); + } + + @Test + public void untypedParametersAreInferred() throws Exception { + ParameterDecoder d = new ParameterDecoder(BERLIN, TimestampPrecision.MS); + assertTrue(text(d, 0, "12") instanceof LongLiteral); + assertTrue(text(d, 0, "1.5") instanceof DoubleLiteral); + assertTrue(text(d, 0, "2026-10-25 00:00:00+00") instanceof LongLiteral); + assertTrue(text(d, 0, "m1") instanceof StringLiteral); + } + + @Test + public void invalidValuesAre22P02() { + ParameterDecoder d = new ParameterDecoder(BERLIN, TimestampPrecision.MS); + try { + text(d, 1184, "yesterday"); + fail("invalid"); + } catch (PgException e) { + assertEquals("22P02", e.sqlState()); + } + } +} diff --git a/external-service-impl/postgresql/src/test/java/org/apache/iotdb/postgresql/types/PgValueEncoderTest.java b/external-service-impl/postgresql/src/test/java/org/apache/iotdb/postgresql/types/PgValueEncoderTest.java new file mode 100644 index 0000000000000..f029f94e921c1 --- /dev/null +++ b/external-service-impl/postgresql/src/test/java/org/apache/iotdb/postgresql/types/PgValueEncoderTest.java @@ -0,0 +1,173 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.types; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.read.common.block.TsBlockBuilder; +import org.apache.tsfile.utils.Binary; +import org.junit.Test; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.time.ZoneId; +import java.util.Collections; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + +public class PgValueEncoderTest { + + private static final ZoneId BERLIN = ZoneId.of("Europe/Berlin"); + + /** One value of one type in a column. */ + private static Column column(TSDataType type, Object value) { + TsBlockBuilder builder = new TsBlockBuilder(Collections.singletonList(type)); + builder.getTimeColumnBuilder().writeLong(0); + builder.getColumnBuilder(0).writeObject(value); + builder.declarePosition(); + TsBlock block = builder.build(); + return block.getColumn(0); + } + + private static String text(PgType type, TSDataType dataType, Object value, TimestampPrecision p) { + return new String( + new PgValueEncoder(BERLIN, p).text(type, column(dataType, value), 0), + StandardCharsets.UTF_8); + } + + private static String text(PgType type, TSDataType dataType, Object value) { + return text(type, dataType, value, TimestampPrecision.MS); + } + + private static byte[] binary( + PgType type, TSDataType dataType, Object value, TimestampPrecision p) { + return new PgValueEncoder(BERLIN, p).binary(type, column(dataType, value), 0); + } + + @Test + public void everyExposedTypeHasItsOid() { + assertEquals(16, PgType.of(TSDataType.BOOLEAN).get().oid()); + assertEquals(23, PgType.of(TSDataType.INT32).get().oid()); + assertEquals(20, PgType.of(TSDataType.INT64).get().oid()); + assertEquals(700, PgType.of(TSDataType.FLOAT).get().oid()); + assertEquals(701, PgType.of(TSDataType.DOUBLE).get().oid()); + assertEquals(25, PgType.of(TSDataType.TEXT).get().oid()); + assertEquals(25, PgType.of(TSDataType.STRING).get().oid()); + assertEquals(17, PgType.of(TSDataType.BLOB).get().oid()); + assertEquals(1082, PgType.of(TSDataType.DATE).get().oid()); + assertEquals(1184, PgType.of(TSDataType.TIMESTAMP).get().oid()); + assertEquals(1185, PgType.TIMESTAMPTZ.arrayOid()); + assertEquals(false, PgType.of(TSDataType.OBJECT).isPresent()); + } + + @Test + public void simpleTextValues() { + assertEquals("t", text(PgType.BOOL, TSDataType.BOOLEAN, true)); + assertEquals("f", text(PgType.BOOL, TSDataType.BOOLEAN, false)); + assertEquals("-42", text(PgType.INT4, TSDataType.INT32, -42)); + assertEquals("9007199254740993", text(PgType.INT8, TSDataType.INT64, 9007199254740993L)); + assertEquals( + "Grüße", text(PgType.TEXT, TSDataType.STRING, new Binary("Grüße", StandardCharsets.UTF_8))); + assertEquals( + "\\x00ff10", + text(PgType.BYTEA, TSDataType.BLOB, new Binary(new byte[] {0, (byte) 0xff, 0x10}))); + assertEquals("2026-10-25", text(PgType.DATE, TSDataType.DATE, 20261025)); + } + + @Test + public void floatTextFollowsPostgresShortestOutput() { + assertEquals("1.5", PgValueEncoder.float8Text(1.5)); + assertEquals("1", PgValueEncoder.float8Text(1.0)); + assertEquals("0", PgValueEncoder.float8Text(0.0)); + assertEquals("-0", PgValueEncoder.float8Text(-0.0)); + assertEquals("0.1", PgValueEncoder.float8Text(0.1)); + assertEquals("0.0001", PgValueEncoder.float8Text(0.0001)); + assertEquals("1e-05", PgValueEncoder.float8Text(0.00001)); + assertEquals("123456789012345", PgValueEncoder.float8Text(123456789012345.0)); + assertEquals("1e+15", PgValueEncoder.float8Text(1e15)); + assertEquals("1.7976931348623157e+308", PgValueEncoder.float8Text(Double.MAX_VALUE)); + assertEquals("-2.5e-07", PgValueEncoder.float8Text(-2.5e-7)); + assertEquals("NaN", PgValueEncoder.float8Text(Double.NaN)); + assertEquals("Infinity", PgValueEncoder.float8Text(Double.POSITIVE_INFINITY)); + assertEquals("-Infinity", PgValueEncoder.float8Text(Double.NEGATIVE_INFINITY)); + assertEquals("123456", PgValueEncoder.float4Text(123456f)); + assertEquals("1.234567e+06", PgValueEncoder.float4Text(1234567f)); + assertEquals("0.1", PgValueEncoder.float4Text(0.1f)); + assertEquals("NaN", PgValueEncoder.float4Text(Float.NaN)); + } + + @Test + public void timestampTextUsesTheSessionZoneLikePostgres() { + // 2026-10-25 00:30 UTC is still summer time in Berlin, 01:30 UTC already winter time + assertEquals( + "2026-10-25 02:30:00+02", text(PgType.TIMESTAMPTZ, TSDataType.TIMESTAMP, 1792888200000L)); + assertEquals( + "2026-10-25 02:30:00+01", text(PgType.TIMESTAMPTZ, TSDataType.TIMESTAMP, 1792891800000L)); + assertEquals( + "2026-10-25 02:30:00.5+01", text(PgType.TIMESTAMPTZ, TSDataType.TIMESTAMP, 1792891800500L)); + assertEquals( + "2026-10-25 02:30:00.000001+01", + text(PgType.TIMESTAMPTZ, TSDataType.TIMESTAMP, 1792891800000001L, TimestampPrecision.US)); + assertEquals( + "1969-12-31 23:59:59.999+00", PgValueEncoder.timestamptzText(-1000L, ZoneId.of("UTC"))); + assertEquals( + "2026-01-01 05:30:00+05:30", + PgValueEncoder.timestamptzText(1767225600000000L, ZoneId.of("Asia/Kolkata"))); + } + + @Test + public void binaryValues() { + assertArrayEquals( + new byte[] {1}, binary(PgType.BOOL, TSDataType.BOOLEAN, true, TimestampPrecision.MS)); + assertArrayEquals( + ByteBuffer.allocate(4).putInt(-42).array(), + binary(PgType.INT4, TSDataType.INT32, -42, TimestampPrecision.MS)); + assertArrayEquals( + ByteBuffer.allocate(8).putDouble(Double.NaN).array(), + binary(PgType.FLOAT8, TSDataType.DOUBLE, Double.NaN, TimestampPrecision.MS)); + // 2000-01-01 is day 0, 2026-10-25 is day 9794 + assertArrayEquals( + ByteBuffer.allocate(4).putInt(9794).array(), + binary(PgType.DATE, TSDataType.DATE, 20261025, TimestampPrecision.MS)); + assertArrayEquals( + ByteBuffer.allocate(4).putInt(0).array(), + binary(PgType.DATE, TSDataType.DATE, 20000101, TimestampPrecision.MS)); + } + + @Test + public void binaryTimestampsAreMicrosSince2000InEveryPrecision() { + long micros = 1792891800000001L - PgValueEncoder.PG_EPOCH_MICROS; + assertArrayEquals( + ByteBuffer.allocate(8).putLong(micros).array(), + binary(PgType.TIMESTAMPTZ, TSDataType.TIMESTAMP, 1792891800000001L, TimestampPrecision.US)); + assertArrayEquals( + ByteBuffer.allocate(8).putLong(micros).array(), + binary( + PgType.TIMESTAMPTZ, TSDataType.TIMESTAMP, 1792891800000001999L, TimestampPrecision.NS)); + assertArrayEquals( + ByteBuffer.allocate(8).putLong(micros - 1).array(), + binary(PgType.TIMESTAMPTZ, TSDataType.TIMESTAMP, 1792891800000000L, TimestampPrecision.US)); + assertArrayEquals( + ByteBuffer.allocate(8).putLong(-PgValueEncoder.PG_EPOCH_MICROS).array(), + binary(PgType.TIMESTAMPTZ, TSDataType.TIMESTAMP, 0L, TimestampPrecision.MS)); + } +} diff --git a/external-service-impl/postgresql/src/test/resources/corpus/README.md b/external-service-impl/postgresql/src/test/resources/corpus/README.md new file mode 100644 index 0000000000000..ece4678b15551 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/README.md @@ -0,0 +1,91 @@ + + +# Client SQL corpus (spec 012, T001) + +What PostgreSQL clients actually send, recorded against a real PostgreSQL 14 with +`log_statement=all`. The SQL translation and the catalog tables are built and tested against these +statements (golden tests, T034/T044), not against guesses. + +One file per client, `.sql`: a header with client and server versions and the capture +date, then the statements in order, each after a `-- next` line. Bound parameters of the extended +protocol follow as a `-- parameters: ...` comment; repeated identical executions (portal fetches, +repeated prepared statements) are collapsed into one entry with a count. + +| File | Client | What was done | +|---|---|---| +| `psql.sql` | psql 14.25 | `\l`, `\dn`, `\dt refdb.*`, `\d refdb.sensors`, `\d+ refdb.events`, `\dt *.*`, one aggregation | +| `pgjdbc.sql` | pgJDBC 42.7.4 (DBeaver's driver) | `DatabaseMetaData` catalogs, schemas, tables, columns, primary keys, indexes, imported keys, type info; a plain query; a prepared statement executed 6 times; `setFetchSize(1000)` over 100,000 rows | +| `psycopg.sql` | psycopg 3.3.9 | `version()`, plain query, parameterised query 6 times, a binary-result query, a server-side cursor over 100,000 rows | +| `powerbi.sql` | Power BI Desktop | **to be recorded** (owner, Windows), see below | +| `excel.sql`, `metabase.sql` | Excel, Metabase | to be recorded | + +## Expected outcomes + +`expected//.sql` holds, for the n-th statement of `.sql`, what the adapter does +with it: the table-model SQL it runs, `-- session command`, or `-- refused : `. +`TranslatorGoldenTest` (T034) compares them; after a deliberate change run it with +`-Dgolden.update=true` and review the diff. `EmbeddedPostgreSqlIT` runs every translated statement +without parameters on the engine. + +## Repeating a recording + +The scripts are in `services/postgresql/src/test/corpus-capture/`: + +```shell +docker run -d --name pg14-corpus -e POSTGRES_PASSWORD= -p 127.0.0.1:54314:5432 \ + postgres:14-alpine -c log_statement=all \ + -c "log_line_prefix=%m [%p] app=%a user=%u db=%d " +docker exec -i pg14-corpus psql -U postgres < services/postgresql/src/test/corpus-capture/setup.sql +# clients: give each its own application_name (psql: PGAPPNAME=psql) +java -cp postgresql-42.7.4.jar services/postgresql/src/test/corpus-capture/PgJdbcCapture.java +python3 services/postgresql/src/test/corpus-capture/psycopg_capture.py # psycopg 3 installed +docker logs pg14-corpus > pg14.log 2>&1 +python3 services/postgresql/src/test/corpus-capture/extract.py pg14.log \ + services/postgresql/src/test/resources/corpus 14.25 psql=14.25 pgjdbc=42.7.4 psycopg=3.3.9 +``` + +`extract.py` skips everything up to the final `ANALYZE` of `setup.sql` and writes one file per +`application_name` given on the command line. + +## Power BI and Excel (Windows) + +Power BI's connector sets no `application_name` of its own that we can rely on, so record it alone: +start a fresh container (as above, but published on an address the Windows machine reaches, for +example `-p :54314:5432`), run `setup.sql`, then on Windows: + +1. Power BI Desktop → Get data → PostgreSQL database → server `:54314`, database + `iotdb`, Import → user `postgres` and the password. When asked about encryption, accept the + unencrypted connection (the recording container has no certificate). +2. Navigator: expand `refdb` and `other`, select `refdb.sensors` and `refdb.events`, load. +3. A second report with **DirectQuery** on `refdb.sensors`: card with the last `temperature`; line + chart of average `temperature` by `time` with the date hierarchy, drilled down year → quarter → + month → day; clustered bar of average `temperature` by `machine`; table and matrix by `plant`, + `line`, `machine`; slicer on `machine`; relative date filter (last 7 days) on `time`; a Top N + filter (top 3 machines by average `temperature`). +4. Refresh once, then close Power BI. +5. Excel: Data → Get Data → From Database → From PostgreSQL Database, same server, load + `refdb.events`. + +Then `docker logs` into a file and run `extract.py` with the application names found in the log +(`grep -o 'app=[^ ]*' pg14.log | sort | uniq -c`). An empty name works too (`=`); it is +written to `.sql`, so rename that file to `powerbi.sql` or `excel.sql`. Record the Power BI Desktop +and Excel versions in the header. diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/1.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/1.sql new file mode 100644 index 0000000000000..5d64e61816087 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/1.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT "datname" AS "table_cat" FROM information_schema."pg_database" WHERE "datallowconn" = TRUE ORDER BY "datname" NULLS LAST diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/10.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/10.sql new file mode 100644 index 0000000000000..d731a673d8515 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/10.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +-- refused 0A000: table functions in FROM is not supported diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/11.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/11.sql new file mode 100644 index 0000000000000..2866183213ea7 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/11.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT * FROM "refdb"."sensors" LIMIT 5 diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/12.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/12.sql new file mode 100644 index 0000000000000..ca2cbc04970b9 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/12.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT "time", "machine", "temperature" FROM "refdb"."sensors" WHERE "machine" = ? AND "time" >= ? ORDER BY "time" NULLS LAST LIMIT ? diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/13.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/13.sql new file mode 100644 index 0000000000000..ca2cbc04970b9 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/13.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT "time", "machine", "temperature" FROM "refdb"."sensors" WHERE "machine" = ? AND "time" >= ? ORDER BY "time" NULLS LAST LIMIT ? diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/14.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/14.sql new file mode 100644 index 0000000000000..ca2cbc04970b9 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/14.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT "time", "machine", "temperature" FROM "refdb"."sensors" WHERE "machine" = ? AND "time" >= ? ORDER BY "time" NULLS LAST LIMIT ? diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/15.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/15.sql new file mode 100644 index 0000000000000..ca2cbc04970b9 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/15.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT "time", "machine", "temperature" FROM "refdb"."sensors" WHERE "machine" = ? AND "time" >= ? ORDER BY "time" NULLS LAST LIMIT ? diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/16.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/16.sql new file mode 100644 index 0000000000000..ca2cbc04970b9 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/16.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT "time", "machine", "temperature" FROM "refdb"."sensors" WHERE "machine" = ? AND "time" >= ? ORDER BY "time" NULLS LAST LIMIT ? diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/17.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/17.sql new file mode 100644 index 0000000000000..ca2cbc04970b9 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/17.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT "time", "machine", "temperature" FROM "refdb"."sensors" WHERE "machine" = ? AND "time" >= ? ORDER BY "time" NULLS LAST LIMIT ? diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/18.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/18.sql new file mode 100644 index 0000000000000..81c84bd1dfe17 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/18.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +-- session command diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/19.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/19.sql new file mode 100644 index 0000000000000..39157ed3a1cfa --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/19.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT * FROM "refdb"."events" diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/2.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/2.sql new file mode 100644 index 0000000000000..e5987086f656f --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/2.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT "nspname" AS "table_schem", CAST(NULL AS STRING) AS "table_catalog" FROM information_schema."pg_namespace" WHERE "nspname" <> 'pg_toast' AND (NOT ("nspname" LIKE 'pg\_temp\_%' ESCAPE '\') OR "nspname" = 'pg_catalog') AND (NOT ("nspname" LIKE 'pg\_toast\_temp\_%' ESCAPE '\') OR "nspname" = replace('pg_catalog', 'pg_temp_', 'pg_toast_temp_')) ORDER BY "table_schem" NULLS LAST diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/20.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/20.sql new file mode 100644 index 0000000000000..81c84bd1dfe17 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/20.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +-- session command diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/3.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/3.sql new file mode 100644 index 0000000000000..0c90c16f9568d --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/3.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT CAST(NULL AS STRING) AS "table_cat", "n"."nspname" AS "table_schem", "c"."relname" AS "table_name", CASE ("n"."nspname" LIKE 'pg\_%' ESCAPE '\') OR "n"."nspname" = 'information_schema' WHEN TRUE THEN CASE WHEN "n"."nspname" = 'pg_catalog' OR "n"."nspname" = 'information_schema' THEN CASE "c"."relkind" WHEN 'r' THEN 'SYSTEM TABLE' WHEN 'v' THEN 'SYSTEM VIEW' WHEN 'i' THEN 'SYSTEM INDEX' ELSE CAST(NULL AS STRING) END WHEN "n"."nspname" = 'pg_toast' THEN CASE "c"."relkind" WHEN 'r' THEN 'SYSTEM TOAST TABLE' WHEN 'i' THEN 'SYSTEM TOAST INDEX' ELSE CAST(NULL AS STRING) END ELSE CASE "c"."relkind" WHEN 'r' THEN 'TEMPORARY TABLE' WHEN 'p' THEN 'TEMPORARY TABLE' WHEN 'i' THEN 'TEMPORARY INDEX' WHEN 'S' THEN 'TEMPORARY SEQUENCE' WHEN 'v' THEN 'TEMPORARY VIEW' ELSE CAST(NULL AS STRING) END END WHEN FALSE THEN CASE "c"."relkind" WHEN 'r' THEN 'TABLE' WHEN 'p' THEN 'PARTITIONED TABLE' WHEN 'i' THEN 'INDEX' WHEN 'P' THEN 'PARTITIONED INDEX' WHEN 'S' THEN 'SEQUENCE' WHEN 'v' THEN 'VIEW' WHEN 'c' THEN 'TYPE' WHEN 'f' THEN 'FOREIGN TABLE' WHEN 'm' THEN 'MATERIALIZED VIEW' ELSE CAST(NULL AS STRING) END ELSE CAST(NULL AS STRING) END AS "table_type", "d"."description" AS "remarks", '' AS "type_cat", '' AS "type_schem", '' AS "type_name", '' AS "self_referencing_col_name", '' AS "ref_generation" FROM information_schema."pg_namespace" AS "n", information_schema."pg_class" AS "c" LEFT JOIN information_schema."pg_description" AS "d" ON ("c"."oid" = "d"."objoid" AND "d"."objsubid" = 0 AND "d"."classoid" = 1259) WHERE "c"."relnamespace" = "n"."oid" AND "n"."nspname" LIKE 'refdb' AND "c"."relname" LIKE '%' AND (FALSE OR ("c"."relkind" = 'r' AND NOT ("n"."nspname" LIKE 'pg\_%' ESCAPE '\') AND "n"."nspname" <> 'information_schema') OR ("c"."relkind" = 'v' AND "n"."nspname" <> 'pg_catalog' AND "n"."nspname" <> 'information_schema')) ORDER BY "table_type" NULLS LAST, "table_schem" NULLS LAST, "table_name" NULLS LAST diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/4.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/4.sql new file mode 100644 index 0000000000000..c2d25fa944c7d --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/4.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT * FROM (SELECT "n"."nspname", "c"."relname", "a"."attname", "a"."atttypid", "a"."attnotnull" OR ("t"."typtype" = 'd' AND "t"."typnotnull") AS "attnotnull", "a"."atttypmod", "a"."attlen", "t"."typtypmod", row_number() OVER (PARTITION BY "a"."attrelid" ORDER BY "a"."attnum" NULLS LAST) AS "attnum", CASE WHEN "a"."attidentity" <> '' OR "a"."attidentity" IS NULL OR '' IS NULL THEN "a"."attidentity" END AS "attidentity", CASE WHEN "a"."attgenerated" <> '' OR "a"."attgenerated" IS NULL OR '' IS NULL THEN "a"."attgenerated" END AS "attgenerated", CAST(NULL AS STRING) AS "adsrc", "dsc"."description", "t"."typbasetype", "t"."typtype" FROM information_schema."pg_namespace" AS "n" JOIN information_schema."pg_class" AS "c" ON ("c"."relnamespace" = "n"."oid") JOIN information_schema."pg_attribute" AS "a" ON ("a"."attrelid" = "c"."oid") JOIN information_schema."pg_type" AS "t" ON ("a"."atttypid" = "t"."oid") LEFT JOIN information_schema."pg_attrdef" AS "def" ON ("a"."attrelid" = "def"."adrelid" AND "a"."attnum" = "def"."adnum") LEFT JOIN information_schema."pg_description" AS "dsc" ON ("c"."oid" = "dsc"."objoid" AND "a"."attnum" = "dsc"."objsubid") LEFT JOIN information_schema."pg_class" AS "dc" ON ("dc"."oid" = "dsc"."classoid" AND "dc"."relname" = 'pg_class') LEFT JOIN information_schema."pg_namespace" AS "dn" ON ("dc"."relnamespace" = "dn"."oid" AND "dn"."nspname" = 'pg_catalog') WHERE "c"."relkind" IN ('r', 'p', 'v', 'f', 'm') AND "a"."attnum" > 0 AND NOT "a"."attisdropped" AND "n"."nspname" LIKE 'refdb' AND "c"."relname" LIKE 'sensors') c WHERE TRUE AND "attname" LIKE '%' ORDER BY "nspname" NULLS LAST, "c"."relname" NULLS LAST, "attnum" NULLS LAST diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/5.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/5.sql new file mode 100644 index 0000000000000..d829f4c5987c4 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/5.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT CAST(NULL AS STRING) AS "table_cat", CAST(NULL AS STRING) AS "table_schem", CAST(NULL AS STRING) AS "table_name", CAST(NULL AS STRING) AS "column_name", CAST(NULL AS STRING) AS "key_seq", CAST(NULL AS STRING) AS "pk_name" FROM (SELECT CAST(NULL AS STRING) AS "table_cat", CAST(NULL AS STRING) AS "table_schem", CAST(NULL AS STRING) AS "table_name", CAST(NULL AS STRING) AS "column_name", CAST(NULL AS STRING) AS "key_seq", CAST(NULL AS STRING) AS "pk_name", CAST(NULL AS STRING) AS "keys", CAST(NULL AS STRING) AS "a_attnum" FROM information_schema."pg_index" WHERE FALSE) result WHERE FALSE diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/6.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/6.sql new file mode 100644 index 0000000000000..1e055e4c65d4d --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/6.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT CAST(NULL AS STRING) AS "table_cat", CAST(NULL AS STRING) AS "table_schem", CAST(NULL AS STRING) AS "table_name", CAST(NULL AS STRING) AS "non_unique", CAST(NULL AS STRING) AS "index_qualifier", CAST(NULL AS STRING) AS "index_name", CAST(NULL AS STRING) AS "type", CAST(NULL AS STRING) AS "ordinal_position", CAST(NULL AS STRING) AS "column_name", CAST(NULL AS STRING) AS "asc_or_desc", CAST(NULL AS STRING) AS "cardinality", CAST(NULL AS STRING) AS "pages", CAST(NULL AS STRING) AS "filter_condition" FROM (SELECT CAST(NULL AS STRING) AS "table_cat", CAST(NULL AS STRING) AS "table_schem", CAST(NULL AS STRING) AS "table_name", CAST(NULL AS STRING) AS "non_unique", CAST(NULL AS STRING) AS "index_qualifier", CAST(NULL AS STRING) AS "index_name", CAST(NULL AS STRING) AS "type", CAST(NULL AS STRING) AS "ordinal_position", CAST(NULL AS STRING) AS "cardinality", CAST(NULL AS STRING) AS "pages", CAST(NULL AS STRING) AS "filter_condition", CAST(NULL AS STRING) AS "ci_oid", CAST(NULL AS STRING) AS "i_indoption", CAST(NULL AS STRING) AS "am_name" FROM information_schema."pg_index" WHERE FALSE) AS tmp WHERE FALSE diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/7.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/7.sql new file mode 100644 index 0000000000000..8923ed0b721e0 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/7.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT "setting" FROM information_schema."pg_settings" WHERE "name" = 'max_index_keys' diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/8.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/8.sql new file mode 100644 index 0000000000000..36059d5c7d151 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/8.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT CAST(NULL AS STRING) AS "pktable_cat", CAST(NULL AS STRING) AS "pktable_schem", CAST(NULL AS STRING) AS "pktable_name", CAST(NULL AS STRING) AS "pkcolumn_name", CAST(NULL AS STRING) AS "fktable_cat", CAST(NULL AS STRING) AS "fktable_schem", CAST(NULL AS STRING) AS "fktable_name", CAST(NULL AS STRING) AS "fkcolumn_name", CAST(NULL AS STRING) AS "key_seq", CAST(NULL AS STRING) AS "update_rule", CAST(NULL AS STRING) AS "delete_rule", CAST(NULL AS STRING) AS "fk_name", CAST(NULL AS STRING) AS "pk_name", CAST(NULL AS STRING) AS "deferrability" FROM information_schema."pg_constraint" WHERE FALSE diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/9.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/9.sql new file mode 100644 index 0000000000000..01abf39616662 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/pgjdbc/9.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT "t"."typname", "t"."oid" FROM information_schema."pg_type" AS "t" JOIN information_schema."pg_namespace" AS "n" ON ("t"."typnamespace" = "n"."oid") WHERE "n"."nspname" != 'pg_toast' AND ("t"."typrelid" = 0 OR (SELECT "c"."relkind" = 'c' FROM information_schema."pg_class" AS "c" WHERE "c"."oid" = "t"."typrelid")) diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/psql/1.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/psql/1.sql new file mode 100644 index 0000000000000..7566732ec0897 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/psql/1.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT "d"."datname" AS "Name", 'admin' AS "Owner", CASE CAST("d"."encoding" AS INT32) WHEN 0 THEN 'SQL_ASCII' WHEN 1 THEN 'EUC_JP' WHEN 2 THEN 'EUC_CN' WHEN 3 THEN 'EUC_KR' WHEN 4 THEN 'EUC_TW' WHEN 5 THEN 'EUC_JIS_2004' WHEN 6 THEN 'UTF8' WHEN 7 THEN 'MULE_INTERNAL' WHEN 8 THEN 'LATIN1' WHEN 9 THEN 'LATIN2' WHEN 10 THEN 'LATIN3' WHEN 11 THEN 'LATIN4' WHEN 12 THEN 'LATIN5' WHEN 13 THEN 'LATIN6' WHEN 14 THEN 'LATIN7' WHEN 15 THEN 'LATIN8' WHEN 16 THEN 'LATIN9' WHEN 17 THEN 'LATIN10' WHEN 18 THEN 'WIN1256' WHEN 19 THEN 'WIN1258' WHEN 20 THEN 'WIN866' WHEN 21 THEN 'WIN874' WHEN 22 THEN 'KOI8R' WHEN 23 THEN 'WIN1251' WHEN 24 THEN 'WIN1252' WHEN 25 THEN 'ISO_8859_5' WHEN 26 THEN 'ISO_8859_6' WHEN 27 THEN 'ISO_8859_7' WHEN 28 THEN 'ISO_8859_8' WHEN 29 THEN 'WIN1250' WHEN 30 THEN 'WIN1253' WHEN 31 THEN 'WIN1254' WHEN 32 THEN 'WIN1255' WHEN 33 THEN 'WIN1257' WHEN 34 THEN 'KOI8U' WHEN 35 THEN 'SJIS' WHEN 36 THEN 'BIG5' WHEN 37 THEN 'GBK' WHEN 38 THEN 'UHC' WHEN 39 THEN 'GB18030' WHEN 40 THEN 'JOHAB' WHEN 41 THEN 'SHIFT_JIS_2004' ELSE CASE WHEN "d"."encoding" IS NULL THEN CAST(NULL AS STRING) ELSE '' END END AS "Encoding", "d"."datcollate" AS "Collate", "d"."datctype" AS "Ctype", CAST(NULL AS STRING) AS "Access privileges" FROM information_schema."pg_database" AS "d" ORDER BY 1 NULLS LAST diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/psql/10.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/psql/10.sql new file mode 100644 index 0000000000000..29fb390cd2991 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/psql/10.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT CAST(NULL AS STRING) AS "oid" FROM information_schema."pg_inherits" WHERE FALSE diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/psql/11.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/psql/11.sql new file mode 100644 index 0000000000000..bd0af996ae700 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/psql/11.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT CAST(NULL AS STRING) AS "oid", CAST(NULL AS STRING) AS "relkind", CAST(NULL AS BOOLEAN) AS "inhdetachpending", CAST(NULL AS STRING) AS "pg_get_expr" FROM information_schema."pg_inherits" WHERE FALSE diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/psql/12.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/psql/12.sql new file mode 100644 index 0000000000000..60b4fc5c0d5a6 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/psql/12.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT "c"."oid", "n"."nspname", "c"."relname" FROM information_schema."pg_class" AS "c" LEFT JOIN information_schema."pg_namespace" AS "n" ON "n"."oid" = "c"."relnamespace" WHERE ("c"."relname" = 'events') AND ("n"."nspname" = 'refdb') ORDER BY 2 NULLS LAST, 3 NULLS LAST diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/psql/13.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/psql/13.sql new file mode 100644 index 0000000000000..d731a673d8515 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/psql/13.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +-- refused 0A000: table functions in FROM is not supported diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/psql/14.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/psql/14.sql new file mode 100644 index 0000000000000..fcb78995e94c2 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/psql/14.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT "a"."attname", CASE CAST("a"."atttypid" AS INT32) WHEN 16 THEN 'boolean' WHEN 17 THEN 'bytea' WHEN 18 THEN '"char"' WHEN 19 THEN 'name' WHEN 20 THEN 'bigint' WHEN 21 THEN 'smallint' WHEN 23 THEN 'integer' WHEN 24 THEN 'regproc' WHEN 25 THEN 'text' WHEN 26 THEN 'oid' WHEN 700 THEN 'real' WHEN 701 THEN 'double precision' WHEN 1043 THEN 'character varying' WHEN 1082 THEN 'date' WHEN 1114 THEN 'timestamp without time zone' WHEN 1184 THEN 'timestamp with time zone' WHEN 1700 THEN 'numeric' WHEN 2206 THEN 'regtype' WHEN 705 THEN 'unknown' WHEN 2249 THEN 'record' WHEN 2278 THEN 'void' WHEN 1000 THEN 'boolean[]' WHEN 1001 THEN 'bytea[]' WHEN 1002 THEN '"char"[]' WHEN 1003 THEN 'name[]' WHEN 1016 THEN 'bigint[]' WHEN 1005 THEN 'smallint[]' WHEN 1007 THEN 'integer[]' WHEN 1008 THEN 'regproc[]' WHEN 1009 THEN 'text[]' WHEN 1028 THEN 'oid[]' WHEN 1021 THEN 'real[]' WHEN 1022 THEN 'double precision[]' WHEN 1015 THEN 'character varying[]' WHEN 1182 THEN 'date[]' WHEN 1115 THEN 'timestamp without time zone[]' WHEN 1185 THEN 'timestamp with time zone[]' WHEN 1231 THEN 'numeric[]' WHEN 2211 THEN 'regtype[]' WHEN 2287 THEN 'record[]' ELSE CASE WHEN "a"."atttypid" IS NULL THEN CAST(NULL AS STRING) ELSE '???' END END AS "format_type", CAST(NULL AS STRING), "a"."attnotnull", (SELECT "c"."collname" FROM information_schema."pg_collation" AS "c", information_schema."pg_type" AS "t" WHERE "c"."oid" = "a"."attcollation" AND "t"."oid" = "a"."atttypid" AND "a"."attcollation" <> "t"."typcollation") AS "attcollation", "a"."attidentity", "a"."attgenerated", "a"."attstorage", "a"."attcompression" AS "attcompression", CASE WHEN "a"."attstattarget" = -1 THEN CAST(NULL AS INT32) ELSE "a"."attstattarget" END AS "attstattarget", (SELECT max(pg_d."description") FROM information_schema."pg_description" pg_d WHERE pg_d."objoid" = "a"."attrelid" AND pg_d."objsubid" = "a"."attnum" AND pg_d."classoid" = 1259) AS "col_description" FROM information_schema."pg_attribute" AS "a" WHERE "a"."attrelid" = 16391 AND "a"."attnum" > 0 AND NOT "a"."attisdropped" ORDER BY "a"."attnum" NULLS LAST diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/psql/15.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/psql/15.sql new file mode 100644 index 0000000000000..767274f1b86dd --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/psql/15.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT CAST(NULL AS STRING) AS "polname", CAST(NULL AS BOOLEAN) AS "polpermissive", CAST(NULL AS STRING) AS "?column?", CAST(NULL AS STRING) AS "pg_get_expr", CAST(NULL AS STRING) AS "pg_get_expr", CAST(NULL AS STRING) AS "cmd" FROM information_schema."pg_policy" WHERE FALSE diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/psql/16.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/psql/16.sql new file mode 100644 index 0000000000000..8107944b88c21 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/psql/16.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT CAST(NULL AS INT64) AS "oid", CAST(NULL AS STRING) AS "stxrelid", CAST(NULL AS STRING) AS "nsp", CAST(NULL AS STRING) AS "stxname", CAST(NULL AS STRING) AS "columns", CAST(NULL AS STRING) AS "ndist_enabled", CAST(NULL AS STRING) AS "deps_enabled", CAST(NULL AS STRING) AS "mcv_enabled", CAST(NULL AS INT32) AS "stxstattarget" FROM information_schema."pg_statistic_ext" WHERE FALSE diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/psql/17.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/psql/17.sql new file mode 100644 index 0000000000000..610a32884935b --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/psql/17.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT CAST(NULL AS STRING) AS "pubname" FROM information_schema."pg_publication" WHERE FALSE UNION ALL SELECT CAST(NULL AS STRING) AS "pubname" FROM information_schema."pg_publication" WHERE FALSE ORDER BY 1 NULLS LAST diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/psql/18.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/psql/18.sql new file mode 100644 index 0000000000000..29fb390cd2991 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/psql/18.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT CAST(NULL AS STRING) AS "oid" FROM information_schema."pg_inherits" WHERE FALSE diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/psql/19.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/psql/19.sql new file mode 100644 index 0000000000000..bd0af996ae700 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/psql/19.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT CAST(NULL AS STRING) AS "oid", CAST(NULL AS STRING) AS "relkind", CAST(NULL AS BOOLEAN) AS "inhdetachpending", CAST(NULL AS STRING) AS "pg_get_expr" FROM information_schema."pg_inherits" WHERE FALSE diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/psql/2.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/psql/2.sql new file mode 100644 index 0000000000000..b247ac9e8e61c --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/psql/2.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT "n"."nspname" AS "Name", 'admin' AS "Owner" FROM information_schema."pg_namespace" AS "n" WHERE NOT ("n"."nspname" LIKE 'pg\_%' ESCAPE '\') AND "n"."nspname" <> 'information_schema' ORDER BY 1 NULLS LAST diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/psql/20.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/psql/20.sql new file mode 100644 index 0000000000000..a052178bd94e2 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/psql/20.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT "n"."nspname" AS "Schema", "c"."relname" AS "Name", CASE "c"."relkind" WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 't' THEN 'TOAST table' WHEN 'f' THEN 'foreign table' WHEN 'p' THEN 'partitioned table' WHEN 'I' THEN 'partitioned index' END AS "Type", 'admin' AS "Owner" FROM information_schema."pg_class" AS "c" LEFT JOIN information_schema."pg_namespace" AS "n" ON "n"."oid" = "c"."relnamespace" LEFT JOIN information_schema."pg_am" AS "am" ON "am"."oid" = "c"."relam" WHERE "c"."relkind" IN ('r', 'p', 't', 's', '') ORDER BY 1 NULLS LAST, 2 NULLS LAST diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/psql/21.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/psql/21.sql new file mode 100644 index 0000000000000..66e5db06e717b --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/psql/21.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT "machine", avg("temperature") AS "avg" FROM "refdb"."sensors" WHERE "time" >= (now() - 2592000000) GROUP BY "machine" ORDER BY "machine" NULLS LAST LIMIT 3 diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/psql/3.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/psql/3.sql new file mode 100644 index 0000000000000..e0de97e0d1719 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/psql/3.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT "n"."nspname" AS "Schema", "c"."relname" AS "Name", CASE "c"."relkind" WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 't' THEN 'TOAST table' WHEN 'f' THEN 'foreign table' WHEN 'p' THEN 'partitioned table' WHEN 'I' THEN 'partitioned index' END AS "Type", 'admin' AS "Owner" FROM information_schema."pg_class" AS "c" LEFT JOIN information_schema."pg_namespace" AS "n" ON "n"."oid" = "c"."relnamespace" LEFT JOIN information_schema."pg_am" AS "am" ON "am"."oid" = "c"."relam" WHERE "c"."relkind" IN ('r', 'p', 't', 's', '') AND ("n"."nspname" = 'refdb') ORDER BY 1 NULLS LAST, 2 NULLS LAST diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/psql/4.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/psql/4.sql new file mode 100644 index 0000000000000..ed409c6163bb0 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/psql/4.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT "c"."oid", "n"."nspname", "c"."relname" FROM information_schema."pg_class" AS "c" LEFT JOIN information_schema."pg_namespace" AS "n" ON "n"."oid" = "c"."relnamespace" WHERE ("c"."relname" = 'sensors') AND ("n"."nspname" = 'refdb') ORDER BY 2 NULLS LAST, 3 NULLS LAST diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/psql/5.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/psql/5.sql new file mode 100644 index 0000000000000..e637a3e46a1d8 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/psql/5.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT "c"."relchecks", "c"."relkind", "c"."relhasindex", "c"."relhasrules", "c"."relhastriggers", "c"."relrowsecurity", "c"."relforcerowsecurity", FALSE AS "relhasoids", "c"."relispartition", '' AS "?column?", "c"."reltablespace", CASE WHEN "c"."reloftype" = 0 THEN '' ELSE CASE CAST("c"."reloftype" AS INT32) WHEN 16 THEN 'boolean' WHEN 17 THEN 'bytea' WHEN 18 THEN '"char"' WHEN 19 THEN 'name' WHEN 20 THEN 'bigint' WHEN 21 THEN 'smallint' WHEN 23 THEN 'integer' WHEN 24 THEN 'regproc' WHEN 25 THEN 'text' WHEN 26 THEN 'oid' WHEN 700 THEN 'real' WHEN 701 THEN 'double precision' WHEN 1043 THEN 'character varying' WHEN 1082 THEN 'date' WHEN 1114 THEN 'timestamp without time zone' WHEN 1184 THEN 'timestamp with time zone' WHEN 1700 THEN 'numeric' WHEN 2206 THEN 'regtype' WHEN 705 THEN 'unknown' WHEN 2249 THEN 'record' WHEN 2278 THEN 'void' WHEN 1000 THEN 'boolean[]' WHEN 1001 THEN 'bytea[]' WHEN 1002 THEN '"char"[]' WHEN 1003 THEN 'name[]' WHEN 1016 THEN 'bigint[]' WHEN 1005 THEN 'smallint[]' WHEN 1007 THEN 'integer[]' WHEN 1008 THEN 'regproc[]' WHEN 1009 THEN 'text[]' WHEN 1028 THEN 'oid[]' WHEN 1021 THEN 'real[]' WHEN 1022 THEN 'double precision[]' WHEN 1015 THEN 'character varying[]' WHEN 1182 THEN 'date[]' WHEN 1115 THEN 'timestamp without time zone[]' WHEN 1185 THEN 'timestamp with time zone[]' WHEN 1231 THEN 'numeric[]' WHEN 2211 THEN 'regtype[]' WHEN 2287 THEN 'record[]' ELSE CAST("c"."reloftype" AS STRING) END END AS "case", "c"."relpersistence", "c"."relreplident", "am"."amname" FROM information_schema."pg_class" AS "c" LEFT JOIN information_schema."pg_class" AS "tc" ON ("c"."reltoastrelid" = "tc"."oid") LEFT JOIN information_schema."pg_am" AS "am" ON ("c"."relam" = "am"."oid") WHERE "c"."oid" = 16386 diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/psql/6.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/psql/6.sql new file mode 100644 index 0000000000000..1a5e9681faa61 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/psql/6.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT "a"."attname", CASE CAST("a"."atttypid" AS INT32) WHEN 16 THEN 'boolean' WHEN 17 THEN 'bytea' WHEN 18 THEN '"char"' WHEN 19 THEN 'name' WHEN 20 THEN 'bigint' WHEN 21 THEN 'smallint' WHEN 23 THEN 'integer' WHEN 24 THEN 'regproc' WHEN 25 THEN 'text' WHEN 26 THEN 'oid' WHEN 700 THEN 'real' WHEN 701 THEN 'double precision' WHEN 1043 THEN 'character varying' WHEN 1082 THEN 'date' WHEN 1114 THEN 'timestamp without time zone' WHEN 1184 THEN 'timestamp with time zone' WHEN 1700 THEN 'numeric' WHEN 2206 THEN 'regtype' WHEN 705 THEN 'unknown' WHEN 2249 THEN 'record' WHEN 2278 THEN 'void' WHEN 1000 THEN 'boolean[]' WHEN 1001 THEN 'bytea[]' WHEN 1002 THEN '"char"[]' WHEN 1003 THEN 'name[]' WHEN 1016 THEN 'bigint[]' WHEN 1005 THEN 'smallint[]' WHEN 1007 THEN 'integer[]' WHEN 1008 THEN 'regproc[]' WHEN 1009 THEN 'text[]' WHEN 1028 THEN 'oid[]' WHEN 1021 THEN 'real[]' WHEN 1022 THEN 'double precision[]' WHEN 1015 THEN 'character varying[]' WHEN 1182 THEN 'date[]' WHEN 1115 THEN 'timestamp without time zone[]' WHEN 1185 THEN 'timestamp with time zone[]' WHEN 1231 THEN 'numeric[]' WHEN 2211 THEN 'regtype[]' WHEN 2287 THEN 'record[]' ELSE CASE WHEN "a"."atttypid" IS NULL THEN CAST(NULL AS STRING) ELSE '???' END END AS "format_type", CAST(NULL AS STRING), "a"."attnotnull", (SELECT "c"."collname" FROM information_schema."pg_collation" AS "c", information_schema."pg_type" AS "t" WHERE "c"."oid" = "a"."attcollation" AND "t"."oid" = "a"."atttypid" AND "a"."attcollation" <> "t"."typcollation") AS "attcollation", "a"."attidentity", "a"."attgenerated" FROM information_schema."pg_attribute" AS "a" WHERE "a"."attrelid" = 16386 AND "a"."attnum" > 0 AND NOT "a"."attisdropped" ORDER BY "a"."attnum" NULLS LAST diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/psql/7.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/psql/7.sql new file mode 100644 index 0000000000000..767274f1b86dd --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/psql/7.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT CAST(NULL AS STRING) AS "polname", CAST(NULL AS BOOLEAN) AS "polpermissive", CAST(NULL AS STRING) AS "?column?", CAST(NULL AS STRING) AS "pg_get_expr", CAST(NULL AS STRING) AS "pg_get_expr", CAST(NULL AS STRING) AS "cmd" FROM information_schema."pg_policy" WHERE FALSE diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/psql/8.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/psql/8.sql new file mode 100644 index 0000000000000..8107944b88c21 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/psql/8.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT CAST(NULL AS INT64) AS "oid", CAST(NULL AS STRING) AS "stxrelid", CAST(NULL AS STRING) AS "nsp", CAST(NULL AS STRING) AS "stxname", CAST(NULL AS STRING) AS "columns", CAST(NULL AS STRING) AS "ndist_enabled", CAST(NULL AS STRING) AS "deps_enabled", CAST(NULL AS STRING) AS "mcv_enabled", CAST(NULL AS INT32) AS "stxstattarget" FROM information_schema."pg_statistic_ext" WHERE FALSE diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/psql/9.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/psql/9.sql new file mode 100644 index 0000000000000..610a32884935b --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/psql/9.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT CAST(NULL AS STRING) AS "pubname" FROM information_schema."pg_publication" WHERE FALSE UNION ALL SELECT CAST(NULL AS STRING) AS "pubname" FROM information_schema."pg_publication" WHERE FALSE ORDER BY 1 NULLS LAST diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/psycopg/1.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/psycopg/1.sql new file mode 100644 index 0000000000000..81c84bd1dfe17 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/psycopg/1.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +-- session command diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/psycopg/10.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/psycopg/10.sql new file mode 100644 index 0000000000000..685cc856b2ded --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/psycopg/10.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT * FROM "refdb"."sensors" WHERE "running" = ? AND "level" > ? LIMIT 3 diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/psycopg/11.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/psycopg/11.sql new file mode 100644 index 0000000000000..1fb380773751a --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/psycopg/11.sql @@ -0,0 +1,20 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +-- cursor DECLARE big FOR +SELECT * FROM "refdb"."events" diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/psycopg/12.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/psycopg/12.sql new file mode 100644 index 0000000000000..7b731d4bb96f5 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/psycopg/12.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +-- cursor FETCH big diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/psycopg/13.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/psycopg/13.sql new file mode 100644 index 0000000000000..e1a47839517cd --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/psycopg/13.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +-- cursor CLOSE big diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/psycopg/14.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/psycopg/14.sql new file mode 100644 index 0000000000000..81c84bd1dfe17 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/psycopg/14.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +-- session command diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/psycopg/2.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/psycopg/2.sql new file mode 100644 index 0000000000000..5218753e4ce43 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/psycopg/2.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT 'PostgreSQL 14.0 (IoTDB), read-only adapter' AS "version" diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/psycopg/3.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/psycopg/3.sql new file mode 100644 index 0000000000000..2866183213ea7 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/psycopg/3.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT * FROM "refdb"."sensors" LIMIT 5 diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/psycopg/4.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/psycopg/4.sql new file mode 100644 index 0000000000000..b065b680a494d --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/psycopg/4.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT "time", "temperature" FROM "refdb"."sensors" WHERE "machine" = ? AND "time" >= ? LIMIT ? diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/psycopg/5.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/psycopg/5.sql new file mode 100644 index 0000000000000..b065b680a494d --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/psycopg/5.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT "time", "temperature" FROM "refdb"."sensors" WHERE "machine" = ? AND "time" >= ? LIMIT ? diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/psycopg/6.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/psycopg/6.sql new file mode 100644 index 0000000000000..b065b680a494d --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/psycopg/6.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT "time", "temperature" FROM "refdb"."sensors" WHERE "machine" = ? AND "time" >= ? LIMIT ? diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/psycopg/7.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/psycopg/7.sql new file mode 100644 index 0000000000000..b065b680a494d --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/psycopg/7.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT "time", "temperature" FROM "refdb"."sensors" WHERE "machine" = ? AND "time" >= ? LIMIT ? diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/psycopg/8.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/psycopg/8.sql new file mode 100644 index 0000000000000..b065b680a494d --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/psycopg/8.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT "time", "temperature" FROM "refdb"."sensors" WHERE "machine" = ? AND "time" >= ? LIMIT ? diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/psycopg/9.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/psycopg/9.sql new file mode 100644 index 0000000000000..b065b680a494d --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/psycopg/9.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT "time", "temperature" FROM "refdb"."sensors" WHERE "machine" = ? AND "time" >= ? LIMIT ? diff --git a/external-service-impl/postgresql/src/test/resources/corpus/pgjdbc.sql b/external-service-impl/postgresql/src/test/resources/corpus/pgjdbc.sql new file mode 100644 index 0000000000000..ffd522fccad6c --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/pgjdbc.sql @@ -0,0 +1,68 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +-- Client: pgjdbc 42.7.4 +-- Server: PostgreSQL 14.25 (docker postgres:14-alpine), log_statement=all +-- Captured: 2026-11-19 (T001, spec 012); schema refdb/other from corpus/README.md +-- next +SELECT datname AS TABLE_CAT FROM pg_catalog.pg_database WHERE datallowconn = true ORDER BY datname +-- next +SELECT nspname AS TABLE_SCHEM, NULL AS TABLE_CATALOG FROM pg_catalog.pg_namespace WHERE nspname <> 'pg_toast' AND (nspname !~ '^pg_temp_' OR nspname = (pg_catalog.current_schemas(true))[1]) AND (nspname !~ '^pg_toast_temp_' OR nspname = replace((pg_catalog.current_schemas(true))[1], 'pg_temp_', 'pg_toast_temp_')) ORDER BY TABLE_SCHEM +-- next +SELECT NULL AS TABLE_CAT, n.nspname AS TABLE_SCHEM, c.relname AS TABLE_NAME, CASE n.nspname ~ '^pg_' OR n.nspname = 'information_schema' WHEN true THEN CASE WHEN n.nspname = 'pg_catalog' OR n.nspname = 'information_schema' THEN CASE c.relkind WHEN 'r' THEN 'SYSTEM TABLE' WHEN 'v' THEN 'SYSTEM VIEW' WHEN 'i' THEN 'SYSTEM INDEX' ELSE NULL END WHEN n.nspname = 'pg_toast' THEN CASE c.relkind WHEN 'r' THEN 'SYSTEM TOAST TABLE' WHEN 'i' THEN 'SYSTEM TOAST INDEX' ELSE NULL END ELSE CASE c.relkind WHEN 'r' THEN 'TEMPORARY TABLE' WHEN 'p' THEN 'TEMPORARY TABLE' WHEN 'i' THEN 'TEMPORARY INDEX' WHEN 'S' THEN 'TEMPORARY SEQUENCE' WHEN 'v' THEN 'TEMPORARY VIEW' ELSE NULL END END WHEN false THEN CASE c.relkind WHEN 'r' THEN 'TABLE' WHEN 'p' THEN 'PARTITIONED TABLE' WHEN 'i' THEN 'INDEX' WHEN 'P' then 'PARTITIONED INDEX' WHEN 'S' THEN 'SEQUENCE' WHEN 'v' THEN 'VIEW' WHEN 'c' THEN 'TYPE' WHEN 'f' THEN 'FOREIGN TABLE' WHEN 'm' THEN 'MATERIALIZED VIEW' ELSE NULL END ELSE NULL END AS TABLE_TYPE, d.description AS REMARKS, '' as TYPE_CAT, '' as TYPE_SCHEM, '' as TYPE_NAME, '' AS SELF_REFERENCING_COL_NAME, '' AS REF_GENERATION FROM pg_catalog.pg_namespace n, pg_catalog.pg_class c LEFT JOIN pg_catalog.pg_description d ON (c.oid = d.objoid AND d.objsubid = 0 and d.classoid = 'pg_class'::regclass) WHERE c.relnamespace = n.oid AND n.nspname LIKE 'refdb' AND c.relname LIKE '%' AND (false OR ( c.relkind = 'r' AND n.nspname !~ '^pg_' AND n.nspname <> 'information_schema' ) OR ( c.relkind = 'v' AND n.nspname <> 'pg_catalog' AND n.nspname <> 'information_schema' ) ) ORDER BY TABLE_TYPE,TABLE_SCHEM,TABLE_NAME +-- next +SELECT * FROM (SELECT n.nspname,c.relname,a.attname,a.atttypid,a.attnotnull OR (t.typtype = 'd' AND t.typnotnull) AS attnotnull,a.atttypmod,a.attlen,t.typtypmod,row_number() OVER (PARTITION BY a.attrelid ORDER BY a.attnum) AS attnum, nullif(a.attidentity, '') as attidentity,nullif(a.attgenerated, '') as attgenerated,pg_catalog.pg_get_expr(def.adbin, def.adrelid) AS adsrc,dsc.description,t.typbasetype,t.typtype FROM pg_catalog.pg_namespace n JOIN pg_catalog.pg_class c ON (c.relnamespace = n.oid) JOIN pg_catalog.pg_attribute a ON (a.attrelid=c.oid) JOIN pg_catalog.pg_type t ON (a.atttypid = t.oid) LEFT JOIN pg_catalog.pg_attrdef def ON (a.attrelid=def.adrelid AND a.attnum = def.adnum) LEFT JOIN pg_catalog.pg_description dsc ON (c.oid=dsc.objoid AND a.attnum = dsc.objsubid) LEFT JOIN pg_catalog.pg_class dc ON (dc.oid=dsc.classoid AND dc.relname='pg_class') LEFT JOIN pg_catalog.pg_namespace dn ON (dc.relnamespace=dn.oid AND dn.nspname='pg_catalog') WHERE c.relkind in ('r','p','v','f','m') and a.attnum > 0 AND NOT a.attisdropped AND n.nspname LIKE 'refdb' AND c.relname LIKE 'sensors') c WHERE true AND attname LIKE '%' ORDER BY nspname,c.relname,attnum +-- next +SELECT result.TABLE_CAT, result.TABLE_SCHEM, result.TABLE_NAME, result.COLUMN_NAME, result.KEY_SEQ, result.PK_NAME FROM (SELECT NULL AS TABLE_CAT, n.nspname AS TABLE_SCHEM, ct.relname AS TABLE_NAME, a.attname AS COLUMN_NAME, (information_schema._pg_expandarray(i.indkey)).n AS KEY_SEQ, ci.relname AS PK_NAME, information_schema._pg_expandarray(i.indkey) AS KEYS, a.attnum AS A_ATTNUM FROM pg_catalog.pg_class ct JOIN pg_catalog.pg_attribute a ON (ct.oid = a.attrelid) JOIN pg_catalog.pg_namespace n ON (ct.relnamespace = n.oid) JOIN pg_catalog.pg_index i ON ( a.attrelid = i.indrelid) JOIN pg_catalog.pg_class ci ON (ci.oid = i.indexrelid) WHERE true AND n.nspname = 'refdb' AND ct.relname = 'sensors' AND i.indisprimary ) result where result.A_ATTNUM = (result.KEYS).x ORDER BY result.table_name, result.pk_name, result.key_seq +-- next +SELECT tmp.TABLE_CAT, tmp.TABLE_SCHEM, tmp.TABLE_NAME, tmp.NON_UNIQUE, tmp.INDEX_QUALIFIER, tmp.INDEX_NAME, tmp.TYPE, tmp.ORDINAL_POSITION, trim(both '"' from pg_catalog.pg_get_indexdef(tmp.CI_OID, tmp.ORDINAL_POSITION, false)) AS COLUMN_NAME, CASE tmp.AM_NAME WHEN 'btree' THEN CASE tmp.I_INDOPTION[tmp.ORDINAL_POSITION - 1] & 1::smallint WHEN 1 THEN 'D' ELSE 'A' END ELSE NULL END AS ASC_OR_DESC, tmp.CARDINALITY, tmp.PAGES, tmp.FILTER_CONDITION FROM (SELECT NULL AS TABLE_CAT, n.nspname AS TABLE_SCHEM, ct.relname AS TABLE_NAME, NOT i.indisunique AS NON_UNIQUE, NULL AS INDEX_QUALIFIER, ci.relname AS INDEX_NAME, CASE i.indisclustered WHEN true THEN 1 ELSE CASE am.amname WHEN 'hash' THEN 2 ELSE 3 END END AS TYPE, (information_schema._pg_expandarray(i.indkey)).n AS ORDINAL_POSITION, ci.reltuples AS CARDINALITY, ci.relpages AS PAGES, pg_catalog.pg_get_expr(i.indpred, i.indrelid) AS FILTER_CONDITION, ci.oid AS CI_OID, i.indoption AS I_INDOPTION, am.amname AS AM_NAME FROM pg_catalog.pg_class ct JOIN pg_catalog.pg_namespace n ON (ct.relnamespace = n.oid) JOIN pg_catalog.pg_index i ON (ct.oid = i.indrelid) JOIN pg_catalog.pg_class ci ON (ci.oid = i.indexrelid) JOIN pg_catalog.pg_am am ON (ci.relam = am.oid) WHERE true AND n.nspname = 'refdb' AND ct.relname = 'sensors') AS tmp ORDER BY NON_UNIQUE, TYPE, INDEX_NAME, ORDINAL_POSITION +-- next +SELECT setting FROM pg_catalog.pg_settings WHERE name='max_index_keys' +-- next +SELECT NULL::text AS PKTABLE_CAT, pkn.nspname AS PKTABLE_SCHEM, pkc.relname AS PKTABLE_NAME, pka.attname AS PKCOLUMN_NAME, NULL::text AS FKTABLE_CAT, fkn.nspname AS FKTABLE_SCHEM, fkc.relname AS FKTABLE_NAME, fka.attname AS FKCOLUMN_NAME, pos.n AS KEY_SEQ, CASE con.confupdtype WHEN 'c' THEN 0 WHEN 'n' THEN 2 WHEN 'd' THEN 4 WHEN 'r' THEN 1 WHEN 'p' THEN 1 WHEN 'a' THEN 3 ELSE NULL END AS UPDATE_RULE, CASE con.confdeltype WHEN 'c' THEN 0 WHEN 'n' THEN 2 WHEN 'd' THEN 4 WHEN 'r' THEN 1 WHEN 'p' THEN 1 WHEN 'a' THEN 3 ELSE NULL END AS DELETE_RULE, con.conname AS FK_NAME, pkic.relname AS PK_NAME, CASE WHEN con.condeferrable AND con.condeferred THEN 5 WHEN con.condeferrable THEN 6 ELSE 7 END AS DEFERRABILITY FROM pg_catalog.pg_namespace pkn, pg_catalog.pg_class pkc, pg_catalog.pg_attribute pka, pg_catalog.pg_namespace fkn, pg_catalog.pg_class fkc, pg_catalog.pg_attribute fka, pg_catalog.pg_constraint con, pg_catalog.generate_series(1, 32) pos(n), pg_catalog.pg_class pkic WHERE pkn.oid = pkc.relnamespace AND pkc.oid = pka.attrelid AND pka.attnum = con.confkey[pos.n] AND con.confrelid = pkc.oid AND fkn.oid = fkc.relnamespace AND fkc.oid = fka.attrelid AND fka.attnum = con.conkey[pos.n] AND con.conrelid = fkc.oid AND con.contype = 'f' AND (pkic.relkind = 'i' OR pkic.relkind = 'I') AND pkic.oid = con.conindid AND fkn.nspname = 'refdb' AND fkc.relname = 'sensors' ORDER BY pkn.nspname,pkc.relname, con.conname,pos.n +-- next +SELECT t.typname,t.oid FROM pg_catalog.pg_type t JOIN pg_catalog.pg_namespace n ON (t.typnamespace = n.oid) WHERE n.nspname != 'pg_toast' AND (t.typrelid = 0 OR (SELECT c.relkind = 'c' FROM pg_catalog.pg_class c WHERE c.oid = t.typrelid)) +-- next +SELECT typinput='pg_catalog.array_in'::regproc as is_array, typtype, typname, pg_type.oid FROM pg_catalog.pg_type LEFT JOIN (select ns.oid as nspoid, ns.nspname, r.r from pg_namespace as ns join ( select s.r, (current_schemas(false))[s.r] as nspname from generate_series(1, array_upper(current_schemas(false), 1)) as s(r) ) as r using ( nspname ) ) as sp ON sp.nspoid = typnamespace ORDER BY sp.r, pg_type.oid DESC +-- next +SELECT * FROM refdb.sensors LIMIT 5 +-- next +-- parameters: $1 = 'm2', $2 = '2026-10-25 02:00:00+02', $3 = '10' +SELECT "time", machine, temperature FROM refdb.sensors WHERE machine = $1 AND "time" >= $2 ORDER BY "time" LIMIT $3 +-- next +-- parameters: $1 = 'm3', $2 = '2026-10-25 02:00:00+02', $3 = '10' +SELECT "time", machine, temperature FROM refdb.sensors WHERE machine = $1 AND "time" >= $2 ORDER BY "time" LIMIT $3 +-- next +-- parameters: $1 = 'm4', $2 = '2026-10-25 02:00:00+02', $3 = '10' +SELECT "time", machine, temperature FROM refdb.sensors WHERE machine = $1 AND "time" >= $2 ORDER BY "time" LIMIT $3 +-- next +-- parameters: $1 = 'm5', $2 = '2026-10-25 02:00:00+02', $3 = '10' +SELECT "time", machine, temperature FROM refdb.sensors WHERE machine = $1 AND "time" >= $2 ORDER BY "time" LIMIT $3 +-- next +-- parameters: $1 = 'm1', $2 = '2026-10-25 02:00:00+02', $3 = '10' +SELECT "time", machine, temperature FROM refdb.sensors WHERE machine = $1 AND "time" >= $2 ORDER BY "time" LIMIT $3 +-- next +-- parameters: $1 = 'm2', $2 = '2026-10-25 02:00:00+02', $3 = '10' +SELECT "time", machine, temperature FROM refdb.sensors WHERE machine = $1 AND "time" >= $2 ORDER BY "time" LIMIT $3 +-- next +BEGIN +-- next +-- executed 101 times in a row (portal fetches or repeated executions) +SELECT * FROM refdb.events +-- next +COMMIT diff --git a/external-service-impl/postgresql/src/test/resources/corpus/psql.sql b/external-service-impl/postgresql/src/test/resources/corpus/psql.sql new file mode 100644 index 0000000000000..a74b2499006ba --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/psql.sql @@ -0,0 +1,210 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +-- Client: psql 14.25 +-- Server: PostgreSQL 14.25 (docker postgres:14-alpine), log_statement=all +-- Captured: 2026-11-19 (T001, spec 012); schema refdb/other from corpus/README.md +-- next +SELECT d.datname as "Name", + pg_catalog.pg_get_userbyid(d.datdba) as "Owner", + pg_catalog.pg_encoding_to_char(d.encoding) as "Encoding", + d.datcollate as "Collate", + d.datctype as "Ctype", + pg_catalog.array_to_string(d.datacl, E'\n') AS "Access privileges" + FROM pg_catalog.pg_database d + ORDER BY 1; +-- next +SELECT n.nspname AS "Name", + pg_catalog.pg_get_userbyid(n.nspowner) AS "Owner" + FROM pg_catalog.pg_namespace n + WHERE n.nspname !~ '^pg_' AND n.nspname <> 'information_schema' + ORDER BY 1; +-- next +SELECT n.nspname as "Schema", + c.relname as "Name", + CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 't' THEN 'TOAST table' WHEN 'f' THEN 'foreign table' WHEN 'p' THEN 'partitioned table' WHEN 'I' THEN 'partitioned index' END as "Type", + pg_catalog.pg_get_userbyid(c.relowner) as "Owner" + FROM pg_catalog.pg_class c + LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace + LEFT JOIN pg_catalog.pg_am am ON am.oid = c.relam + WHERE c.relkind IN ('r','p','t','s','') + AND n.nspname OPERATOR(pg_catalog.~) '^(refdb)$' COLLATE pg_catalog.default + ORDER BY 1,2; +-- next +SELECT c.oid, + n.nspname, + c.relname + FROM pg_catalog.pg_class c + LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace + WHERE c.relname OPERATOR(pg_catalog.~) '^(sensors)$' COLLATE pg_catalog.default + AND n.nspname OPERATOR(pg_catalog.~) '^(refdb)$' COLLATE pg_catalog.default + ORDER BY 2, 3; +-- next +SELECT c.relchecks, c.relkind, c.relhasindex, c.relhasrules, c.relhastriggers, c.relrowsecurity, c.relforcerowsecurity, false AS relhasoids, c.relispartition, '', c.reltablespace, CASE WHEN c.reloftype = 0 THEN '' ELSE c.reloftype::pg_catalog.regtype::pg_catalog.text END, c.relpersistence, c.relreplident, am.amname + FROM pg_catalog.pg_class c + LEFT JOIN pg_catalog.pg_class tc ON (c.reltoastrelid = tc.oid) + LEFT JOIN pg_catalog.pg_am am ON (c.relam = am.oid) + WHERE c.oid = '16386'; +-- next +SELECT a.attname, + pg_catalog.format_type(a.atttypid, a.atttypmod), + (SELECT pg_catalog.pg_get_expr(d.adbin, d.adrelid, true) + FROM pg_catalog.pg_attrdef d + WHERE d.adrelid = a.attrelid AND d.adnum = a.attnum AND a.atthasdef), + a.attnotnull, + (SELECT c.collname FROM pg_catalog.pg_collation c, pg_catalog.pg_type t + WHERE c.oid = a.attcollation AND t.oid = a.atttypid AND a.attcollation <> t.typcollation) AS attcollation, + a.attidentity, + a.attgenerated + FROM pg_catalog.pg_attribute a + WHERE a.attrelid = '16386' AND a.attnum > 0 AND NOT a.attisdropped + ORDER BY a.attnum; +-- next +SELECT pol.polname, pol.polpermissive, + CASE WHEN pol.polroles = '{0}' THEN NULL ELSE pg_catalog.array_to_string(array(select rolname from pg_catalog.pg_roles where oid = any (pol.polroles) order by 1),',') END, + pg_catalog.pg_get_expr(pol.polqual, pol.polrelid), + pg_catalog.pg_get_expr(pol.polwithcheck, pol.polrelid), + CASE pol.polcmd + WHEN 'r' THEN 'SELECT' + WHEN 'a' THEN 'INSERT' + WHEN 'w' THEN 'UPDATE' + WHEN 'd' THEN 'DELETE' + END AS cmd + FROM pg_catalog.pg_policy pol + WHERE pol.polrelid = '16386' ORDER BY 1; +-- next +SELECT oid, stxrelid::pg_catalog.regclass, stxnamespace::pg_catalog.regnamespace::pg_catalog.text AS nsp, stxname, + pg_catalog.pg_get_statisticsobjdef_columns(oid) AS columns, + 'd' = any(stxkind) AS ndist_enabled, + 'f' = any(stxkind) AS deps_enabled, + 'm' = any(stxkind) AS mcv_enabled, + stxstattarget + FROM pg_catalog.pg_statistic_ext + WHERE stxrelid = '16386' + ORDER BY nsp, stxname; +-- next +SELECT pubname + FROM pg_catalog.pg_publication p + JOIN pg_catalog.pg_publication_rel pr ON p.oid = pr.prpubid + WHERE pr.prrelid = '16386' + UNION ALL + SELECT pubname + FROM pg_catalog.pg_publication p + WHERE p.puballtables AND pg_catalog.pg_relation_is_publishable('16386') + ORDER BY 1; +-- next +SELECT c.oid::pg_catalog.regclass + FROM pg_catalog.pg_class c, pg_catalog.pg_inherits i + WHERE c.oid = i.inhparent AND i.inhrelid = '16386' + AND c.relkind != 'p' AND c.relkind != 'I' + ORDER BY inhseqno; +-- next +SELECT c.oid::pg_catalog.regclass, c.relkind, inhdetachpending, pg_catalog.pg_get_expr(c.relpartbound, c.oid) + FROM pg_catalog.pg_class c, pg_catalog.pg_inherits i + WHERE c.oid = i.inhrelid AND i.inhparent = '16386' + ORDER BY pg_catalog.pg_get_expr(c.relpartbound, c.oid) = 'DEFAULT', c.oid::pg_catalog.regclass::pg_catalog.text; +-- next +SELECT c.oid, + n.nspname, + c.relname + FROM pg_catalog.pg_class c + LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace + WHERE c.relname OPERATOR(pg_catalog.~) '^(events)$' COLLATE pg_catalog.default + AND n.nspname OPERATOR(pg_catalog.~) '^(refdb)$' COLLATE pg_catalog.default + ORDER BY 2, 3; +-- next +SELECT c.relchecks, c.relkind, c.relhasindex, c.relhasrules, c.relhastriggers, c.relrowsecurity, c.relforcerowsecurity, false AS relhasoids, c.relispartition, pg_catalog.array_to_string(c.reloptions || array(select 'toast.' || x from pg_catalog.unnest(tc.reloptions) x), ', ') + , c.reltablespace, CASE WHEN c.reloftype = 0 THEN '' ELSE c.reloftype::pg_catalog.regtype::pg_catalog.text END, c.relpersistence, c.relreplident, am.amname + FROM pg_catalog.pg_class c + LEFT JOIN pg_catalog.pg_class tc ON (c.reltoastrelid = tc.oid) + LEFT JOIN pg_catalog.pg_am am ON (c.relam = am.oid) + WHERE c.oid = '16391'; +-- next +SELECT a.attname, + pg_catalog.format_type(a.atttypid, a.atttypmod), + (SELECT pg_catalog.pg_get_expr(d.adbin, d.adrelid, true) + FROM pg_catalog.pg_attrdef d + WHERE d.adrelid = a.attrelid AND d.adnum = a.attnum AND a.atthasdef), + a.attnotnull, + (SELECT c.collname FROM pg_catalog.pg_collation c, pg_catalog.pg_type t + WHERE c.oid = a.attcollation AND t.oid = a.atttypid AND a.attcollation <> t.typcollation) AS attcollation, + a.attidentity, + a.attgenerated, + a.attstorage, + a.attcompression AS attcompression, + CASE WHEN a.attstattarget=-1 THEN NULL ELSE a.attstattarget END AS attstattarget, + pg_catalog.col_description(a.attrelid, a.attnum) + FROM pg_catalog.pg_attribute a + WHERE a.attrelid = '16391' AND a.attnum > 0 AND NOT a.attisdropped + ORDER BY a.attnum; +-- next +SELECT pol.polname, pol.polpermissive, + CASE WHEN pol.polroles = '{0}' THEN NULL ELSE pg_catalog.array_to_string(array(select rolname from pg_catalog.pg_roles where oid = any (pol.polroles) order by 1),',') END, + pg_catalog.pg_get_expr(pol.polqual, pol.polrelid), + pg_catalog.pg_get_expr(pol.polwithcheck, pol.polrelid), + CASE pol.polcmd + WHEN 'r' THEN 'SELECT' + WHEN 'a' THEN 'INSERT' + WHEN 'w' THEN 'UPDATE' + WHEN 'd' THEN 'DELETE' + END AS cmd + FROM pg_catalog.pg_policy pol + WHERE pol.polrelid = '16391' ORDER BY 1; +-- next +SELECT oid, stxrelid::pg_catalog.regclass, stxnamespace::pg_catalog.regnamespace::pg_catalog.text AS nsp, stxname, + pg_catalog.pg_get_statisticsobjdef_columns(oid) AS columns, + 'd' = any(stxkind) AS ndist_enabled, + 'f' = any(stxkind) AS deps_enabled, + 'm' = any(stxkind) AS mcv_enabled, + stxstattarget + FROM pg_catalog.pg_statistic_ext + WHERE stxrelid = '16391' + ORDER BY nsp, stxname; +-- next +SELECT pubname + FROM pg_catalog.pg_publication p + JOIN pg_catalog.pg_publication_rel pr ON p.oid = pr.prpubid + WHERE pr.prrelid = '16391' + UNION ALL + SELECT pubname + FROM pg_catalog.pg_publication p + WHERE p.puballtables AND pg_catalog.pg_relation_is_publishable('16391') + ORDER BY 1; +-- next +SELECT c.oid::pg_catalog.regclass + FROM pg_catalog.pg_class c, pg_catalog.pg_inherits i + WHERE c.oid = i.inhparent AND i.inhrelid = '16391' + AND c.relkind != 'p' AND c.relkind != 'I' + ORDER BY inhseqno; +-- next +SELECT c.oid::pg_catalog.regclass, c.relkind, inhdetachpending, pg_catalog.pg_get_expr(c.relpartbound, c.oid) + FROM pg_catalog.pg_class c, pg_catalog.pg_inherits i + WHERE c.oid = i.inhrelid AND i.inhparent = '16391' + ORDER BY pg_catalog.pg_get_expr(c.relpartbound, c.oid) = 'DEFAULT', c.oid::pg_catalog.regclass::pg_catalog.text; +-- next +SELECT n.nspname as "Schema", + c.relname as "Name", + CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 't' THEN 'TOAST table' WHEN 'f' THEN 'foreign table' WHEN 'p' THEN 'partitioned table' WHEN 'I' THEN 'partitioned index' END as "Type", + pg_catalog.pg_get_userbyid(c.relowner) as "Owner" + FROM pg_catalog.pg_class c + LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace + LEFT JOIN pg_catalog.pg_am am ON am.oid = c.relam + WHERE c.relkind IN ('r','p','t','s','') + ORDER BY 1,2; +-- next +SELECT machine, avg(temperature) FROM refdb.sensors WHERE "time" >= now() - interval '30 days' GROUP BY machine ORDER BY machine LIMIT 3 diff --git a/external-service-impl/postgresql/src/test/resources/corpus/psycopg.sql b/external-service-impl/postgresql/src/test/resources/corpus/psycopg.sql new file mode 100644 index 0000000000000..78e5264b7ed0b --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/psycopg.sql @@ -0,0 +1,57 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +-- Client: psycopg 3.3.9 +-- Server: PostgreSQL 14.25 (docker postgres:14-alpine), log_statement=all +-- Captured: 2026-11-19 (T001, spec 012); schema refdb/other from corpus/README.md +-- next +BEGIN +-- next +SELECT version() +-- next +SELECT * FROM refdb.sensors LIMIT 5 +-- next +-- parameters: $1 = 'm1', $2 = '2026-10-25 00:00:00+00', $3 = '10' +SELECT "time", temperature FROM refdb.sensors WHERE machine = $1 AND "time" >= $2 LIMIT $3 +-- next +-- parameters: $1 = 'm2', $2 = '2026-10-25 00:00:00+00', $3 = '10' +SELECT "time", temperature FROM refdb.sensors WHERE machine = $1 AND "time" >= $2 LIMIT $3 +-- next +-- parameters: $1 = 'm3', $2 = '2026-10-25 00:00:00+00', $3 = '10' +SELECT "time", temperature FROM refdb.sensors WHERE machine = $1 AND "time" >= $2 LIMIT $3 +-- next +-- parameters: $1 = 'm4', $2 = '2026-10-25 00:00:00+00', $3 = '10' +SELECT "time", temperature FROM refdb.sensors WHERE machine = $1 AND "time" >= $2 LIMIT $3 +-- next +-- parameters: $1 = 'm5', $2 = '2026-10-25 00:00:00+00', $3 = '10' +SELECT "time", temperature FROM refdb.sensors WHERE machine = $1 AND "time" >= $2 LIMIT $3 +-- next +-- parameters: $1 = 'm1', $2 = '2026-10-25 00:00:00+00', $3 = '10' +SELECT "time", temperature FROM refdb.sensors WHERE machine = $1 AND "time" >= $2 LIMIT $3 +-- next +-- parameters: $1 = 't', $2 = '50' +SELECT * FROM refdb.sensors WHERE running = $1 AND level > $2 LIMIT 3 +-- next +DECLARE "big" CURSOR FOR SELECT * FROM refdb.events +-- next +-- executed 21 times in a row (portal fetches or repeated executions) +FETCH FORWARD 5000 FROM "big" +-- next +CLOSE "big" +-- next +COMMIT diff --git a/integration-test/pom.xml b/integration-test/pom.xml index f9045c99508e8..311323170007a 100644 --- a/integration-test/pom.xml +++ b/integration-test/pom.xml @@ -264,6 +264,19 @@ provided + + org.apache.iotdb + postgresql + 2.0.13-SNAPSHOT + + provided + + + + org.postgresql + postgresql + test + diff --git a/integration-test/src/assembly/mpp-share.xml b/integration-test/src/assembly/mpp-share.xml index 70072e8282ec6..578518bb95ecc 100644 --- a/integration-test/src/assembly/mpp-share.xml +++ b/integration-test/src/assembly/mpp-share.xml @@ -39,5 +39,10 @@ ${project.basedir}/../external-service-impl/rest/target/rest-${project.version}-jar-with-dependencies.jar lib + + + ${project.basedir}/../external-service-impl/postgresql/target/postgresql-${project.version}-jar-with-dependencies.jar + lib + diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppDataNodeConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppDataNodeConfig.java index a15feddb4e038..558d49e1ea712 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppDataNodeConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppDataNodeConfig.java @@ -140,6 +140,36 @@ public DataNodeConfig setEnableMQTTService(boolean enableMQTTService) { return this; } + @Override + public DataNodeConfig setEnablePostgreSqlService(boolean enablePostgreSqlService) { + setProperty("enable_postgresql_service", String.valueOf(enablePostgreSqlService)); + return this; + } + + @Override + public DataNodeConfig setPostgreSqlServiceAddress(String address) { + setProperty("postgresql_service_address", address); + return this; + } + + @Override + public DataNodeConfig setPostgreSqlServicePort(int port) { + setProperty("postgresql_service_port", String.valueOf(port)); + return this; + } + + @Override + public DataNodeConfig setPostgreSqlServiceMaxConnections(int maxConnections) { + setProperty("postgresql_service_max_connections", String.valueOf(maxConnections)); + return this; + } + + @Override + public DataNodeConfig setPostgreSqlServiceWriteTimeoutInMs(long writeTimeoutInMs) { + setProperty("postgresql_service_write_timeout_in_ms", String.valueOf(writeTimeoutInMs)); + return this; + } + @Override public DataNodeConfig setMqttPayloadFormatter(String mqttPayloadFormatter) { setProperty("mqtt_payload_formatter", String.valueOf(mqttPayloadFormatter)); diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteDataNodeConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteDataNodeConfig.java index a1db5cfc04e91..2e2a6b7a71e36 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteDataNodeConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteDataNodeConfig.java @@ -91,6 +91,31 @@ public DataNodeConfig setEnableMQTTService(boolean enableMQTTService) { return this; } + @Override + public DataNodeConfig setEnablePostgreSqlService(boolean enablePostgreSqlService) { + return this; + } + + @Override + public DataNodeConfig setPostgreSqlServiceAddress(String address) { + return this; + } + + @Override + public DataNodeConfig setPostgreSqlServicePort(int port) { + return this; + } + + @Override + public DataNodeConfig setPostgreSqlServiceMaxConnections(int maxConnections) { + return this; + } + + @Override + public DataNodeConfig setPostgreSqlServiceWriteTimeoutInMs(long writeTimeoutInMs) { + return this; + } + @Override public DataNodeConfig setMqttPayloadFormatter(String mqttPayloadFormatter) { return this; diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/env/DataNodeConfig.java b/integration-test/src/main/java/org/apache/iotdb/itbase/env/DataNodeConfig.java index 183eadcac8b10..e39e20e5607b2 100644 --- a/integration-test/src/main/java/org/apache/iotdb/itbase/env/DataNodeConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/itbase/env/DataNodeConfig.java @@ -52,6 +52,16 @@ DataNodeConfig setLoadActiveListeningCheckIntervalSeconds( DataNodeConfig setEnableMQTTService(boolean enableMQTTService); + DataNodeConfig setEnablePostgreSqlService(boolean enablePostgreSqlService); + + DataNodeConfig setPostgreSqlServiceAddress(String address); + + DataNodeConfig setPostgreSqlServicePort(int port); + + DataNodeConfig setPostgreSqlServiceMaxConnections(int maxConnections); + + DataNodeConfig setPostgreSqlServiceWriteTimeoutInMs(long writeTimeoutInMs); + DataNodeConfig setMqttPayloadFormatter(String mqttPayloadFormatter); DataNodeConfig setMqttHandlerPoolSize(int mqttHandlerPoolSize); diff --git a/integration-test/src/test/java/org/apache/iotdb/postgresql/it/IoTDBPostgreSqlClientsIT.java b/integration-test/src/test/java/org/apache/iotdb/postgresql/it/IoTDBPostgreSqlClientsIT.java new file mode 100644 index 0000000000000..7fa600c1a3504 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/postgresql/it/IoTDBPostgreSqlClientsIT.java @@ -0,0 +1,217 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.it; + +import org.apache.iotdb.isession.ITableSession; +import org.apache.iotdb.isession.SessionConfig; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.net.ServerSocket; +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.Timestamp; +import java.time.LocalDate; +import java.util.Properties; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** pgJDBC as DBeaver and other JDBC tools use it. */ +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class}) +public class IoTDBPostgreSqlClientsIT { + + private static final long START = 1_767_225_600_000L; // 2026-01-01T00:00:00Z + private static int port; + + @BeforeClass + public static void setUp() throws Exception { + try (ServerSocket socket = new ServerSocket(0)) { + port = socket.getLocalPort(); + } + EnvFactory.getEnv() + .getConfig() + .getDataNodeConfig() + .setEnablePostgreSqlService(true) + .setPostgreSqlServicePort(port); + EnvFactory.getEnv().initClusterEnvironment(); + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("CREATE DATABASE plant"); + session.executeNonQueryStatement("USE plant"); + session.executeNonQueryStatement( + "CREATE TABLE sensors (device STRING TAG, ok BOOLEAN FIELD, n INT32 FIELD," + + " big INT64 FIELD, f FLOAT FIELD, temp DOUBLE FIELD, label STRING FIELD," + + " raw BLOB FIELD, day DATE FIELD, at TIMESTAMP FIELD)"); + for (int i = 0; i < 100; i++) { + session.executeNonQueryStatement( + String.format( + "INSERT INTO sensors (time, device, ok, n, big, f, temp, label, raw, day, at)" + + " VALUES (%d, 'd%d', %s, %d, %d, %s, %s, 'l%d', X'0%d', '2026-01-%02d', %d)", + START + i * 1000L, + i % 3, + i % 2 == 0 ? "true" : "false", + i, + i * 10_000_000_000L, + i / 2.0f, + i / 4.0, + i, + i % 10, + i % 28 + 1, + START + i)); + } + } + } + + @AfterClass + public static void tearDown() { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + private static Connection connect(String mode) throws SQLException { + Properties props = new Properties(); + props.setProperty("user", SessionConfig.DEFAULT_USER); + props.setProperty("password", SessionConfig.DEFAULT_PASSWORD); + props.setProperty("sslmode", "disable"); + props.setProperty("preferQueryMode", mode); + return DriverManager.getConnection("jdbc:postgresql://127.0.0.1:" + port + "/any", props); + } + + @Test + public void databaseMetaData() throws Exception { + try (Connection c = connect("extended")) { + DatabaseMetaData meta = c.getMetaData(); + StringBuilder tables = new StringBuilder(); + try (ResultSet rs = meta.getTables(null, "plant", "%", new String[] {"TABLE"})) { + while (rs.next()) { + tables.append(rs.getString("TABLE_SCHEM")).append('.').append(rs.getString("TABLE_NAME")); + } + } + assertEquals("plant.sensors", tables.toString()); + StringBuilder columns = new StringBuilder(); + try (ResultSet rs = meta.getColumns(null, "plant", "sensors", "%")) { + while (rs.next()) { + columns.append(rs.getString("COLUMN_NAME")).append(':').append(rs.getString("TYPE_NAME")); + columns.append(' '); + } + } + assertEquals( + "time:timestamptz device:text ok:bool n:int4 big:int8 f:float4 temp:float8 label:text" + + " raw:bytea day:date at:timestamptz ", + columns.toString()); + try (ResultSet rs = meta.getPrimaryKeys(null, "plant", "sensors")) { + assertFalse("no keys in the table model", rs.next()); + } + } + } + + @Test + public void preparedStatementsWithEveryParameterType() throws Exception { + try (Connection c = connect("extended"); + PreparedStatement p = + c.prepareStatement( + "SELECT n, big, label, raw, day FROM plant.sensors WHERE device = ? AND ok = ?" + + " AND n >= ? AND big >= ? AND f >= ? AND temp >= ? AND day >= ?" + + " AND time >= ? ORDER BY time LIMIT 1")) { + // executed repeatedly with different values (pgJDBC switches to a server-side statement) + for (int round = 0; round < 8; round++) { + int i = round * 6; // device d0, ok = true + p.setString(1, "d0"); + p.setBoolean(2, true); + p.setInt(3, i); + p.setLong(4, i * 10_000_000_000L); + p.setFloat(5, i / 2.0f); + p.setDouble(6, i / 4.0); + p.setObject(7, LocalDate.of(2026, 1, 1)); + p.setTimestamp(8, new Timestamp(START + i * 1000L)); + try (ResultSet rs = p.executeQuery()) { + assertTrue("round " + round, rs.next()); + assertEquals(i, rs.getInt(1)); + assertEquals(i * 10_000_000_000L, rs.getLong(2)); + assertEquals("l" + i, rs.getString(3)); + assertArrayEquals(new byte[] {(byte) (i % 10)}, rs.getBytes(4)); + assertEquals(LocalDate.of(2026, 1, i % 28 + 1), rs.getObject(5, LocalDate.class)); + } + } + } + } + + @Test + public void fetchSizeStreamsInPortions() throws Exception { + try (Connection c = connect("extended")) { + c.setAutoCommit(false); + try (Statement s = c.createStatement()) { + s.setFetchSize(7); + int rows = 0; + try (ResultSet rs = s.executeQuery("SELECT time FROM plant.sensors ORDER BY time")) { + while (rs.next()) { + assertEquals(START + rows * 1000L, rs.getTimestamp(1).getTime()); + rows++; + } + } + assertEquals(100, rows); + } + c.commit(); + } + } + + @Test + public void simpleModeMultiStatementsTransactionsAndWrites() throws Exception { + try (Connection c = connect("simple"); + Statement s = c.createStatement()) { + // BEGIN, the query and COMMIT: the query's result is the second one + s.execute("BEGIN; SELECT count(*) FROM plant.sensors; COMMIT"); + long count = -1; + do { + ResultSet rs = s.getResultSet(); + if (rs != null) { + assertTrue(rs.next()); + count = rs.getLong(1); + } + } while (s.getMoreResults() || s.getUpdateCount() != -1); + assertEquals(100L, count); + try (ResultSet rs = s.executeQuery("SELECT count(*) FROM plant.sensors WHERE ok")) { + assertTrue(rs.next()); + assertEquals(50L, rs.getLong(1)); + } + try { + s.execute("INSERT INTO plant.sensors (time, device, n) VALUES (1, 'x', 1)"); + fail("read-only"); + } catch (SQLException e) { + assertEquals("25006", e.getSQLState()); + } + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/postgresql/it/IoTDBPostgreSqlEngineIT.java b/integration-test/src/test/java/org/apache/iotdb/postgresql/it/IoTDBPostgreSqlEngineIT.java new file mode 100644 index 0000000000000..edb140c93b65a --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/postgresql/it/IoTDBPostgreSqlEngineIT.java @@ -0,0 +1,855 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.it; + +import org.apache.iotdb.isession.ISession; +import org.apache.iotdb.isession.SessionConfig; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.net.ServerSocket; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * The PostgreSQL service on loopback against the engine: the recorded client corpus, psql's catalog + * queries, Npgsql's type loading, date rules, pgJDBC metadata, cursors, cancel and parameters. + */ +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class}) +public class IoTDBPostgreSqlEngineIT { + + private static final String PASSWORD = "Pg-Pass-2026a"; + private static final String ROOT_PASSWORD = SessionConfig.DEFAULT_PASSWORD; + private static int port; + + @BeforeClass + public static void start() throws Exception { + try (ServerSocket socket = new ServerSocket(0)) { + port = socket.getLocalPort(); + } + EnvFactory.getEnv() + .getConfig() + .getDataNodeConfig() + .setEnablePostgreSqlService(true) + .setPostgreSqlServicePort(port); + EnvFactory.getEnv().initClusterEnvironment(); + try (ISession admin = EnvFactory.getEnv().getSessionConnection()) { + admin.executeNonQueryStatement("CREATE USER pguser '" + PASSWORD + "'"); + } + try (org.apache.iotdb.isession.ITableSession table = + EnvFactory.getEnv().getTableSessionConnection()) { + table.executeNonQueryStatement("CREATE DATABASE plant"); + table.executeNonQueryStatement("USE plant"); + table.executeNonQueryStatement( + "CREATE TABLE sensors (device STRING TAG, temp DOUBLE FIELD, ok BOOLEAN FIELD," + + " n INT32 FIELD, label STRING FIELD)"); + table.executeNonQueryStatement( + "INSERT INTO sensors (time, device, temp, ok, n, label) VALUES" + + " (1000, 'd1', 20.5, true, 1, 'a'), (2000, 'd1', 21.5, false, 2, 'b')," + + " (3000, 'd2', 19.0, true, 3, null)"); + table.executeNonQueryStatement("GRANT SELECT ON TABLE plant.sensors TO USER pguser"); + // not readable for pguser: hidden from its catalog + table.executeNonQueryStatement("CREATE TABLE secret (v INT32 FIELD)"); + table.executeNonQueryStatement("CREATE DATABASE other"); + table.executeNonQueryStatement("CREATE TABLE other.t (v INT32 FIELD)"); + // the reference schema of the client corpus + // (external-service-impl/postgresql/src/test/corpus-capture) + table.executeNonQueryStatement("CREATE DATABASE refdb"); + table.executeNonQueryStatement( + "CREATE TABLE refdb.sensors (plant STRING TAG, line STRING TAG, machine STRING TAG," + + " model STRING ATTRIBUTE, temperature DOUBLE FIELD, pressure FLOAT FIELD," + + " \"count\" INT64 FIELD, level INT32 FIELD, running BOOLEAN FIELD," + + " state STRING FIELD, note STRING FIELD, blob BLOB FIELD, day DATE FIELD," + + " ts TIMESTAMP FIELD)"); + table.executeNonQueryStatement( + "INSERT INTO refdb.sensors (time, plant, line, machine, model, temperature, level," + + " running, day) VALUES (1000, 'plant1', 'line1', 'm1', 'model1', 20.5, 3, true," + + " '2026-01-05')"); + // every parameter type + table.executeNonQueryStatement( + "CREATE TABLE refdb.ptypes (device STRING TAG, ok BOOLEAN FIELD, n INT32 FIELD," + + " big INT64 FIELD, f FLOAT FIELD, temp DOUBLE FIELD, label STRING FIELD," + + " raw BLOB FIELD, day DATE FIELD, at TIMESTAMP FIELD)"); + table.executeNonQueryStatement( + "INSERT INTO refdb.ptypes (time, device, ok, n, big, f, temp, label, raw, day, at)" + + " VALUES (1000, 'd0', true, 6, 60000000000, 3.0, 1.5, 'l6', X'06', '2026-01-07'," + + " 1006)"); + // enough rows for a slow cross join + table.executeNonQueryStatement("CREATE TABLE refdb.many (v INT32 FIELD)"); + for (int batch = 0; batch < 4; batch++) { + StringBuilder insert = new StringBuilder("INSERT INTO refdb.many (time, v) VALUES "); + for (int i = 0; i < 500; i++) { + int n = batch * 500 + i; + insert + .append(i > 0 ? ", " : "") + .append('(') + .append(n + 1) + .append(", ") + .append(n) + .append(')'); + } + table.executeNonQueryStatement(insert.toString()); + } + // instants around the DST changes in Europe/Berlin + table.executeNonQueryStatement("CREATE TABLE refdb.dates (v INT32 FIELD, s STRING FIELD)"); + long[] dates = { + java.time.Instant.parse("2026-01-15T09:20:30.250Z").toEpochMilli(), + java.time.Instant.parse("2026-03-28T23:30:00Z").toEpochMilli(), + java.time.Instant.parse("2026-03-29T10:00:00Z").toEpochMilli(), + java.time.Instant.parse("2026-10-24T22:30:00Z").toEpochMilli(), + java.time.Instant.parse("2026-10-25T11:00:00Z").toEpochMilli(), + java.time.Instant.parse("2026-12-31T23:30:00Z").toEpochMilli() + }; + for (int i = 0; i < dates.length; i++) { + table.executeNonQueryStatement( + "INSERT INTO refdb.dates (time, v, s) VALUES (" + + dates[i] + + ", " + + i + + ", " + + (i == 2 ? "null" : "'s" + i + "'") + + ")"); + } + table.executeNonQueryStatement( + "CREATE TABLE refdb.events (machine STRING TAG, code INT32 FIELD, message STRING FIELD)"); + table.executeNonQueryStatement( + "INSERT INTO refdb.events (time, machine, code, message) VALUES (1000, 'm1', 1, 'e')"); + } + } + + /** Npgsql 4.0.17's type-loading queries (PostgresDatabaseInfo.GenerateTypesQuery), verbatim. */ + private static final String[] NPGSQL_TYPE_LOADING = { + "/*** Load all supported types ***/\n" + + "SELECT ns.nspname, a.typname, a.oid, a.typrelid, a.typbasetype,\n" + + "CASE WHEN pg_proc.proname='array_recv' THEN 'a' ELSE a.typtype END AS type,\n" + + "CASE\n" + + " WHEN pg_proc.proname='array_recv' THEN a.typelem\n" + + " WHEN a.typtype='r' THEN rngsubtype\n" + + " ELSE 0\n" + + "END AS elemoid,\n" + + "CASE\n" + + " WHEN pg_proc.proname IN ('array_recv','oidvectorrecv') THEN 3 /* Arrays last */\n" + + " WHEN a.typtype='r' THEN 2 /* Ranges before */\n" + + " WHEN a.typtype='d' THEN 1 /* Domains before */\n" + + " ELSE 0 /* Base types first */\n" + + "END AS ord\n" + + "FROM pg_type AS a\n" + + "JOIN pg_namespace AS ns ON (ns.oid = a.typnamespace)\n" + + "JOIN pg_proc ON pg_proc.oid = a.typreceive\n" + + "LEFT OUTER JOIN pg_class AS cls ON (cls.oid = a.typrelid)\n" + + "LEFT OUTER JOIN pg_type AS b ON (b.oid = a.typelem)\n" + + "LEFT OUTER JOIN pg_class AS elemcls ON (elemcls.oid = b.typrelid)\n" + + "LEFT OUTER JOIN pg_range ON (pg_range.rngtypid = a.oid) \n" + + "WHERE\n" + + " a.typtype IN ('b', 'r', 'e', 'd') OR /* Base, range, enum, domain */\n" + + " (a.typtype = 'c' AND cls.relkind='c') OR /* User-defined free-standing composites (not table composites) by default */\n" + + " (pg_proc.proname='array_recv' AND (\n" + + " b.typtype IN ('b', 'r', 'e', 'd') OR /* Array of base, range, enum, domain */\n" + + " (b.typtype = 'p' AND b.typname IN ('record', 'void')) OR /* Arrays of special supported pseudo-types */\n" + + " (b.typtype = 'c' AND elemcls.relkind='c') /* Array of user-defined free-standing composites (not table composites) */\n" + + " )) OR\n" + + " (a.typtype = 'p' AND a.typname IN ('record', 'void')) /* Some special supported pseudo-types */\n" + + "ORDER BY ord", + "/*** Load field definitions for (free-standing) composite types ***/\n" + + "SELECT typ.oid, att.attname, att.atttypid\n" + + "FROM pg_type AS typ\n" + + "JOIN pg_namespace AS ns ON (ns.oid = typ.typnamespace)\n" + + "JOIN pg_class AS cls ON (cls.oid = typ.typrelid)\n" + + "JOIN pg_attribute AS att ON (att.attrelid = typ.typrelid)\n" + + "WHERE\n" + + " (typ.typtype = 'c' AND cls.relkind='c') AND\n" + + " attnum > 0 AND /* Don't load system attributes */\n" + + " NOT attisdropped\n" + + "ORDER BY typ.oid, att.attnum", + "/*** Load enum fields ***/\n" + + "SELECT pg_type.oid, enumlabel\n" + + "FROM pg_enum\n" + + "JOIN pg_type ON pg_type.oid=enumtypid\n" + + "ORDER BY oid, enumsortorder" + }; + + @Test + public void npgsqlLoadsTheTypes() throws Exception { + try (Connection c = connect("pguser", PASSWORD, "extended"); + Statement s = c.createStatement()) { + int rows = 0; + int lastOrd = 0; + Map kinds = new HashMap<>(); + try (ResultSet rs = s.executeQuery(NPGSQL_TYPE_LOADING[0])) { + while (rs.next()) { + rows++; + assertEquals("pg_catalog", rs.getString("nspname")); + assertTrue("arrays last", rs.getInt("ord") >= lastOrd); + lastOrd = rs.getInt("ord"); + kinds.put(rs.getString("typname"), rs.getString("type") + ":" + rs.getLong("elemoid")); + } + } + // 18 base types, their 18 arrays, record, _record and void + assertEquals(kinds.toString(), 39, rows); + assertEquals("b:0", kinds.get("timestamptz")); + assertEquals("a:1184", kinds.get("_timestamptz")); + assertEquals("a:2249", kinds.get("_record")); + assertEquals("p:0", kinds.get("void")); + for (int i = 1; i < NPGSQL_TYPE_LOADING.length; i++) { + try (ResultSet rs = s.executeQuery(NPGSQL_TYPE_LOADING[i])) { + assertTrue("no composites or enums", !rs.next()); + } + } + } + } + + /** + * Every corpus statement that the golden files record as translated runs on the engine. + * Statements with bound parameters are left to the extended-protocol tests. + */ + @Test + public void translatedCorpusStatementsRunOnTheEngine() throws Exception { + java.nio.file.Path corpus = + java.nio.file.Paths.get( + IoTDBPostgreSqlEngineIT.class + .getProtectionDomain() + .getCodeSource() + .getLocation() + .toURI()) + .getParent() + .getParent() + .resolve("../external-service-impl/postgresql/src/test/resources/corpus") + .normalize(); + // translated, but the engine cannot run them yet; each with the reason + Map known = new HashMap<>(); + java.util.List failures = new java.util.ArrayList<>(); + int ran = 0; + try (Connection c = connect("root", ROOT_PASSWORD); + Statement s = c.createStatement()) { + // the recordings (and the golden files) were made in UTC + s.execute("SET TimeZone = 'UTC'"); + java.io.File[] files = corpus.toFile().listFiles((dir, name) -> name.endsWith(".sql")); + assertTrue("corpus at " + corpus, files != null && files.length > 0); + java.util.Arrays.sort(files); + for (java.io.File file : files) { + String client = file.getName().replace(".sql", ""); + java.util.List statements = new java.util.ArrayList<>(); + StringBuilder current = null; + for (String line : java.nio.file.Files.readAllLines(file.toPath())) { + if (line.equals("-- next")) { + if (current != null) { + statements.add(current.toString().trim()); + } + current = new StringBuilder(); + } else if (current != null && !line.startsWith("--")) { + current.append(line).append('\n'); + } + } + if (current != null) { + statements.add(current.toString().trim()); + } + for (int i = 0; i < statements.size(); i++) { + java.util.List expected = + java.nio.file.Files.readAllLines( + corpus.resolve("expected").resolve(client).resolve((i + 1) + ".sql")); + String outcome = expected.get(expected.size() - 1); + String statement = statements.get(i); + // cursor statements need a transaction: sqlCursorsStreamInPortions covers them + if (outcome.startsWith("--") + || expected.stream().anyMatch(l -> l.startsWith("-- cursor")) + || statement.contains("$1") + || known.containsKey(client + "/" + (i + 1))) { + continue; + } + ran++; + try (ResultSet rs = s.executeQuery(statement)) { + while (rs.next()) { + // drain + } + } catch (SQLException e) { + failures.add(client + "/" + (i + 1) + ": " + e.getMessage()); + } + } + } + } + assertEquals(String.join("\n", failures), 0, failures.size()); + assertTrue(ran > 5); + } + + /** psql's {@code \\dn} and the table lookup of {@code \\d refdb.sensors}, from the corpus. */ + @Test + public void psqlCatalogQueriesReturnTheRightRows() throws Exception { + java.util.List psql = corpusStatements("psql"); + try (Connection c = connect("root", ROOT_PASSWORD); + Statement s = c.createStatement()) { + StringBuilder schemas = new StringBuilder(); + try (ResultSet rs = s.executeQuery(psql.get(1))) { + while (rs.next()) { + schemas.append(rs.getString("Name")).append(' '); + } + } + assertEquals("\\dn", "other plant refdb ", schemas.toString()); + try (ResultSet rs = s.executeQuery(psql.get(3))) { + assertTrue(rs.next()); + assertEquals("refdb", rs.getString("nspname")); + assertEquals("sensors", rs.getString("relname")); + long oid = rs.getLong("oid"); + assertTrue(oid >= 16384); + assertTrue("one table", !rs.next()); + // \d then lists the columns by that OID (corpus statement 6 used PostgreSQL's OID) + String columns = psql.get(5).replace("'16386'", "'" + oid + "'"); + StringBuilder names = new StringBuilder(); + try (ResultSet cols = s.executeQuery(columns)) { + while (cols.next()) { + names.append(cols.getString(1)).append(':').append(cols.getString(2)).append(' '); + } + } + assertTrue( + names.toString(), + names + .toString() + .startsWith( + "time:timestamp with time zone plant:text line:text machine:text model:text" + + " temperature:double precision pressure:real count:bigint level:integer" + + " running:boolean ")); + } + } + } + + private static java.util.List corpusStatements(String client) throws Exception { + java.nio.file.Path file = + java.nio.file.Paths.get( + IoTDBPostgreSqlEngineIT.class + .getProtectionDomain() + .getCodeSource() + .getLocation() + .toURI()) + .getParent() + .getParent() + .resolve( + "../external-service-impl/postgresql/src/test/resources/corpus/" + client + ".sql") + .normalize(); + java.util.List statements = new java.util.ArrayList<>(); + StringBuilder current = null; + for (String line : java.nio.file.Files.readAllLines(file)) { + if (line.equals("-- next")) { + if (current != null) { + statements.add(current.toString().trim()); + } + current = new StringBuilder(); + } else if (current != null && !line.startsWith("--")) { + current.append(line).append('\n'); + } + } + if (current != null) { + statements.add(current.toString().trim()); + } + return statements; + } + + /** date_trunc, EXTRACT and intervals give PostgreSQL's results in Europe/Berlin. */ + @Test + public void dateRulesMatchPostgreSqlAcrossDaylightSavingTime() throws Exception { + java.time.ZoneId berlin = java.time.ZoneId.of("Europe/Berlin"); + try (Connection c = connect("root", ROOT_PASSWORD); + Statement s = c.createStatement()) { + s.execute("SET TimeZone = 'Europe/Berlin'"); + try (ResultSet rs = + s.executeQuery( + "SELECT time, date_trunc('day', time), date_trunc('month', time)," + + " date_trunc('quarter', time), date_trunc('year', time)," + + " date_trunc('hour', time), extract(dow from time), extract(second from time)," + + " extract(epoch from time), date_part('doy', time), extract(hour from time)" + + " FROM refdb.dates ORDER BY time")) { + int rows = 0; + while (rs.next()) { + rows++; + java.time.ZonedDateTime t = rs.getTimestamp(1).toInstant().atZone(berlin); + String at = t.toString(); + java.time.ZonedDateTime day = t.toLocalDate().atStartOfDay(berlin); + assertEquals(at, day.toInstant(), rs.getTimestamp(2).toInstant()); + assertEquals(at, day.withDayOfMonth(1).toInstant(), rs.getTimestamp(3).toInstant()); + java.time.ZonedDateTime quarter = + t.toLocalDate() + .withDayOfMonth(1) + .withMonth((t.getMonthValue() - 1) / 3 * 3 + 1) + .atStartOfDay(berlin); + assertEquals(at, quarter.toInstant(), rs.getTimestamp(4).toInstant()); + assertEquals( + at, + t.toLocalDate().withDayOfYear(1).atStartOfDay(berlin).toInstant(), + rs.getTimestamp(5).toInstant()); + assertEquals( + at, + t.truncatedTo(java.time.temporal.ChronoUnit.HOURS).toInstant(), + rs.getTimestamp(6).toInstant()); + assertEquals(at, t.getDayOfWeek().getValue() % 7, rs.getLong(7)); + assertEquals(at, t.getSecond() + t.getNano() / 1e9, rs.getDouble(8), 1e-9); + assertEquals(at, t.toInstant().toEpochMilli() / 1000.0, rs.getDouble(9), 1e-9); + assertEquals(at, t.getDayOfYear(), rs.getDouble(10), 0); + assertEquals(at, t.getHour(), rs.getLong(11)); + } + assertEquals(6, rows); + } + // GROUP BY a truncated time, as Power BI's date hierarchy does + try (ResultSet rs = + s.executeQuery( + "SELECT date_trunc('month', time) AS m, count(*) FROM refdb.dates" + + " GROUP BY date_trunc('month', time) ORDER BY m")) { + StringBuilder months = new StringBuilder(); + while (rs.next()) { + months + .append(rs.getTimestamp(1).toInstant().atZone(berlin).toLocalDate()) + .append('=') + .append(rs.getLong(2)) + .append(' '); + } + assertEquals("2026-01-01=1 2026-03-01=2 2026-10-01=2 2027-01-01=1 ", months.toString()); + } + try { + s.executeQuery("SELECT now() - interval '7 days'"); + fail("days have no fixed length in Europe/Berlin"); + } catch (SQLException e) { + assertEquals("0A000", e.getSQLState()); + } + s.execute("SET TimeZone = 'UTC'"); + try (ResultSet rs = + s.executeQuery( + "SELECT count(*) FROM refdb.dates WHERE time >= TIMESTAMP '2026-10-25 00:00:00'" + + " - interval '1 hour 30 minutes'")) { + assertTrue(rs.next()); + assertEquals(3, rs.getLong(1)); + } + try (ResultSet rs = + s.executeQuery( + "SELECT v, s || '-x', concat(s, '-y') FROM refdb.dates WHERE v IN (1, 2)" + + " ORDER BY v")) { + assertTrue(rs.next()); + assertEquals("s1-x", rs.getString(2)); + assertEquals("s1-y", rs.getString(3)); + assertTrue(rs.next()); + assertEquals("NULL || text is NULL", null, rs.getString(2)); + assertEquals("concat skips NULL", "-y", rs.getString(3)); + } + } + } + + /** pgJDBC's DatabaseMetaData, as DBeaver and other JDBC tools use it. */ + @Test + public void pgJdbcDatabaseMetaData() throws Exception { + try (Connection c = connect("pguser", PASSWORD, "extended")) { + java.sql.DatabaseMetaData meta = c.getMetaData(); + StringBuilder tables = new StringBuilder(); + try (ResultSet rs = meta.getTables(null, "plant", "%", new String[] {"TABLE"})) { + while (rs.next()) { + tables.append(rs.getString("TABLE_SCHEM")).append('.').append(rs.getString("TABLE_NAME")); + } + } + assertEquals("plant.sensors", tables.toString()); + StringBuilder columns = new StringBuilder(); + try (ResultSet rs = meta.getColumns(null, "plant", "sensors", "%")) { + while (rs.next()) { + columns + .append(rs.getString("COLUMN_NAME")) + .append(':') + .append(rs.getString("TYPE_NAME")) + .append(':') + .append(rs.getInt("DATA_TYPE")) + .append(':') + .append(rs.getInt("ORDINAL_POSITION")) + .append(':') + .append(rs.getString("REMARKS")) + .append(' '); + } + } + assertEquals( + "time:timestamptz:93:1:TIME device:text:12:2:TAG temp:float8:8:3:FIELD" + + " ok:bool:-7:4:FIELD n:int4:4:5:FIELD label:text:12:6:FIELD ", + columns.toString()); + } + } + + private static final String SLOW = + "SELECT count(*) FROM refdb.many a, refdb.many b, refdb.many c WHERE a.v + b.v + c.v = -1"; + + /** CancelRequest from a second connection stops a running query with 57014. */ + @Test + public void cancelRequestStopsARunningQuery() throws Exception { + try (Connection c = connect("root", ROOT_PASSWORD, "extended"); + Statement s = c.createStatement()) { + java.util.concurrent.ScheduledExecutorService timer = + java.util.concurrent.Executors.newSingleThreadScheduledExecutor(); + try { + timer.schedule( + () -> { + try { + s.cancel(); + } catch (SQLException e) { + throw new IllegalStateException(e); + } + }, + 1, + java.util.concurrent.TimeUnit.SECONDS); + long start = System.nanoTime(); + try { + s.executeQuery(SLOW); + fail("cancelled"); + } catch (SQLException e) { + assertEquals(e.getMessage(), "57014", e.getSQLState()); + } + long seconds = (System.nanoTime() - start) / 1_000_000_000L; + assertTrue("stopped within a few seconds, took " + seconds, seconds < 10); + } finally { + timer.shutdownNow(); + } + // the connection is usable afterwards + try (ResultSet rs = s.executeQuery("SELECT count(*) FROM refdb.many")) { + assertTrue(rs.next()); + assertEquals(2000L, rs.getLong(1)); + } + // statement_timeout ends a query the same way + s.execute("SET statement_timeout = '1s'"); + long start = System.nanoTime(); + try { + s.executeQuery(SLOW); + fail("timed out"); + } catch (SQLException e) { + assertEquals(e.getMessage(), "57014", e.getSQLState()); + } + assertTrue((System.nanoTime() - start) / 1_000_000_000L < 10); + } + } + + /** psycopg's server-side cursors: DECLARE, FETCH, MOVE, CLOSE in a transaction. */ + @Test + public void sqlCursorsStreamInPortions() throws Exception { + for (String mode : new String[] {"simple", "extended"}) { + try (Connection c = connect("pguser", PASSWORD, mode); + Statement s = c.createStatement()) { + try { + s.execute("DECLARE outside CURSOR FOR SELECT 1"); + fail("only in a transaction block"); + } catch (SQLException e) { + assertEquals("25P01", e.getSQLState()); + } + s.execute("BEGIN"); + s.execute("DECLARE \"big\" CURSOR FOR SELECT time, temp FROM plant.sensors ORDER BY time"); + assertEquals(mode, "1000:20.5 2000:21.5 ", fetch(s, "FETCH FORWARD 2 FROM \"big\"")); + assertEquals(mode, "3000:19 ", fetch(s, "FETCH FORWARD 2 FROM \"big\"")); + assertEquals(mode, "", fetch(s, "FETCH ALL FROM \"big\"")); + s.execute("CLOSE \"big\""); + try { + s.executeQuery("FETCH 1 FROM \"big\""); + fail("closed"); + } catch (SQLException e) { + assertEquals("34000", e.getSQLState()); + } + s.execute("ROLLBACK"); + + s.execute("BEGIN"); + s.execute("DECLARE c2 CURSOR FOR SELECT time FROM plant.sensors ORDER BY time"); + s.execute("MOVE FORWARD 2 IN c2"); + assertEquals(mode, "3000 ", fetch(s, "FETCH NEXT FROM c2")); + s.execute("COMMIT"); + try { + s.executeQuery("FETCH 1 FROM c2"); + fail("a cursor without HOLD ends with the transaction"); + } catch (SQLException e) { + assertEquals("34000", e.getSQLState()); + } + + s.execute("BEGIN"); + s.execute("DECLARE kept CURSOR WITH HOLD FOR SELECT time FROM plant.sensors ORDER BY time"); + s.execute("COMMIT"); + assertEquals(mode, "1000 ", fetch(s, "FETCH 1 FROM kept")); + s.execute("CLOSE ALL"); + } + } + } + + private static String fetch(Statement s, String sql) throws SQLException { + StringBuilder rows = new StringBuilder(); + try (ResultSet rs = s.executeQuery(sql)) { + int columns = rs.getMetaData().getColumnCount(); + while (rs.next()) { + rows.append(rs.getTimestamp(1).getTime()); + for (int i = 2; i <= columns; i++) { + rows.append(':').append(rs.getString(i)); + } + rows.append(' '); + } + } + return rows.toString(); + } + + @Test + public void theCatalogShowsOnlyWhatTheUserMayRead() throws Exception { + try (Connection c = connect("pguser", PASSWORD); + Statement s = c.createStatement()) { + Map tables = new HashMap<>(); + try (ResultSet rs = + s.executeQuery( + "SELECT table_schema, table_name, table_type FROM information_schema.tables" + + " WHERE table_schema NOT IN ('pg_catalog', 'information_schema')")) { + while (rs.next()) { + tables.put(rs.getString(1) + "." + rs.getString(2), rs.getString(3)); + } + } + assertEquals("{plant.sensors=BASE TABLE}", tables.toString()); + + StringBuilder columns = new StringBuilder(); + try (ResultSet rs = + s.executeQuery( + "SELECT a.attname, t.typname, a.attnotnull FROM pg_catalog.pg_attribute a" + + " JOIN pg_catalog.pg_class c ON a.attrelid = c.oid" + + " JOIN pg_catalog.pg_namespace n ON c.relnamespace = n.oid" + + " JOIN pg_catalog.pg_type t ON a.atttypid = t.oid" + + " WHERE n.nspname = 'plant' AND c.relname = 'sensors' ORDER BY a.attnum")) { + while (rs.next()) { + columns.append(rs.getString(1)).append(':').append(rs.getString(2)); + columns.append(rs.getBoolean(3) ? "! " : " "); + } + } + assertEquals( + "time:timestamptz! device:text! temp:float8 ok:bool n:int4 label:text ", + columns.toString()); + + try (ResultSet rs = + s.executeQuery("SELECT nspname FROM pg_catalog.pg_namespace ORDER BY nspname")) { + StringBuilder names = new StringBuilder(); + while (rs.next()) { + names.append(rs.getString(1)).append(' '); + } + assertEquals("information_schema pg_catalog plant ", names.toString()); + } + } + } + + @AfterClass + public static void stop() { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + private static Connection connect(String user, String password) throws SQLException { + return connect(user, password, "simple"); + } + + private static Connection connect(String user, String password, String queryMode) + throws SQLException { + Properties props = new Properties(); + props.setProperty("user", user); + props.setProperty("password", password); + props.setProperty("preferQueryMode", queryMode); + props.setProperty("ApplicationName", "pg-it"); + return DriverManager.getConnection("jdbc:postgresql://127.0.0.1:" + port + "/anydb", props); + } + + @Test + public void userLogsInAndUsesSessionCommands() throws Exception { + try (Connection c = connect("pguser", PASSWORD); + Statement s = c.createStatement()) { + s.execute("SET application_name = 'pg-it-2'"); + s.execute("BEGIN"); + s.execute("COMMIT"); + try (ResultSet rs = s.executeQuery("SHOW server_version")) { + assertTrue(rs.next()); + assertTrue(rs.getString(1), rs.getString(1).startsWith("14.0 (IoTDB ")); + } + try (ResultSet rs = s.executeQuery("SHOW application_name")) { + assertTrue(rs.next()); + assertEquals("pg-it-2", rs.getString(1)); + } + s.execute("SET TimeZone = 'Europe/Berlin'"); + try (ResultSet rs = s.executeQuery("SHOW TimeZone")) { + assertTrue(rs.next()); + assertEquals("Europe/Berlin", rs.getString(1)); + } + String startZone = java.time.ZoneId.systemDefault().getId(); + s.execute("SET TimeZone = 'Pacific/Auckland'"); + s.execute("RESET TimeZone"); + try (ResultSet rs = s.executeQuery("SHOW TimeZone")) { + assertTrue(rs.next()); + assertEquals("back to the zone of the start", startZone, rs.getString(1)); + } + s.execute("SET statement_timeout = '30s'"); + try (ResultSet rs = s.executeQuery("SELECT count(*) FROM plant.sensors")) { + assertTrue(rs.next()); + } + s.execute("RESET ALL"); + try (ResultSet rs = s.executeQuery("SHOW statement_timeout")) { + assertTrue(rs.next()); + assertEquals("0", rs.getString(1)); + } + try { + s.execute("SET statement_timeout = 'soon'"); + fail("invalid timeout"); + } catch (SQLException e) { + assertEquals("22023", e.getSQLState()); + } + try { + s.execute("SET client_encoding = 'LATIN1'"); + fail("only UTF8 is accepted"); + } catch (SQLException e) { + assertEquals("0A000", e.getSQLState()); + } + } + } + + @Test + public void queriesRunThroughTheSimpleProtocol() throws Exception { + try (Connection c = connect("pguser", PASSWORD); + Statement s = c.createStatement(); + ResultSet rs = + s.executeQuery( + "SELECT time, device, temp, ok, n, label FROM plant.sensors" + + " WHERE device = 'd1' ORDER BY time")) { + assertEquals("timestamptz", rs.getMetaData().getColumnTypeName(1)); + assertEquals("float8", rs.getMetaData().getColumnTypeName(3)); + assertTrue(rs.next()); + assertEquals(1000L, rs.getTimestamp(1).getTime()); + assertEquals("d1", rs.getString(2)); + assertEquals(20.5, rs.getDouble(3), 0.0); + assertTrue(rs.getBoolean(4)); + assertEquals(1, rs.getInt(5)); + assertEquals("a", rs.getString(6)); + assertTrue(rs.next()); + assertEquals(2000L, rs.getTimestamp(1).getTime()); + assertTrue(!rs.next()); + } + } + + @Test + public void queriesRunThroughTheExtendedProtocolWithParameters() throws Exception { + try (Connection c = connect("pguser", PASSWORD, "extended"); + java.sql.PreparedStatement p = + c.prepareStatement( + "SELECT device, avg(temp) AS t, count(*) FROM plant.sensors" + + " WHERE time >= ? AND device <> ? GROUP BY device ORDER BY device")) { + for (int round = 0; round < 6; round++) { + // pgJDBC switches to a named server-side statement and binary results after 5 runs + p.setTimestamp(1, new java.sql.Timestamp(1000L)); + p.setString(2, "x"); + try (ResultSet rs = p.executeQuery()) { + assertTrue(rs.next()); + assertEquals("d1", rs.getString(1)); + assertEquals(21.0, rs.getDouble(2), 1e-9); + assertEquals(2L, rs.getLong(3)); + assertTrue(rs.next()); + assertEquals("d2", rs.getString(1)); + assertTrue(!rs.next()); + } + } + } + } + + @Test + public void fetchSizeStreamsInPortions() throws Exception { + try (Connection c = connect("pguser", PASSWORD, "extended")) { + c.setAutoCommit(false); // pgJDBC uses portals with a row limit only inside a transaction + try (Statement s = c.createStatement()) { + s.setFetchSize(1); + try (ResultSet rs = s.executeQuery("SELECT time FROM plant.sensors ORDER BY time")) { + int rows = 0; + while (rs.next()) { + rows++; + } + assertEquals(3, rows); + } + } + c.commit(); + } + } + + @Test + public void untranslatableAndWritingStatementsAreRefused() throws Exception { + try (Connection c = connect("pguser", PASSWORD); + Statement s = c.createStatement()) { + try { + s.executeQuery("SELECT DISTINCT ON (device) * FROM plant.sensors"); + fail("refused"); + } catch (SQLException e) { + assertEquals("0A000", e.getSQLState()); + assertTrue(e.getMessage(), e.getMessage().contains("DISTINCT ON")); + } + try { + s.execute("INSERT INTO plant.sensors (time, device) VALUES (5, 'x')"); + fail("refused"); + } catch (SQLException e) { + assertEquals("25006", e.getSQLState()); + } + try { + s.executeQuery("SELECT * FROM plant.nothing_here"); + fail("refused"); + } catch (SQLException e) { + // the engine checks the privilege before the table exists, so a user without the + // privilege cannot tell absent tables from hidden ones + assertEquals(e.getMessage(), "42501", e.getSQLState()); + } + // the connection still works after errors + try (ResultSet rs = s.executeQuery("SELECT count(*) FROM plant.sensors")) { + assertTrue(rs.next()); + assertEquals(3L, rs.getLong(1)); + } + } + } + + /** Prepared statements with parameters of every type. */ + @Test + public void preparedStatementsWithEveryParameterType() throws Exception { + try (Connection c = connect("root", ROOT_PASSWORD, "extended"); + java.sql.PreparedStatement p = + c.prepareStatement( + "SELECT n, big, label, raw, day FROM refdb.ptypes WHERE device = ? AND ok = ?" + + " AND n >= ? AND big >= ? AND f >= ? AND temp >= ? AND day >= ?" + + " AND time >= ? ORDER BY time LIMIT 1")) { + for (int round = 0; round < 6; round++) { + p.setString(1, "d0"); + p.setBoolean(2, true); + p.setInt(3, 6); + p.setLong(4, 60_000_000_000L); + p.setFloat(5, 3.0f); + p.setDouble(6, 1.5); + p.setObject(7, java.time.LocalDate.of(2026, 1, 1)); + p.setTimestamp(8, new java.sql.Timestamp(1000)); + try (ResultSet rs = p.executeQuery()) { + assertTrue("round " + round, rs.next()); + assertEquals(6, rs.getInt(1)); + assertEquals(60_000_000_000L, rs.getLong(2)); + assertEquals("l6", rs.getString(3)); + assertEquals( + java.time.LocalDate.of(2026, 1, 7), rs.getObject(5, java.time.LocalDate.class)); + } + } + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/postgresql/it/IoTDBPostgreSqlImportIT.java b/integration-test/src/test/java/org/apache/iotdb/postgresql/it/IoTDBPostgreSqlImportIT.java new file mode 100644 index 0000000000000..fed9a275846ba --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/postgresql/it/IoTDBPostgreSqlImportIT.java @@ -0,0 +1,425 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.it; + +import org.apache.iotdb.isession.ITableSession; +import org.apache.iotdb.isession.SessionConfig; +import org.apache.iotdb.isession.SessionDataSet; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.Field; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.write.record.Tablet; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import javax.net.SocketFactory; + +import java.io.IOException; +import java.io.InputStream; +import java.net.InetAddress; +import java.net.ServerSocket; +import java.net.Socket; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.time.LocalDate; +import java.util.Arrays; +import java.util.List; +import java.util.Properties; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +/** + * A large table with every data type is read completely through pgJDBC (extended protocol, binary + * results, fetch size) and equals what the session API (the CLI) returns; rows stream; a client + * that stops reading loses its query after {@code write_timeout}. Rows: {@code -Dpg.import.rows} + * (default 1,000,000; 10,000,000 for manual runs). + */ +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class}) +public class IoTDBPostgreSqlImportIT { + + private static final int ROWS = Integer.getInteger("pg.import.rows", 1_000_000); + private static final long START = 1_767_225_600_000L; // 2026-01-01T00:00:00Z + private static final String QUERY = + "SELECT time, device, site, b, i, l, f, d, t, s, bl, dt, ts FROM imp.all_types ORDER BY time"; + + private static int port; + + @BeforeClass + public static void setUp() throws Exception { + try (ServerSocket socket = new ServerSocket(0)) { + port = socket.getLocalPort(); + } + EnvFactory.getEnv() + .getConfig() + .getDataNodeConfig() + .setEnablePostgreSqlService(true) + .setPostgreSqlServicePort(port) + .setPostgreSqlServiceWriteTimeoutInMs(2000); + EnvFactory.getEnv().initClusterEnvironment(); + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("CREATE DATABASE imp"); + session.executeNonQueryStatement("USE imp"); + session.executeNonQueryStatement( + "CREATE TABLE all_types (device STRING TAG, site STRING ATTRIBUTE, b BOOLEAN FIELD," + + " i INT32 FIELD, l INT64 FIELD, f FLOAT FIELD, d DOUBLE FIELD, t TEXT FIELD," + + " s STRING FIELD, bl BLOB FIELD, dt DATE FIELD, ts TIMESTAMP FIELD)"); + List names = + Arrays.asList("device", "site", "b", "i", "l", "f", "d", "t", "s", "bl", "dt", "ts"); + List types = + Arrays.asList( + TSDataType.STRING, + TSDataType.STRING, + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.STRING, + TSDataType.BLOB, + TSDataType.DATE, + TSDataType.TIMESTAMP); + List categories = + Arrays.asList( + ColumnCategory.TAG, + ColumnCategory.ATTRIBUTE, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD); + int batch = 10_000; + Tablet tablet = new Tablet("all_types", names, types, categories, batch); + for (int n = 0; n < ROWS; n++) { + int row = tablet.getRowSize(); + tablet.addTimestamp(row, START + n); + tablet.addValue(row, 0, "d" + (n % 5)); + tablet.addValue(row, 1, "site" + (n % 5)); + tablet.addValue(row, 2, n % 3 == 0); + tablet.addValue(row, 3, n * 7); + tablet.addValue(row, 4, (long) n * 1_000_003L); + tablet.addValue(row, 5, n / 4.0f); + tablet.addValue(row, 6, n / 3.0); + tablet.addValue(row, 7, "text " + n); + tablet.addValue(row, 8, "s" + (n % 1000)); + tablet.addValue(row, 9, new byte[] {(byte) n, (byte) (n >> 8), 0x7f}); + tablet.addValue(row, 10, LocalDate.of(2026, 1, 1).plusDays(n % 365)); + tablet.addValue(row, 11, START + n * 1000L); + if (tablet.getRowSize() == batch) { + session.insert(tablet); + tablet.reset(); + } + } + if (tablet.getRowSize() > 0) { + session.insert(tablet); + } + } + } + + @AfterClass + public static void tearDown() { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + private static Properties properties() { + Properties props = new Properties(); + props.setProperty("user", SessionConfig.DEFAULT_USER); + props.setProperty("password", SessionConfig.DEFAULT_PASSWORD); + props.setProperty("sslmode", "disable"); + return props; + } + + private static String url() { + return "jdbc:postgresql://127.0.0.1:" + port + "/any"; + } + + /** One row as text, the same for both clients. */ + private static String canonical( + long time, + String device, + String site, + boolean b, + int i, + long l, + float f, + double d, + String t, + String s, + byte[] bl, + LocalDate dt, + long ts) { + StringBuilder hex = new StringBuilder(); + for (byte x : bl) { + hex.append(String.format("%02x", x)); + } + return time + "|" + device + "|" + site + "|" + b + "|" + i + "|" + l + "|" + f + "|" + d + "|" + + t + "|" + s + "|" + hex + "|" + dt + "|" + ts; + } + + @Test + public void everyRowArrivesAsTheSessionApiReturnsIt() throws Exception { + long expectedHash = 0; + long expectedRows = 0; + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection(); + SessionDataSet rows = session.executeQueryStatement(QUERY)) { + rows.setFetchSize(10_000); + while (rows.hasNext()) { + RowRecord record = rows.next(); + List f = record.getFields(); + String line = + canonical( + f.get(0).getLongV(), + f.get(1).getStringValue(), + f.get(2).getStringValue(), + f.get(3).getBoolV(), + f.get(4).getIntV(), + f.get(5).getLongV(), + f.get(6).getFloatV(), + f.get(7).getDoubleV(), + f.get(8).getStringValue(), + f.get(9).getStringValue(), + f.get(10).getBinaryV().getValues(), + f.get(11).getDateV(), + f.get(12).getLongV()); + expectedHash = expectedHash * 31 + line.hashCode(); + expectedRows++; + } + } + assertEquals(ROWS, expectedRows); + + Properties props = properties(); + // server-side prepared statement from the first execution: binary results + props.setProperty("prepareThreshold", "-1"); + props.setProperty("binaryTransfer", "true"); + long actualHash = 0; + long actualRows = 0; + long started = System.nanoTime(); + long firstRow = -1; + try (Connection c = DriverManager.getConnection(url(), props)) { + c.setAutoCommit(false); // pgJDBC streams with a fetch size only inside a transaction + try (PreparedStatement p = c.prepareStatement(QUERY)) { + p.setFetchSize(10_000); + try (ResultSet rs = p.executeQuery()) { + while (rs.next()) { + if (firstRow < 0) { + firstRow = System.nanoTime() - started; + } + String line = + canonical( + rs.getTimestamp(1).getTime(), + rs.getString(2), + rs.getString(3), + rs.getBoolean(4), + rs.getInt(5), + rs.getLong(6), + rs.getFloat(7), + rs.getDouble(8), + rs.getString(9), + rs.getString(10), + rs.getBytes(11), + rs.getObject(12, LocalDate.class), + rs.getTimestamp(13).getTime()); + actualHash = actualHash * 31 + line.hashCode(); + actualRows++; + } + } + } + c.commit(); + } + long total = System.nanoTime() - started; + assertEquals("no row limit (REST has one)", ROWS, actualRows); + assertEquals("the same rows and values", expectedHash, actualHash); + assertTrue( + "rows stream: first after " + firstRow / 1_000_000 + " ms of " + total / 1_000_000, + firstRow < total / 2); + } + + @Test + public void aTimeWindowIsAnsweredFromTheWindow() throws Exception { + try (Connection c = DriverManager.getConnection(url(), properties()); + PreparedStatement p = + c.prepareStatement( + "SELECT count(*), min(i), max(i) FROM imp.all_types" + + " WHERE time >= ? AND time < ?")) { + p.setTimestamp(1, new java.sql.Timestamp(START + 1000)); + p.setTimestamp(2, new java.sql.Timestamp(START + 2000)); + try (ResultSet rs = p.executeQuery()) { + assertTrue(rs.next()); + assertEquals(1000L, rs.getLong(1)); + assertEquals(7000, rs.getInt(2)); + assertEquals(1999 * 7, rs.getInt(3)); + } + } + } + + /** A socket whose reads stop while {@code stalled} is set: a client that stops reading. */ + public static final class StallingSocketFactory extends SocketFactory { + static volatile boolean stalled; + static final Object LOCK = new Object(); + + @Override + public Socket createSocket() { + return new Socket() { + @Override + public InputStream getInputStream() throws IOException { + InputStream in = super.getInputStream(); + return new InputStream() { + private void await() { + synchronized (LOCK) { + while (stalled) { + try { + LOCK.wait(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } + } + } + } + + @Override + public int read() throws IOException { + await(); + return in.read(); + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + await(); + return in.read(b, off, len); + } + + @Override + public int available() throws IOException { + return in.available(); + } + + @Override + public void close() throws IOException { + in.close(); + } + }; + } + }; + } + + @Override + public Socket createSocket(String host, int port) throws IOException { + Socket socket = createSocket(); + socket.connect(new java.net.InetSocketAddress(host, port)); + return socket; + } + + @Override + public Socket createSocket(String host, int port, InetAddress local, int localPort) + throws IOException { + return createSocket(host, port); + } + + @Override + public Socket createSocket(InetAddress host, int port) throws IOException { + return createSocket(host.getHostAddress(), port); + } + + @Override + public Socket createSocket(InetAddress host, int port, InetAddress local, int localPort) + throws IOException { + return createSocket(host.getHostAddress(), port); + } + + static void release() { + synchronized (LOCK) { + stalled = false; + LOCK.notifyAll(); + } + } + } + + @Test + public void aClientThatStopsReadingLosesItsQuery() throws Exception { + Properties props = properties(); + props.setProperty("socketFactory", StallingSocketFactory.class.getName()); + props.setProperty("preferQueryMode", "simple"); + try (Connection c = DriverManager.getConnection(url(), props); + Statement s = c.createStatement()) { + StallingSocketFactory.stalled = true; + AtomicReference outcome = new AtomicReference<>(); + CountDownLatch done = new CountDownLatch(1); + Thread reader = + new Thread( + () -> { + try (ResultSet rs = s.executeQuery(QUERY)) { + while (rs.next()) { + // never reached while stalled + } + } catch (Throwable e) { + outcome.set(e); + } finally { + done.countDown(); + } + }); + reader.start(); + // the server fills the socket buffers, waits write_timeout (2 s) and closes the connection + Thread.sleep(8_000); + StallingSocketFactory.release(); + assertTrue("the query ended", done.await(60, TimeUnit.SECONDS)); + assertNotNull("the query failed", outcome.get()); + String message = String.valueOf(outcome.get().getMessage()); + assertTrue( + message, + (outcome.get() instanceof SQLException + && "57P01".equals(((SQLException) outcome.get()).getSQLState())) + || message.toLowerCase(java.util.Locale.ROOT).contains("connection") + || message.contains("I/O")); + } finally { + StallingSocketFactory.release(); + } + // the server is fine + try (Connection c = DriverManager.getConnection(url(), properties()); + Statement s = c.createStatement(); + ResultSet rs = s.executeQuery("SELECT count(*) FROM imp.all_types")) { + assertTrue(rs.next()); + assertEquals(ROWS, rs.getLong(1)); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/postgresql/it/IoTDBPostgreSqlNoKeyStoreIT.java b/integration-test/src/test/java/org/apache/iotdb/postgresql/it/IoTDBPostgreSqlNoKeyStoreIT.java new file mode 100644 index 0000000000000..c45b364ec9e07 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/postgresql/it/IoTDBPostgreSqlNoKeyStoreIT.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.it; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.net.ServerSocket; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * A non-loopback bind without a key store does not open the port; the failure names {@code + * postgresql_service_address}. + */ +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class}) +public class IoTDBPostgreSqlNoKeyStoreIT { + + private static int port; + + @BeforeClass + public static void setUp() throws Exception { + try (ServerSocket socket = new ServerSocket(0)) { + port = socket.getLocalPort(); + } + EnvFactory.getEnv() + .getConfig() + .getDataNodeConfig() + .setEnablePostgreSqlService(true) + .setPostgreSqlServiceAddress("0.0.0.0") + .setPostgreSqlServicePort(port); + EnvFactory.getEnv().initClusterEnvironment(); + } + + @AfterClass + public static void tearDown() { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void noPortAndTheAddressIsNamed() { + assertFalse("no listener without TLS off loopback", PgItSupport.listening(port)); + String log = PgItSupport.dataNodeLog(); + assertTrue( + log.contains("PostgreSQL endpoint not started") + && log.contains("postgresql_service_address=0.0.0.0")); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/postgresql/it/IoTDBPostgreSqlPortInUseIT.java b/integration-test/src/test/java/org/apache/iotdb/postgresql/it/IoTDBPostgreSqlPortInUseIT.java new file mode 100644 index 0000000000000..f432e27ba53f5 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/postgresql/it/IoTDBPostgreSqlPortInUseIT.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.it; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.net.InetAddress; +import java.net.ServerSocket; + +import static org.junit.Assert.assertTrue; + +/** A port already in use fails the service, naming the port key; the database starts anyway. */ +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class}) +public class IoTDBPostgreSqlPortInUseIT { + + private static ServerSocket occupied; + + @BeforeClass + public static void setUp() throws Exception { + occupied = new ServerSocket(0, 50, InetAddress.getByName("127.0.0.1")); + EnvFactory.getEnv() + .getConfig() + .getDataNodeConfig() + .setEnablePostgreSqlService(true) + .setPostgreSqlServicePort(occupied.getLocalPort()); + EnvFactory.getEnv().initClusterEnvironment(); + } + + @AfterClass + public static void tearDown() throws Exception { + try { + EnvFactory.getEnv().cleanClusterEnvironment(); + } finally { + occupied.close(); + } + } + + @Test + public void theServiceFailsNamingThePort() throws Exception { + String log = PgItSupport.dataNodeLog(); + assertTrue( + "log names the port key", + log.contains("PostgreSQL endpoint not started") && log.contains("postgresql_service_port")); + // the database itself runs + try (org.apache.iotdb.isession.ITableSession session = + EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("CREATE DATABASE still_up"); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/postgresql/it/IoTDBPostgreSqlSecurityIT.java b/integration-test/src/test/java/org/apache/iotdb/postgresql/it/IoTDBPostgreSqlSecurityIT.java new file mode 100644 index 0000000000000..2e04be5f37b2e --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/postgresql/it/IoTDBPostgreSqlSecurityIT.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.it; + +import org.apache.iotdb.isession.SessionConfig; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.net.ServerSocket; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Properties; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * Password login on loopback, refusals (28P01), the connection limit and read-only statements. + * Failed PostgreSQL logins feed the database's login lock like any other login. + */ +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class}) +public class IoTDBPostgreSqlSecurityIT { + + private static int port; + + @BeforeClass + public static void setUp() throws Exception { + try (ServerSocket socket = new ServerSocket(0)) { + port = socket.getLocalPort(); + } + EnvFactory.getEnv() + .getConfig() + .getDataNodeConfig() + .setEnablePostgreSqlService(true) + .setPostgreSqlServicePort(port) + .setPostgreSqlServiceMaxConnections(2); + EnvFactory.getEnv().initClusterEnvironment(); + } + + @AfterClass + public static void tearDown() { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + private static Connection connect(String user, String password) throws SQLException { + Properties props = new Properties(); + props.setProperty("user", user); + props.setProperty("password", password); + props.setProperty("sslmode", "disable"); + return DriverManager.getConnection("jdbc:postgresql://127.0.0.1:" + port + "/any", props); + } + + private static SQLException refused(String user, String password) { + try (Connection ignored = connect(user, password)) { + fail("refused: " + user); + return null; + } catch (SQLException e) { + return e; + } + } + + @Test + public void loginQueryAndRefusals() throws Exception { + try (Connection c = connect(SessionConfig.DEFAULT_USER, SessionConfig.DEFAULT_PASSWORD); + Statement s = c.createStatement()) { + try (ResultSet rs = s.executeQuery("SELECT current_user")) { + assertTrue(rs.next()); + assertEquals(SessionConfig.DEFAULT_USER, rs.getString(1)); + } + try { + s.execute("CREATE DATABASE x"); + fail("read-only"); + } catch (SQLException e) { + assertEquals("25006", e.getSQLState()); + } + } + SQLException wrong = refused(SessionConfig.DEFAULT_USER, "Wrong-Pass-2026"); + SQLException unknown = refused("nobody", "Wrong-Pass-2026"); + assertEquals("28P01", wrong.getSQLState()); + assertEquals("28P01", unknown.getSQLState()); + // the messages are the database's own login messages + assertTrue(wrong.getMessage(), wrong.getMessage().contains("Authentication failed.")); + assertTrue(unknown.getMessage(), unknown.getMessage().contains("nobody")); + } + + @Test + public void theConnectionLimitHolds() throws Exception { + try (Connection first = connect(SessionConfig.DEFAULT_USER, SessionConfig.DEFAULT_PASSWORD); + Connection second = connect(SessionConfig.DEFAULT_USER, SessionConfig.DEFAULT_PASSWORD)) { + SQLException third = refused(SessionConfig.DEFAULT_USER, SessionConfig.DEFAULT_PASSWORD); + assertEquals(third.getMessage(), "53300", third.getSQLState()); + assertTrue(first.isValid(5) && second.isValid(5)); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/postgresql/it/IoTDBPostgreSqlTlsIT.java b/integration-test/src/test/java/org/apache/iotdb/postgresql/it/IoTDBPostgreSqlTlsIT.java new file mode 100644 index 0000000000000..061928eabff2e --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/postgresql/it/IoTDBPostgreSqlTlsIT.java @@ -0,0 +1,209 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.it; + +import org.apache.iotdb.isession.SessionConfig; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLSocket; +import javax.net.ssl.TrustManager; +import javax.net.ssl.X509TrustManager; + +import java.io.DataOutputStream; +import java.io.File; +import java.net.ServerSocket; +import java.net.Socket; +import java.nio.file.Files; +import java.security.cert.X509Certificate; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Properties; +import java.util.concurrent.TimeUnit; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * With a key store and a non-loopback bind the endpoint is TLS-only (TLS 1.2/1.3, AEAD suites); + * logins are recorded with the protocol. + */ +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class}) +public class IoTDBPostgreSqlTlsIT { + + private static final String STORE_PASSWORD = "PgTls-Store-2026"; + private static int port; + + @BeforeClass + public static void setUp() throws Exception { + try (ServerSocket socket = new ServerSocket(0)) { + port = socket.getLocalPort(); + } + File store = Files.createTempDirectory("pg-tls").resolve("server.p12").toFile(); + Process keytool = + new ProcessBuilder( + System.getProperty("java.home") + + File.separator + + "bin" + + File.separator + + "keytool", + "-genkeypair", + "-alias", + "pg", + "-keyalg", + "RSA", + "-keysize", + "2048", + "-validity", + "30", + "-dname", + "CN=localhost", + "-ext", + "SAN=dns:localhost,ip:127.0.0.1", + "-storetype", + "PKCS12", + "-keystore", + store.getAbsolutePath(), + "-storepass", + STORE_PASSWORD, + "-keypass", + STORE_PASSWORD) + .redirectErrorStream(true) + .start(); + assertTrue(keytool.waitFor(60, TimeUnit.SECONDS)); + assertEquals(0, keytool.exitValue()); + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setKeyStorePath(store.getAbsolutePath()) + .setKeyStorePwd(STORE_PASSWORD); + EnvFactory.getEnv() + .getConfig() + .getDataNodeConfig() + .setEnablePostgreSqlService(true) + .setPostgreSqlServiceAddress("0.0.0.0") + .setPostgreSqlServicePort(port); + EnvFactory.getEnv().initClusterEnvironment(); + } + + @AfterClass + public static void tearDown() { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + private static Connection connect(boolean tls) throws SQLException { + Properties props = new Properties(); + props.setProperty("user", SessionConfig.DEFAULT_USER); + props.setProperty("password", SessionConfig.DEFAULT_PASSWORD); + props.setProperty("sslmode", tls ? "require" : "disable"); + if (tls) { + props.setProperty("sslfactory", "org.postgresql.ssl.NonValidatingFactory"); + } + return DriverManager.getConnection("jdbc:postgresql://127.0.0.1:" + port + "/any", props); + } + + @Test + public void plainConnectionsAreRefused() { + try (Connection ignored = connect(false)) { + fail("off loopback the endpoint is TLS-only"); + } catch (SQLException e) { + assertEquals(e.getMessage(), "28000", e.getSQLState()); + assertTrue(e.getMessage(), e.getMessage().contains("encryption required")); + } + } + + @Test + public void tlsLoginWorksAndIsRecorded() throws Exception { + try (Connection c = connect(true); + Statement s = c.createStatement(); + ResultSet rs = s.executeQuery("SELECT current_user")) { + assertTrue(rs.next()); + assertEquals(SessionConfig.DEFAULT_USER, rs.getString(1)); + } + assertTrue(PgItSupport.dataNodeLog().contains(":postgresql:127.0.0.1:")); + } + + @Test + public void onlyModernTlsWithAeadSuites() throws Exception { + SSLContext tls = SSLContext.getInstance("TLS"); + tls.init( + null, + new TrustManager[] { + new X509TrustManager() { + @Override + public void checkClientTrusted(X509Certificate[] chain, String authType) {} + + @Override + public void checkServerTrusted(X509Certificate[] chain, String authType) {} + + @Override + public X509Certificate[] getAcceptedIssuers() { + return new X509Certificate[0]; + } + } + }, + null); + for (String[] protocols : new String[][] {{"TLSv1.3"}, {"TLSv1.2"}, {"TLSv1.2", "TLSv1.3"}}) { + try (Socket plain = new Socket("127.0.0.1", port)) { + DataOutputStream out = new DataOutputStream(plain.getOutputStream()); + out.writeInt(8); + out.writeInt(80877103); // SSLRequest + out.flush(); + assertEquals('S', plain.getInputStream().read()); + try (SSLSocket secure = + (SSLSocket) tls.getSocketFactory().createSocket(plain, "127.0.0.1", port, false)) { + secure.setEnabledProtocols(protocols); + secure.startHandshake(); + String suite = secure.getSession().getCipherSuite(); + assertTrue(suite, suite.contains("GCM") || suite.contains("CHACHA20")); + } + } + } + // TLS 1.1 is refused + try (Socket plain = new Socket("127.0.0.1", port)) { + DataOutputStream out = new DataOutputStream(plain.getOutputStream()); + out.writeInt(8); + out.writeInt(80877103); + out.flush(); + assertEquals('S', plain.getInputStream().read()); + try (SSLSocket secure = + (SSLSocket) tls.getSocketFactory().createSocket(plain, "127.0.0.1", port, false)) { + secure.setEnabledProtocols(new String[] {"TLSv1.1"}); + secure.startHandshake(); + fail("TLS 1.1 must not be accepted"); + } catch (javax.net.ssl.SSLException expected) { + // refused (or not even offered by this JVM) + } + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/postgresql/it/PgItSupport.java b/integration-test/src/test/java/org/apache/iotdb/postgresql/it/PgItSupport.java new file mode 100644 index 0000000000000..cbdf69ab41b9c --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/postgresql/it/PgItSupport.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.postgresql.it; + +import org.apache.iotdb.it.env.EnvFactory; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** Helpers for the PostgreSQL wire protocol ITs . */ +final class PgItSupport { + + private PgItSupport() {} + + /** The DataNode's log output so far. */ + static String dataNodeLog() { + Path dir = Paths.get(EnvFactory.getEnv().getDataNodeWrapperList().get(0).getLogDirPath()); + try (Stream files = Files.list(dir)) { + return files + .filter(p -> p.getFileName().toString().startsWith("DataNode")) + .map( + p -> { + try { + return new String(Files.readAllBytes(p), StandardCharsets.UTF_8); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }) + .collect(Collectors.joining("\n")); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + /** True if something accepts connections on 127.0.0.1:port. */ + static boolean listening(int port) { + try (java.net.Socket socket = new java.net.Socket()) { + socket.connect(new java.net.InetSocketAddress("127.0.0.1", port), 1000); + return true; + } catch (IOException e) { + return false; + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index c6a0d4878969f..c9c0554a704bf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -113,6 +113,9 @@ public class IoTDBConfig implements DefaultEncodingProvider { /** Whether to enable the mqtt service. */ private boolean enableMQTTService = false; + /** Whether to enable the PostgreSQL wire protocol service. */ + private boolean enablePostgreSqlService = false; + /** The mqtt service binding host. */ private String mqttHost = "127.0.0.1"; @@ -2850,6 +2853,14 @@ public void setEnableMQTTService(boolean enableMQTTService) { this.enableMQTTService = enableMQTTService; } + public boolean isEnablePostgreSqlService() { + return enablePostgreSqlService; + } + + public void setEnablePostgreSqlService(boolean enablePostgreSqlService) { + this.enablePostgreSqlService = enablePostgreSqlService; + } + public String getMqttHost() { return mqttHost; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index 04fe3381b7541..f6ba42cc659e4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -915,6 +915,12 @@ public void loadProperties(TrimProperties properties) throws BadNodeUrlException // mqtt loadMqttProps(properties); + conf.setEnablePostgreSqlService( + Boolean.parseBoolean( + properties + .getProperty( + "enable_postgresql_service", Boolean.toString(conf.isEnablePostgreSqlService())) + .trim())); conf.setIntoOperationBufferSizeInByte( Long.parseLong( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/session/ExternalServiceClientSession.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/session/ExternalServiceClientSession.java new file mode 100644 index 0000000000000..8cd9d345f3fd3 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/session/ExternalServiceClientSession.java @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.protocol.session; + +import org.apache.iotdb.service.rpc.thrift.TSConnectionType; + +import java.net.InetSocketAddress; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +/** + * The session of a client that reaches the database through an external service speaking another + * wire protocol (for example the PostgreSQL wire protocol). Like a client-port session it carries + * statement and query ids and prepared statements, and it logs in through {@link + * SessionManager#login} like any other client. + */ +public final class ExternalServiceClientSession extends IClientSession { + + private final String protocol; + private final InetSocketAddress remote; + private final Map> statementIdToQueryId = new ConcurrentHashMap<>(); + private final Map preparedStatements = new ConcurrentHashMap<>(); + + /** + * @param protocol the service's protocol, e.g. {@code postgresql} + * @param remote the client's address and port + */ + public ExternalServiceClientSession(String protocol, InetSocketAddress remote) { + this.protocol = Objects.requireNonNull(protocol, "protocol"); + this.remote = Objects.requireNonNull(remote, "remote"); + } + + /** The service's protocol, e.g. {@code postgresql}. */ + public String getProtocol() { + return protocol; + } + + @Override + public String getClientAddress() { + return remote.getAddress() != null + ? remote.getAddress().getHostAddress() + : remote.getHostString(); + } + + @Override + public int getClientPort() { + return remote.getPort(); + } + + /** + * A network client like the client port's. The Thrift enum has no value for other protocols, and + * adding one would change the client IDL; the connection id carries the protocol name. + */ + @Override + TSConnectionType getConnectionType() { + return TSConnectionType.THRIFT_BASED; + } + + @Override + String getConnectionId() { + return protocol + ':' + getClientAddress() + ':' + getClientPort(); + } + + @Override + public Set getStatementIds() { + return statementIdToQueryId.keySet(); + } + + @Override + public void addStatementId(long statementId) { + statementIdToQueryId.computeIfAbsent(statementId, sid -> ConcurrentHashMap.newKeySet()); + } + + @Override + public Set removeStatementId(long statementId) { + return statementIdToQueryId.remove(statementId); + } + + @Override + public void addQueryId(Long statementId, long queryId) { + statementIdToQueryId + .computeIfAbsent(statementId, sid -> ConcurrentHashMap.newKeySet()) + .add(queryId); + } + + @Override + public boolean containsQueryId(Long statementId, long queryId) { + return ClientSession.containsQueryId(statementIdToQueryId, statementId, queryId); + } + + @Override + public void removeQueryId(Long statementId, Long queryId) { + ClientSession.removeQueryId(statementIdToQueryId, statementId, queryId); + } + + @Override + public void addPreparedStatement(String statementName, PreparedStatementInfo info) { + preparedStatements.put(statementName, info); + } + + @Override + public PreparedStatementInfo removePreparedStatement(String statementName) { + return preparedStatements.remove(statementName); + } + + @Override + public PreparedStatementInfo getPreparedStatement(String statementName) { + return preparedStatements.get(statementName); + } + + @Override + public Set getPreparedStatementNames() { + return preparedStatements.keySet(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/externalservice/BuiltinExternalServices.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/externalservice/BuiltinExternalServices.java index 157734aef56ce..11fb0eeb493f5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/externalservice/BuiltinExternalServices.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/externalservice/BuiltinExternalServices.java @@ -32,7 +32,11 @@ public enum BuiltinExternalServices { REST( "REST", "org.apache.iotdb.rest.RestService", - IoTDBRestServiceDescriptor.getInstance().getConfig()::isEnableRestService); + IoTDBRestServiceDescriptor.getInstance().getConfig()::isEnableRestService), + POSTGRESQL( + "POSTGRESQL", + "org.apache.iotdb.postgresql.PostgreSqlService", + IoTDBDescriptor.getInstance().getConfig()::isEnablePostgreSqlService); private final String serviceName; private final String className; diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template index c5117185e7040..da472470d04ce 100644 --- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template +++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template @@ -659,6 +659,47 @@ client_auth=false # Datatype: int idle_timeout_in_seconds=50000 +#################### +### PostgreSQL Service Configuration +#################### + +# Is the PostgreSQL wire protocol service enabled. PostgreSQL clients (psql, JDBC, psycopg, +# Npgsql, Power BI, ...) can then query the table model, read-only. +# effectiveMode: restart +# Datatype: boolean +enable_postgresql_service=false + +# The bind address. Any address other than loopback accepts only TLS connections and needs +# key_store_path / key_store_pwd. Must be an IP address of this machine (or 0.0.0.0 / ::). +# effectiveMode: restart +# Datatype: String +postgresql_service_address=127.0.0.1 + +# The port of the PostgreSQL service +# effectiveMode: restart +# Datatype: int +postgresql_service_port=5432 + +# Connections above this number are refused (SQLSTATE 53300) +# effectiveMode: restart +# Datatype: int +postgresql_service_max_connections=32 + +# Connections without a running query are closed after this time +# effectiveMode: restart +# Datatype: long +postgresql_service_idle_timeout_in_ms=1800000 + +# Time allowed from connecting to a completed login +# effectiveMode: restart +# Datatype: long +postgresql_service_startup_timeout_in_ms=10000 + +# A client that reads no result data for this long loses its query and its connection +# effectiveMode: restart +# Datatype: long +postgresql_service_write_timeout_in_ms=300000 + #################### ### Load balancing configuration #################### diff --git a/pom.xml b/pom.xml index 425530778fe66..58f7b361c8478 100644 --- a/pom.xml +++ b/pom.xml @@ -101,6 +101,8 @@ 12.0.41 3.30.17 5.14.0 + + 5.4 3.1 4.13.2 1.5.34 @@ -115,6 +117,8 @@ 0.18.0 4.1.137.Final 7.10.0 + + 42.7.14 2.0.9 1.1.0 3.3.1 @@ -632,6 +636,17 @@ jackson-core ${jackson.version} + + com.github.jsqlparser + jsqlparser + ${jsqlparser.version} + + + org.postgresql + postgresql + ${postgresql-jdbc.version} + test + From 7adfa9ec3a01bc2acb11717e76d6a1b02924cd07 Mon Sep 17 00:00:00 2001 From: Christofer Dutz Date: Fri, 20 Nov 2026 12:03:40 +0100 Subject: [PATCH 2/7] Document querying tree-model data through table views --- external-service-impl/postgresql/README.md | 38 +++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/external-service-impl/postgresql/README.md b/external-service-impl/postgresql/README.md index e07f11ac5d108..3b1baee490587 100644 --- a/external-service-impl/postgresql/README.md +++ b/external-service-impl/postgresql/README.md @@ -122,6 +122,42 @@ libpq-based clients only from libpq 15 on, which matches IP-address names). `information_schema.pg_is_*`. IoTDB clients see them in `SHOW TABLES FROM information_schema`. They list only what the user may read. +## Tree-model data + +The service reads the table model only. Tree-model data becomes visible through a **table view** +over a path pattern: the path levels below the prefix become tags, the measurements become fields. +For devices `root.factory..` with the measurements `temperature` and `running`: + +```sql +-- in the table model (CLI: -sql_dialect table) +CREATE DATABASE factory; +CREATE VIEW factory.machines ( + line STRING TAG, + machine STRING TAG, + temperature DOUBLE FIELD, + running BOOLEAN FIELD +) AS root.factory.**; +``` + +PostgreSQL clients then see the view like a table (`relkind` `v`), for example in psql: + +``` +iotdb=> \dv factory.* + List of relations + Schema | Name | Type | Owner +---------+----------+------+------- + factory | machines | view | root + +iotdb=> SELECT line, avg(temperature) FROM factory.machines GROUP BY line ORDER BY line; + line | avg +-------+------- + line1 | 21.25 + line2 | 19.5 +``` + +Clients that list views show it next to the tables. One view is needed per device pattern; +the service does not create views by itself. + ## Supported SQL The rules are documented in `PgToTableSql`, `DateTimeRules` and `ConstructCheck`. In short: @@ -147,7 +183,7 @@ The rules are documented in `PgToTableSql`, `DateTimeRules` and `ConstructCheck` ## Limitations - Read-only: `INSERT`, `UPDATE`, `DELETE`, DDL and `COPY` are refused with 25006. -- Table model only; tree-model data is visible through table views. +- Table model only; tree-model data is visible through table views (see "Tree-model data"). - Constructs without an exact equivalent are refused with 0A000 naming them, among them arrays, `numeric(p,s)`, `INTERVAL` months/years, explicit window frames, `lpad`/`rpad`. - Timestamps with nanosecond precision are sent with microsecond precision. From 1e6853edfe886dd1cefb52ab6a5c0986ff920aeb Mon Sep 17 00:00:00 2001 From: Christofer Dutz Date: Fri, 20 Nov 2026 12:38:58 +0100 Subject: [PATCH 3/7] Add JSqlParser to the dependency reference list --- dependencies.json | 1 + 1 file changed, 1 insertion(+) diff --git a/dependencies.json b/dependencies.json index 609615c8a3ff8..7ce28d06ce80c 100644 --- a/dependencies.json +++ b/dependencies.json @@ -14,6 +14,7 @@ "com.fasterxml.jackson.jakarta.rs:jackson-jakarta-rs-json-provider", "com.fasterxml.jackson.module:jackson-module-jakarta-xmlbind-annotations", "com.github.ben-manes.caffeine:caffeine", + "com.github.jsqlparser:jsqlparser", "com.github.luben:zstd-jni", "com.github.moquette-io.moquette:moquette-broker", "com.github.wendykierp:JTransforms", From b492ba0eb83608a61d7b7cbb027d2e6cdf1d79a3 Mon Sep 17 00:00:00 2001 From: Christofer Dutz Date: Fri, 20 Nov 2026 13:28:35 +0100 Subject: [PATCH 4/7] Run queries that read only information_schema with the highest priority Catalog queries of PostgreSQL clients (Npgsql's type loading joins seven pg_* tables) were refused at small memory budgets with "There is not enough memory to execute current fragment instance": every sort of a join is estimated at a full sort buffer, however small the inputs are. Queries that read only information_schema tables now get the existing highest priority (so far only information_schema.queries had it), so they use the memory fallback and are scheduled first. QueryExecution also sets the flag on the query context after distribution planning, because table-model analysis decides it after the context was created. --- .../plan/execution/QueryExecution.java | 3 + .../plan/relational/analyzer/Analysis.java | 15 +++- .../relational/planner/RelationPlanner.java | 3 + .../analyzer/AnalysisHighestPriorityTest.java | 82 +++++++++++++++++++ 4 files changed, 102 insertions(+), 1 deletion(-) create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AnalysisHighestPriorityTest.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/QueryExecution.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/QueryExecution.java index f6ba26a0fb716..1f38ddd9df9f3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/QueryExecution.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/QueryExecution.java @@ -417,6 +417,9 @@ public void doDistributedPlan() { // that the sub-tasks of the ShowQueries instances could be executed first. if (analysis.needSetHighestPriority()) { distributedPlan.getInstances().forEach(instance -> instance.setHighestPriority(true)); + // table-model analysis decides this while planning, after the constructor copied it, so the + // result handle needs it here as well + context.setNeedSetHighestPriority(true); } if (LOGGER.isDebugEnabled() && isQuery()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java index 2e02d5c94dc8b..89146d7107379 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java @@ -125,6 +125,13 @@ public class Analysis implements IAnalysis { @Nullable private Statement root; private boolean needSetHighestPriority; + // A query that reads only information_schema tables (catalog queries of BI tools join up to + // seven of them) gets the highest priority too: its memory estimate (one sort buffer per sort of + // every join) is far above what its tiny inputs need and would refuse it at small memory budgets. + private boolean readsInformationSchema; + + private boolean readsOtherTables; + private final Map, Expression> parameters; private String updateType; @@ -282,6 +289,7 @@ public void setInternalRpcSession(final boolean internalRpcSession) { } public void updateNeedSetHighestPriority(QualifiedObjectName tableName) { + readsInformationSchema = true; if (needSetHighestPriority) { return; } @@ -1006,9 +1014,14 @@ public void setQuery(boolean query) { isQuery = query; } + /** Called for every scanned table outside information_schema. */ + public void markNonInformationSchemaTableRead() { + readsOtherTables = true; + } + @Override public boolean needSetHighestPriority() { - return needSetHighestPriority; + return needSetHighestPriority || (readsInformationSchema && !readsOtherTables); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java index 6f5f3c278ed3d..8a75eb5ef3be2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java @@ -351,6 +351,9 @@ private RelationPlan processPhysicalTable(Table table, Scope scope) { final ImmutableMap.Builder symbolToColumnSchema = ImmutableMap.builder(); final Collection fields = scope.getRelationType().getAllFields(); final QualifiedObjectName qualifiedObjectName = getQualifiedObjectName(table, analysis); + if (!qualifiedObjectName.getDatabaseName().equals(INFORMATION_DATABASE)) { + analysis.markNonInformationSchemaTableRead(); + } // on the basis of that the order of fields is same with the column category order of segments // in DeviceEntry diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AnalysisHighestPriorityTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AnalysisHighestPriorityTest.java new file mode 100644 index 0000000000000..73538c6755482 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AnalysisHighestPriorityTest.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.analyzer; + +import org.apache.iotdb.commons.queryengine.plan.relational.metadata.QualifiedObjectName; +import org.apache.iotdb.commons.schema.table.InformationSchema; + +import org.junit.Test; + +import java.util.Collections; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** Queries reading only information_schema run with the highest priority (memory fallback). */ +public class AnalysisHighestPriorityTest { + + private static final QualifiedObjectName TABLES = + new QualifiedObjectName(InformationSchema.INFORMATION_DATABASE, InformationSchema.TABLES); + private static final QualifiedObjectName COLUMNS = + new QualifiedObjectName(InformationSchema.INFORMATION_DATABASE, InformationSchema.COLUMNS); + private static final QualifiedObjectName QUERIES = + new QualifiedObjectName(InformationSchema.INFORMATION_DATABASE, InformationSchema.QUERIES); + + private static Analysis analysis() { + return new Analysis(null, Collections.emptyMap()); + } + + @Test + public void noTablesKeepsNormalPriority() { + assertFalse(analysis().needSetHighestPriority()); + } + + @Test + public void onlyInformationSchemaTablesGetTheHighestPriority() { + Analysis analysis = analysis(); + analysis.updateNeedSetHighestPriority(TABLES); + analysis.updateNeedSetHighestPriority(COLUMNS); + assertTrue(analysis.needSetHighestPriority()); + } + + @Test + public void anyOtherTableKeepsNormalPriority() { + Analysis analysis = analysis(); + analysis.updateNeedSetHighestPriority(TABLES); + analysis.markNonInformationSchemaTableRead(); + assertFalse(analysis.needSetHighestPriority()); + } + + @Test + public void otherTableFirstKeepsNormalPriorityToo() { + Analysis analysis = analysis(); + analysis.markNonInformationSchemaTableRead(); + analysis.updateNeedSetHighestPriority(COLUMNS); + assertFalse(analysis.needSetHighestPriority()); + } + + @Test + public void queriesTableKeepsItsUpstreamRuleEvenWhenMixed() { + Analysis analysis = analysis(); + analysis.markNonInformationSchemaTableRead(); + analysis.updateNeedSetHighestPriority(QUERIES); + assertTrue(analysis.needSetHighestPriority()); + } +} From 64ed17b25257db0895f0b4cca609910606ea9867 Mon Sep 17 00:00:00 2001 From: Christofer Dutz Date: Fri, 20 Nov 2026 13:28:35 +0100 Subject: [PATCH 5/7] Fix SonarCloud reliability findings in the PostgreSQL service --- .../iotdb/postgresql/PostgreSqlService.java | 2 +- .../postgresql/config/BindAddresses.java | 44 ++++++++++++++++--- .../postgresql/execution/RunningQuery.java | 6 ++- .../postgresql/session/CursorCommand.java | 2 +- .../postgresql/session/SessionCommands.java | 9 ++-- .../iotdb/postgresql/session/SqlProtocol.java | 13 +++--- .../iotdb/postgresql/sql/DateTimeRules.java | 4 +- .../iotdb/postgresql/sql/PgToTableSql.java | 7 +-- .../postgresql/types/ParameterDecoder.java | 2 +- 9 files changed, 62 insertions(+), 27 deletions(-) diff --git a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/PostgreSqlService.java b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/PostgreSqlService.java index 3370dd0724f73..adaca4ff2175c 100644 --- a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/PostgreSqlService.java +++ b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/PostgreSqlService.java @@ -59,7 +59,7 @@ public class PostgreSqlService implements IExternalService { /** The reported server version: 14, so that clients take their standard paths. */ static final String SERVER_VERSION = "14.0 (IoTDB " + IoTDBConstant.VERSION + ")"; - private volatile PgServerContext context; + private PgServerContext context; private PgServer server; private ExecutorService workers; diff --git a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/config/BindAddresses.java b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/config/BindAddresses.java index ae25054655bfb..5a0ff015d48c4 100644 --- a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/config/BindAddresses.java +++ b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/config/BindAddresses.java @@ -37,10 +37,6 @@ final class BindAddresses { Pattern.compile( "^(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}$"); - /** Hex digits, colons and an optional embedded IPv4 tail or zone id. */ - private static final Pattern IPV6 = - Pattern.compile("^[0-9A-Fa-f:]*:[0-9A-Fa-f:.]*(%[\\w.\\-]+)?$"); - private BindAddresses() {} /** @@ -85,7 +81,7 @@ static InetAddress parseLiteral(String value) { if (candidate.startsWith("[") && candidate.endsWith("]")) { candidate = candidate.substring(1, candidate.length() - 1); } - if (!IPV4.matcher(candidate).matches() && !IPV6.matcher(candidate).matches()) { + if (!IPV4.matcher(candidate).matches() && !looksLikeIpv6(candidate)) { return null; } try { @@ -95,4 +91,42 @@ static InetAddress parseLiteral(String value) { return null; } } + + /** Hex digits and colons (at least one), an optional embedded IPv4 tail and zone id. */ + private static boolean looksLikeIpv6(String value) { + int percent = value.indexOf('%'); + String address = percent < 0 ? value : value.substring(0, percent); + if (address.indexOf(':') < 0) { + return false; + } + for (int i = 0; i < address.length(); i++) { + char c = address.charAt(i); + if (!(c == ':' || c == '.' || isAsciiHexDigit(c))) { + return false; + } + } + if (percent < 0) { + return true; + } + String zone = value.substring(percent + 1); + if (zone.isEmpty()) { + return false; + } + for (int i = 0; i < zone.length(); i++) { + char c = zone.charAt(i); + if (!(isAsciiHexDigit(c) + || (c >= 'g' && c <= 'z') + || (c >= 'G' && c <= 'Z') + || c == '_' + || c == '.' + || c == '-')) { + return false; + } + } + return true; + } + + private static boolean isAsciiHexDigit(char c) { + return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); + } } diff --git a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/execution/RunningQuery.java b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/execution/RunningQuery.java index 12b1b070808cc..0a0460ab2f71c 100644 --- a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/execution/RunningQuery.java +++ b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/execution/RunningQuery.java @@ -20,9 +20,11 @@ package org.apache.iotdb.postgresql.execution; import org.apache.iotdb.commons.exception.IoTDBException; +import org.apache.iotdb.commons.exception.SemanticException; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Literal; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Statement; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.StringLiteral; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.parser.ParsingException; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.protocol.session.IClientSession; import org.apache.iotdb.db.protocol.session.SessionManager; @@ -152,10 +154,10 @@ private static int errorCode(Throwable error) { return ((org.apache.iotdb.commons.exception.IoTDBRuntimeException) t).getErrorCode(); } } - if (error.getClass().getSimpleName().equals("SemanticException")) { + if (error instanceof SemanticException) { return TSStatusCode.SEMANTIC_ERROR.getStatusCode(); } - if (error.getClass().getSimpleName().equals("ParsingException")) { + if (error instanceof ParsingException) { return TSStatusCode.SQL_PARSE_ERROR.getStatusCode(); } return TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode(); diff --git a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/session/CursorCommand.java b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/session/CursorCommand.java index 99a658919fd14..f9e8cb9265210 100644 --- a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/session/CursorCommand.java +++ b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/session/CursorCommand.java @@ -43,7 +43,7 @@ public enum Kind { } private static final int FLAGS = Pattern.CASE_INSENSITIVE | Pattern.DOTALL; - private static final String NAME = "(\"(?:[^\"]|\"\")+\"|[A-Za-z_][\\w$]*)"; + private static final String NAME = "(\"(?:[^\"]++|\"\")++\"|[A-Za-z_][\\w$]*+)"; private static final Pattern DECLARE = Pattern.compile( diff --git a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/session/SessionCommands.java b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/session/SessionCommands.java index 9f0e0f6698621..af4e4792ea184 100644 --- a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/session/SessionCommands.java +++ b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/session/SessionCommands.java @@ -124,14 +124,15 @@ public Map parameterStatus() { Pattern.compile("SET\\s+SESSION\\s+AUTHORIZATION\\s+DEFAULT", FLAGS); private static final Pattern SET = Pattern.compile( - "SET\\s+(?:SESSION\\s+|LOCAL\\s+)?([A-Za-z_][\\w.]*)\\s*(?:=|\\s+TO\\s+)\\s*(.+)", FLAGS); - private static final Pattern RESET = Pattern.compile("RESET\\s+(ALL|[A-Za-z_][\\w.]*)", FLAGS); - private static final Pattern SHOW = Pattern.compile("SHOW\\s+([A-Za-z_][\\w.]*|ALL)", FLAGS); + "SET\\s++(?:SESSION\\s++|LOCAL\\s++)?+([A-Za-z_][\\w.]*+)(?:\\s*+=|\\s++TO\\s)\\s*+(.+)", + FLAGS); + private static final Pattern RESET = Pattern.compile("RESET\\s+([A-Za-z_][\\w.]*)", FLAGS); + private static final Pattern SHOW = Pattern.compile("SHOW\\s+([A-Za-z_][\\w.]*)", FLAGS); private static final Pattern DISCARD = Pattern.compile("DISCARD\\s+(ALL|TEMP|TEMPORARY|SEQUENCES|PLANS)", FLAGS); private static final Pattern UNLISTEN = Pattern.compile("UNLISTEN\\s+\\*", FLAGS); private static final Pattern DEALLOCATE = - Pattern.compile("DEALLOCATE\\s+(?:PREPARE\\s+)?(ALL|[A-Za-z_\"][\\w\"]*)", FLAGS); + Pattern.compile("DEALLOCATE\\s+(?:PREPARE\\s+)?([A-Za-z_\"][\\w\"]*)", FLAGS); private static final Pattern ADVISORY_UNLOCK_ALL = Pattern.compile("SELECT\\s+(?:pg_catalog\\.)?pg_advisory_unlock_all\\s*\\(\\s*\\)", FLAGS); diff --git a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/session/SqlProtocol.java b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/session/SqlProtocol.java index 4342a50410cef..0c123f93aa45b 100644 --- a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/session/SqlProtocol.java +++ b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/session/SqlProtocol.java @@ -50,6 +50,7 @@ import java.util.Locale; import java.util.Map; import java.util.Optional; +import java.util.concurrent.atomic.AtomicReference; /** * The query layer: session commands, and queries translated to table-model SQL and run by the @@ -126,7 +127,7 @@ private static final class Portal { private final Map portals = new HashMap<>(); private final Map cursors = new HashMap<>(); private boolean skipUntilSync; - private volatile ResultCursor running; + private final AtomicReference running = new AtomicReference<>(); private volatile boolean cancelRequested; private volatile boolean busy; @@ -647,7 +648,7 @@ private void run(Portal portal, Channel channel, ByteBuf out, long maxRows, bool if (describe) { BackendMessages.rowDescription(out, portal.cursor.fields()); } - running = portal.cursor; + running.set(portal.cursor); long sent; try { sent = portal.cursor.send(channel, out, maxRows, writeTimeout); @@ -658,7 +659,7 @@ private void run(Portal portal, Channel channel, ByteBuf out, long maxRows, bool } throw e; } finally { - running = null; + running.set(null); } if (portal.cursor.exhausted()) { closeCursor(portal); @@ -795,7 +796,7 @@ private void declare(Portal portal, CursorCommand command, ByteBuf out) throws P /** Sends ({@code out != null}) or skips up to {@code count} rows (0 = all). */ private long moveOrFetch(String name, SqlCursor cursor, Channel channel, ByteBuf out, long count) throws PgException, ResultCursor.WriteTimeoutException { - running = cursor.rows; + running.set(cursor.rows); try { return out == null ? cursor.rows.skip(count) @@ -808,7 +809,7 @@ private long moveOrFetch(String name, SqlCursor cursor, Channel channel, ByteBuf } throw e; } finally { - running = null; + running.set(null); } } @@ -914,7 +915,7 @@ public boolean busy() { @Override public void cancel() { - ResultCursor cursor = running; + ResultCursor cursor = running.get(); if (cursor != null) { cancelRequested = true; cursor.cancel(); diff --git a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/sql/DateTimeRules.java b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/sql/DateTimeRules.java index 57339f9882a46..8fbf2425b2e0f 100644 --- a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/sql/DateTimeRules.java +++ b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/sql/DateTimeRules.java @@ -293,9 +293,9 @@ static String extract(String rawField, ZoneId zone, TimestampPrecision precision // ---------------------------------------------------------------------------- intervals private static final Pattern PART = - Pattern.compile("([+-]?\\d+(?:\\.\\d+)?)\\s*([a-z]+)", Pattern.CASE_INSENSITIVE); + Pattern.compile("([+-]?+\\d++(?:\\.\\d++)?+)\\s*+([a-z]++)", Pattern.CASE_INSENSITIVE); private static final Pattern CLOCK = - Pattern.compile("([+-]?)(\\d+):(\\d{2})(?::(\\d{2}(?:\\.\\d+)?))?"); + Pattern.compile("([+-]?+)(\\d++):(\\d{2})(?::(\\d{2}(?:\\.\\d++)?+))?+"); /** * The length of an interval literal in timestamp units, for arithmetic with timestamps. Only diff --git a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/sql/PgToTableSql.java b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/sql/PgToTableSql.java index 92f2e68cbb875..43b649e3a95d1 100644 --- a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/sql/PgToTableSql.java +++ b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/sql/PgToTableSql.java @@ -1668,11 +1668,8 @@ public StringBuilder visit( } private void appendCurrentSchema() { - for (String schema : writer.context.searchPath()) { - builder.append(Identifiers.literal(schema)); - return; - } - builder.append("NULL"); + List searchPath = writer.context.searchPath(); + builder.append(searchPath.isEmpty() ? "NULL" : Identifiers.literal(searchPath.get(0))); } private static boolean isQuoted(String identifier) { diff --git a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/types/ParameterDecoder.java b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/types/ParameterDecoder.java index 31f4dce7d674e..36005cc51636d 100644 --- a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/types/ParameterDecoder.java +++ b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/types/ParameterDecoder.java @@ -54,7 +54,7 @@ public final class ParameterDecoder { private static final Pattern DECIMAL = Pattern.compile("[+-]?(\\d+\\.\\d*|\\.\\d+|\\d+)([eE][+-]?\\d+)?"); private static final Pattern TIMESTAMP = - Pattern.compile("\\d{4}-\\d{2}-\\d{2}[ T]\\d{2}:\\d{2}(:\\d{2}(\\.\\d+)?)?.*"); + Pattern.compile("\\d{4}-\\d{2}-\\d{2}[ T]\\d{2}:\\d{2}(?::\\d{2}(?:\\.\\d++)?+)?+.*"); /** PostgreSQL's text form: {@code 2026-11-19 10:15:30.123456+01}, offset optional. */ private static final DateTimeFormatter PG_TIMESTAMP = From 5231d966c44fdae2b0e2ede4183c76c7b980bd08 Mon Sep 17 00:00:00 2001 From: Christofer Dutz Date: Sat, 21 Nov 2026 14:14:02 +0100 Subject: [PATCH 6/7] Stabilize the PostgreSQL ITs and expect the service in existing ITs - IoTDBServicesIT lists the POSTGRESQL built-in service. - IoTDBDatabaseIT expects the information_schema tables that an InformationSchemaExtension (the PostgreSQL catalog) registers. - The PostgreSQL ITs wait until the endpoint listens: the service starts after the DataNode reports itself ready (refused connections on Windows). - Log checks wait for the expected line instead of reading the log once. - The write-timeout IT starts its countdown when the first result bytes arrive, so slow machines that sort the result for a while do not finish the query before the stall. --- .../it/IoTDBPostgreSqlClientsIT.java | 1 + .../it/IoTDBPostgreSqlEngineIT.java | 1 + .../it/IoTDBPostgreSqlImportIT.java | 23 +++++- .../it/IoTDBPostgreSqlNoKeyStoreIT.java | 8 +- .../it/IoTDBPostgreSqlPortInUseIT.java | 6 +- .../it/IoTDBPostgreSqlSecurityIT.java | 1 + .../postgresql/it/IoTDBPostgreSqlTlsIT.java | 1 + .../iotdb/postgresql/it/PgItSupport.java | 31 +++++++ .../informationschema/IoTDBServicesIT.java | 2 +- .../relational/it/schema/IoTDBDatabaseIT.java | 80 ++++++++++++------- 10 files changed, 116 insertions(+), 38 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/postgresql/it/IoTDBPostgreSqlClientsIT.java b/integration-test/src/test/java/org/apache/iotdb/postgresql/it/IoTDBPostgreSqlClientsIT.java index 7fa600c1a3504..885d27804cc82 100644 --- a/integration-test/src/test/java/org/apache/iotdb/postgresql/it/IoTDBPostgreSqlClientsIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/postgresql/it/IoTDBPostgreSqlClientsIT.java @@ -68,6 +68,7 @@ public static void setUp() throws Exception { .setEnablePostgreSqlService(true) .setPostgreSqlServicePort(port); EnvFactory.getEnv().initClusterEnvironment(); + PgItSupport.awaitListening(port); try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { session.executeNonQueryStatement("CREATE DATABASE plant"); session.executeNonQueryStatement("USE plant"); diff --git a/integration-test/src/test/java/org/apache/iotdb/postgresql/it/IoTDBPostgreSqlEngineIT.java b/integration-test/src/test/java/org/apache/iotdb/postgresql/it/IoTDBPostgreSqlEngineIT.java index edb140c93b65a..bd836e57ab87a 100644 --- a/integration-test/src/test/java/org/apache/iotdb/postgresql/it/IoTDBPostgreSqlEngineIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/postgresql/it/IoTDBPostgreSqlEngineIT.java @@ -68,6 +68,7 @@ public static void start() throws Exception { .setEnablePostgreSqlService(true) .setPostgreSqlServicePort(port); EnvFactory.getEnv().initClusterEnvironment(); + PgItSupport.awaitListening(port); try (ISession admin = EnvFactory.getEnv().getSessionConnection()) { admin.executeNonQueryStatement("CREATE USER pguser '" + PASSWORD + "'"); } diff --git a/integration-test/src/test/java/org/apache/iotdb/postgresql/it/IoTDBPostgreSqlImportIT.java b/integration-test/src/test/java/org/apache/iotdb/postgresql/it/IoTDBPostgreSqlImportIT.java index fed9a275846ba..c3e51e660202d 100644 --- a/integration-test/src/test/java/org/apache/iotdb/postgresql/it/IoTDBPostgreSqlImportIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/postgresql/it/IoTDBPostgreSqlImportIT.java @@ -91,6 +91,7 @@ public static void setUp() throws Exception { .setPostgreSqlServicePort(port) .setPostgreSqlServiceWriteTimeoutInMs(2000); EnvFactory.getEnv().initClusterEnvironment(); + PgItSupport.awaitListening(port); try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { session.executeNonQueryStatement("CREATE DATABASE imp"); session.executeNonQueryStatement("USE imp"); @@ -295,6 +296,10 @@ public void aTimeWindowIsAnsweredFromTheWindow() throws Exception { /** A socket whose reads stop while {@code stalled} is set: a client that stops reading. */ public static final class StallingSocketFactory extends SocketFactory { static volatile boolean stalled; + + /** Set once result bytes wait unread: the server has started sending. */ + static volatile boolean dataArrived; + static final Object LOCK = new Object(); @Override @@ -304,11 +309,14 @@ public Socket createSocket() { public InputStream getInputStream() throws IOException { InputStream in = super.getInputStream(); return new InputStream() { - private void await() { + private void await() throws IOException { synchronized (LOCK) { while (stalled) { + if (in.available() > 0) { + dataArrived = true; + } try { - LOCK.wait(); + LOCK.wait(50); } catch (InterruptedException e) { Thread.currentThread().interrupt(); return; @@ -382,6 +390,7 @@ public void aClientThatStopsReadingLosesItsQuery() throws Exception { props.setProperty("preferQueryMode", "simple"); try (Connection c = DriverManager.getConnection(url(), props); Statement s = c.createStatement()) { + StallingSocketFactory.dataArrived = false; StallingSocketFactory.stalled = true; AtomicReference outcome = new AtomicReference<>(); CountDownLatch done = new CountDownLatch(1); @@ -399,8 +408,14 @@ public void aClientThatStopsReadingLosesItsQuery() throws Exception { } }); reader.start(); - // the server fills the socket buffers, waits write_timeout (2 s) and closes the connection - Thread.sleep(8_000); + // the server fills the socket buffers, waits write_timeout (2 s) and closes the connection; + // the countdown starts with the first result bytes (slow machines take a while to sort) + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(120); + while (!StallingSocketFactory.dataArrived && System.nanoTime() < deadline) { + Thread.sleep(100); + } + assertTrue("the server started sending", StallingSocketFactory.dataArrived); + Thread.sleep(10_000); StallingSocketFactory.release(); assertTrue("the query ended", done.await(60, TimeUnit.SECONDS)); assertNotNull("the query failed", outcome.get()); diff --git a/integration-test/src/test/java/org/apache/iotdb/postgresql/it/IoTDBPostgreSqlNoKeyStoreIT.java b/integration-test/src/test/java/org/apache/iotdb/postgresql/it/IoTDBPostgreSqlNoKeyStoreIT.java index c45b364ec9e07..f88ab2c73fa03 100644 --- a/integration-test/src/test/java/org/apache/iotdb/postgresql/it/IoTDBPostgreSqlNoKeyStoreIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/postgresql/it/IoTDBPostgreSqlNoKeyStoreIT.java @@ -64,9 +64,13 @@ public static void tearDown() { } @Test - public void noPortAndTheAddressIsNamed() { + public void noPortAndTheAddressIsNamed() throws Exception { + String log = + PgItSupport.awaitDataNodeLog( + l -> + l.contains("PostgreSQL endpoint not started") + && l.contains("postgresql_service_address=0.0.0.0")); assertFalse("no listener without TLS off loopback", PgItSupport.listening(port)); - String log = PgItSupport.dataNodeLog(); assertTrue( log.contains("PostgreSQL endpoint not started") && log.contains("postgresql_service_address=0.0.0.0")); diff --git a/integration-test/src/test/java/org/apache/iotdb/postgresql/it/IoTDBPostgreSqlPortInUseIT.java b/integration-test/src/test/java/org/apache/iotdb/postgresql/it/IoTDBPostgreSqlPortInUseIT.java index f432e27ba53f5..81104160cf00a 100644 --- a/integration-test/src/test/java/org/apache/iotdb/postgresql/it/IoTDBPostgreSqlPortInUseIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/postgresql/it/IoTDBPostgreSqlPortInUseIT.java @@ -63,7 +63,11 @@ public static void tearDown() throws Exception { @Test public void theServiceFailsNamingThePort() throws Exception { - String log = PgItSupport.dataNodeLog(); + String log = + PgItSupport.awaitDataNodeLog( + l -> + l.contains("PostgreSQL endpoint not started") + && l.contains("postgresql_service_port")); assertTrue( "log names the port key", log.contains("PostgreSQL endpoint not started") && log.contains("postgresql_service_port")); diff --git a/integration-test/src/test/java/org/apache/iotdb/postgresql/it/IoTDBPostgreSqlSecurityIT.java b/integration-test/src/test/java/org/apache/iotdb/postgresql/it/IoTDBPostgreSqlSecurityIT.java index 2e04be5f37b2e..c856d904b4778 100644 --- a/integration-test/src/test/java/org/apache/iotdb/postgresql/it/IoTDBPostgreSqlSecurityIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/postgresql/it/IoTDBPostgreSqlSecurityIT.java @@ -64,6 +64,7 @@ public static void setUp() throws Exception { .setPostgreSqlServicePort(port) .setPostgreSqlServiceMaxConnections(2); EnvFactory.getEnv().initClusterEnvironment(); + PgItSupport.awaitListening(port); } @AfterClass diff --git a/integration-test/src/test/java/org/apache/iotdb/postgresql/it/IoTDBPostgreSqlTlsIT.java b/integration-test/src/test/java/org/apache/iotdb/postgresql/it/IoTDBPostgreSqlTlsIT.java index 061928eabff2e..df4cabadb1628 100644 --- a/integration-test/src/test/java/org/apache/iotdb/postgresql/it/IoTDBPostgreSqlTlsIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/postgresql/it/IoTDBPostgreSqlTlsIT.java @@ -114,6 +114,7 @@ public static void setUp() throws Exception { .setPostgreSqlServiceAddress("0.0.0.0") .setPostgreSqlServicePort(port); EnvFactory.getEnv().initClusterEnvironment(); + PgItSupport.awaitListening(port); } @AfterClass diff --git a/integration-test/src/test/java/org/apache/iotdb/postgresql/it/PgItSupport.java b/integration-test/src/test/java/org/apache/iotdb/postgresql/it/PgItSupport.java index cbdf69ab41b9c..4e5e9092fabd4 100644 --- a/integration-test/src/test/java/org/apache/iotdb/postgresql/it/PgItSupport.java +++ b/integration-test/src/test/java/org/apache/iotdb/postgresql/it/PgItSupport.java @@ -27,12 +27,15 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; /** Helpers for the PostgreSQL wire protocol ITs . */ final class PgItSupport { + private static final long WAIT_NANOS = java.util.concurrent.TimeUnit.SECONDS.toNanos(60); + private PgItSupport() {} /** The DataNode's log output so far. */ @@ -55,6 +58,34 @@ static String dataNodeLog() { } } + /** + * Waits until the DataNode's log satisfies {@code condition}: the service starts after the + * DataNode reports itself ready, and slow machines write the log later. + */ + static String awaitDataNodeLog(Predicate condition) throws InterruptedException { + long deadline = System.nanoTime() + WAIT_NANOS; + String log = dataNodeLog(); + while (!condition.test(log) && System.nanoTime() < deadline) { + Thread.sleep(200); + log = dataNodeLog(); + } + return log; + } + + /** + * Waits until the PostgreSQL endpoint accepts connections: the service starts after the DataNode + * reports itself ready. + */ + static void awaitListening(int port) throws InterruptedException { + long deadline = System.nanoTime() + WAIT_NANOS; + while (!listening(port)) { + if (System.nanoTime() > deadline) { + throw new AssertionError("the PostgreSQL endpoint does not listen on port " + port); + } + Thread.sleep(200); + } + } + /** True if something accepts connections on 127.0.0.1:port. */ static boolean listening(int port) { try (java.net.Socket socket = new java.net.Socket()) { diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/informationschema/IoTDBServicesIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/informationschema/IoTDBServicesIT.java index 014586c5e1206..f654bdf696c5f 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/informationschema/IoTDBServicesIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/informationschema/IoTDBServicesIT.java @@ -67,7 +67,7 @@ public static void tearDown() throws Exception { public void testQueryResult() { String[] retArray = new String[] { - "MQTT,1,STOPPED,", "REST,1,STOPPED,", + "MQTT,1,STOPPED,", "POSTGRESQL,1,STOPPED,", "REST,1,STOPPED,", }; // TableModel diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java index ce3123dc8faa9..318a8ca707843 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java @@ -48,6 +48,8 @@ import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.showDBColumnHeaders; import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.showDBDetailsColumnHeaders; @@ -564,6 +566,13 @@ public void testDatabaseWithSpecificCharacters() throws SQLException { } } + /** information_schema tables added by an InformationSchemaExtension (PostgreSQL service). */ + private static List extensionTables() { + return InformationSchema.getSchemaTables().keySet().stream() + .filter(name -> name.startsWith("pg_")) + .collect(Collectors.toList()); + } + /** * Check read-only system metadata and ordinary-user visibility, then verify administrator * metadata for tables, views, pipes, functions, configuration, and the running cluster. @@ -614,32 +623,37 @@ public void testInformationSchema() throws SQLException { TestUtils.assertResultSetEqual( statement.executeQuery("show tables"), "TableName,TTL(ms),", - Arrays.asList( - "columns,INF,", - "config_nodes,INF,", - "configurations,INF,", - "connections,INF,", - "current_queries,INF,", - "data_nodes,INF,", - "databases,INF,", - "drop_table_column_procedures,INF,", - "drop_table_procedures,INF,", - "functions,INF,", - "keywords,INF,", - "nodes,INF,", - "pipe_memory,INF,", - "pipe_plugins,INF,", - "pipes,INF,", - "queries,INF,", - "queries_costs_histogram,INF,", - "receivers,INF,", - "regions,INF,", - "services,INF,", - "subscriptions,INF,", - "table_disk_usage,INF,", - "tables,INF,", - "topics,INF,", - "views,INF,")); + Stream.concat( + Stream.of( + "columns", + "config_nodes", + "configurations", + "connections", + "current_queries", + "data_nodes", + "databases", + "drop_table_column_procedures", + "drop_table_procedures", + "functions", + "keywords", + "nodes", + "pipe_memory", + "pipe_plugins", + "pipes", + "queries", + "queries_costs_histogram", + "receivers", + "regions", + "services", + "subscriptions", + "table_disk_usage", + "tables", + "topics", + "views"), + extensionTables().stream()) + .sorted() + .map(name -> name + ",INF,") + .collect(Collectors.toList())); TestUtils.assertResultSetEqual( statement.executeQuery("desc databases"), @@ -978,9 +992,7 @@ public void testInformationSchema() throws SQLException { } assertEquals(2, cnt); } - TestUtils.assertResultSetEqual( - statement.executeQuery("show devices from tables where status = 'USING'"), - "database,table_name,ttl(ms),status,comment,table_type,need_last_cache,", + final Set expectedDevices = new HashSet<>( Arrays.asList( "information_schema,databases,INF,USING,null,SYSTEM VIEW,false,", @@ -1009,7 +1021,15 @@ public void testInformationSchema() throws SQLException { "information_schema,current_queries,INF,USING,null,SYSTEM VIEW,false,", "information_schema,queries_costs_histogram,INF,USING,null,SYSTEM VIEW,false,", "test,test,INF,USING,test,BASE TABLE,true,", - "test,view_table,100,USING,null,VIEW FROM TREE,true,"))); + "test,view_table,100,USING,null,VIEW FROM TREE,true,")); + expectedDevices.addAll( + extensionTables().stream() + .map(name -> "information_schema," + name + ",INF,USING,null,SYSTEM VIEW,false,") + .collect(Collectors.toList())); + TestUtils.assertResultSetEqual( + statement.executeQuery("show devices from tables where status = 'USING'"), + "database,table_name,ttl(ms),status,comment,table_type,need_last_cache,", + expectedDevices); // Include all system tables and the table and view created in the test database. TestUtils.assertResultSetEqual( statement.executeQuery("count devices from tables where status = 'USING'"), From 0b2fe71c1f2a1da155cb81c713ed087098a50509 Mon Sep 17 00:00:00 2001 From: Christofer Dutz Date: Sat, 21 Nov 2026 17:05:10 +0100 Subject: [PATCH 7/7] Record Power BI's SQL and answer information_schema.character_sets Power BI Desktop 2.149 recorded against PostgreSQL 14: connect (Npgsql type loading), navigator, Import and three DirectQuery visuals, now in the client corpus with the expected translations. Power BI reads information_schema.character_sets on every connection; the catalog now answers it as PostgreSQL 14 does for a UTF8 database with collation C (UTF8 / UCS, default collation pg_catalog.ucs_basic), and pg_collation lists ucs_basic. pg_total_relation_size stays refused. extract.py accepts application names with spaces (Power BI sends "Mashup Engine") and takes the capture date from the server log. --- .../iotdb/postgresql/catalog/CatalogRows.java | 19 + .../postgresql/catalog/PgCatalogTables.java | 9 + .../postgresql/catalog/StaticCatalog.java | 2 + .../src/test/corpus-capture/extract.py | 9 +- .../postgresql/catalog/CatalogRowsTest.java | 23 + .../src/test/resources/corpus/README.md | 5 +- .../resources/corpus/expected/powerbi/1.sql | 19 + .../resources/corpus/expected/powerbi/10.sql | 19 + .../resources/corpus/expected/powerbi/11.sql | 19 + .../resources/corpus/expected/powerbi/12.sql | 19 + .../resources/corpus/expected/powerbi/13.sql | 19 + .../resources/corpus/expected/powerbi/14.sql | 19 + .../resources/corpus/expected/powerbi/15.sql | 19 + .../resources/corpus/expected/powerbi/16.sql | 19 + .../resources/corpus/expected/powerbi/17.sql | 19 + .../resources/corpus/expected/powerbi/18.sql | 19 + .../resources/corpus/expected/powerbi/19.sql | 19 + .../resources/corpus/expected/powerbi/2.sql | 19 + .../resources/corpus/expected/powerbi/20.sql | 19 + .../resources/corpus/expected/powerbi/21.sql | 19 + .../resources/corpus/expected/powerbi/22.sql | 19 + .../resources/corpus/expected/powerbi/23.sql | 19 + .../resources/corpus/expected/powerbi/24.sql | 19 + .../resources/corpus/expected/powerbi/25.sql | 19 + .../resources/corpus/expected/powerbi/26.sql | 19 + .../resources/corpus/expected/powerbi/27.sql | 19 + .../resources/corpus/expected/powerbi/28.sql | 19 + .../resources/corpus/expected/powerbi/29.sql | 19 + .../resources/corpus/expected/powerbi/3.sql | 19 + .../resources/corpus/expected/powerbi/30.sql | 19 + .../resources/corpus/expected/powerbi/31.sql | 19 + .../resources/corpus/expected/powerbi/32.sql | 19 + .../resources/corpus/expected/powerbi/33.sql | 19 + .../resources/corpus/expected/powerbi/34.sql | 19 + .../resources/corpus/expected/powerbi/35.sql | 19 + .../resources/corpus/expected/powerbi/36.sql | 19 + .../resources/corpus/expected/powerbi/37.sql | 19 + .../resources/corpus/expected/powerbi/38.sql | 19 + .../resources/corpus/expected/powerbi/39.sql | 19 + .../resources/corpus/expected/powerbi/4.sql | 19 + .../resources/corpus/expected/powerbi/40.sql | 19 + .../resources/corpus/expected/powerbi/41.sql | 19 + .../resources/corpus/expected/powerbi/42.sql | 19 + .../resources/corpus/expected/powerbi/43.sql | 19 + .../resources/corpus/expected/powerbi/44.sql | 19 + .../resources/corpus/expected/powerbi/45.sql | 19 + .../resources/corpus/expected/powerbi/46.sql | 19 + .../resources/corpus/expected/powerbi/47.sql | 19 + .../resources/corpus/expected/powerbi/48.sql | 19 + .../resources/corpus/expected/powerbi/49.sql | 19 + .../resources/corpus/expected/powerbi/5.sql | 19 + .../resources/corpus/expected/powerbi/50.sql | 19 + .../resources/corpus/expected/powerbi/51.sql | 19 + .../resources/corpus/expected/powerbi/52.sql | 19 + .../resources/corpus/expected/powerbi/53.sql | 19 + .../resources/corpus/expected/powerbi/54.sql | 19 + .../resources/corpus/expected/powerbi/55.sql | 19 + .../resources/corpus/expected/powerbi/6.sql | 19 + .../resources/corpus/expected/powerbi/7.sql | 19 + .../resources/corpus/expected/powerbi/8.sql | 19 + .../resources/corpus/expected/powerbi/9.sql | 19 + .../src/test/resources/corpus/powerbi.sql | 520 ++++++++++++++++++ 62 files changed, 1627 insertions(+), 5 deletions(-) create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/1.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/10.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/11.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/12.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/13.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/14.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/15.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/16.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/17.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/18.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/19.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/2.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/20.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/21.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/22.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/23.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/24.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/25.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/26.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/27.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/28.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/29.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/3.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/30.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/31.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/32.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/33.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/34.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/35.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/36.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/37.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/38.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/39.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/4.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/40.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/41.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/42.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/43.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/44.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/45.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/46.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/47.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/48.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/49.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/5.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/50.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/51.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/52.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/53.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/54.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/55.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/6.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/7.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/8.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/9.sql create mode 100644 external-service-impl/postgresql/src/test/resources/corpus/powerbi.sql diff --git a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/catalog/CatalogRows.java b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/catalog/CatalogRows.java index 8b00d359359f1..bdcde271debaa 100644 --- a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/catalog/CatalogRows.java +++ b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/catalog/CatalogRows.java @@ -96,6 +96,8 @@ public List> rows(String table) { return users(); case "pg_is_schemata": return schemata(); + case "pg_is_character_sets": + return characterSets(); case "pg_is_tables": return tables(); case "pg_is_columns": @@ -379,6 +381,23 @@ private List> schemata() { return rows; } + /** + * The database encoding, as PostgreSQL 14 shows it for a UTF8 database with collation C: the + * default collation is the longest collation name with that locale, {@code ucs_basic}. + */ + private static List> characterSets() { + Map row = new HashMap<>(); + row.put("character_set_catalog", null); + row.put("character_set_schema", null); + row.put("character_set_name", "UTF8"); + row.put("character_repertoire", "UCS"); + row.put("form_of_use", "UTF8"); + row.put("default_collate_catalog", DATABASE); + row.put("default_collate_schema", "pg_catalog"); + row.put("default_collate_name", "ucs_basic"); + return Collections.singletonList(row); + } + private List> tables() { List> rows = new ArrayList<>(); for (Listed relation : listed()) { diff --git a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/catalog/PgCatalogTables.java b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/catalog/PgCatalogTables.java index 9cc993637aacb..0031609ba0959 100644 --- a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/catalog/PgCatalogTables.java +++ b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/catalog/PgCatalogTables.java @@ -389,6 +389,15 @@ public TSDataType type() { .col("default_character_set_schema", TEXT) .col("default_character_set_name", TEXT) .col("sql_path", TEXT); + table("pg_is_character_sets") + .col("character_set_catalog", TEXT) + .col("character_set_schema", TEXT) + .col("character_set_name", TEXT) + .col("character_repertoire", TEXT) + .col("form_of_use", TEXT) + .col("default_collate_catalog", TEXT) + .col("default_collate_schema", TEXT) + .col("default_collate_name", TEXT); table("pg_is_tables") .col("table_catalog", TEXT) .col("table_schema", TEXT) diff --git a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/catalog/StaticCatalog.java b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/catalog/StaticCatalog.java index 2022e04541492..4e58ca8b761f9 100644 --- a/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/catalog/StaticCatalog.java +++ b/external-service-impl/postgresql/src/main/java/org/apache/iotdb/postgresql/catalog/StaticCatalog.java @@ -741,6 +741,8 @@ static List> collations() { collation(rows, DEFAULT_COLLATION, "default", "d", -1, null); collation(rows, 950, "C", "c", -1, "C"); collation(rows, 951, "POSIX", "c", -1, "POSIX"); + // UTF8 only; information_schema.character_sets names it as the default collation + collation(rows, 12545, "ucs_basic", "c", 6, "C"); return rows; } diff --git a/external-service-impl/postgresql/src/test/corpus-capture/extract.py b/external-service-impl/postgresql/src/test/corpus-capture/extract.py index 0934416efa04a..849eb168397f0 100644 --- a/external-service-impl/postgresql/src/test/corpus-capture/extract.py +++ b/external-service-impl/postgresql/src/test/corpus-capture/extract.py @@ -41,13 +41,16 @@ # Usage: extract.py =... log, outdir, pgv = sys.argv[1], sys.argv[2], sys.argv[3] versions = dict(a.split('=',1) for a in sys.argv[4:]) -start = re.compile(r'^\d{4}-\d\d-\d\d \d\d:\d\d:\d\d\.\d+ \w+ \[\d+\] app=(\S*) user=\S* db=\S* (LOG|DETAIL|ERROR|STATEMENT|FATAL): (.*)$') +# application names may contain spaces (Power BI: "Mashup Engine") +start = re.compile(r'^(\d{4}-\d\d-\d\d) \d\d:\d\d:\d\d\.\d+ \w+ \[\d+\] app=(.*?) user=\S* db=\S* (LOG|DETAIL|ERROR|STATEMENT|FATAL): (.*)$') +captured = None records = [] for line in open(log, encoding='utf-8', errors='replace'): line = line.rstrip('\n') m = start.match(line) if m: - records.append([m.group(1), m.group(2), m.group(3)]) + captured = captured or m.group(1) + records.append([m.group(2), m.group(3), m.group(4)]) elif records and not re.match(r'^\d{4}-\d\d-\d\d ', line): records[-1][2] += '\n' + line # skip the setup (it ran as psql too): everything up to the final ANALYZE @@ -67,7 +70,7 @@ with open(f'{outdir}/{app}.sql', 'w') as f: f.write(LICENSE) f.write(f'-- Client: {app} {versions[app]}\n-- Server: PostgreSQL {pgv} (docker postgres:14-alpine), log_statement=all\n') - f.write('-- Captured: 2026-11-19 (T001, spec 012); schema refdb/other from corpus/README.md\n') + f.write(f'-- Captured: {captured} (T001, spec 012); schema refdb/other from corpus/README.md\n') collapsed = [] for sql, params in stmts: if collapsed and collapsed[-1][0] == sql and collapsed[-1][1] == params: diff --git a/external-service-impl/postgresql/src/test/java/org/apache/iotdb/postgresql/catalog/CatalogRowsTest.java b/external-service-impl/postgresql/src/test/java/org/apache/iotdb/postgresql/catalog/CatalogRowsTest.java index a1b797aad9aa6..56580c933b655 100644 --- a/external-service-impl/postgresql/src/test/java/org/apache/iotdb/postgresql/catalog/CatalogRowsTest.java +++ b/external-service-impl/postgresql/src/test/java/org/apache/iotdb/postgresql/catalog/CatalogRowsTest.java @@ -183,6 +183,29 @@ public void classesAttributesAndDescriptionsAgree() { .anyMatch(r -> "pg_type".equals(r.get("relname")) && r.get("oid").equals(1247L))); } + @Test + public void characterSetsNameTheEncodingAndAnExistingDefaultCollation() { + // PostgreSQL 14, UTF8 database with collation C (Power BI reads character_set_name) + CatalogRows rows = new CatalogRows("u", CatalogRowsTest::userSnapshot); + List> sets = rows.rows("pg_is_character_sets"); + assertEquals(1, sets.size()); + Map set = sets.get(0); + assertEquals(null, set.get("character_set_catalog")); + assertEquals("UTF8", set.get("character_set_name")); + assertEquals("UCS", set.get("character_repertoire")); + assertEquals("UTF8", set.get("form_of_use")); + assertEquals("pg_catalog", set.get("default_collate_schema")); + assertEquals("ucs_basic", set.get("default_collate_name")); + Map collation = + rows.rows("pg_collation").stream() + .filter(r -> "ucs_basic".equals(r.get("collname"))) + .findFirst() + .get(); + assertEquals(12545L, collation.get("oid")); + assertEquals(6, collation.get("collencoding")); + assertEquals("C", collation.get("collcollate")); + } + @Test public void oidsDoNotDependOnTheUser() { CatalogSnapshot all = CatalogSnapshot.of(tables(), db -> true, (db, table) -> true); diff --git a/external-service-impl/postgresql/src/test/resources/corpus/README.md b/external-service-impl/postgresql/src/test/resources/corpus/README.md index ece4678b15551..ca845a3c2afeb 100644 --- a/external-service-impl/postgresql/src/test/resources/corpus/README.md +++ b/external-service-impl/postgresql/src/test/resources/corpus/README.md @@ -35,7 +35,7 @@ repeated prepared statements) are collapsed into one entry with a count. | `psql.sql` | psql 14.25 | `\l`, `\dn`, `\dt refdb.*`, `\d refdb.sensors`, `\d+ refdb.events`, `\dt *.*`, one aggregation | | `pgjdbc.sql` | pgJDBC 42.7.4 (DBeaver's driver) | `DatabaseMetaData` catalogs, schemas, tables, columns, primary keys, indexes, imported keys, type info; a plain query; a prepared statement executed 6 times; `setFetchSize(1000)` over 100,000 rows | | `psycopg.sql` | psycopg 3.3.9 | `version()`, plain query, parameterised query 6 times, a binary-result query, a server-side cursor over 100,000 rows | -| `powerbi.sql` | Power BI Desktop | **to be recorded** (owner, Windows), see below | +| `powerbi.sql` | Power BI Desktop 2.149 (Npgsql, `application_name` `Mashup Engine`) | connect (Npgsql type loading), navigator over `refdb`, Import of `refdb.sensors` (with `pg_total_relation_size`), DirectQuery: three visuals (`sum` by `time`); date hierarchy, slicer, relative date and Top N filters still to be recorded | | `excel.sql`, `metabase.sql` | Excel, Metabase | to be recorded | ## Expected outcomes @@ -86,6 +86,7 @@ example `-p :54314:5432`), run `setup.sql`, then on Windows: `refdb.events`. Then `docker logs` into a file and run `extract.py` with the application names found in the log -(`grep -o 'app=[^ ]*' pg14.log | sort | uniq -c`). An empty name works too (`=`); it is +(`grep -o 'app=.* user=' pg14.log | sort | uniq -c`; Power BI 2.149 sends `Mashup Engine`, so pass +`"Mashup Engine=2.149"` and rename the output to `powerbi.sql`). An empty name works too (`=`); it is written to `.sql`, so rename that file to `powerbi.sql` or `excel.sql`. Record the Power BI Desktop and Excel versions in the header. diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/1.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/1.sql new file mode 100644 index 0000000000000..ebb5353bd5ac8 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/1.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT "ns"."nspname", "a"."typname", "a"."oid", "a"."typrelid", "a"."typbasetype", CASE WHEN "pg_proc"."proname" = 'array_recv' THEN 'a' ELSE "a"."typtype" END AS "type", CASE WHEN "pg_proc"."proname" = 'array_recv' THEN "a"."typelem" WHEN "a"."typtype" = 'r' THEN "rngsubtype" ELSE 0 END AS "elemoid", CASE WHEN "pg_proc"."proname" IN ('array_recv', 'oidvectorrecv') THEN 3 WHEN "a"."typtype" = 'r' THEN 2 WHEN "a"."typtype" = 'd' THEN 1 ELSE 0 END AS "ord" FROM information_schema."pg_type" AS "a" JOIN information_schema."pg_namespace" AS "ns" ON ("ns"."oid" = "a"."typnamespace") JOIN information_schema."pg_proc" ON "pg_proc"."oid" = "a"."typreceive" LEFT OUTER JOIN information_schema."pg_class" AS "cls" ON ("cls"."oid" = "a"."typrelid") LEFT OUTER JOIN information_schema."pg_type" AS "b" ON ("b"."oid" = "a"."typelem") LEFT OUTER JOIN information_schema."pg_class" AS "elemcls" ON ("elemcls"."oid" = "b"."typrelid") LEFT OUTER JOIN information_schema."pg_range" ON ("pg_range"."rngtypid" = "a"."oid") WHERE "a"."typtype" IN ('b', 'r', 'e', 'd') OR ("a"."typtype" = 'c' AND "cls"."relkind" = 'c') OR ("pg_proc"."proname" = 'array_recv' AND ("b"."typtype" IN ('b', 'r', 'e', 'd') OR ("b"."typtype" = 'p' AND "b"."typname" IN ('record', 'void')) OR ("b"."typtype" = 'c' AND "elemcls"."relkind" = 'c'))) OR ("a"."typtype" = 'p' AND "a"."typname" IN ('record', 'void')) ORDER BY "ord" NULLS LAST diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/10.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/10.sql new file mode 100644 index 0000000000000..02b36f4d8f788 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/10.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT CAST(NULL AS STRING) AS "pk_column_name", CAST(NULL AS STRING) AS "fk_table_schema", CAST(NULL AS STRING) AS "fk_table_name", CAST(NULL AS STRING) AS "fk_column_name", CAST(NULL AS INT32) AS "ordinal", CAST(NULL AS STRING) AS "fk_name" FROM information_schema."pg_is_key_column_usage" WHERE FALSE diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/11.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/11.sql new file mode 100644 index 0000000000000..81c84bd1dfe17 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/11.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +-- session command diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/12.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/12.sql new file mode 100644 index 0000000000000..aa0ac715d3509 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/12.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT CAST(NULL AS STRING) AS "pk_table_schema", CAST(NULL AS STRING) AS "pk_table_name", CAST(NULL AS STRING) AS "pk_column_name", CAST(NULL AS STRING) AS "fk_column_name", CAST(NULL AS INT32) AS "ordinal", CAST(NULL AS STRING) AS "fk_name" FROM information_schema."pg_is_key_column_usage" WHERE FALSE diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/13.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/13.sql new file mode 100644 index 0000000000000..81c84bd1dfe17 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/13.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +-- session command diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/14.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/14.sql new file mode 100644 index 0000000000000..4df63104a5cb1 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/14.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT CAST(NULL AS STRING) AS "index_name", CAST(NULL AS STRING) AS "column_name", CAST(NULL AS STRING) AS "ordinal_position", CAST(NULL AS STRING) AS "primary_key" FROM information_schema."pg_is_table_constraints" WHERE FALSE diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/15.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/15.sql new file mode 100644 index 0000000000000..81c84bd1dfe17 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/15.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +-- session command diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/16.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/16.sql new file mode 100644 index 0000000000000..f220f2dd20e93 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/16.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT "$Table"."time" AS "time", "$Table"."plant" AS "plant", "$Table"."line" AS "line", "$Table"."machine" AS "machine", "$Table"."model" AS "model", "$Table"."temperature" AS "temperature", "$Table"."pressure" AS "pressure", "$Table"."count" AS "count", "$Table"."level" AS "level", "$Table"."running" AS "running", "$Table"."state" AS "state", "$Table"."note" AS "note", "$Table"."blob" AS "blob", "$Table"."day" AS "day", "$Table"."ts" AS "ts" FROM "refdb"."sensors" AS "$Table" LIMIT 100 diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/17.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/17.sql new file mode 100644 index 0000000000000..81c84bd1dfe17 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/17.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +-- session command diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/18.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/18.sql new file mode 100644 index 0000000000000..2aeb44b61cfb7 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/18.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT "$Table"."time" AS "time", "$Table"."plant" AS "plant", "$Table"."line" AS "line", "$Table"."machine" AS "machine", "$Table"."model" AS "model", "$Table"."temperature" AS "temperature", "$Table"."pressure" AS "pressure", "$Table"."count" AS "count", "$Table"."level" AS "level", "$Table"."running" AS "running", "$Table"."state" AS "state", "$Table"."note" AS "note", "$Table"."blob" AS "blob", "$Table"."day" AS "day", "$Table"."ts" AS "ts" FROM "refdb"."sensors" AS "$Table" LIMIT 200 diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/19.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/19.sql new file mode 100644 index 0000000000000..81c84bd1dfe17 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/19.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +-- session command diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/2.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/2.sql new file mode 100644 index 0000000000000..7de1fcb21365a --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/2.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT "typ"."oid", "att"."attname", "att"."atttypid" FROM information_schema."pg_type" AS "typ" JOIN information_schema."pg_namespace" AS "ns" ON ("ns"."oid" = "typ"."typnamespace") JOIN information_schema."pg_class" AS "cls" ON ("cls"."oid" = "typ"."typrelid") JOIN information_schema."pg_attribute" AS "att" ON ("att"."attrelid" = "typ"."typrelid") WHERE ("typ"."typtype" = 'c' AND "cls"."relkind" = 'c') AND "attnum" > 0 AND NOT "attisdropped" ORDER BY "typ"."oid" NULLS LAST, "att"."attnum" NULLS LAST diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/20.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/20.sql new file mode 100644 index 0000000000000..73d7656391289 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/20.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +-- refused 0A000: function pg_total_relation_size() is not supported diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/21.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/21.sql new file mode 100644 index 0000000000000..81c84bd1dfe17 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/21.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +-- session command diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/22.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/22.sql new file mode 100644 index 0000000000000..f220f2dd20e93 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/22.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT "$Table"."time" AS "time", "$Table"."plant" AS "plant", "$Table"."line" AS "line", "$Table"."machine" AS "machine", "$Table"."model" AS "model", "$Table"."temperature" AS "temperature", "$Table"."pressure" AS "pressure", "$Table"."count" AS "count", "$Table"."level" AS "level", "$Table"."running" AS "running", "$Table"."state" AS "state", "$Table"."note" AS "note", "$Table"."blob" AS "blob", "$Table"."day" AS "day", "$Table"."ts" AS "ts" FROM "refdb"."sensors" AS "$Table" LIMIT 100 diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/23.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/23.sql new file mode 100644 index 0000000000000..81c84bd1dfe17 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/23.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +-- session command diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/24.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/24.sql new file mode 100644 index 0000000000000..3ac1fe8a4c245 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/24.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT "$Table"."time" AS "time", "$Table"."plant" AS "plant", "$Table"."line" AS "line", "$Table"."machine" AS "machine", "$Table"."model" AS "model", "$Table"."temperature" AS "temperature", "$Table"."pressure" AS "pressure", "$Table"."count" AS "count", "$Table"."level" AS "level", "$Table"."running" AS "running", "$Table"."state" AS "state", "$Table"."note" AS "note", "$Table"."blob" AS "blob", "$Table"."day" AS "day", "$Table"."ts" AS "ts" FROM "refdb"."sensors" AS "$Table" LIMIT 1000 diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/25.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/25.sql new file mode 100644 index 0000000000000..81c84bd1dfe17 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/25.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +-- session command diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/26.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/26.sql new file mode 100644 index 0000000000000..3ac1fe8a4c245 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/26.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT "$Table"."time" AS "time", "$Table"."plant" AS "plant", "$Table"."line" AS "line", "$Table"."machine" AS "machine", "$Table"."model" AS "model", "$Table"."temperature" AS "temperature", "$Table"."pressure" AS "pressure", "$Table"."count" AS "count", "$Table"."level" AS "level", "$Table"."running" AS "running", "$Table"."state" AS "state", "$Table"."note" AS "note", "$Table"."blob" AS "blob", "$Table"."day" AS "day", "$Table"."ts" AS "ts" FROM "refdb"."sensors" AS "$Table" LIMIT 1000 diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/27.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/27.sql new file mode 100644 index 0000000000000..81c84bd1dfe17 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/27.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +-- session command diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/28.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/28.sql new file mode 100644 index 0000000000000..3ac1fe8a4c245 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/28.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT "$Table"."time" AS "time", "$Table"."plant" AS "plant", "$Table"."line" AS "line", "$Table"."machine" AS "machine", "$Table"."model" AS "model", "$Table"."temperature" AS "temperature", "$Table"."pressure" AS "pressure", "$Table"."count" AS "count", "$Table"."level" AS "level", "$Table"."running" AS "running", "$Table"."state" AS "state", "$Table"."note" AS "note", "$Table"."blob" AS "blob", "$Table"."day" AS "day", "$Table"."ts" AS "ts" FROM "refdb"."sensors" AS "$Table" LIMIT 1000 diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/29.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/29.sql new file mode 100644 index 0000000000000..81c84bd1dfe17 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/29.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +-- session command diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/3.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/3.sql new file mode 100644 index 0000000000000..bb4a81261abef --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/3.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT CAST(NULL AS INT64) AS "oid", CAST(NULL AS STRING) AS "enumlabel" FROM information_schema."pg_enum" WHERE FALSE diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/30.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/30.sql new file mode 100644 index 0000000000000..5f05884cba08f --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/30.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT "$Table"."time" AS "time", "$Table"."plant" AS "plant", "$Table"."line" AS "line", "$Table"."machine" AS "machine", "$Table"."model" AS "model", "$Table"."temperature" AS "temperature", "$Table"."pressure" AS "pressure", "$Table"."count" AS "count", "$Table"."level" AS "level", "$Table"."running" AS "running", "$Table"."state" AS "state", "$Table"."note" AS "note", "$Table"."blob" AS "blob", "$Table"."day" AS "day", "$Table"."ts" AS "ts" FROM "refdb"."sensors" AS "$Table" LIMIT 20 diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/31.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/31.sql new file mode 100644 index 0000000000000..81c84bd1dfe17 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/31.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +-- session command diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/32.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/32.sql new file mode 100644 index 0000000000000..3ac1fe8a4c245 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/32.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT "$Table"."time" AS "time", "$Table"."plant" AS "plant", "$Table"."line" AS "line", "$Table"."machine" AS "machine", "$Table"."model" AS "model", "$Table"."temperature" AS "temperature", "$Table"."pressure" AS "pressure", "$Table"."count" AS "count", "$Table"."level" AS "level", "$Table"."running" AS "running", "$Table"."state" AS "state", "$Table"."note" AS "note", "$Table"."blob" AS "blob", "$Table"."day" AS "day", "$Table"."ts" AS "ts" FROM "refdb"."sensors" AS "$Table" LIMIT 1000 diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/33.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/33.sql new file mode 100644 index 0000000000000..ebb5353bd5ac8 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/33.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT "ns"."nspname", "a"."typname", "a"."oid", "a"."typrelid", "a"."typbasetype", CASE WHEN "pg_proc"."proname" = 'array_recv' THEN 'a' ELSE "a"."typtype" END AS "type", CASE WHEN "pg_proc"."proname" = 'array_recv' THEN "a"."typelem" WHEN "a"."typtype" = 'r' THEN "rngsubtype" ELSE 0 END AS "elemoid", CASE WHEN "pg_proc"."proname" IN ('array_recv', 'oidvectorrecv') THEN 3 WHEN "a"."typtype" = 'r' THEN 2 WHEN "a"."typtype" = 'd' THEN 1 ELSE 0 END AS "ord" FROM information_schema."pg_type" AS "a" JOIN information_schema."pg_namespace" AS "ns" ON ("ns"."oid" = "a"."typnamespace") JOIN information_schema."pg_proc" ON "pg_proc"."oid" = "a"."typreceive" LEFT OUTER JOIN information_schema."pg_class" AS "cls" ON ("cls"."oid" = "a"."typrelid") LEFT OUTER JOIN information_schema."pg_type" AS "b" ON ("b"."oid" = "a"."typelem") LEFT OUTER JOIN information_schema."pg_class" AS "elemcls" ON ("elemcls"."oid" = "b"."typrelid") LEFT OUTER JOIN information_schema."pg_range" ON ("pg_range"."rngtypid" = "a"."oid") WHERE "a"."typtype" IN ('b', 'r', 'e', 'd') OR ("a"."typtype" = 'c' AND "cls"."relkind" = 'c') OR ("pg_proc"."proname" = 'array_recv' AND ("b"."typtype" IN ('b', 'r', 'e', 'd') OR ("b"."typtype" = 'p' AND "b"."typname" IN ('record', 'void')) OR ("b"."typtype" = 'c' AND "elemcls"."relkind" = 'c'))) OR ("a"."typtype" = 'p' AND "a"."typname" IN ('record', 'void')) ORDER BY "ord" NULLS LAST diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/34.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/34.sql new file mode 100644 index 0000000000000..7de1fcb21365a --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/34.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT "typ"."oid", "att"."attname", "att"."atttypid" FROM information_schema."pg_type" AS "typ" JOIN information_schema."pg_namespace" AS "ns" ON ("ns"."oid" = "typ"."typnamespace") JOIN information_schema."pg_class" AS "cls" ON ("cls"."oid" = "typ"."typrelid") JOIN information_schema."pg_attribute" AS "att" ON ("att"."attrelid" = "typ"."typrelid") WHERE ("typ"."typtype" = 'c' AND "cls"."relkind" = 'c') AND "attnum" > 0 AND NOT "attisdropped" ORDER BY "typ"."oid" NULLS LAST, "att"."attnum" NULLS LAST diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/35.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/35.sql new file mode 100644 index 0000000000000..bb4a81261abef --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/35.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT CAST(NULL AS INT64) AS "oid", CAST(NULL AS STRING) AS "enumlabel" FROM information_schema."pg_enum" WHERE FALSE diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/36.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/36.sql new file mode 100644 index 0000000000000..17dbf05f37294 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/36.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT "character_set_name" FROM information_schema."pg_is_character_sets" diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/37.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/37.sql new file mode 100644 index 0000000000000..81c84bd1dfe17 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/37.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +-- session command diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/38.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/38.sql new file mode 100644 index 0000000000000..f634d7fb849d0 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/38.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT "table_schema", "table_name", "table_type" FROM information_schema."pg_is_tables" WHERE "table_schema" NOT IN ('information_schema', 'pg_catalog') ORDER BY "table_schema" NULLS LAST, "table_name" NULLS LAST diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/39.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/39.sql new file mode 100644 index 0000000000000..81c84bd1dfe17 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/39.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +-- session command diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/4.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/4.sql new file mode 100644 index 0000000000000..17dbf05f37294 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/4.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT "character_set_name" FROM information_schema."pg_is_character_sets" diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/40.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/40.sql new file mode 100644 index 0000000000000..1fa03c2b2a6ba --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/40.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT "column_name", "ordinal_position", "is_nullable", CASE WHEN ("data_type" LIKE '%unsigned%') THEN CASE WHEN "data_type" IS NULL OR ' unsigned' IS NULL THEN CAST(NULL AS STRING) ELSE concat("data_type", ' unsigned') END ELSE "data_type" END AS "data_type" FROM information_schema."pg_is_columns" WHERE "table_schema" = 'refdb' AND "table_name" = 'sensors' ORDER BY "table_schema" NULLS LAST, "table_name" NULLS LAST, "ordinal_position" NULLS LAST diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/41.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/41.sql new file mode 100644 index 0000000000000..81c84bd1dfe17 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/41.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +-- session command diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/42.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/42.sql new file mode 100644 index 0000000000000..02b36f4d8f788 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/42.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT CAST(NULL AS STRING) AS "pk_column_name", CAST(NULL AS STRING) AS "fk_table_schema", CAST(NULL AS STRING) AS "fk_table_name", CAST(NULL AS STRING) AS "fk_column_name", CAST(NULL AS INT32) AS "ordinal", CAST(NULL AS STRING) AS "fk_name" FROM information_schema."pg_is_key_column_usage" WHERE FALSE diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/43.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/43.sql new file mode 100644 index 0000000000000..81c84bd1dfe17 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/43.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +-- session command diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/44.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/44.sql new file mode 100644 index 0000000000000..aa0ac715d3509 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/44.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT CAST(NULL AS STRING) AS "pk_table_schema", CAST(NULL AS STRING) AS "pk_table_name", CAST(NULL AS STRING) AS "pk_column_name", CAST(NULL AS STRING) AS "fk_column_name", CAST(NULL AS INT32) AS "ordinal", CAST(NULL AS STRING) AS "fk_name" FROM information_schema."pg_is_key_column_usage" WHERE FALSE diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/45.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/45.sql new file mode 100644 index 0000000000000..81c84bd1dfe17 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/45.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +-- session command diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/46.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/46.sql new file mode 100644 index 0000000000000..4df63104a5cb1 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/46.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT CAST(NULL AS STRING) AS "index_name", CAST(NULL AS STRING) AS "column_name", CAST(NULL AS STRING) AS "ordinal_position", CAST(NULL AS STRING) AS "primary_key" FROM information_schema."pg_is_table_constraints" WHERE FALSE diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/47.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/47.sql new file mode 100644 index 0000000000000..ebb5353bd5ac8 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/47.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT "ns"."nspname", "a"."typname", "a"."oid", "a"."typrelid", "a"."typbasetype", CASE WHEN "pg_proc"."proname" = 'array_recv' THEN 'a' ELSE "a"."typtype" END AS "type", CASE WHEN "pg_proc"."proname" = 'array_recv' THEN "a"."typelem" WHEN "a"."typtype" = 'r' THEN "rngsubtype" ELSE 0 END AS "elemoid", CASE WHEN "pg_proc"."proname" IN ('array_recv', 'oidvectorrecv') THEN 3 WHEN "a"."typtype" = 'r' THEN 2 WHEN "a"."typtype" = 'd' THEN 1 ELSE 0 END AS "ord" FROM information_schema."pg_type" AS "a" JOIN information_schema."pg_namespace" AS "ns" ON ("ns"."oid" = "a"."typnamespace") JOIN information_schema."pg_proc" ON "pg_proc"."oid" = "a"."typreceive" LEFT OUTER JOIN information_schema."pg_class" AS "cls" ON ("cls"."oid" = "a"."typrelid") LEFT OUTER JOIN information_schema."pg_type" AS "b" ON ("b"."oid" = "a"."typelem") LEFT OUTER JOIN information_schema."pg_class" AS "elemcls" ON ("elemcls"."oid" = "b"."typrelid") LEFT OUTER JOIN information_schema."pg_range" ON ("pg_range"."rngtypid" = "a"."oid") WHERE "a"."typtype" IN ('b', 'r', 'e', 'd') OR ("a"."typtype" = 'c' AND "cls"."relkind" = 'c') OR ("pg_proc"."proname" = 'array_recv' AND ("b"."typtype" IN ('b', 'r', 'e', 'd') OR ("b"."typtype" = 'p' AND "b"."typname" IN ('record', 'void')) OR ("b"."typtype" = 'c' AND "elemcls"."relkind" = 'c'))) OR ("a"."typtype" = 'p' AND "a"."typname" IN ('record', 'void')) ORDER BY "ord" NULLS LAST diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/48.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/48.sql new file mode 100644 index 0000000000000..7de1fcb21365a --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/48.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT "typ"."oid", "att"."attname", "att"."atttypid" FROM information_schema."pg_type" AS "typ" JOIN information_schema."pg_namespace" AS "ns" ON ("ns"."oid" = "typ"."typnamespace") JOIN information_schema."pg_class" AS "cls" ON ("cls"."oid" = "typ"."typrelid") JOIN information_schema."pg_attribute" AS "att" ON ("att"."attrelid" = "typ"."typrelid") WHERE ("typ"."typtype" = 'c' AND "cls"."relkind" = 'c') AND "attnum" > 0 AND NOT "attisdropped" ORDER BY "typ"."oid" NULLS LAST, "att"."attnum" NULLS LAST diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/49.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/49.sql new file mode 100644 index 0000000000000..bb4a81261abef --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/49.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT CAST(NULL AS INT64) AS "oid", CAST(NULL AS STRING) AS "enumlabel" FROM information_schema."pg_enum" WHERE FALSE diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/5.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/5.sql new file mode 100644 index 0000000000000..81c84bd1dfe17 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/5.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +-- session command diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/50.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/50.sql new file mode 100644 index 0000000000000..91037b112ab15 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/50.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT "rows"."time" AS "time" FROM "refdb"."sensors" AS "rows" GROUP BY "time" LIMIT 1000001 diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/51.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/51.sql new file mode 100644 index 0000000000000..81c84bd1dfe17 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/51.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +-- session command diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/52.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/52.sql new file mode 100644 index 0000000000000..90f4f10dda368 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/52.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT "_"."time", "_"."a0" FROM (SELECT "rows"."time" AS "time", sum("rows"."level") AS "a0" FROM "refdb"."sensors" AS "rows" GROUP BY "time") "_" WHERE NOT "_"."a0" IS NULL LIMIT 1000001 diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/53.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/53.sql new file mode 100644 index 0000000000000..81c84bd1dfe17 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/53.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +-- session command diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/54.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/54.sql new file mode 100644 index 0000000000000..985361ee7b9ba --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/54.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT "_"."time", "_"."a0", "_"."a1" FROM (SELECT "rows"."time" AS "time", sum("rows"."temperature") AS "a0", sum("rows"."level") AS "a1" FROM "refdb"."sensors" AS "rows" GROUP BY "time") "_" WHERE NOT "_"."a0" IS NULL OR NOT "_"."a1" IS NULL LIMIT 1000001 diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/55.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/55.sql new file mode 100644 index 0000000000000..81c84bd1dfe17 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/55.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +-- session command diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/6.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/6.sql new file mode 100644 index 0000000000000..f634d7fb849d0 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/6.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT "table_schema", "table_name", "table_type" FROM information_schema."pg_is_tables" WHERE "table_schema" NOT IN ('information_schema', 'pg_catalog') ORDER BY "table_schema" NULLS LAST, "table_name" NULLS LAST diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/7.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/7.sql new file mode 100644 index 0000000000000..81c84bd1dfe17 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/7.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +-- session command diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/8.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/8.sql new file mode 100644 index 0000000000000..1fa03c2b2a6ba --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/8.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +SELECT "column_name", "ordinal_position", "is_nullable", CASE WHEN ("data_type" LIKE '%unsigned%') THEN CASE WHEN "data_type" IS NULL OR ' unsigned' IS NULL THEN CAST(NULL AS STRING) ELSE concat("data_type", ' unsigned') END ELSE "data_type" END AS "data_type" FROM information_schema."pg_is_columns" WHERE "table_schema" = 'refdb' AND "table_name" = 'sensors' ORDER BY "table_schema" NULLS LAST, "table_name" NULLS LAST, "ordinal_position" NULLS LAST diff --git a/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/9.sql b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/9.sql new file mode 100644 index 0000000000000..81c84bd1dfe17 --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/expected/powerbi/9.sql @@ -0,0 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +-- session command diff --git a/external-service-impl/postgresql/src/test/resources/corpus/powerbi.sql b/external-service-impl/postgresql/src/test/resources/corpus/powerbi.sql new file mode 100644 index 0000000000000..814cdcc6b682c --- /dev/null +++ b/external-service-impl/postgresql/src/test/resources/corpus/powerbi.sql @@ -0,0 +1,520 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +-- Client: Power BI Desktop 2.149 (application_name "Mashup Engine", Npgsql), Windows x64 +-- Server: PostgreSQL 14.25 (docker postgres:14-alpine), log_statement=all +-- Captured: 2026-11-21 (T001, spec 012); schema refdb/other from corpus/README.md +-- next +/*** Load all supported types ***/ + SELECT ns.nspname, a.typname, a.oid, a.typrelid, a.typbasetype, + CASE WHEN pg_proc.proname='array_recv' THEN 'a' ELSE a.typtype END AS type, + CASE + WHEN pg_proc.proname='array_recv' THEN a.typelem + WHEN a.typtype='r' THEN rngsubtype + ELSE 0 + END AS elemoid, + CASE + WHEN pg_proc.proname IN ('array_recv','oidvectorrecv') THEN 3 /* Arrays last */ + WHEN a.typtype='r' THEN 2 /* Ranges before */ + WHEN a.typtype='d' THEN 1 /* Domains before */ + ELSE 0 /* Base types first */ + END AS ord + FROM pg_type AS a + JOIN pg_namespace AS ns ON (ns.oid = a.typnamespace) + JOIN pg_proc ON pg_proc.oid = a.typreceive + LEFT OUTER JOIN pg_class AS cls ON (cls.oid = a.typrelid) + LEFT OUTER JOIN pg_type AS b ON (b.oid = a.typelem) + LEFT OUTER JOIN pg_class AS elemcls ON (elemcls.oid = b.typrelid) + LEFT OUTER JOIN pg_range ON (pg_range.rngtypid = a.oid) + WHERE + a.typtype IN ('b', 'r', 'e', 'd') OR /* Base, range, enum, domain */ + (a.typtype = 'c' AND cls.relkind='c') OR /* User-defined free-standing composites (not table composites) by default */ + (pg_proc.proname='array_recv' AND ( + b.typtype IN ('b', 'r', 'e', 'd') OR /* Array of base, range, enum, domain */ + (b.typtype = 'p' AND b.typname IN ('record', 'void')) OR /* Arrays of special supported pseudo-types */ + (b.typtype = 'c' AND elemcls.relkind='c') /* Array of user-defined free-standing composites (not table composites) */ + )) OR + (a.typtype = 'p' AND a.typname IN ('record', 'void')) /* Some special supported pseudo-types */ + ORDER BY ord +-- next +/*** Load field definitions for (free-standing) composite types ***/ + SELECT typ.oid, att.attname, att.atttypid + FROM pg_type AS typ + JOIN pg_namespace AS ns ON (ns.oid = typ.typnamespace) + JOIN pg_class AS cls ON (cls.oid = typ.typrelid) + JOIN pg_attribute AS att ON (att.attrelid = typ.typrelid) + WHERE + (typ.typtype = 'c' AND cls.relkind='c') AND + attnum > 0 AND /* Don't load system attributes */ + NOT attisdropped + ORDER BY typ.oid, att.attnum +-- next +/*** Load enum fields ***/ + SELECT pg_type.oid, enumlabel + FROM pg_enum + JOIN pg_type ON pg_type.oid=enumtypid + ORDER BY oid, enumsortorder +-- next +select character_set_name from INFORMATION_SCHEMA.character_sets +-- next +DISCARD ALL +-- next +select TABLE_SCHEMA, TABLE_NAME, TABLE_TYPE + from INFORMATION_SCHEMA.tables + where TABLE_SCHEMA not in ('information_schema', 'pg_catalog') + order by TABLE_SCHEMA, TABLE_NAME +-- next +DISCARD ALL +-- next +select COLUMN_NAME, ORDINAL_POSITION, IS_NULLABLE, case when (data_type like '%unsigned%') then DATA_TYPE || ' unsigned' else DATA_TYPE end as DATA_TYPE + from INFORMATION_SCHEMA.columns + where TABLE_SCHEMA = 'refdb' and TABLE_NAME = 'sensors' + order by TABLE_SCHEMA, TABLE_NAME, ORDINAL_POSITION +-- next +DISCARD ALL +-- next +select + pkcol.COLUMN_NAME as PK_COLUMN_NAME, + fkcol.TABLE_SCHEMA AS FK_TABLE_SCHEMA, + fkcol.TABLE_NAME AS FK_TABLE_NAME, + fkcol.COLUMN_NAME as FK_COLUMN_NAME, + fkcol.ORDINAL_POSITION as ORDINAL, + fkcon.CONSTRAINT_SCHEMA || '_' || fkcol.TABLE_NAME || '_' || 'sensors' || '_' || fkcon.CONSTRAINT_NAME as FK_NAME + from + (select distinct constraint_catalog, constraint_schema, unique_constraint_schema, constraint_name, unique_constraint_name + from INFORMATION_SCHEMA.REFERENTIAL_CONSTRAINTS) fkcon + inner join + INFORMATION_SCHEMA.KEY_COLUMN_USAGE fkcol + on fkcon.CONSTRAINT_SCHEMA = fkcol.CONSTRAINT_SCHEMA + and fkcon.CONSTRAINT_NAME = fkcol.CONSTRAINT_NAME + inner join + INFORMATION_SCHEMA.KEY_COLUMN_USAGE pkcol + on fkcon.UNIQUE_CONSTRAINT_SCHEMA = pkcol.CONSTRAINT_SCHEMA + and fkcon.UNIQUE_CONSTRAINT_NAME = pkcol.CONSTRAINT_NAME + where pkcol.TABLE_SCHEMA = 'refdb' and pkcol.TABLE_NAME = 'sensors' + and pkcol.ORDINAL_POSITION = fkcol.ORDINAL_POSITION + order by FK_NAME, fkcol.ORDINAL_POSITION +-- next +DISCARD ALL +-- next +select + pkcol.TABLE_SCHEMA AS PK_TABLE_SCHEMA, + pkcol.TABLE_NAME AS PK_TABLE_NAME, + pkcol.COLUMN_NAME as PK_COLUMN_NAME, + fkcol.COLUMN_NAME as FK_COLUMN_NAME, + fkcol.ORDINAL_POSITION as ORDINAL, + fkcon.CONSTRAINT_SCHEMA || '_' || 'sensors' || '_' || pkcol.TABLE_NAME || '_' || fkcon.CONSTRAINT_NAME as FK_NAME + from + (select distinct constraint_catalog, constraint_schema, unique_constraint_schema, constraint_name, unique_constraint_name + from INFORMATION_SCHEMA.REFERENTIAL_CONSTRAINTS) fkcon + inner join + INFORMATION_SCHEMA.KEY_COLUMN_USAGE fkcol + on fkcon.CONSTRAINT_SCHEMA = fkcol.CONSTRAINT_SCHEMA + and fkcon.CONSTRAINT_NAME = fkcol.CONSTRAINT_NAME + inner join + INFORMATION_SCHEMA.KEY_COLUMN_USAGE pkcol + on fkcon.UNIQUE_CONSTRAINT_SCHEMA = pkcol.CONSTRAINT_SCHEMA + and fkcon.UNIQUE_CONSTRAINT_NAME = pkcol.CONSTRAINT_NAME + where fkcol.TABLE_SCHEMA = 'refdb' and fkcol.TABLE_NAME = 'sensors' + and pkcol.ORDINAL_POSITION = fkcol.ORDINAL_POSITION + order by FK_NAME, fkcol.ORDINAL_POSITION +-- next +DISCARD ALL +-- next +select i.CONSTRAINT_SCHEMA || '_' || i.CONSTRAINT_NAME as INDEX_NAME, ii.COLUMN_NAME, ii.ORDINAL_POSITION, case when i.CONSTRAINT_TYPE = 'PRIMARY KEY' then 'Y' else 'N' end as PRIMARY_KEY + from INFORMATION_SCHEMA.table_constraints i inner join INFORMATION_SCHEMA.key_column_usage ii on i.CONSTRAINT_SCHEMA = ii.CONSTRAINT_SCHEMA and i.CONSTRAINT_NAME = ii.CONSTRAINT_NAME and i.TABLE_SCHEMA = ii.TABLE_SCHEMA and i.TABLE_NAME = ii.TABLE_NAME + where i.TABLE_SCHEMA = 'refdb' and i.TABLE_NAME = 'sensors' + and i.CONSTRAINT_TYPE in ('PRIMARY KEY', 'UNIQUE') + order by i.CONSTRAINT_SCHEMA || '_' || i.CONSTRAINT_NAME, ii.TABLE_SCHEMA, ii.TABLE_NAME, ii.ORDINAL_POSITION +-- next +DISCARD ALL +-- next +select "$Table"."time" as "time", + "$Table"."plant" as "plant", + "$Table"."line" as "line", + "$Table"."machine" as "machine", + "$Table"."model" as "model", + "$Table"."temperature" as "temperature", + "$Table"."pressure" as "pressure", + "$Table"."count" as "count", + "$Table"."level" as "level", + "$Table"."running" as "running", + "$Table"."state" as "state", + "$Table"."note" as "note", + "$Table"."blob" as "blob", + "$Table"."day" as "day", + "$Table"."ts" as "ts" + from "refdb"."sensors" "$Table" + limit 100 +-- next +DISCARD ALL +-- next +select "$Table"."time" as "time", + "$Table"."plant" as "plant", + "$Table"."line" as "line", + "$Table"."machine" as "machine", + "$Table"."model" as "model", + "$Table"."temperature" as "temperature", + "$Table"."pressure" as "pressure", + "$Table"."count" as "count", + "$Table"."level" as "level", + "$Table"."running" as "running", + "$Table"."state" as "state", + "$Table"."note" as "note", + "$Table"."blob" as "blob", + "$Table"."day" as "day", + "$Table"."ts" as "ts" + from "refdb"."sensors" "$Table" + limit 200 +-- next +DISCARD ALL +-- next +SELECT PG_TOTAL_RELATION_SIZE(C.OID) AS TOTAL_BYTES + FROM PG_CLASS C JOIN PG_NAMESPACE N + ON (N.OID = C.RELNAMESPACE) + WHERE N.NSPNAME = 'refdb' AND C.RELNAME = 'sensors' +-- next +DISCARD ALL +-- next +select "$Table"."time" as "time", + "$Table"."plant" as "plant", + "$Table"."line" as "line", + "$Table"."machine" as "machine", + "$Table"."model" as "model", + "$Table"."temperature" as "temperature", + "$Table"."pressure" as "pressure", + "$Table"."count" as "count", + "$Table"."level" as "level", + "$Table"."running" as "running", + "$Table"."state" as "state", + "$Table"."note" as "note", + "$Table"."blob" as "blob", + "$Table"."day" as "day", + "$Table"."ts" as "ts" + from "refdb"."sensors" "$Table" + limit 100 +-- next +DISCARD ALL +-- next +select "$Table"."time" as "time", + "$Table"."plant" as "plant", + "$Table"."line" as "line", + "$Table"."machine" as "machine", + "$Table"."model" as "model", + "$Table"."temperature" as "temperature", + "$Table"."pressure" as "pressure", + "$Table"."count" as "count", + "$Table"."level" as "level", + "$Table"."running" as "running", + "$Table"."state" as "state", + "$Table"."note" as "note", + "$Table"."blob" as "blob", + "$Table"."day" as "day", + "$Table"."ts" as "ts" + from "refdb"."sensors" "$Table" + limit 1000 +-- next +DISCARD ALL +-- next +select "$Table"."time" as "time", + "$Table"."plant" as "plant", + "$Table"."line" as "line", + "$Table"."machine" as "machine", + "$Table"."model" as "model", + "$Table"."temperature" as "temperature", + "$Table"."pressure" as "pressure", + "$Table"."count" as "count", + "$Table"."level" as "level", + "$Table"."running" as "running", + "$Table"."state" as "state", + "$Table"."note" as "note", + "$Table"."blob" as "blob", + "$Table"."day" as "day", + "$Table"."ts" as "ts" + from "refdb"."sensors" "$Table" + limit 1000 +-- next +DISCARD ALL +-- next +select "$Table"."time" as "time", + "$Table"."plant" as "plant", + "$Table"."line" as "line", + "$Table"."machine" as "machine", + "$Table"."model" as "model", + "$Table"."temperature" as "temperature", + "$Table"."pressure" as "pressure", + "$Table"."count" as "count", + "$Table"."level" as "level", + "$Table"."running" as "running", + "$Table"."state" as "state", + "$Table"."note" as "note", + "$Table"."blob" as "blob", + "$Table"."day" as "day", + "$Table"."ts" as "ts" + from "refdb"."sensors" "$Table" + limit 1000 +-- next +DISCARD ALL +-- next +select "$Table"."time" as "time", + "$Table"."plant" as "plant", + "$Table"."line" as "line", + "$Table"."machine" as "machine", + "$Table"."model" as "model", + "$Table"."temperature" as "temperature", + "$Table"."pressure" as "pressure", + "$Table"."count" as "count", + "$Table"."level" as "level", + "$Table"."running" as "running", + "$Table"."state" as "state", + "$Table"."note" as "note", + "$Table"."blob" as "blob", + "$Table"."day" as "day", + "$Table"."ts" as "ts" + from "refdb"."sensors" "$Table" + limit 20 +-- next +DISCARD ALL +-- next +select "$Table"."time" as "time", + "$Table"."plant" as "plant", + "$Table"."line" as "line", + "$Table"."machine" as "machine", + "$Table"."model" as "model", + "$Table"."temperature" as "temperature", + "$Table"."pressure" as "pressure", + "$Table"."count" as "count", + "$Table"."level" as "level", + "$Table"."running" as "running", + "$Table"."state" as "state", + "$Table"."note" as "note", + "$Table"."blob" as "blob", + "$Table"."day" as "day", + "$Table"."ts" as "ts" + from "refdb"."sensors" "$Table" + limit 1000 +-- next +/*** Load all supported types ***/ + SELECT ns.nspname, a.typname, a.oid, a.typrelid, a.typbasetype, + CASE WHEN pg_proc.proname='array_recv' THEN 'a' ELSE a.typtype END AS type, + CASE + WHEN pg_proc.proname='array_recv' THEN a.typelem + WHEN a.typtype='r' THEN rngsubtype + ELSE 0 + END AS elemoid, + CASE + WHEN pg_proc.proname IN ('array_recv','oidvectorrecv') THEN 3 /* Arrays last */ + WHEN a.typtype='r' THEN 2 /* Ranges before */ + WHEN a.typtype='d' THEN 1 /* Domains before */ + ELSE 0 /* Base types first */ + END AS ord + FROM pg_type AS a + JOIN pg_namespace AS ns ON (ns.oid = a.typnamespace) + JOIN pg_proc ON pg_proc.oid = a.typreceive + LEFT OUTER JOIN pg_class AS cls ON (cls.oid = a.typrelid) + LEFT OUTER JOIN pg_type AS b ON (b.oid = a.typelem) + LEFT OUTER JOIN pg_class AS elemcls ON (elemcls.oid = b.typrelid) + LEFT OUTER JOIN pg_range ON (pg_range.rngtypid = a.oid) + WHERE + a.typtype IN ('b', 'r', 'e', 'd') OR /* Base, range, enum, domain */ + (a.typtype = 'c' AND cls.relkind='c') OR /* User-defined free-standing composites (not table composites) by default */ + (pg_proc.proname='array_recv' AND ( + b.typtype IN ('b', 'r', 'e', 'd') OR /* Array of base, range, enum, domain */ + (b.typtype = 'p' AND b.typname IN ('record', 'void')) OR /* Arrays of special supported pseudo-types */ + (b.typtype = 'c' AND elemcls.relkind='c') /* Array of user-defined free-standing composites (not table composites) */ + )) OR + (a.typtype = 'p' AND a.typname IN ('record', 'void')) /* Some special supported pseudo-types */ + ORDER BY ord +-- next +/*** Load field definitions for (free-standing) composite types ***/ + SELECT typ.oid, att.attname, att.atttypid + FROM pg_type AS typ + JOIN pg_namespace AS ns ON (ns.oid = typ.typnamespace) + JOIN pg_class AS cls ON (cls.oid = typ.typrelid) + JOIN pg_attribute AS att ON (att.attrelid = typ.typrelid) + WHERE + (typ.typtype = 'c' AND cls.relkind='c') AND + attnum > 0 AND /* Don't load system attributes */ + NOT attisdropped + ORDER BY typ.oid, att.attnum +-- next +/*** Load enum fields ***/ + SELECT pg_type.oid, enumlabel + FROM pg_enum + JOIN pg_type ON pg_type.oid=enumtypid + ORDER BY oid, enumsortorder +-- next +select character_set_name from INFORMATION_SCHEMA.character_sets +-- next +DISCARD ALL +-- next +select TABLE_SCHEMA, TABLE_NAME, TABLE_TYPE + from INFORMATION_SCHEMA.tables + where TABLE_SCHEMA not in ('information_schema', 'pg_catalog') + order by TABLE_SCHEMA, TABLE_NAME +-- next +DISCARD ALL +-- next +select COLUMN_NAME, ORDINAL_POSITION, IS_NULLABLE, case when (data_type like '%unsigned%') then DATA_TYPE || ' unsigned' else DATA_TYPE end as DATA_TYPE + from INFORMATION_SCHEMA.columns + where TABLE_SCHEMA = 'refdb' and TABLE_NAME = 'sensors' + order by TABLE_SCHEMA, TABLE_NAME, ORDINAL_POSITION +-- next +DISCARD ALL +-- next +select + pkcol.COLUMN_NAME as PK_COLUMN_NAME, + fkcol.TABLE_SCHEMA AS FK_TABLE_SCHEMA, + fkcol.TABLE_NAME AS FK_TABLE_NAME, + fkcol.COLUMN_NAME as FK_COLUMN_NAME, + fkcol.ORDINAL_POSITION as ORDINAL, + fkcon.CONSTRAINT_SCHEMA || '_' || fkcol.TABLE_NAME || '_' || 'sensors' || '_' || fkcon.CONSTRAINT_NAME as FK_NAME + from + (select distinct constraint_catalog, constraint_schema, unique_constraint_schema, constraint_name, unique_constraint_name + from INFORMATION_SCHEMA.REFERENTIAL_CONSTRAINTS) fkcon + inner join + INFORMATION_SCHEMA.KEY_COLUMN_USAGE fkcol + on fkcon.CONSTRAINT_SCHEMA = fkcol.CONSTRAINT_SCHEMA + and fkcon.CONSTRAINT_NAME = fkcol.CONSTRAINT_NAME + inner join + INFORMATION_SCHEMA.KEY_COLUMN_USAGE pkcol + on fkcon.UNIQUE_CONSTRAINT_SCHEMA = pkcol.CONSTRAINT_SCHEMA + and fkcon.UNIQUE_CONSTRAINT_NAME = pkcol.CONSTRAINT_NAME + where pkcol.TABLE_SCHEMA = 'refdb' and pkcol.TABLE_NAME = 'sensors' + and pkcol.ORDINAL_POSITION = fkcol.ORDINAL_POSITION + order by FK_NAME, fkcol.ORDINAL_POSITION +-- next +DISCARD ALL +-- next +select + pkcol.TABLE_SCHEMA AS PK_TABLE_SCHEMA, + pkcol.TABLE_NAME AS PK_TABLE_NAME, + pkcol.COLUMN_NAME as PK_COLUMN_NAME, + fkcol.COLUMN_NAME as FK_COLUMN_NAME, + fkcol.ORDINAL_POSITION as ORDINAL, + fkcon.CONSTRAINT_SCHEMA || '_' || 'sensors' || '_' || pkcol.TABLE_NAME || '_' || fkcon.CONSTRAINT_NAME as FK_NAME + from + (select distinct constraint_catalog, constraint_schema, unique_constraint_schema, constraint_name, unique_constraint_name + from INFORMATION_SCHEMA.REFERENTIAL_CONSTRAINTS) fkcon + inner join + INFORMATION_SCHEMA.KEY_COLUMN_USAGE fkcol + on fkcon.CONSTRAINT_SCHEMA = fkcol.CONSTRAINT_SCHEMA + and fkcon.CONSTRAINT_NAME = fkcol.CONSTRAINT_NAME + inner join + INFORMATION_SCHEMA.KEY_COLUMN_USAGE pkcol + on fkcon.UNIQUE_CONSTRAINT_SCHEMA = pkcol.CONSTRAINT_SCHEMA + and fkcon.UNIQUE_CONSTRAINT_NAME = pkcol.CONSTRAINT_NAME + where fkcol.TABLE_SCHEMA = 'refdb' and fkcol.TABLE_NAME = 'sensors' + and pkcol.ORDINAL_POSITION = fkcol.ORDINAL_POSITION + order by FK_NAME, fkcol.ORDINAL_POSITION +-- next +DISCARD ALL +-- next +select i.CONSTRAINT_SCHEMA || '_' || i.CONSTRAINT_NAME as INDEX_NAME, ii.COLUMN_NAME, ii.ORDINAL_POSITION, case when i.CONSTRAINT_TYPE = 'PRIMARY KEY' then 'Y' else 'N' end as PRIMARY_KEY + from INFORMATION_SCHEMA.table_constraints i inner join INFORMATION_SCHEMA.key_column_usage ii on i.CONSTRAINT_SCHEMA = ii.CONSTRAINT_SCHEMA and i.CONSTRAINT_NAME = ii.CONSTRAINT_NAME and i.TABLE_SCHEMA = ii.TABLE_SCHEMA and i.TABLE_NAME = ii.TABLE_NAME + where i.TABLE_SCHEMA = 'refdb' and i.TABLE_NAME = 'sensors' + and i.CONSTRAINT_TYPE in ('PRIMARY KEY', 'UNIQUE') + order by i.CONSTRAINT_SCHEMA || '_' || i.CONSTRAINT_NAME, ii.TABLE_SCHEMA, ii.TABLE_NAME, ii.ORDINAL_POSITION +-- next +/*** Load all supported types ***/ + SELECT ns.nspname, a.typname, a.oid, a.typrelid, a.typbasetype, + CASE WHEN pg_proc.proname='array_recv' THEN 'a' ELSE a.typtype END AS type, + CASE + WHEN pg_proc.proname='array_recv' THEN a.typelem + WHEN a.typtype='r' THEN rngsubtype + ELSE 0 + END AS elemoid, + CASE + WHEN pg_proc.proname IN ('array_recv','oidvectorrecv') THEN 3 /* Arrays last */ + WHEN a.typtype='r' THEN 2 /* Ranges before */ + WHEN a.typtype='d' THEN 1 /* Domains before */ + ELSE 0 /* Base types first */ + END AS ord + FROM pg_type AS a + JOIN pg_namespace AS ns ON (ns.oid = a.typnamespace) + JOIN pg_proc ON pg_proc.oid = a.typreceive + LEFT OUTER JOIN pg_class AS cls ON (cls.oid = a.typrelid) + LEFT OUTER JOIN pg_type AS b ON (b.oid = a.typelem) + LEFT OUTER JOIN pg_class AS elemcls ON (elemcls.oid = b.typrelid) + LEFT OUTER JOIN pg_range ON (pg_range.rngtypid = a.oid) + WHERE + a.typtype IN ('b', 'r', 'e', 'd') OR /* Base, range, enum, domain */ + (a.typtype = 'c' AND cls.relkind='c') OR /* User-defined free-standing composites (not table composites) by default */ + (pg_proc.proname='array_recv' AND ( + b.typtype IN ('b', 'r', 'e', 'd') OR /* Array of base, range, enum, domain */ + (b.typtype = 'p' AND b.typname IN ('record', 'void')) OR /* Arrays of special supported pseudo-types */ + (b.typtype = 'c' AND elemcls.relkind='c') /* Array of user-defined free-standing composites (not table composites) */ + )) OR + (a.typtype = 'p' AND a.typname IN ('record', 'void')) /* Some special supported pseudo-types */ + ORDER BY ord +-- next +/*** Load field definitions for (free-standing) composite types ***/ + SELECT typ.oid, att.attname, att.atttypid + FROM pg_type AS typ + JOIN pg_namespace AS ns ON (ns.oid = typ.typnamespace) + JOIN pg_class AS cls ON (cls.oid = typ.typrelid) + JOIN pg_attribute AS att ON (att.attrelid = typ.typrelid) + WHERE + (typ.typtype = 'c' AND cls.relkind='c') AND + attnum > 0 AND /* Don't load system attributes */ + NOT attisdropped + ORDER BY typ.oid, att.attnum +-- next +/*** Load enum fields ***/ + SELECT pg_type.oid, enumlabel + FROM pg_enum + JOIN pg_type ON pg_type.oid=enumtypid + ORDER BY oid, enumsortorder +-- next +select "rows"."time" as "time" + from "refdb"."sensors" "rows" + group by "time" + limit 1000001 +-- next +DISCARD ALL +-- next +select "_"."time", + "_"."a0" + from + ( + select "rows"."time" as "time", + sum("rows"."level") as "a0" + from "refdb"."sensors" "rows" + group by "time" + ) "_" + where not "_"."a0" is null + limit 1000001 +-- next +DISCARD ALL +-- next +select "_"."time", + "_"."a0", + "_"."a1" + from + ( + select "rows"."time" as "time", + sum("rows"."temperature") as "a0", + sum("rows"."level") as "a1" + from "refdb"."sensors" "rows" + group by "time" + ) "_" + where not "_"."a0" is null or not "_"."a1" is null + limit 1000001 +-- next +-- executed 3 times in a row (portal fetches or repeated executions) +DISCARD ALL