diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index dd46f8697..1369f65bb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -56,7 +56,7 @@ jobs: 21 cache: "maven" - name: Build and install libraries - run: mvn --batch-mode --no-transfer-progress --show-version --strict-checksums --threads 2 -Dmaven.wagon.rto=30000 -Dj8 -DskipITs install + run: mvn --batch-mode --no-transfer-progress --show-version --strict-checksums --threads 2 -Dmaven.wagon.rto=30000 -Dj8 -DskipITs -Dspotless.skip=true install - name: Copy Artifacts to Build dir run: | mkdir clickhouse-jdbc-artifacts @@ -135,7 +135,7 @@ jobs: java-version: "21" github-token: ${{ secrets.GITHUB_TOKEN }} - name: Build native image - run: mvn --batch-mode --no-transfer-progress -Pnative -Dj8 -DskipTests install + run: mvn --batch-mode --no-transfer-progress -Pnative -Dj8 -DskipTests -Dspotless.skip=true install - name: Test native image run: ./clickhouse-jdbc/target/clickhouse-jdbc-bin - name: Compress binary @@ -194,10 +194,10 @@ jobs: EOF - name: Build and install libraries - run: mvn --batch-mode --no-transfer-progress --show-version --strict-checksums --threads 2 -Dmaven.wagon.rto=30000 -Dj8 -DskipTests=true -Dmaven.javadoc.skip=true install + run: mvn --batch-mode --no-transfer-progress --show-version --strict-checksums --threads 2 -Dmaven.wagon.rto=30000 -Dj8 -DskipTests=true -Dmaven.javadoc.skip=true -Dspotless.skip=true install - name: Test Java client run: | - mvn --also-make --batch-mode --no-transfer-progress --projects ${{ matrix.project }} -DclickhouseVersion=${{ matrix.clickhouse }} -Dmaven.javadoc.skip=true verify + mvn --also-make --batch-mode --no-transfer-progress --projects ${{ matrix.project }} -DclickhouseVersion=${{ matrix.clickhouse }} -Dmaven.javadoc.skip=true -Dspotless.skip=true verify - name: Upload test results uses: actions/upload-artifact@v4 if: failure() @@ -257,14 +257,14 @@ jobs: EOF - name: Install Java client - run: mvn --also-make --batch-mode --no-transfer-progress -DskipTests=true -Dmaven.javadoc.skip=true install + run: mvn --also-make --batch-mode --no-transfer-progress -DskipTests=true -Dmaven.javadoc.skip=true -Dspotless.skip=true install - name: Test http client env: CLICKHOUSE_CLOUD_HOST: ${{ secrets.INTEGRATIONS_TEAM_TESTS_CLOUD_HOST_SMT }} CLICKHOUSE_CLOUD_PASSWORD: ${{ secrets.INTEGRATIONS_TEAM_TESTS_CLOUD_PASSWORD_SMT }} CLIENT_JWT: ${{ secrets.INTEGRATIONS_TEAM_TESTS_CLOUD_JWT_DESERT_VM_43 }} run: | - mvn --batch-mode --no-transfer-progress --projects ${{ matrix.project }} -DclickhouseVersion=${{ matrix.clickhouse }} -Dprotocol=http -Dmaven.javadoc.skip=true verify + mvn --batch-mode --no-transfer-progress --projects ${{ matrix.project }} -DclickhouseVersion=${{ matrix.clickhouse }} -Dprotocol=http -Dmaven.javadoc.skip=true -Dspotless.skip=true verify - name: Upload test results uses: actions/upload-artifact@v4 if: failure() @@ -321,13 +321,13 @@ jobs: EOF - name: Install Java client - run: mvn --also-make --batch-mode --no-transfer-progress --projects clickhouse-http-client,client-v2 -DskipTests=true -Dmaven.javadoc.skip=true install + run: mvn --also-make --batch-mode --no-transfer-progress --projects clickhouse-http-client,client-v2 -DskipTests=true -Dmaven.javadoc.skip=true -Dspotless.skip=true install - name: Test JDBC driver env: CLICKHOUSE_CLOUD_HOST: ${{ secrets.INTEGRATIONS_TEAM_TESTS_CLOUD_HOST_SMT }} CLICKHOUSE_CLOUD_PASSWORD: ${{ secrets.INTEGRATIONS_TEAM_TESTS_CLOUD_PASSWORD_SMT }} run: | - mvn --batch-mode --no-transfer-progress --projects clickhouse-jdbc,jdbc-v2 -DclickhouseVersion=${{ matrix.clickhouse }} -Dprotocol=${{ matrix.protocol }} -Dmaven.javadoc.skip=true verify + mvn --batch-mode --no-transfer-progress --projects clickhouse-jdbc,jdbc-v2 -DclickhouseVersion=${{ matrix.clickhouse }} -Dprotocol=${{ matrix.protocol }} -Dmaven.javadoc.skip=true -Dspotless.skip=true verify - name: Upload test results uses: actions/upload-artifact@v4 if: failure() @@ -384,11 +384,11 @@ jobs: EOF - name: Install Java client - run: mvn --also-make --no-transfer-progress --batch-mode --projects clickhouse-jdbc -DskipTests=true -Dmaven.javadoc.skip=true install + run: mvn --also-make --no-transfer-progress --batch-mode --projects clickhouse-jdbc -DskipTests=true -Dmaven.javadoc.skip=true -Dspotless.skip=true install - name: Test R2DBC ${{ matrix.r2dbc }} run: | mvn --batch-mode --no-transfer-progress --projects clickhouse-r2dbc -DclickhouseVersion=${{ matrix.clickhouse }} \ - -D'r2dbc-spi.version=${{ matrix.r2dbc }}' -Dprotocol=${{ matrix.protocol }} -Dmaven.javadoc.skip=true verify + -D'r2dbc-spi.version=${{ matrix.r2dbc }}' -Dprotocol=${{ matrix.protocol }} -Dmaven.javadoc.skip=true -Dspotless.skip=true verify - name: Upload test results uses: actions/upload-artifact@v4 if: failure() diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 1846891dd..5290f26b7 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -59,9 +59,6 @@ jobs: EOF - name: Update Configuration run: | - # find . -type f -name "pom.xml" -exec sed -i -e 's|${revision}|${{ env.CHC_VERSION }}-SNAPSHOT|g' \ - # -e 's|^\( \).*\(\)$|\1${{ env.CHC_VERSION }}-SNAPSHOT\2|' \ - # -e 's|${parent.groupId}|com.clickhouse|g' -e 's|${project.parent.groupId}|com.clickhouse|g' '{}' \; find . -type f -name "simplelogger.*" -exec rm -fv '{}' \; - name: Release Snapshot uses: samuelmeuli/action-maven-publish@v1 @@ -78,7 +75,7 @@ jobs: with: directory: clickhouse-r2dbc maven_profiles: release - maven_args: -q --batch-mode -Dr2dbc-spi.version=0.9.1.RELEASE -DclickhouseVersion=${{ env.CH_VERSION }} + maven_args: -q --batch-mode -Dr2dbc-spi.version=0.9.1.RELEASE -DclickhouseVersion=${{ env.CH_VERSION }} -Dspotless.skip=true server_id: central gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} gpg_passphrase: ${{ secrets.GPG_PASSPHRASE }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 022d27155..1662109de 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -65,7 +65,7 @@ jobs: with: directory: clickhouse-r2dbc maven_profiles: release - maven_args: -q --batch-mode -Dr2dbc-spi.version=0.9.1.RELEASE -DclickhouseVersion=${{ env.CH_VERSION }} + maven_args: -q --batch-mode -Dr2dbc-spi.version=0.9.1.RELEASE -DclickhouseVersion=${{ env.CH_VERSION }} -Dspotless.skip=true server_id: central gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} gpg_passphrase: ${{ secrets.GPG_PASSPHRASE }} diff --git a/.github/workflows/test_head.yml b/.github/workflows/test_head.yml index f1129021b..5292a9774 100644 --- a/.github/workflows/test_head.yml +++ b/.github/workflows/test_head.yml @@ -38,7 +38,7 @@ jobs: architecture: x64 - name: Test Java client run: | - mvn --also-make --batch-mode --no-transfer-progress --projects ${{ matrix.project }} -DclickhouseVersion=$CH_VERSION -Dmaven.javadoc.skip=true clean install + mvn --also-make --batch-mode --no-transfer-progress --projects ${{ matrix.project }} -DclickhouseVersion=$CH_VERSION -Dmaven.javadoc.skip=true -Dspotless.skip=true clean install - name: Upload test results uses: actions/upload-artifact@v4 if: failure() @@ -68,7 +68,7 @@ jobs: architecture: x64 - name: Build run: | - mvn --also-make --batch-mode --no-transfer-progress -DclickhouseVersion=$CH_VERSION -Dmaven.javadoc.skip=true -DskipTests=true clean install + mvn --also-make --batch-mode --no-transfer-progress -DclickhouseVersion=$CH_VERSION -Dmaven.javadoc.skip=true -DskipTests=true -Dspotless.skip=true clean install - name: Test R2DBC ${{ matrix.r2dbc }} run: | mvn --batch-mode --no-transfer-progress --projects clickhouse-r2dbc -DclickhouseVersion=$CH_VERSION \ diff --git a/.spotless/formatter.xml b/.spotless/formatter.xml new file mode 100644 index 000000000..f09f1a267 --- /dev/null +++ b/.spotless/formatter.xml @@ -0,0 +1,380 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/clickhouse-r2dbc/src/test/java/com/clickhouse/r2dbc/spi/test/R2DBCTestKitImplTest.java b/clickhouse-r2dbc/src/test/java/com/clickhouse/r2dbc/spi/test/R2DBCTestKitImplTest.java index 38f8b6ace..510bd614f 100644 --- a/clickhouse-r2dbc/src/test/java/com/clickhouse/r2dbc/spi/test/R2DBCTestKitImplTest.java +++ b/clickhouse-r2dbc/src/test/java/com/clickhouse/r2dbc/spi/test/R2DBCTestKitImplTest.java @@ -100,7 +100,7 @@ private static JdbcTemplate jdbcTemplate(String database) throws SQLException { .map(Duration::toMillis).orElse(0L)); ZoneId zoneId = ZoneId.systemDefault(); - source.addDataSourceProperty("serverTimezone", TimeZone.getTimeZone(zoneId).getID()); + source.addDataSourceProperty("server_time_zone", TimeZone.getTimeZone(zoneId).getID()); return new JdbcTemplate(source); } diff --git a/client-v2/src/main/java/com/clickhouse/client/api/Client.java b/client-v2/src/main/java/com/clickhouse/client/api/Client.java index f7cc187d4..e75f45527 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/Client.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/Client.java @@ -345,6 +345,7 @@ public Builder setOption(String key, String value) { if (key.equals(ClientConfigProperties.BEARERTOKEN_AUTH.getKey())) { useBearerTokenAuth(value); } + return this; } diff --git a/client-v2/src/main/java/com/clickhouse/client/api/ClientConfigProperties.java b/client-v2/src/main/java/com/clickhouse/client/api/ClientConfigProperties.java index 131fc0f20..4a024971a 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/ClientConfigProperties.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/ClientConfigProperties.java @@ -1,435 +1,452 @@ -package com.clickhouse.client.api; - -import com.clickhouse.client.api.data_formats.internal.AbstractBinaryFormatReader; -import com.clickhouse.client.api.internal.ClickHouseLZ4OutputStream; -import com.clickhouse.data.ClickHouseDataType; -import com.clickhouse.data.ClickHouseFormat; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.TimeZone; -import java.util.function.Consumer; -import java.util.function.Function; -import java.util.stream.Collectors; - -/** - * Enumerates all client properties that are known at release. - */ -public enum ClientConfigProperties { - - SESSION_DB_ROLES("session_db_roles", List.class), - - SETTING_LOG_COMMENT(serverSetting("log_comment"), String.class), - - HTTP_USE_BASIC_AUTH("http_use_basic_auth", Boolean.class, "true"), - - USER("user", String.class, "default"), - - PASSWORD("password", String.class), - - /** - * Maximum number of active connection in internal connection pool. - */ - HTTP_MAX_OPEN_CONNECTIONS("max_open_connections", Integer.class, "10"), - - /** - * HTTP keep-alive timeout override. - */ - HTTP_KEEP_ALIVE_TIMEOUT("http_keep_alive_timeout", Long.class), - - USE_SERVER_TIMEZONE("use_server_time_zone", Boolean.class, "true"), - - USE_TIMEZONE("use_time_zone", TimeZone.class), - - SERVER_VERSION("server_version", String.class), - - SERVER_TIMEZONE("server_time_zone", TimeZone.class, "UTC"), - - ASYNC_OPERATIONS("async", Boolean.class, "false"), - - CONNECTION_TTL("connection_ttl", Long.class, "-1"), - - CONNECTION_TIMEOUT("connection_timeout", Long.class), - - CONNECTION_REUSE_STRATEGY("connection_reuse_strategy", ConnectionReuseStrategy.class, String.valueOf(ConnectionReuseStrategy.FIFO)), - - SOCKET_OPERATION_TIMEOUT("socket_timeout", Integer.class, "0"), - - SOCKET_RCVBUF_OPT("socket_rcvbuf", Integer.class, "804800"), - - SOCKET_SNDBUF_OPT("socket_sndbuf", Integer.class,"804800"), - - SOCKET_REUSEADDR_OPT("socket_reuseaddr", Boolean.class), - - SOCKET_KEEPALIVE_OPT("socket_keepalive", Boolean.class), - - SOCKET_TCP_NO_DELAY_OPT("socket_tcp_nodelay", Boolean.class), - - SOCKET_LINGER_OPT("socket_linger", Integer.class), - - DATABASE("database", String.class, "default"), - - COMPRESS_SERVER_RESPONSE("compress", Boolean.class, "true"), // actually a server setting, but has client effect too - - COMPRESS_CLIENT_REQUEST("decompress", Boolean.class, "false"), // actually a server setting, but has client effect too - - USE_HTTP_COMPRESSION("client.use_http_compression", Boolean.class, "false"), - - COMPRESSION_LZ4_UNCOMPRESSED_BUF_SIZE("compression.lz4.uncompressed_buffer_size", Integer.class, String.valueOf(ClickHouseLZ4OutputStream.UNCOMPRESSED_BUFF_SIZE)), - - DISABLE_NATIVE_COMPRESSION("disable_native_compression", Boolean.class, "false"), - - PROXY_TYPE("proxy_type", String.class), // "http" - - PROXY_HOST("proxy_host", String.class), - - PROXY_PORT("proxy_port", Integer.class), - - PROXY_USER("proxy_user", String.class), - - PROXY_PASSWORD("proxy_password", String.class), - - MAX_EXECUTION_TIME("max_execution_time", Integer.class,"0"), - - SSL_TRUST_STORE("trust_store", String.class), - - SSL_KEYSTORE_TYPE("key_store_type", String.class), - - SSL_KEY_STORE("ssl_key_store", String.class), - - SSL_KEY_STORE_PASSWORD("key_store_password", String.class), - - SSL_KEY("ssl_key", String.class), - - CA_CERTIFICATE("sslrootcert", String.class), - - SSL_CERTIFICATE("sslcert", String.class), - - RETRY_ON_FAILURE("retry", Integer.class, "3"), - - INPUT_OUTPUT_FORMAT("format", ClickHouseFormat.class), - - MAX_THREADS_PER_CLIENT("max_threads_per_client", Integer.class, "0"), - - QUERY_ID("query_id", String.class), // actually a server setting, but has client effect too - - CLIENT_NETWORK_BUFFER_SIZE("client_network_buffer_size", Integer.class, "300000"), - - ACCESS_TOKEN("access_token", String.class), - - SSL_AUTH("ssl_authentication", Boolean.class, "false"), - - CONNECTION_POOL_ENABLED("connection_pool_enabled", Boolean.class, "true"), - - CONNECTION_REQUEST_TIMEOUT("connection_request_timeout", Long.class, "10000"), - - CLIENT_RETRY_ON_FAILURE("client_retry_on_failures", List.class, - String.join(",", ClientFaultCause.NoHttpResponse.name(), ClientFaultCause.ConnectTimeout.name(), - ClientFaultCause.ConnectionRequestTimeout.name(), ClientFaultCause.ServerRetryable.name())) { - @Override - public Object parseValue(String value) { - List strValues = (List) super.parseValue(value); - List failures = new ArrayList(); - if (strValues != null) { - for (String strValue : strValues) { - failures.add(ClientFaultCause.valueOf(strValue)); - } - } - return failures; - } - }, - - CLIENT_NAME("client_name", String.class, ""), - - /** - * An old alias to {@link ClientConfigProperties#CLIENT_NAME}. Using the last one is preferred. - */ - @Deprecated - PRODUCT_NAME("product_name", String.class), - - BEARERTOKEN_AUTH ("bearer_token", String.class), - /** - * Indicates that data provided for write operation is compressed by application. - */ - APP_COMPRESSED_DATA("app_compressed_data", Boolean.class, "false"), - - /** - * Name of the group under which client metrics appear - */ - METRICS_GROUP_NAME("metrics_name", String.class, "ch-http-pool"), - - HTTP_SAVE_COOKIES("client.http.cookies_enabled", Boolean.class, "false"), - - BINARY_READER_USE_PREALLOCATED_BUFFERS("client_allow_binary_reader_to_reuse_buffers", Boolean.class, "false"), - - /** - * Defines mapping between ClickHouse data type and target Java type - * Used by binary readers to convert values into desired Java type. - */ - TYPE_HINT_MAPPING("type_hint_mapping", Map.class), - - /** - * SNI SSL parameter that will be set for each outbound SSL socket. - */ - SSL_SOCKET_SNI("ssl_socket_sni", String.class,""), - ; - - private static final Logger LOG = LoggerFactory.getLogger(ClientConfigProperties.class); - - private final String key; - - private final Class valueType; - - private final String defaultValue; - - private final Object defaultObjValue; - - ClientConfigProperties(String key, Class valueType) { - this(key, valueType, null); - } - - ClientConfigProperties(String key, Class valueType, String defaultValue) { - this.key = key; - this.valueType = valueType; - this.defaultValue = defaultValue; - this.defaultObjValue = parseValue(defaultValue); - } - - public String getKey() { - return key; - } - - public String getDefaultValue() { - return defaultValue; - } - - @SuppressWarnings("unchecked") - public T getDefObjVal() { - return (T) defaultObjValue; - } - - public static final String HTTP_HEADER_PREFIX = "http_header_"; - - public static final String SERVER_SETTING_PREFIX = "clickhouse_setting_"; - - // Key used to identify default value in configuration map - public static final String DEFAULT_KEY = "_default_"; - - public static String serverSetting(String key) { - return SERVER_SETTING_PREFIX + key; - } - - public static String httpHeader(String key) { - return HTTP_HEADER_PREFIX + key.toUpperCase(Locale.US); - } - - public static String commaSeparated(Collection values) { - StringBuilder sb = new StringBuilder(); - for (Object value : values) { - sb.append(value.toString().replaceAll(",", "\\\\,")).append(","); - } - - if (sb.length() > 0) { - sb.setLength(sb.length() - 1); - } - return sb.toString(); - } - - public static List valuesFromCommaSeparated(String value) { - if (value == null || value.isEmpty()) { - return Collections.emptyList(); - } - - return Arrays.stream(value.split("(? s.replaceAll("\\\\,", ",")) - .collect(Collectors.toList()); - } - - public Object parseValue(String value) { - if (value == null) { - return null; - } - - if (valueType.equals(String.class)) { - return value; - } - - if (valueType.equals(Boolean.class)) { - if (value.equals("1")) return true; - if (value.equals("0")) return false; - return Boolean.parseBoolean(value); - } - - if (valueType.equals(Integer.class)) { - return Integer.parseInt(value); - } - - if (valueType.equals(Long.class)) { - return Long.parseLong(value); - } - - if (valueType.equals(List.class)) { - return valuesFromCommaSeparated(value); - } - - if (valueType.isEnum()) { - Object[] constants = valueType.getEnumConstants(); - for (Object constant : constants) { - if (constant.toString().equals(value)) { - return constant; - } - } - throw new IllegalArgumentException("Invalid constant name '" + value + "' for enum " + valueType.getName()); - } - - if (valueType.equals(TimeZone.class)) { - return TimeZone.getTimeZone(value); - } - - if (valueType.equals(Map.class)) { - return toKeyValuePairs(value); - } - - return null; - } - - @SuppressWarnings("unchecked") - public T getOrDefault(Map configMap) { - return (T) configMap.getOrDefault(getKey(), getDefObjVal()); - } - - public void applyIfSet(Map configMap, Consumer consumer) { - T value = (T) configMap.get(getKey()); - if (value != null) { - consumer.accept(value); - } - } - - public static Map parseConfigMap(Map configMap) { - Map parsedConfig = new HashMap<>(); - - Map tmpMap = new HashMap<>(configMap); - - for (ClientConfigProperties config : ClientConfigProperties.values()) { - String value = tmpMap.remove(config.getKey()); - if (value != null) { - Object parsedValue; - switch (config) { - case TYPE_HINT_MAPPING: - parsedValue = translateTypeHintMapping(value); - break; - default: - parsedValue = config.parseValue(value); - } - parsedConfig.put(config.getKey(), parsedValue); - } - } - - for (String key : new HashSet<>(tmpMap.keySet())) { - if (key.startsWith(HTTP_HEADER_PREFIX) || key.startsWith(SERVER_SETTING_PREFIX)) { - parsedConfig.put(key, tmpMap.remove(key)); - } - } - - if (!tmpMap.isEmpty()) { - LOG.warn("Unknown and unmapped config properties: {}", tmpMap); - } - - return parsedConfig; - } - - - /** - * Converts given string to key value pairs. - * This is very simple implementation that do not handle edge cases like - * {@code k1=v1, ,k2=v2} - * - * @param str string - * @return non-null key value pairs - */ - public static Map toKeyValuePairs(String str) { - if (str == null || str.isEmpty()) { - return Collections.emptyMap(); - } - - Map map = new LinkedHashMap<>(); - String key = null; - StringBuilder builder = new StringBuilder(); - for (int i = 0, len = str.length(); i < len; i++) { - char ch = str.charAt(i); - if (ch == '\\' && i + 1 < len) { - ch = str.charAt(++i); - builder.append(ch); - continue; - } - - if (Character.isWhitespace(ch)) { - if (builder.length() > 0) { - builder.append(ch); - } - } else if (ch == '=' && key == null) { - key = builder.toString().trim(); - builder.setLength(0); - } else if (ch == ',' && key != null) { - String value = builder.toString().trim(); - builder.setLength(0); - if (!key.isEmpty() && !value.isEmpty()) { - map.put(key, value); - } - key = null; - } else { - builder.append(ch); - } - } - - if (key != null && builder.length() > 0) { - String value = builder.toString().trim(); - if (!key.isEmpty() && !value.isEmpty()) { - map.put(key, value); - } - } - - return Collections.unmodifiableMap(map); - } - - - - public static String mapToString(Map map, Function valueConverter) { - StringBuilder sb = new StringBuilder(); - for (Map.Entry entry : map.entrySet()) { - sb.append(entry.getKey()).append("=").append(valueConverter.apply(entry.getValue())).append(","); - } - - if (sb.length() > 0) { - sb.setLength(sb.length() - 1); - } - return sb.toString(); - } - - public static Map> translateTypeHintMapping(String mappingStr) { - if (mappingStr == null || mappingStr.isEmpty()) { - return AbstractBinaryFormatReader.NO_TYPE_HINT_MAPPING; - } - - Map mapping= ClientConfigProperties.toKeyValuePairs(mappingStr); - Map> hintMapping = new HashMap<>(); - try { - for (Map.Entry entry : mapping.entrySet()) { - hintMapping.put(ClickHouseDataType.of(entry.getKey()), - Class.forName(entry.getValue())); - } - } catch (ClassNotFoundException e) { - throw new ClientMisconfigurationException("Failed to translate type-hint mapping", e); - } - return hintMapping; - } -} +package com.clickhouse.client.api; + +import com.clickhouse.client.api.data_formats.internal.AbstractBinaryFormatReader; +import com.clickhouse.client.api.internal.ClickHouseLZ4OutputStream; +import com.clickhouse.data.ClickHouseDataType; +import com.clickhouse.data.ClickHouseFormat; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.TimeZone; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * Enumerates all client properties that are known at release. + */ +public enum ClientConfigProperties { + + SESSION_DB_ROLES("session_db_roles", List.class), + + SETTING_LOG_COMMENT(serverSetting("log_comment"), String.class), + + HTTP_USE_BASIC_AUTH("http_use_basic_auth", Boolean.class, "true"), + + USER("user", String.class, "default"), + + PASSWORD("password", String.class), + + /** + * Maximum number of active connection in internal connection pool. + */ + HTTP_MAX_OPEN_CONNECTIONS("max_open_connections", Integer.class, "10"), + + /** + * HTTP keep-alive timeout override. + */ + HTTP_KEEP_ALIVE_TIMEOUT("http_keep_alive_timeout", Long.class), + + USE_SERVER_TIMEZONE("use_server_time_zone", Boolean.class, "true"), + + USE_TIMEZONE("use_time_zone", TimeZone.class), + + SERVER_VERSION("server_version", String.class), + + SERVER_TIMEZONE("server_time_zone", TimeZone.class, "UTC"), + + ASYNC_OPERATIONS("async", Boolean.class, "false"), + + CONNECTION_TTL("connection_ttl", Long.class, "-1"), + + CONNECTION_TIMEOUT("connection_timeout", Long.class), + + CONNECTION_REUSE_STRATEGY("connection_reuse_strategy", ConnectionReuseStrategy.class, String.valueOf(ConnectionReuseStrategy.FIFO)), + + SOCKET_OPERATION_TIMEOUT("socket_timeout", Integer.class, "0"), + + SOCKET_RCVBUF_OPT("socket_rcvbuf", Integer.class, "804800"), + + SOCKET_SNDBUF_OPT("socket_sndbuf", Integer.class,"804800"), + + SOCKET_REUSEADDR_OPT("socket_reuseaddr", Boolean.class), + + SOCKET_KEEPALIVE_OPT("socket_keepalive", Boolean.class), + + SOCKET_TCP_NO_DELAY_OPT("socket_tcp_nodelay", Boolean.class), + + SOCKET_LINGER_OPT("socket_linger", Integer.class), + + DATABASE("database", String.class, "default"), + + COMPRESS_SERVER_RESPONSE("compress", Boolean.class, "true"), // actually a server setting, but has client effect too + + COMPRESS_CLIENT_REQUEST("decompress", Boolean.class, "false"), // actually a server setting, but has client effect too + + USE_HTTP_COMPRESSION("client.use_http_compression", Boolean.class, "false"), + + COMPRESSION_LZ4_UNCOMPRESSED_BUF_SIZE("compression.lz4.uncompressed_buffer_size", Integer.class, String.valueOf(ClickHouseLZ4OutputStream.UNCOMPRESSED_BUFF_SIZE)), + + DISABLE_NATIVE_COMPRESSION("disable_native_compression", Boolean.class, "false"), + + PROXY_TYPE("proxy_type", String.class), // "http" + + PROXY_HOST("proxy_host", String.class), + + PROXY_PORT("proxy_port", Integer.class), + + PROXY_USER("proxy_user", String.class), + + PROXY_PASSWORD("proxy_password", String.class), + + MAX_EXECUTION_TIME("max_execution_time", Integer.class,"0"), + + SSL_TRUST_STORE("trust_store", String.class), + + SSL_KEYSTORE_TYPE("key_store_type", String.class), + + SSL_KEY_STORE("ssl_key_store", String.class), + + SSL_KEY_STORE_PASSWORD("key_store_password", String.class), + + SSL_KEY("ssl_key", String.class), + + CA_CERTIFICATE("sslrootcert", String.class), + + SSL_CERTIFICATE("sslcert", String.class), + + RETRY_ON_FAILURE("retry", Integer.class, "3"), + + INPUT_OUTPUT_FORMAT("format", ClickHouseFormat.class), + + MAX_THREADS_PER_CLIENT("max_threads_per_client", Integer.class, "0"), + + QUERY_ID("query_id", String.class), // actually a server setting, but has client effect too + + CLIENT_NETWORK_BUFFER_SIZE("client_network_buffer_size", Integer.class, "300000"), + + ACCESS_TOKEN("access_token", String.class), + + SSL_AUTH("ssl_authentication", Boolean.class, "false"), + + CONNECTION_POOL_ENABLED("connection_pool_enabled", Boolean.class, "true"), + + CONNECTION_REQUEST_TIMEOUT("connection_request_timeout", Long.class, "10000"), + + CLIENT_RETRY_ON_FAILURE("client_retry_on_failures", List.class, + String.join(",", ClientFaultCause.NoHttpResponse.name(), ClientFaultCause.ConnectTimeout.name(), + ClientFaultCause.ConnectionRequestTimeout.name(), ClientFaultCause.ServerRetryable.name())) { + @Override + public Object parseValue(String value) { + List strValues = (List) super.parseValue(value); + List failures = new ArrayList(); + if (strValues != null) { + for (String strValue : strValues) { + failures.add(ClientFaultCause.valueOf(strValue)); + } + } + return failures; + } + }, + + CLIENT_NAME("client_name", String.class, ""), + + /** + * An old alias to {@link ClientConfigProperties#CLIENT_NAME}. Using the last one is preferred. + */ + @Deprecated + PRODUCT_NAME("product_name", String.class), + + BEARERTOKEN_AUTH ("bearer_token", String.class), + /** + * Indicates that data provided for write operation is compressed by application. + */ + APP_COMPRESSED_DATA("app_compressed_data", Boolean.class, "false"), + + /** + * Name of the group under which client metrics appear + */ + METRICS_GROUP_NAME("metrics_name", String.class, "ch-http-pool"), + + HTTP_SAVE_COOKIES("client.http.cookies_enabled", Boolean.class, "false"), + + BINARY_READER_USE_PREALLOCATED_BUFFERS("client_allow_binary_reader_to_reuse_buffers", Boolean.class, "false"), + + /** + * Defines mapping between ClickHouse data type and target Java type + * Used by binary readers to convert values into desired Java type. + */ + TYPE_HINT_MAPPING("type_hint_mapping", Map.class), + + /** + * SNI SSL parameter that will be set for each outbound SSL socket. + */ + SSL_SOCKET_SNI("ssl_socket_sni", String.class,""), + + /** + * Prefix for custom settings. Should be aligned with server configuration. + * See ClickHouse Docs + */ + CUSTOM_SETTINGS_PREFIX("clickhouse_setting_", String.class, "custom_"), + + ; + + public static final String NO_THROW_ON_UNKNOWN_CONFIG = "no_throw_on_unknown_config"; + + private static final Logger LOG = LoggerFactory.getLogger(ClientConfigProperties.class); + + private final String key; + + private final Class valueType; + + private final String defaultValue; + + private final Object defaultObjValue; + + ClientConfigProperties(String key, Class valueType) { + this(key, valueType, null); + } + + ClientConfigProperties(String key, Class valueType, String defaultValue) { + this.key = key; + this.valueType = valueType; + this.defaultValue = defaultValue; + this.defaultObjValue = parseValue(defaultValue); + } + + public String getKey() { + return key; + } + + public String getDefaultValue() { + return defaultValue; + } + + @SuppressWarnings("unchecked") + public T getDefObjVal() { + return (T) defaultObjValue; + } + + public static final String HTTP_HEADER_PREFIX = "http_header_"; + + public static final String SERVER_SETTING_PREFIX = "clickhouse_setting_"; + + // Key used to identify default value in configuration map + public static final String DEFAULT_KEY = "_default_"; + + public static String serverSetting(String key) { + return SERVER_SETTING_PREFIX + key; + } + + public static String httpHeader(String key) { + return HTTP_HEADER_PREFIX + key.toUpperCase(Locale.US); + } + + public static String commaSeparated(Collection values) { + StringBuilder sb = new StringBuilder(); + for (Object value : values) { + sb.append(value.toString().replaceAll(",", "\\\\,")).append(","); + } + + if (sb.length() > 0) { + sb.setLength(sb.length() - 1); + } + return sb.toString(); + } + + public static List valuesFromCommaSeparated(String value) { + if (value == null || value.isEmpty()) { + return Collections.emptyList(); + } + + return Arrays.stream(value.split("(? s.replaceAll("\\\\,", ",")) + .collect(Collectors.toList()); + } + + public Object parseValue(String value) { + if (value == null) { + return null; + } + + if (valueType.equals(String.class)) { + return value; + } + + if (valueType.equals(Boolean.class)) { + if (value.equals("1")) return true; + if (value.equals("0")) return false; + return Boolean.parseBoolean(value); + } + + if (valueType.equals(Integer.class)) { + return Integer.parseInt(value); + } + + if (valueType.equals(Long.class)) { + return Long.parseLong(value); + } + + if (valueType.equals(List.class)) { + return valuesFromCommaSeparated(value); + } + + if (valueType.isEnum()) { + Object[] constants = valueType.getEnumConstants(); + for (Object constant : constants) { + if (constant.toString().equals(value)) { + return constant; + } + } + throw new IllegalArgumentException("Invalid constant name '" + value + "' for enum " + valueType.getName()); + } + + if (valueType.equals(TimeZone.class)) { + return TimeZone.getTimeZone(value); + } + + if (valueType.equals(Map.class)) { + return toKeyValuePairs(value); + } + + return null; + } + + @SuppressWarnings("unchecked") + public T getOrDefault(Map configMap) { + return (T) configMap.getOrDefault(getKey(), getDefObjVal()); + } + + public void applyIfSet(Map configMap, Consumer consumer) { + T value = (T) configMap.get(getKey()); + if (value != null) { + consumer.accept(value); + } + } + + public static Map parseConfigMap(Map configMap) { + Map parsedConfig = new HashMap<>(); + + Map tmpMap = new HashMap<>(configMap); + + for (ClientConfigProperties config : ClientConfigProperties.values()) { + String value = tmpMap.remove(config.getKey()); + if (value != null) { + Object parsedValue; + switch (config) { + case TYPE_HINT_MAPPING: + parsedValue = translateTypeHintMapping(value); + break; + default: + parsedValue = config.parseValue(value); + } + parsedConfig.put(config.getKey(), parsedValue); + } + } + + final String customSettingsPrefix = configMap.getOrDefault(ClientConfigProperties.CUSTOM_SETTINGS_PREFIX.getKey(), + CUSTOM_SETTINGS_PREFIX.getDefaultValue()); + for (String key : new HashSet<>(tmpMap.keySet())) { + if (key.startsWith(HTTP_HEADER_PREFIX) || key.startsWith(SERVER_SETTING_PREFIX)) { + parsedConfig.put(key, tmpMap.remove(key)); + } else if (key.startsWith(customSettingsPrefix)) { + parsedConfig.put(serverSetting(key), tmpMap.remove(key)); + } + } + + tmpMap.remove(ClientConfigProperties.NO_THROW_ON_UNKNOWN_CONFIG); + if (!tmpMap.isEmpty()) { + String msg = "Unknown and unmapped config properties: " + tmpMap.keySet(); + if (configMap.containsKey(NO_THROW_ON_UNKNOWN_CONFIG)) { + LOG.warn(msg); + } else { + throw new ClientMisconfigurationException(msg); + } + } + + return parsedConfig; + } + + + /** + * Converts given string to key value pairs. + * This is very simple implementation that do not handle edge cases like + * {@code k1=v1, ,k2=v2} + * + * @param str string + * @return non-null key value pairs + */ + public static Map toKeyValuePairs(String str) { + if (str == null || str.isEmpty()) { + return Collections.emptyMap(); + } + + Map map = new LinkedHashMap<>(); + String key = null; + StringBuilder builder = new StringBuilder(); + for (int i = 0, len = str.length(); i < len; i++) { + char ch = str.charAt(i); + if (ch == '\\' && i + 1 < len) { + ch = str.charAt(++i); + builder.append(ch); + continue; + } + + if (Character.isWhitespace(ch)) { + if (builder.length() > 0) { + builder.append(ch); + } + } else if (ch == '=' && key == null) { + key = builder.toString().trim(); + builder.setLength(0); + } else if (ch == ',' && key != null) { + String value = builder.toString().trim(); + builder.setLength(0); + if (!key.isEmpty() && !value.isEmpty()) { + map.put(key, value); + } + key = null; + } else { + builder.append(ch); + } + } + + if (key != null && builder.length() > 0) { + String value = builder.toString().trim(); + if (!key.isEmpty() && !value.isEmpty()) { + map.put(key, value); + } + } + + return Collections.unmodifiableMap(map); + } + + public static String mapToString(Map map, Function valueConverter) { + StringBuilder sb = new StringBuilder(); + for (Map.Entry entry : map.entrySet()) { + sb.append(entry.getKey()).append("=").append(valueConverter.apply(entry.getValue())).append(","); + } + + if (sb.length() > 0) { + sb.setLength(sb.length() - 1); + } + return sb.toString(); + } + + public static Map> translateTypeHintMapping(String mappingStr) { + if (mappingStr == null || mappingStr.isEmpty()) { + return AbstractBinaryFormatReader.NO_TYPE_HINT_MAPPING; + } + + Map mapping= ClientConfigProperties.toKeyValuePairs(mappingStr); + Map> hintMapping = new HashMap<>(); + try { + for (Map.Entry entry : mapping.entrySet()) { + hintMapping.put(ClickHouseDataType.of(entry.getKey()), + Class.forName(entry.getValue())); + } + } catch (ClassNotFoundException e) { + throw new ClientMisconfigurationException("Failed to translate type-hint mapping", e); + } + return hintMapping; + } +} diff --git a/client-v2/src/main/java/com/clickhouse/client/api/ServerException.java b/client-v2/src/main/java/com/clickhouse/client/api/ServerException.java index 0c6ed7574..53dd43429 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/ServerException.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/ServerException.java @@ -6,6 +6,8 @@ public class ServerException extends ClickHouseException { public static final int TABLE_NOT_FOUND = 60; + public static final int UNKNOWN_SETTING = 115; + private final int code; private final int transportProtocolCode; diff --git a/client-v2/src/test/java/com/clickhouse/client/ClientTests.java b/client-v2/src/test/java/com/clickhouse/client/ClientTests.java index d78953cac..714cc25f3 100644 --- a/client-v2/src/test/java/com/clickhouse/client/ClientTests.java +++ b/client-v2/src/test/java/com/clickhouse/client/ClientTests.java @@ -1,461 +1,489 @@ -package com.clickhouse.client; - -import com.clickhouse.client.api.Client; -import com.clickhouse.client.api.ClientConfigProperties; -import com.clickhouse.client.api.ClientException; -import com.clickhouse.client.api.ClientFaultCause; -import com.clickhouse.client.api.ConnectionReuseStrategy; -import com.clickhouse.client.api.command.CommandResponse; -import com.clickhouse.client.api.enums.Protocol; -import com.clickhouse.client.api.insert.InsertSettings; -import com.clickhouse.client.api.internal.ClickHouseLZ4OutputStream; -import com.clickhouse.client.api.internal.ServerSettings; -import com.clickhouse.client.api.metadata.DefaultColumnToMethodMatchingStrategy; -import com.clickhouse.client.api.query.GenericRecord; -import com.clickhouse.client.api.query.QueryResponse; -import com.clickhouse.client.api.query.QuerySettings; -import com.clickhouse.client.api.query.Records; -import com.clickhouse.client.config.ClickHouseClientOption; -import com.clickhouse.client.query.QueryTests; -import com.clickhouse.data.ClickHouseColumn; -import com.clickhouse.data.ClickHouseFormat; -import com.clickhouse.data.ClickHouseVersion; -import org.apache.commons.lang3.RandomStringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.testng.Assert; -import org.testng.annotations.DataProvider; -import org.testng.annotations.Test; -import org.testng.util.Strings; - -import java.io.ByteArrayInputStream; -import java.net.ConnectException; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; - -import static java.time.temporal.ChronoUnit.MILLIS; -import static java.time.temporal.ChronoUnit.SECONDS; - -public class ClientTests extends BaseIntegrationTest { - private static final Logger LOGGER = LoggerFactory.getLogger(ClientTests.class); - - @Test(groups = {"integration"}, dataProvider = "secureClientProvider") - public void testAddSecureEndpoint(Client client) { - if (isCloud()) { - return; // will fail in other tests - } - try { - Optional genericRecord = client - .queryAll("SELECT hostname()").stream().findFirst(); - Assert.assertTrue(genericRecord.isPresent()); - } catch (ClientException e) { - e.printStackTrace(); - if (e.getCause().getCause() instanceof ClickHouseException) { - Exception cause = (Exception) e.getCause().getCause().getCause(); - Assert.assertTrue(cause instanceof ConnectException); - // TODO: correct when SSL support is fully implemented. - Assert.assertTrue(cause.getMessage() - .startsWith("HTTP request failed: PKIX path building failed")); - return; - } - Assert.fail(e.getMessage()); - } finally { - client.close(); - } - } - - @DataProvider - public static Object[][] secureClientProvider() throws Exception { - ClickHouseNode node = ClickHouseServerForTest.getClickHouseNode(ClickHouseProtocol.HTTP, - true, ClickHouseNode.builder() - .addOption(ClickHouseClientOption.SSL_MODE.getKey(), "none") - .addOption(ClickHouseClientOption.SSL.getKey(), "true").build()); - return new Client[][]{ - { - new Client.Builder() - .addEndpoint("https://" + node.getHost() + ":" + node.getPort()) - .setUsername("default") - .setPassword("") - .setRootCertificate("containers/clickhouse-server/certs/localhost.crt") - .build() - }, - { - new Client.Builder() - .addEndpoint(Protocol.HTTP, node.getHost(), node.getPort(), true) - .setUsername("default") - .setPassword("") - .setRootCertificate("containers/clickhouse-server/certs/localhost.crt") - .setClientKey("user.key") - .setClientCertificate("user.crt") - .build() - } - }; - } - - @Test(groups = {"integration"}) - public void testRawSettings() { - Client client = newClient() - .setOption("custom_setting_1", "value_1") - .build(); - - client.execute("SELECT 1"); - - QuerySettings querySettings = new QuerySettings(); - querySettings.serverSetting("session_timezone", "Europe/Zurich"); - - try (Records response = - client.queryRecords("SELECT timeZone(), serverTimeZone()", querySettings).get(10, TimeUnit.SECONDS)) { - - response.forEach(record -> { - System.out.println(record.getString(1) + " " + record.getString(2)); - Assert.assertEquals("Europe/Zurich", record.getString(1)); - Assert.assertEquals("UTC", record.getString(2)); - }); - } catch (Exception e) { - Assert.fail(e.getMessage()); - } finally { - client.close(); - } - } - - @Test(groups = {"integration"}) - public void testPing() { - try (Client client = newClient().build()) { - Assert.assertTrue(client.ping()); - } - } - - @Test(groups = {"integration"}) - public void testPingUnpooled() { - try (Client client = newClient().enableConnectionPool(false).build()) { - Assert.assertTrue(client.ping()); - } - } - - @Test(groups = {"integration"}) - public void testPingFailure() { - try (Client client = new Client.Builder() - .addEndpoint("http://localhost:12345") - .setUsername("default") - .setPassword("") - .build()) { - Assert.assertFalse(client.ping(TimeUnit.SECONDS.toMillis(20))); - } - } - - @Test(groups = {"integration"}) - public void testPingAsync() { - try (Client client = newClient().useAsyncRequests(true).build()) { - Assert.assertTrue(client.ping()); - } - } - - @Test(groups = {"integration"}) - public void testSetOptions() { - Map options = new HashMap<>(); - String productName = "my product_name (version 1.0)"; - options.put(ClickHouseClientOption.PRODUCT_NAME.getKey(), productName); - try (Client client = newClient() - .setOptions(options).build()) { - - Assert.assertEquals(client.getConfiguration().get(ClickHouseClientOption.PRODUCT_NAME.getKey()), productName); - } - } - - @Test(groups = {"integration"}) - public void testProvidedExecutor() throws Exception { - - ExecutorService executorService = Executors.newSingleThreadExecutor(); - try (Client client = newClient().useAsyncRequests(true).setSharedOperationExecutor(executorService).build()) { - QueryResponse response = client.query("SELECT 1").get(); - response.getMetrics(); - } catch (Exception e) { - Assert.fail("unexpected exception", e); - } - - AtomicBoolean flag = new AtomicBoolean(true); - executorService.submit(() -> flag.compareAndSet(true, false)); - executorService.shutdown(); - executorService.awaitTermination(10, TimeUnit.SECONDS); - - Assert.assertFalse(flag.get()); - } - - @Test(groups = {"integration"}) - public void testLoadingServerContext() throws Exception { - long start = System.nanoTime(); - try (Client client = newClient().build()) { - long initTime = (System.nanoTime() - start) / 1_000_000; - Assert.assertTrue(initTime < 100); - Assert.assertEquals(client.getServerVersion(), "unknown"); - client.loadServerInfo(); - Assert.assertNotNull(client.getServerVersion()); - } - } - - @Test(groups = {"integration"}) - public void testDisableNative() { - try (Client client = newClient().disableNativeCompression(true).build()) { - Assert.assertTrue(client.toString().indexOf("JavaUnsafe") != -1); - } - } - - @Test(groups = {"integration"}) - public void testDefaultSettings() { - try (Client client = new Client.Builder().setUsername("default").setPassword("secret") - .addEndpoint("http://localhost:8123").build()) { - Map config = client.getConfiguration(); - for (ClientConfigProperties p : ClientConfigProperties.values()) { - if (p.getDefaultValue() != null) { - Assert.assertTrue(config.containsKey(p.getKey()), "Default value should be set for " + p.getKey()); - Assert.assertEquals(config.get(p.getKey()), p.getDefaultValue(), "Default value doesn't match"); - } - } - Assert.assertEquals(config.size(), 32); // to check everything is set. Increment when new added. - } - - try (Client client = new Client.Builder() - .setUsername("default") - .setPassword("secret") - .addEndpoint("http://localhost:8123") - .setDefaultDatabase("mydb") - .setExecutionTimeout(10, MILLIS) - .setLZ4UncompressedBufferSize(300_000) - .disableNativeCompression(true) - .useServerTimeZone(false) - .setServerTimeZone("America/Los_Angeles") - .useTimeZone("America/Los_Angeles") - .useAsyncRequests(true) - .setMaxConnections(330) - .setConnectionRequestTimeout(20, SECONDS) - .setConnectionReuseStrategy(ConnectionReuseStrategy.LIFO) - .enableConnectionPool(false) - .setConnectionTTL(30, SECONDS) - .retryOnFailures(ClientFaultCause.NoHttpResponse) - .setClientNetworkBufferSize(500_000) - .setMaxRetries(10) - .useHTTPBasicAuth(false) - .compressClientRequest(true) - .compressServerResponse(false) - .useHttpCompression(true) - .appCompressedData(true) - .setSocketTimeout(20, SECONDS) - .setSocketRcvbuf(100000) - .setSocketSndbuf(100000) - .build()) { - Map config = client.getConfiguration(); - Assert.assertEquals(config.size(), 33); // to check everything is set. Increment when new added. - Assert.assertEquals(config.get(ClientConfigProperties.DATABASE.getKey()), "mydb"); - Assert.assertEquals(config.get(ClientConfigProperties.MAX_EXECUTION_TIME.getKey()), "10"); - Assert.assertEquals(config.get(ClientConfigProperties.COMPRESSION_LZ4_UNCOMPRESSED_BUF_SIZE.getKey()), "300000"); - Assert.assertEquals(config.get(ClientConfigProperties.DISABLE_NATIVE_COMPRESSION.getKey()), "true"); - Assert.assertEquals(config.get(ClientConfigProperties.USE_SERVER_TIMEZONE.getKey()), "false"); - Assert.assertEquals(config.get(ClientConfigProperties.SERVER_TIMEZONE.getKey()), "America/Los_Angeles"); - Assert.assertEquals(config.get(ClientConfigProperties.ASYNC_OPERATIONS.getKey()), "true"); - Assert.assertEquals(config.get(ClientConfigProperties.HTTP_MAX_OPEN_CONNECTIONS.getKey()), "330"); - Assert.assertEquals(config.get(ClientConfigProperties.CONNECTION_REQUEST_TIMEOUT.getKey()), "20000"); - Assert.assertEquals(config.get(ClientConfigProperties.CONNECTION_REUSE_STRATEGY.getKey()), "LIFO"); - Assert.assertEquals(config.get(ClientConfigProperties.CONNECTION_POOL_ENABLED.getKey()), "false"); - Assert.assertEquals(config.get(ClientConfigProperties.CONNECTION_TTL.getKey()), "30000"); - Assert.assertEquals(config.get(ClientConfigProperties.CLIENT_RETRY_ON_FAILURE.getKey()), "NoHttpResponse"); - Assert.assertEquals(config.get(ClientConfigProperties.CLIENT_NETWORK_BUFFER_SIZE.getKey()), "500000"); - Assert.assertEquals(config.get(ClientConfigProperties.RETRY_ON_FAILURE.getKey()), "10"); - Assert.assertEquals(config.get(ClientConfigProperties.HTTP_USE_BASIC_AUTH.getKey()), "false"); - Assert.assertEquals(config.get(ClientConfigProperties.COMPRESS_CLIENT_REQUEST.getKey()), "true"); - Assert.assertEquals(config.get(ClientConfigProperties.COMPRESS_SERVER_RESPONSE.getKey()), "false"); - Assert.assertEquals(config.get(ClientConfigProperties.USE_HTTP_COMPRESSION.getKey()), "true"); - Assert.assertEquals(config.get(ClientConfigProperties.APP_COMPRESSED_DATA.getKey()), "true"); - Assert.assertEquals(config.get(ClientConfigProperties.SOCKET_OPERATION_TIMEOUT.getKey()), "20000"); - Assert.assertEquals(config.get(ClientConfigProperties.SOCKET_RCVBUF_OPT.getKey()), "100000"); - Assert.assertEquals(config.get(ClientConfigProperties.SOCKET_SNDBUF_OPT.getKey()), "100000"); - } - } - - @Test(groups = {"integration"}) - public void testWithOldDefaults() { - try (Client client = new Client.Builder() - .setUsername("default") - .setPassword("seceret") - .addEndpoint("http://localhost:8123") - .setDefaultDatabase("default") - .setExecutionTimeout(0, MILLIS) - .setLZ4UncompressedBufferSize(ClickHouseLZ4OutputStream.UNCOMPRESSED_BUFF_SIZE) - .disableNativeCompression(false) - .useServerTimeZone(true) - .setServerTimeZone("UTC") - .useAsyncRequests(false) - .setMaxConnections(10) - .setConnectionRequestTimeout(10, SECONDS) - .setConnectionReuseStrategy(ConnectionReuseStrategy.FIFO) - .enableConnectionPool(true) - .setConnectionTTL(-1, MILLIS) - .retryOnFailures(ClientFaultCause.NoHttpResponse, ClientFaultCause.ConnectTimeout, - ClientFaultCause.ConnectionRequestTimeout, ClientFaultCause.ServerRetryable) - .setClientNetworkBufferSize(300_000) - .setMaxRetries(3) - .allowBinaryReaderToReuseBuffers(false) - .columnToMethodMatchingStrategy(DefaultColumnToMethodMatchingStrategy.INSTANCE) - .useHTTPBasicAuth(true) - .compressClientRequest(false) - .compressServerResponse(true) - .useHttpCompression(false) - .appCompressedData(false) - .setSocketTimeout(0, SECONDS) - .setSocketRcvbuf(804800) - .setSocketSndbuf(804800) - .build()) { - Map config = client.getConfiguration(); - for (ClientConfigProperties p : ClientConfigProperties.values()) { - if (p.getDefaultValue() != null) { - Assert.assertTrue(config.containsKey(p.getKey()), "Default value should be set for " + p.getKey()); - Assert.assertEquals(config.get(p.getKey()), p.getDefaultValue(), "Default value doesn't match"); - } - } - Assert.assertEquals(config.size(), 32); // to check everything is set. Increment when new added. - } - } - - @DataProvider(name = "sessionRoles") - private static Object[][] sessionRoles() { - return new Object[][]{ - {new String[]{"ROL1", "ROL2"}}, - {new String[]{"ROL1", "ROL2"}}, - {new String[]{"ROL1", "ROL2"}}, - {new String[]{"ROL1", "ROL2,☺"}}, - {new String[]{"ROL1", "ROL2"}}, - }; - } - - @Test(groups = {"integration"}, dataProvider = "sessionRoles") - public void testOperationCustomRoles(String[] roles) throws Exception { - if (isVersionMatch("(,24.3]", newClient().build())) { - return; - } - - String password = "^1A" + RandomStringUtils.random(12, true, true) + "3b$"; - final String rolesList = "\"" + Strings.join("\",\"", roles) + "\""; - try (Client client = newClient().build()) { - client.execute("DROP ROLE IF EXISTS " + rolesList).get().close(); - client.execute("CREATE ROLE " + rolesList).get().close(); - client.execute("DROP USER IF EXISTS some_user").get().close(); - client.execute("CREATE USER some_user IDENTIFIED BY '" + password + "'").get().close(); - client.execute("GRANT " + rolesList + " TO some_user").get().close(); - } - - try (Client userClient = newClient().setUsername("some_user").setPassword(password).build()) { - QuerySettings settings = new QuerySettings().setDBRoles(Arrays.asList(roles)); - List resp = userClient.queryAll("SELECT currentRoles()", settings); - Set roleSet = new HashSet<>(Arrays.asList(roles)); - Set currentRoles = new HashSet (resp.get(0).getList(1)); - Assert.assertEquals(currentRoles, roleSet, "Roles " + roleSet + " not found in " + currentRoles); - } - } - - @DataProvider(name = "clientSessionRoles") - private static Object[][] clientSessionRoles() { - return new Object[][]{ - {new String[]{"ROL1", "ROL2"}}, - {new String[]{"ROL1", "ROL2,☺"}}, - }; - } - @Test(groups = {"integration"}, dataProvider = "clientSessionRoles") - public void testClientCustomRoles(String[] roles) throws Exception { - if (isVersionMatch("(,24.3]", newClient().build())) { - return; - } - - String password = "^1A" + RandomStringUtils.random(12, true, true) + "3B$"; - final String rolesList = "\"" + Strings.join("\",\"", roles) + "\""; - try (Client client = newClient().build()) { - client.execute("DROP ROLE IF EXISTS " + rolesList).get().close(); - client.execute("CREATE ROLE " + rolesList).get().close(); - client.execute("DROP USER IF EXISTS some_user").get().close(); - client.execute("CREATE USER some_user IDENTIFIED WITH sha256_password BY '" + password + "'").get().close(); - client.execute("GRANT " + rolesList + " TO some_user").get().close(); - } - - try (Client userClient = newClient().setUsername("some_user").setPassword(password).build()) { - userClient.setDBRoles(Arrays.asList(roles)); - List resp = userClient.queryAll("SELECT currentRoles()"); - Set roleSet = new HashSet<>(Arrays.asList(roles)); - Set currentRoles = new HashSet (resp.get(0).getList(1)); - Assert.assertEquals(currentRoles, roleSet, "Roles " + roleSet + " not found in " + currentRoles); - } - } - - - @Test(groups = {"integration"}) - public void testLogComment() throws Exception { - - String logComment = "Test log comment"; - QuerySettings settings = new QuerySettings() - .setQueryId(UUID.randomUUID().toString()) - .logComment(logComment); - - try (Client client = newClient().build()) { - - try (QueryResponse response = client.query("SELECT 1", settings).get()) { - Assert.assertNotNull(response.getQueryId()); - Assert.assertTrue(response.getQueryId().startsWith(settings.getQueryId())); - } - - client.execute("SYSTEM FLUSH LOGS").get().close(); - - List logRecords = client.queryAll("SELECT query_id, log_comment FROM clusterAllReplicas('default', system.query_log) WHERE query_id = '" + settings.getQueryId() + "'"); - Assert.assertEquals(logRecords.get(0).getString("query_id"), settings.getQueryId()); - Assert.assertEquals(logRecords.get(0).getString("log_comment"), logComment); - } - } - - @Test(groups = {"integration"}) - public void testServerSettings() throws Exception { - try (Client client = newClient().build()) { - client.execute("DROP TABLE IF EXISTS server_settings_test_table"); - client.execute("CREATE TABLE server_settings_test_table (v Float) Engine MergeTree ORDER BY ()"); - - final String queryId = UUID.randomUUID().toString(); - InsertSettings insertSettings = new InsertSettings() - .setQueryId(queryId) - .serverSetting(ServerSettings.ASYNC_INSERT, "1") - .serverSetting(ServerSettings.WAIT_ASYNC_INSERT, "1"); - - String csvData = "0.33\n0.44\n0.55\n"; - client.insert("server_settings_test_table", new ByteArrayInputStream(csvData.getBytes()), ClickHouseFormat.CSV, insertSettings).get().close(); - - client.execute("SYSTEM FLUSH LOGS").get().close(); - - List logRecords = client.queryAll("SELECT * FROM clusterAllReplicas('default', system.query_log) WHERE query_id = '" + queryId + "' AND type = 'QueryFinish'"); - - GenericRecord record = logRecords.get(0); - String settings = record.getString(record.getSchema().nameToColumnIndex("Settings")); - Assert.assertTrue(settings.contains(ServerSettings.ASYNC_INSERT + "=1")); -// Assert.assertTrue(settings.contains(ServerSettings.WAIT_ASYNC_INSERT + "=1")); // uncomment after server fix - } - } - - public boolean isVersionMatch(String versionExpression, Client client) { - List serverVersion = client.queryAll("SELECT version()"); - return ClickHouseVersion.of(serverVersion.get(0).getString(1)).check(versionExpression); - } - - - protected Client.Builder newClient() { - ClickHouseNode node = getServer(ClickHouseProtocol.HTTP); - boolean isSecure = isCloud(); - return new Client.Builder() - .addEndpoint(Protocol.HTTP, node.getHost(), node.getPort(), isSecure) - .setUsername("default") - .setPassword(ClickHouseServerForTest.getPassword()) - .setDefaultDatabase(ClickHouseServerForTest.getDatabase()); - } -} +package com.clickhouse.client; + +import com.clickhouse.client.api.Client; +import com.clickhouse.client.api.ClientConfigProperties; +import com.clickhouse.client.api.ClientException; +import com.clickhouse.client.api.ClientFaultCause; +import com.clickhouse.client.api.ClientMisconfigurationException; +import com.clickhouse.client.api.ConnectionReuseStrategy; +import com.clickhouse.client.api.ServerException; +import com.clickhouse.client.api.enums.Protocol; +import com.clickhouse.client.api.insert.InsertSettings; +import com.clickhouse.client.api.internal.ClickHouseLZ4OutputStream; +import com.clickhouse.client.api.internal.ServerSettings; +import com.clickhouse.client.api.metadata.DefaultColumnToMethodMatchingStrategy; +import com.clickhouse.client.api.query.GenericRecord; +import com.clickhouse.client.api.query.QueryResponse; +import com.clickhouse.client.api.query.QuerySettings; +import com.clickhouse.client.api.query.Records; +import com.clickhouse.client.config.ClickHouseClientOption; +import com.clickhouse.data.ClickHouseFormat; +import com.clickhouse.data.ClickHouseVersion; +import org.apache.commons.lang3.RandomStringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testng.Assert; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; +import org.testng.util.Strings; + +import java.io.ByteArrayInputStream; +import java.net.ConnectException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import static java.time.temporal.ChronoUnit.MILLIS; +import static java.time.temporal.ChronoUnit.SECONDS; + +public class ClientTests extends BaseIntegrationTest { + private static final Logger LOGGER = LoggerFactory.getLogger(ClientTests.class); + + @Test(groups = {"integration"}, dataProvider = "secureClientProvider") + public void testAddSecureEndpoint(Client client) { + if (isCloud()) { + return; // will fail in other tests + } + try { + Optional genericRecord = client + .queryAll("SELECT hostname()").stream().findFirst(); + Assert.assertTrue(genericRecord.isPresent()); + } catch (ClientException e) { + e.printStackTrace(); + if (e.getCause().getCause() instanceof ClickHouseException) { + Exception cause = (Exception) e.getCause().getCause().getCause(); + Assert.assertTrue(cause instanceof ConnectException); + // TODO: correct when SSL support is fully implemented. + Assert.assertTrue(cause.getMessage() + .startsWith("HTTP request failed: PKIX path building failed")); + return; + } + Assert.fail(e.getMessage()); + } finally { + client.close(); + } + } + + @DataProvider + public static Object[][] secureClientProvider() throws Exception { + ClickHouseNode node = ClickHouseServerForTest.getClickHouseNode(ClickHouseProtocol.HTTP, + true, ClickHouseNode.builder() + .addOption(ClickHouseClientOption.SSL_MODE.getKey(), "none") + .addOption(ClickHouseClientOption.SSL.getKey(), "true").build()); + return new Client[][]{ + { + new Client.Builder() + .addEndpoint("https://" + node.getHost() + ":" + node.getPort()) + .setUsername("default") + .setPassword("") + .setRootCertificate("containers/clickhouse-server/certs/localhost.crt") + .build() + }, + { + new Client.Builder() + .addEndpoint(Protocol.HTTP, node.getHost(), node.getPort(), true) + .setUsername("default") + .setPassword("") + .setRootCertificate("containers/clickhouse-server/certs/localhost.crt") + .setClientKey("user.key") + .setClientCertificate("user.crt") + .build() + } + }; + } + + @Test(groups = {"integration"}) + public void testRawSettings() { + Client client = newClient() + .setOption("custom_setting_1", "value_1") + .setOption(ClientConfigProperties.CUSTOM_SETTINGS_PREFIX.getKey(), isCloud()? "SQL_" : + ClientConfigProperties.CUSTOM_SETTINGS_PREFIX.getDefaultValue()) + .build(); + + client.execute("SELECT 1"); + + QuerySettings querySettings = new QuerySettings(); + querySettings.serverSetting("session_timezone", "Europe/Zurich"); + + try (Records response = + client.queryRecords("SELECT timeZone(), serverTimeZone()", querySettings).get(10, TimeUnit.SECONDS)) { + + response.forEach(record -> { + System.out.println(record.getString(1) + " " + record.getString(2)); + Assert.assertEquals("Europe/Zurich", record.getString(1)); + Assert.assertEquals("UTC", record.getString(2)); + }); + } catch (Exception e) { + Assert.fail(e.getMessage()); + } finally { + client.close(); + } + } + + @Test(groups = {"integration"}) + public void testPing() { + try (Client client = newClient().build()) { + Assert.assertTrue(client.ping()); + } + } + + @Test(groups = {"integration"}) + public void testPingUnpooled() { + try (Client client = newClient().enableConnectionPool(false).build()) { + Assert.assertTrue(client.ping()); + } + } + + @Test(groups = {"integration"}) + public void testPingFailure() { + try (Client client = new Client.Builder() + .addEndpoint("http://localhost:12345") + .setUsername("default") + .setPassword("") + .build()) { + Assert.assertFalse(client.ping(TimeUnit.SECONDS.toMillis(20))); + } + } + + @Test(groups = {"integration"}) + public void testPingAsync() { + try (Client client = newClient().useAsyncRequests(true).build()) { + Assert.assertTrue(client.ping()); + } + } + + @Test(groups = {"integration"}) + public void testSetOptions() { + Map options = new HashMap<>(); + String productName = "my product_name (version 1.0)"; + options.put(ClickHouseClientOption.PRODUCT_NAME.getKey(), productName); + try (Client client = newClient() + .setOptions(options).build()) { + + Assert.assertEquals(client.getConfiguration().get(ClickHouseClientOption.PRODUCT_NAME.getKey()), productName); + } + } + + @Test(groups = {"integration"}) + public void testProvidedExecutor() throws Exception { + + ExecutorService executorService = Executors.newSingleThreadExecutor(); + try (Client client = newClient().useAsyncRequests(true).setSharedOperationExecutor(executorService).build()) { + QueryResponse response = client.query("SELECT 1").get(); + response.getMetrics(); + } catch (Exception e) { + Assert.fail("unexpected exception", e); + } + + AtomicBoolean flag = new AtomicBoolean(true); + executorService.submit(() -> flag.compareAndSet(true, false)); + executorService.shutdown(); + executorService.awaitTermination(10, TimeUnit.SECONDS); + + Assert.assertFalse(flag.get()); + } + + @Test(groups = {"integration"}) + public void testLoadingServerContext() throws Exception { + long start = System.nanoTime(); + try (Client client = newClient().build()) { + long initTime = (System.nanoTime() - start) / 1_000_000; + Assert.assertTrue(initTime < 100); + Assert.assertEquals(client.getServerVersion(), "unknown"); + client.loadServerInfo(); + Assert.assertNotNull(client.getServerVersion()); + } + } + + @Test(groups = {"integration"}) + public void testDisableNative() { + try (Client client = newClient().disableNativeCompression(true).build()) { + Assert.assertTrue(client.toString().indexOf("JavaUnsafe") != -1); + } + } + + @Test(groups = {"integration"}) + public void testDefaultSettings() { + try (Client client = new Client.Builder().setUsername("default").setPassword("secret") + .addEndpoint("http://localhost:8123").build()) { + Map config = client.getConfiguration(); + for (ClientConfigProperties p : ClientConfigProperties.values()) { + if (p.getDefaultValue() != null) { + Assert.assertTrue(config.containsKey(p.getKey()), "Default value should be set for " + p.getKey()); + Assert.assertEquals(config.get(p.getKey()), p.getDefaultValue(), "Default value doesn't match"); + } + } + Assert.assertEquals(config.size(), 33); // to check everything is set. Increment when new added. + } + + try (Client client = new Client.Builder() + .setUsername("default") + .setPassword("secret") + .addEndpoint("http://localhost:8123") + .setDefaultDatabase("mydb") + .setExecutionTimeout(10, MILLIS) + .setLZ4UncompressedBufferSize(300_000) + .disableNativeCompression(true) + .useServerTimeZone(false) + .setServerTimeZone("America/Los_Angeles") + .useTimeZone("America/Los_Angeles") + .useAsyncRequests(true) + .setMaxConnections(330) + .setConnectionRequestTimeout(20, SECONDS) + .setConnectionReuseStrategy(ConnectionReuseStrategy.LIFO) + .enableConnectionPool(false) + .setConnectionTTL(30, SECONDS) + .retryOnFailures(ClientFaultCause.NoHttpResponse) + .setClientNetworkBufferSize(500_000) + .setMaxRetries(10) + .useHTTPBasicAuth(false) + .compressClientRequest(true) + .compressServerResponse(false) + .useHttpCompression(true) + .appCompressedData(true) + .setSocketTimeout(20, SECONDS) + .setSocketRcvbuf(100000) + .setSocketSndbuf(100000) + .build()) { + Map config = client.getConfiguration(); + Assert.assertEquals(config.size(), 34); // to check everything is set. Increment when new added. + Assert.assertEquals(config.get(ClientConfigProperties.DATABASE.getKey()), "mydb"); + Assert.assertEquals(config.get(ClientConfigProperties.MAX_EXECUTION_TIME.getKey()), "10"); + Assert.assertEquals(config.get(ClientConfigProperties.COMPRESSION_LZ4_UNCOMPRESSED_BUF_SIZE.getKey()), "300000"); + Assert.assertEquals(config.get(ClientConfigProperties.DISABLE_NATIVE_COMPRESSION.getKey()), "true"); + Assert.assertEquals(config.get(ClientConfigProperties.USE_SERVER_TIMEZONE.getKey()), "false"); + Assert.assertEquals(config.get(ClientConfigProperties.SERVER_TIMEZONE.getKey()), "America/Los_Angeles"); + Assert.assertEquals(config.get(ClientConfigProperties.ASYNC_OPERATIONS.getKey()), "true"); + Assert.assertEquals(config.get(ClientConfigProperties.HTTP_MAX_OPEN_CONNECTIONS.getKey()), "330"); + Assert.assertEquals(config.get(ClientConfigProperties.CONNECTION_REQUEST_TIMEOUT.getKey()), "20000"); + Assert.assertEquals(config.get(ClientConfigProperties.CONNECTION_REUSE_STRATEGY.getKey()), "LIFO"); + Assert.assertEquals(config.get(ClientConfigProperties.CONNECTION_POOL_ENABLED.getKey()), "false"); + Assert.assertEquals(config.get(ClientConfigProperties.CONNECTION_TTL.getKey()), "30000"); + Assert.assertEquals(config.get(ClientConfigProperties.CLIENT_RETRY_ON_FAILURE.getKey()), "NoHttpResponse"); + Assert.assertEquals(config.get(ClientConfigProperties.CLIENT_NETWORK_BUFFER_SIZE.getKey()), "500000"); + Assert.assertEquals(config.get(ClientConfigProperties.RETRY_ON_FAILURE.getKey()), "10"); + Assert.assertEquals(config.get(ClientConfigProperties.HTTP_USE_BASIC_AUTH.getKey()), "false"); + Assert.assertEquals(config.get(ClientConfigProperties.COMPRESS_CLIENT_REQUEST.getKey()), "true"); + Assert.assertEquals(config.get(ClientConfigProperties.COMPRESS_SERVER_RESPONSE.getKey()), "false"); + Assert.assertEquals(config.get(ClientConfigProperties.USE_HTTP_COMPRESSION.getKey()), "true"); + Assert.assertEquals(config.get(ClientConfigProperties.APP_COMPRESSED_DATA.getKey()), "true"); + Assert.assertEquals(config.get(ClientConfigProperties.SOCKET_OPERATION_TIMEOUT.getKey()), "20000"); + Assert.assertEquals(config.get(ClientConfigProperties.SOCKET_RCVBUF_OPT.getKey()), "100000"); + Assert.assertEquals(config.get(ClientConfigProperties.SOCKET_SNDBUF_OPT.getKey()), "100000"); + } + } + + @Test(groups = {"integration"}) + public void testWithOldDefaults() { + try (Client client = new Client.Builder() + .setUsername("default") + .setPassword("seceret") + .addEndpoint("http://localhost:8123") + .setDefaultDatabase("default") + .setExecutionTimeout(0, MILLIS) + .setLZ4UncompressedBufferSize(ClickHouseLZ4OutputStream.UNCOMPRESSED_BUFF_SIZE) + .disableNativeCompression(false) + .useServerTimeZone(true) + .setServerTimeZone("UTC") + .useAsyncRequests(false) + .setMaxConnections(10) + .setConnectionRequestTimeout(10, SECONDS) + .setConnectionReuseStrategy(ConnectionReuseStrategy.FIFO) + .enableConnectionPool(true) + .setConnectionTTL(-1, MILLIS) + .retryOnFailures(ClientFaultCause.NoHttpResponse, ClientFaultCause.ConnectTimeout, + ClientFaultCause.ConnectionRequestTimeout, ClientFaultCause.ServerRetryable) + .setClientNetworkBufferSize(300_000) + .setMaxRetries(3) + .allowBinaryReaderToReuseBuffers(false) + .columnToMethodMatchingStrategy(DefaultColumnToMethodMatchingStrategy.INSTANCE) + .useHTTPBasicAuth(true) + .compressClientRequest(false) + .compressServerResponse(true) + .useHttpCompression(false) + .appCompressedData(false) + .setSocketTimeout(0, SECONDS) + .setSocketRcvbuf(804800) + .setSocketSndbuf(804800) + .build()) { + Map config = client.getConfiguration(); + for (ClientConfigProperties p : ClientConfigProperties.values()) { + if (p.getDefaultValue() != null) { + Assert.assertTrue(config.containsKey(p.getKey()), "Default value should be set for " + p.getKey()); + Assert.assertEquals(config.get(p.getKey()), p.getDefaultValue(), "Default value doesn't match"); + } + } + Assert.assertEquals(config.size(), 33); // to check everything is set. Increment when new added. + } + } + + @DataProvider(name = "sessionRoles") + private static Object[][] sessionRoles() { + return new Object[][]{ + {new String[]{"ROL1", "ROL2"}}, + {new String[]{"ROL1", "ROL2"}}, + {new String[]{"ROL1", "ROL2"}}, + {new String[]{"ROL1", "ROL2,☺"}}, + {new String[]{"ROL1", "ROL2"}}, + }; + } + + @Test(groups = {"integration"}, dataProvider = "sessionRoles") + public void testOperationCustomRoles(String[] roles) throws Exception { + if (isVersionMatch("(,24.3]", newClient().build())) { + return; + } + + String password = "^1A" + RandomStringUtils.random(12, true, true) + "3b$"; + final String rolesList = "\"" + Strings.join("\",\"", roles) + "\""; + try (Client client = newClient().build()) { + client.execute("DROP ROLE IF EXISTS " + rolesList).get().close(); + client.execute("CREATE ROLE " + rolesList).get().close(); + client.execute("DROP USER IF EXISTS some_user").get().close(); + client.execute("CREATE USER some_user IDENTIFIED BY '" + password + "'").get().close(); + client.execute("GRANT " + rolesList + " TO some_user").get().close(); + } + + try (Client userClient = newClient().setUsername("some_user").setPassword(password).build()) { + QuerySettings settings = new QuerySettings().setDBRoles(Arrays.asList(roles)); + List resp = userClient.queryAll("SELECT currentRoles()", settings); + Set roleSet = new HashSet<>(Arrays.asList(roles)); + Set currentRoles = new HashSet (resp.get(0).getList(1)); + Assert.assertEquals(currentRoles, roleSet, "Roles " + roleSet + " not found in " + currentRoles); + } + } + + @DataProvider(name = "clientSessionRoles") + private static Object[][] clientSessionRoles() { + return new Object[][]{ + {new String[]{"ROL1", "ROL2"}}, + {new String[]{"ROL1", "ROL2,☺"}}, + }; + } + @Test(groups = {"integration"}, dataProvider = "clientSessionRoles") + public void testClientCustomRoles(String[] roles) throws Exception { + if (isVersionMatch("(,24.3]", newClient().build())) { + return; + } + + String password = "^1A" + RandomStringUtils.random(12, true, true) + "3B$"; + final String rolesList = "\"" + Strings.join("\",\"", roles) + "\""; + try (Client client = newClient().build()) { + client.execute("DROP ROLE IF EXISTS " + rolesList).get().close(); + client.execute("CREATE ROLE " + rolesList).get().close(); + client.execute("DROP USER IF EXISTS some_user").get().close(); + client.execute("CREATE USER some_user IDENTIFIED WITH sha256_password BY '" + password + "'").get().close(); + client.execute("GRANT " + rolesList + " TO some_user").get().close(); + } + + try (Client userClient = newClient().setUsername("some_user").setPassword(password).build()) { + userClient.setDBRoles(Arrays.asList(roles)); + List resp = userClient.queryAll("SELECT currentRoles()"); + Set roleSet = new HashSet<>(Arrays.asList(roles)); + Set currentRoles = new HashSet (resp.get(0).getList(1)); + Assert.assertEquals(currentRoles, roleSet, "Roles " + roleSet + " not found in " + currentRoles); + } + } + + + @Test(groups = {"integration"}) + public void testLogComment() throws Exception { + + String logComment = "Test log comment"; + QuerySettings settings = new QuerySettings() + .setQueryId(UUID.randomUUID().toString()) + .logComment(logComment); + + try (Client client = newClient().build()) { + + try (QueryResponse response = client.query("SELECT 1", settings).get()) { + Assert.assertNotNull(response.getQueryId()); + Assert.assertTrue(response.getQueryId().startsWith(settings.getQueryId())); + } + + client.execute("SYSTEM FLUSH LOGS").get().close(); + + List logRecords = client.queryAll("SELECT query_id, log_comment FROM clusterAllReplicas('default', system.query_log) WHERE query_id = '" + settings.getQueryId() + "'"); + Assert.assertEquals(logRecords.get(0).getString("query_id"), settings.getQueryId()); + Assert.assertEquals(logRecords.get(0).getString("log_comment"), logComment); + } + } + + @Test(groups = {"integration"}) + public void testServerSettings() throws Exception { + try (Client client = newClient().build()) { + client.execute("DROP TABLE IF EXISTS server_settings_test_table"); + client.execute("CREATE TABLE server_settings_test_table (v Float) Engine MergeTree ORDER BY ()"); + + final String queryId = UUID.randomUUID().toString(); + InsertSettings insertSettings = new InsertSettings() + .setQueryId(queryId) + .serverSetting(ServerSettings.ASYNC_INSERT, "1") + .serverSetting(ServerSettings.WAIT_ASYNC_INSERT, "1"); + + String csvData = "0.33\n0.44\n0.55\n"; + client.insert("server_settings_test_table", new ByteArrayInputStream(csvData.getBytes()), ClickHouseFormat.CSV, insertSettings).get().close(); + + client.execute("SYSTEM FLUSH LOGS").get().close(); + + List logRecords = client.queryAll("SELECT * FROM clusterAllReplicas('default', system.query_log) WHERE query_id = '" + queryId + "' AND type = 'QueryFinish'"); + + GenericRecord record = logRecords.get(0); + String settings = record.getString(record.getSchema().nameToColumnIndex("Settings")); + Assert.assertTrue(settings.contains(ServerSettings.ASYNC_INSERT + "=1")); +// Assert.assertTrue(settings.contains(ServerSettings.WAIT_ASYNC_INSERT + "=1")); // uncomment after server fix + } + } + + @Test(groups = {"integration"}) + public void testUnknownClientSettings() throws Exception { + try (Client client = newClient().setOption("unknown_setting", "value").build()) { + Assert.fail("Exception expected"); + } catch (Exception ex) { + Assert.assertTrue(ex instanceof ClientMisconfigurationException); + Assert.assertTrue(ex.getMessage().contains("unknown_setting")); + } + + try (Client client = newClient().setOption(ClientConfigProperties.NO_THROW_ON_UNKNOWN_CONFIG, "what ever").setOption("unknown_setting", "value").build()) { + Assert.assertTrue(client.ping()); + } + + try (Client client = newClient().setOption(ClientConfigProperties.SERVER_SETTING_PREFIX + "unknown_setting", "value").build()) { + try { + client.execute("SELECT 1"); + Assert.fail("Exception expected"); + } catch (ServerException e) { + Assert.assertEquals(e.getCode(), ServerException.UNKNOWN_SETTING); + } + } + + try (Client client = newClient().setOption(ClientConfigProperties.HTTP_HEADER_PREFIX + "unknown_setting", "value").build()) { + Assert.assertTrue(client.ping()); + } + } + + public boolean isVersionMatch(String versionExpression, Client client) { + List serverVersion = client.queryAll("SELECT version()"); + return ClickHouseVersion.of(serverVersion.get(0).getString(1)).check(versionExpression); + } + + + protected Client.Builder newClient() { + ClickHouseNode node = getServer(ClickHouseProtocol.HTTP); + boolean isSecure = isCloud(); + return new Client.Builder() + .addEndpoint(Protocol.HTTP, node.getHost(), node.getPort(), isSecure) + .setUsername("default") + .setPassword(ClickHouseServerForTest.getPassword()) + .setDefaultDatabase(ClickHouseServerForTest.getDatabase()); + } +} diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/DriverProperties.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/DriverProperties.java index 7c8732187..5b21e4077 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/DriverProperties.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/DriverProperties.java @@ -1,5 +1,6 @@ package com.clickhouse.jdbc; +import com.clickhouse.client.api.ClientConfigProperties; import com.clickhouse.client.api.internal.ServerSettings; import java.util.Arrays; @@ -98,4 +99,12 @@ public String getDefaultValue() { public List getChoices() { return choices; } + + public static String serverSetting(String key) { + return ClientConfigProperties.serverSetting(key); + } + + public static String httpHeader(String key) { + return ClientConfigProperties.httpHeader(key); + } } diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/JdbcConfiguration.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/JdbcConfiguration.java index fe601e51f..c0bb1a51b 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/JdbcConfiguration.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/JdbcConfiguration.java @@ -7,6 +7,7 @@ import com.clickhouse.jdbc.Driver; import com.clickhouse.jdbc.DriverProperties; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -22,6 +23,7 @@ import java.util.List; import java.util.Map; import java.util.Properties; +import java.util.Set; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -57,6 +59,17 @@ public boolean isIgnoreUnsupportedRequests() { return isIgnoreUnsupportedRequests; } + private static final Set DRIVER_PROP_KEYS; + static { + + ImmutableSet.Builder driverPropertiesMapBuilder = ImmutableSet.builder(); + for (DriverProperties prop : DriverProperties.values()) { + driverPropertiesMapBuilder.add(prop.getKey()); + } + + DRIVER_PROP_KEYS = driverPropertiesMapBuilder.build(); + } + /** * Parses URL to get property and target host. * Properties that are passed in the {@code info} parameter will override that are set in the {@code url}. @@ -227,6 +240,12 @@ private void initProperties(Map urlProperties, Properties provid // Copy provided properties Map props = new HashMap<>(); + // Set driver properties defaults (client will do the same) + for (DriverProperties prop : DriverProperties.values()) { + if (prop.getDefaultValue() != null) { + props.put(prop.getKey(), prop.getDefaultValue()); + } + } for (Map.Entry entry : providedProperties.entrySet()) { if (entry.getKey() instanceof String && entry.getValue() instanceof String) { props.put((String) entry.getKey(), (String) entry.getValue()); @@ -251,7 +270,12 @@ private void initProperties(Map urlProperties, Properties provid DriverPropertyInfo propertyInfo = new DriverPropertyInfo(prop.getKey(), prop.getValue()); propertyInfo.description = "(User Defined)"; propertyInfos.put(prop.getKey(), propertyInfo); - clientProperties.put(prop.getKey(), prop.getValue()); + + if (DRIVER_PROP_KEYS.contains(prop.getKey())) { + driverProperties.put(prop.getKey(), prop.getValue()); + } else { + clientProperties.put(prop.getKey(), prop.getValue()); + } } // Fill list of client properties information, add not specified properties (doesn't affect client properties) @@ -272,11 +296,6 @@ private void initProperties(Map urlProperties, Properties provid propertyInfo = new DriverPropertyInfo(driverProp.getKey(), driverProp.getDefaultValue()); propertyInfos.put(driverProp.getKey(), propertyInfo); } - - String value = clientProperties.get(driverProp.getKey()); - if (value != null) { - driverProperties.put(driverProp.getKey(), value); - } } listOfProperties = propertyInfos.values().stream().sorted(Comparator.comparing(o -> o.name)).collect(Collectors.toList()); diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/DriverTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/DriverTest.java index 7e1fb77df..1a023f7dc 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/DriverTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/DriverTest.java @@ -1,17 +1,22 @@ package com.clickhouse.jdbc; import com.clickhouse.client.api.ClientConfigProperties; +import com.clickhouse.client.api.ClientMisconfigurationException; +import com.clickhouse.client.api.ServerException; import org.testng.Assert; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; +import java.sql.Connection; import java.sql.DriverManager; import java.sql.DriverPropertyInfo; import java.sql.SQLException; +import java.sql.Statement; import java.util.HashMap; import java.util.Map; import java.util.Properties; +import static org.testng.Assert.assertEquals; import static org.testng.AssertJUnit.assertTrue; @@ -62,16 +67,16 @@ public void testGetPropertyInfo(String url, Properties props, Map33.4.6-jre 0.8.0 2.17.2 + 2.43.0 17 17 @@ -601,6 +602,11 @@ maven-resources-plugin ${resource-plugin.version} + + com.diffplug.spotless + spotless-maven-plugin + ${spotless.version} + @@ -687,6 +693,39 @@ org.apache.maven.plugins maven-resources-plugin + + com.diffplug.spotless + spotless-maven-plugin + + origin/main + + + + + + + + + + false + ,java,javax,\# + true + + + + google-java-format + + + + + + + apply + + compile + + +