TiKV Client Initialization Fails with "Store Not Match" and "Not Leader" Errors in Rust Where As Go Client is running properly

[divyeshkakadiya-inventyv]

• Rust Tikv Client Where the issue is generate

/// Creates a new TiKV client, initializing it if necessary.
#[async_recursion]
pub async fn create_client(retry: Option<u32>) -> Result<Arc<RwLock<Client>>, String> {
    let retry = retry.unwrap_or_else(|| get_tikv_retry_count());

    // Clone the existing client lock
    let client_lock = TIKV.clone();

    //Check if the prometheus is initialized
    if IS_PROMETHEUS_INIT.read().await.to_owned() == false {
        prometheus::ParometheusService::init();
        let mut is_init = IS_PROMETHEUS_INIT.write().await;
        *is_init = true;
    }

    {
        let client_guard = client_lock.read().await;
        if let Some(ref client) = *client_guard {
            log::info!("Using existing TiKV client");
            return Ok(Arc::new(RwLock::new(client.clone())));
        }
    }

    // Prepare default configuration
    let config = Config::default();

    // Check if TLS is enabled for the cluster
    let tls_cluster_enabled = env::var("TIKV_TLS_ENABLED").map(|v| v.to_lowercase() == "true").unwrap_or(false);

    // Retrieve TiKV host address from environment variables
    let host = env::var("TIKV_SERVER").expect("TIKV_SERVER must be set in .env");

    // Create the client with or without security based on TLS settings
    let client = if tls_cluster_enabled {
        // Retrieve SSL certificate paths from environment variables

        let ssl_ca_cert = env::var("TIKV_SSL_CA_CERT").expect("TIKV_SSL_CA_CERT must be set");
        let ssl_client_cert = env::var("TIKV_SSL_CLIENT_CERT").expect("TIKV_SSL_CLIENT_CERT must be set");
        let ssl_client_key = env::var("TIKV_SSL_CLIENT_KEY").expect("TIKV_SSL_CLIENT_KEY must be set");

        // Configure client with security settings
        let with_security_config = config.with_security(ssl_ca_cert, ssl_client_cert, ssl_client_key).with_timeout(Duration::from_secs(120));
        Client::new_with_config(vec![host], with_security_config).await
    } else {
        // Configure client without security settings
        Client::new_with_config(vec![host], config).await
    };

    match client {
        Ok(client) => {

            // Enable atomic operations for CAS (Compare-And-Swap)
            let new_client = client.with_atomic_for_cas();

            let mut client_guard = client_lock.write().await;
            *client_guard = Some(new_client.clone());

            let client_lock = Arc::new(RwLock::new(new_client));
            log::info!("Created new TiKV client");
            Ok(client_lock)
        }
        Err(error) => {
            if retry <= 0 {
                log::error!("Error creating TiKV client: {}", error);
                return Err(error.to_string());
            }
            reset_client_conn().await;
            time::sleep(Duration::from_millis(10)).await;
            match create_client(Some(retry - 1)).await {
                Ok(client) => Ok(client),
                Err(error) => {
                    log::error!("Error creating TiKV client: {}", error);
                    Err(error.to_string())
                }
            }
        }
    }
}

• Go Client Working Fine

func initializeClient() (err error) {

	if client == nil {

		rawkv.MaxRawKVScanLimit = 50000
		rawkv.ErrMaxScanLimitExceeded = logging.EnrichErrorWithStackTrace(errors.New("limit should be less than MaxRawKVScanLimit"))

		if tlsClusterEnabled {
			security := tikvConfig.NewSecurity(sslCACerti, sslClientCerti, sslClientKeyCerti, []string{""})
			client, err = rawkv.NewClient(context.TODO(), []string{tikvServer}, security)
			if err != nil {
				err = logging.EnrichErrorWithStackTrace(err)
			}
		} else {
			client, err = rawkv.NewClient(context.TODO(), []string{tikvServer}, tikvConfig.DefaultConfig().Security)
			if err != nil {
				err = logging.EnrichErrorWithStackTrace(err)
			}
		}
	}
	return
}

Error

1. Retry failed in get document for this key : CRM-BFF-CONFIG-development error is : Region error: Error { message: "to store id 11001, mine 11002", not_leader: None, region_not_found: None, key_not_in_region: None, epoch_not_match: None, server_is_busy: None, stale_command: None, store_not_match: Some(StoreNotMatch { request_store_id: 11001, actual_store_id: 11002 }), raft_entry_too_large: None, max_timestamp_not_synced: None, read_index_not_ready: None, proposal_in_merging_mode: None, data_is_not_ready: None, region_not_initialized: None, disk_full: None }

2. Retry failed in get document for this key : CRM-BFF-CONFIG-development error is : gRPC api error: status: Unavailable, message: "not leader and follower handling not allowed", details: [ ], metadata: MetadataMap { headers: {"access-control-allow-headers": "accept, content-type, authorization", "access-control-allow-methods": "POST, GET, OPTIONS, PUT, DELETE", "access-control-allow-origin": "*", "content-type": "application/grpc", "trailer": "Grpc-Status", "trailer": "Grpc-Message", "trailer": "Grpc-Status-Details-Bin"} }

Generating Intermittent "Not Leader" and Consistent "Store Not Match" Errors in TiKV

Cargo.toml :
tikv-client = "0.2.0"

[pingyu]

Should be fixed by #484. Please check that, and reopen this issue if it's not. Thanks.

[divyeshkakadiya-inventyv]

Issue still persists