From 8f04dff8557e0bf6b7e46fd5b3f359a47dcd10f2 Mon Sep 17 00:00:00 2001 From: Soby Chacko Date: Mon, 27 Jan 2025 19:39:58 -0500 Subject: [PATCH] Add filter-based deletion to Qdrant vector store Add string-based filter deletion alongside the Filter.Expression-based deletion for Qdrant vector store, providing consistent deletion capabilities with other vector store implementations. Key changes: - Add delete(Filter.Expression) implementation using Qdrant's filter API - Leverage existing QdrantFilterExpressionConverter for filter translation - Use Qdrant's native deleteAsync with filter capabilities - Add comprehensive integration tests for filter deletion - Support both simple and complex filter expressions This maintains consistency with other vector store implementations while utilizing Qdrant's native filtering capabilities for efficient metadata-based deletion. Signed-off-by: Soby Chacko --- .../vectorstore/qdrant/QdrantVectorStore.java | 31 ++++++- .../qdrant/QdrantVectorStoreIT.java | 90 ++++++++++++++++++- 2 files changed, 117 insertions(+), 4 deletions(-) diff --git a/vector-stores/spring-ai-qdrant-store/src/main/java/org/springframework/ai/vectorstore/qdrant/QdrantVectorStore.java b/vector-stores/spring-ai-qdrant-store/src/main/java/org/springframework/ai/vectorstore/qdrant/QdrantVectorStore.java index 9c8733040e7..a780da32a62 100644 --- a/vector-stores/spring-ai-qdrant-store/src/main/java/org/springframework/ai/vectorstore/qdrant/QdrantVectorStore.java +++ b/vector-stores/spring-ai-qdrant-store/src/main/java/org/springframework/ai/vectorstore/qdrant/QdrantVectorStore.java @@ -1,5 +1,5 @@ /* - * Copyright 2023-2024 the original author or authors. + * Copyright 2023-2025 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -32,13 +32,13 @@ import io.qdrant.client.grpc.Points.ScoredPoint; import io.qdrant.client.grpc.Points.SearchPoints; import io.qdrant.client.grpc.Points.UpdateStatus; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.ai.document.Document; import org.springframework.ai.document.DocumentMetadata; -import org.springframework.ai.embedding.BatchingStrategy; import org.springframework.ai.embedding.EmbeddingModel; import org.springframework.ai.embedding.EmbeddingOptionsBuilder; -import org.springframework.ai.embedding.TokenCountBatchingStrategy; import org.springframework.ai.model.EmbeddingUtils; import org.springframework.ai.observation.conventions.VectorStoreProvider; import org.springframework.ai.vectorstore.AbstractVectorStoreBuilder; @@ -126,6 +126,8 @@ */ public class QdrantVectorStore extends AbstractObservationVectorStore implements InitializingBean { + private static final Logger logger = LoggerFactory.getLogger(QdrantVectorStore.class); + public static final String DEFAULT_COLLECTION_NAME = "vector_store"; private static final String CONTENT_FIELD_NAME = "doc_content"; @@ -214,6 +216,29 @@ public Optional doDelete(List documentIds) { } } + @Override + protected void doDelete(org.springframework.ai.vectorstore.filter.Filter.Expression filterExpression) { + Assert.notNull(filterExpression, "Filter expression must not be null"); + + try { + Filter filter = this.filterExpressionConverter.convertExpression(filterExpression); + + io.qdrant.client.grpc.Points.UpdateResult response = this.qdrantClient + .deleteAsync(this.collectionName, filter) + .get(); + + if (response.getStatus() != io.qdrant.client.grpc.Points.UpdateStatus.Completed) { + throw new IllegalStateException("Failed to delete documents by filter: " + response.getStatus()); + } + + logger.debug("Deleted documents matching filter expression"); + } + catch (Exception e) { + logger.error("Failed to delete documents by filter: {}", e.getMessage(), e); + throw new IllegalStateException("Failed to delete documents by filter", e); + } + } + /** * Performs a similarity search on the vector store. * @param request The {@link SearchRequest} object containing the query and other diff --git a/vector-stores/spring-ai-qdrant-store/src/test/java/org/springframework/ai/vectorstore/qdrant/QdrantVectorStoreIT.java b/vector-stores/spring-ai-qdrant-store/src/test/java/org/springframework/ai/vectorstore/qdrant/QdrantVectorStoreIT.java index 8a05e32c549..0c4b06d43f6 100644 --- a/vector-stores/spring-ai-qdrant-store/src/test/java/org/springframework/ai/vectorstore/qdrant/QdrantVectorStoreIT.java +++ b/vector-stores/spring-ai-qdrant-store/src/test/java/org/springframework/ai/vectorstore/qdrant/QdrantVectorStoreIT.java @@ -1,5 +1,5 @@ /* - * Copyright 2023-2024 the original author or authors. + * Copyright 2023-2025 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,6 +21,7 @@ import java.util.Map; import java.util.UUID; import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; import io.qdrant.client.QdrantClient; import io.qdrant.client.QdrantGrpcClient; @@ -41,6 +42,7 @@ import org.springframework.ai.mistralai.api.MistralAiApi; import org.springframework.ai.vectorstore.SearchRequest; import org.springframework.ai.vectorstore.VectorStore; +import org.springframework.ai.vectorstore.filter.Filter; import org.springframework.boot.SpringBootConfiguration; import org.springframework.boot.test.context.runner.ApplicationContextRunner; import org.springframework.context.annotation.Bean; @@ -52,6 +54,7 @@ * @author Josh Long * @author Eddú Meléndez * @author Thomas Vitale + * @author Soby Chacko * @since 0.8.1 */ @Testcontainers @@ -256,6 +259,91 @@ public void searchThresholdTest() { }); } + @Test + void deleteByFilter() { + this.contextRunner.run(context -> { + VectorStore vectorStore = context.getBean(VectorStore.class); + + var bgDocument = new Document("The World is Big and Salvation Lurks Around the Corner", + Map.of("country", "Bulgaria", "number", 3)); + var nlDocument = new Document("The World is Big and Salvation Lurks Around the Corner", + Map.of("country", "Netherlands", "number", 90)); + + vectorStore.add(List.of(bgDocument, nlDocument)); + + Filter.Expression filterExpression = new Filter.Expression(Filter.ExpressionType.EQ, + new Filter.Key("country"), new Filter.Value("Bulgaria")); + + vectorStore.delete(filterExpression); + + List results = vectorStore + .similaritySearch(SearchRequest.builder().query("The World").topK(5).similarityThresholdAll().build()); + + assertThat(results).hasSize(1); + assertThat(results.get(0).getMetadata()).containsEntry("country", "Netherlands"); + + vectorStore.delete(List.of(nlDocument.getId())); + }); + } + + @Test + void deleteWithStringFilterExpression() { + this.contextRunner.run(context -> { + VectorStore vectorStore = context.getBean(VectorStore.class); + + var bgDocument = new Document("The World is Big and Salvation Lurks Around the Corner", + Map.of("country", "Bulgaria", "number", 3)); + var nlDocument = new Document("The World is Big and Salvation Lurks Around the Corner", + Map.of("country", "Netherlands", "number", 90)); + + vectorStore.add(List.of(bgDocument, nlDocument)); + + vectorStore.delete("number > 50"); + + List results = vectorStore + .similaritySearch(SearchRequest.builder().query("The World").topK(5).similarityThresholdAll().build()); + + assertThat(results).hasSize(1); + assertThat(results.get(0).getMetadata()).containsEntry("country", "Bulgaria"); + + vectorStore.delete(List.of(bgDocument.getId())); + }); + } + + @Test + void deleteWithComplexFilterExpression() { + this.contextRunner.run(context -> { + VectorStore vectorStore = context.getBean(VectorStore.class); + + var doc1 = new Document("Content 1", Map.of("type", "A", "priority", 1)); + var doc2 = new Document("Content 2", Map.of("type", "A", "priority", 2)); + var doc3 = new Document("Content 3", Map.of("type", "B", "priority", 1)); + + vectorStore.add(List.of(doc1, doc2, doc3)); + + // Complex filter expression: (type == 'A' AND priority > 1) + Filter.Expression priorityFilter = new Filter.Expression(Filter.ExpressionType.GT, + new Filter.Key("priority"), new Filter.Value(1)); + Filter.Expression typeFilter = new Filter.Expression(Filter.ExpressionType.EQ, new Filter.Key("type"), + new Filter.Value("A")); + Filter.Expression complexFilter = new Filter.Expression(Filter.ExpressionType.AND, typeFilter, + priorityFilter); + + vectorStore.delete(complexFilter); + + var results = vectorStore + .similaritySearch(SearchRequest.builder().query("Content").topK(5).similarityThresholdAll().build()); + + assertThat(results).hasSize(2); + assertThat(results.stream().map(doc -> doc.getMetadata().get("type")).collect(Collectors.toList())) + .containsExactlyInAnyOrder("A", "B"); + assertThat(results.stream().map(doc -> doc.getMetadata().get("priority")).collect(Collectors.toList())) + .containsExactlyInAnyOrder(1L, 1L); + + vectorStore.delete(List.of(doc1.getId(), doc3.getId())); + }); + } + @SpringBootConfiguration public static class TestApplication {