Skip to content

BM25 Release #333

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
xav-db merged 2 commits into from
Aug 6, 2025
Merged

BM25 Release #333

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
33e7d9f
fix clippy errors
lukasnxyz Aug 6, 2025
af0d5b5
uncoment import
lukasnxyz Aug 6, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,4 +23,5 @@ other_data/
*.jsonl
!examples/rag_demo/**
test-store/
helixdb-cfg/

2 changes: 1 addition & 1 deletion clippy_check.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1 +1 @@
cargo clippy --workspace --locked -- -D warnings -A clippy::too_many_arguments -A clippy::let-and-return -A clippy::module-inception -A clippy::new-ret-no-self -A clippy::wrong-self-convention -A clippy::large-enum-variant -A clippy::inherent-to-string -A clippy::inherent_to_string_shadow_display
cargo clippy --workspace --locked -- -D warnings -A clippy::too_many_arguments -A clippy::let-and-return -A clippy::module-inception -A clippy::new-ret-no-self -A clippy::wrong-self-convention -A clippy::large-enum-variant -A clippy::inherent-to-string -A clippy::inherent_to_string_shadow_display
1 change: 1 addition & 0 deletions helix-container/src/queries.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,3 +4,4 @@ use helix_db::helix_engine::graph_core::config::Config;
pub fn config() -> Option<Config> {
None
}

4 changes: 4 additions & 0 deletions helix-db/src/helix_engine/bm25/bm25.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ use crate::{
vector_core::{hnsw::HNSW, vector::HVector},
},
protocol::value::Value,
debug_println,
};

use heed3::{types::*, Database, Env, RoTxn, RwTxn};
Expand Down Expand Up @@ -335,6 +336,8 @@ impl BM25 for HBM25Config {
results.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
results.truncate(limit);

debug_println!("found {} results in bm25 search", results.len());

Ok(results)
}
}
Expand Down Expand Up @@ -429,6 +432,7 @@ impl BM25Flatten for HashMap<String, Value> {
s.push_str(k);
s.push(' ');
s.push_str(&v.to_string());
s.push(' ');
s
})
}
Expand Down
5 changes: 2 additions & 3 deletions helix-db/src/helix_engine/graph_core/ops/source/add_n.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,14 +98,13 @@ impl<'a, 'b, I: Iterator<Item = Result<TraversalVal, GraphError>>> AddNAdapter<'
}
}

if let Some(bm25) = &self.storage.bm25 {
if let Some(props) = node.properties.as_ref() {
if let Some(bm25) = &self.storage.bm25
&& let Some(props) = node.properties.as_ref() {
let mut data = props.flatten_bm25();
data.push_str(&node.label);
if let Err(e) = bm25.insert_doc(self.txn, node.id, &data) {
result = Err(e);
}
}
}

if result.is_ok() {
Expand Down
5 changes: 2 additions & 3 deletions helix-db/src/helix_engine/graph_core/ops/util/drop.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,9 @@ where
match item {
TraversalVal::Node(node) => match storage.drop_node(txn, &node.id) {
Ok(_) => {
if let Some(bm25) = &storage.bm25 {
if let Err(e) = bm25.delete_doc(txn, node.id) {
if let Some(bm25) = &storage.bm25
&& let Err(e) = bm25.delete_doc(txn, node.id) {
println!("failed to delete doc from bm25: {e}");
}
}
println!("Dropped node: {:?}", node.id);
Ok(())
Expand Down
10 changes: 4 additions & 6 deletions helix-db/src/helix_engine/graph_core/ops/util/update.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,8 +58,8 @@ impl<'scope, 'env, I: Iterator<Item = Result<TraversalVal, GraphError>>> UpdateA

if let Some(ref props) = props {
for (key, _new_value) in props.iter() {
if let Some(db) = storage.secondary_indices.get(key) {
if let Some(old_value) = properties.get(key) {
if let Some(db) = storage.secondary_indices.get(key)
&& let Some(old_value) = properties.get(key) {
match bincode::serialize(old_value) {
Ok(old_serialized) => {
if let Err(e) = db.delete_one_duplicate(
Expand All @@ -73,7 +73,6 @@ impl<'scope, 'env, I: Iterator<Item = Result<TraversalVal, GraphError>>> UpdateA
Err(e) => vec.push(Err(GraphError::from(e))),
}
}
}
}
}

Expand Down Expand Up @@ -128,13 +127,12 @@ impl<'scope, 'env, I: Iterator<Item = Result<TraversalVal, GraphError>>> UpdateA
Ok(TraversalVal::Edge(edge)) => match storage.get_edge(self.txn, &edge.id) {
Ok(old_edge) => {
let mut old_edge = old_edge.clone();
if let Some(mut properties) = old_edge.properties.clone() {
if let Some(ref props) = props {
if let Some(mut properties) = old_edge.properties.clone()
&& let Some(ref props) = props {
for (k, v) in props.iter() {
properties.insert(k.clone(), v.clone());
}
old_edge.properties = Some(properties);
}
}
match old_edge.encode_edge() {
Ok(serialized) => {
Expand Down
11 changes: 4 additions & 7 deletions helix-db/src/helix_engine/vector_core/utils.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,14 +124,11 @@ impl VectorFilter for BinaryHeap<HVector> {
None => None, // TODO: maybe should be an error?
};

if SHOULD_CHECK_DELETED {
if let Ok(is_deleted) = item.check_property("is_deleted") {
if let Value::Boolean(is_deleted) = is_deleted.as_ref() {
if *is_deleted {
if SHOULD_CHECK_DELETED
&& let Ok(is_deleted) = item.check_property("is_deleted")
&& let Value::Boolean(is_deleted) = is_deleted.as_ref()
&& *is_deleted {
continue;
}
}
}
}

if item.label() == label
Expand Down
5 changes: 2 additions & 3 deletions helix-db/src/helix_gateway/builtin/all_nodes_and_edges.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,11 +144,10 @@ fn get_all_nodes_edges_json(

if let Some(prop) = &node_label {
let node = Node::decode_node(value, id)?;
if let Some(props) = node.properties {
if let Some(prop_value) = props.get(prop) {
if let Some(props) = node.properties
&& let Some(prop_value) = props.get(prop) {
json_node["label"] = sonic_rs::to_value(&prop_value.to_string())
.unwrap_or_else(|_| sonic_rs::Value::from(""));
}
}
}
nodes.push(json_node);
Expand Down
5 changes: 2 additions & 3 deletions helix-db/src/helixc/analyzer/methods/graph_step_validation.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -359,10 +359,9 @@ pub(crate) fn apply_graph_step<'a>(
cur_ty.kind_str()
);
}
if let Some(ref ty) = sv.vector_type {
if !ctx.vector_set.contains(ty.as_str()) {
if let Some(ref ty) = sv.vector_type
&& !ctx.vector_set.contains(ty.as_str()) {
generate_error!(ctx, original_query, sv.loc.clone(), E103, ty.as_str());
}
}
let vec = match &sv.data {
Some(VectorData::Vector(v)) => {
Expand Down
10 changes: 4 additions & 6 deletions helix-db/src/helixc/analyzer/methods/infer_expr_type.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -715,10 +715,9 @@ pub(crate) fn infer_expr_type<'a>(
// Type::Vector(add.vector_type.as_deref())
// }
SearchVector(sv) => {
if let Some(ref ty) = sv.vector_type {
if !ctx.vector_set.contains(ty.as_str()) {
if let Some(ref ty) = sv.vector_type
&& !ctx.vector_set.contains(ty.as_str()) {
generate_error!(ctx, original_query, sv.loc.clone(), E103, ty.as_str());
}
}
let vec: VecData = match &sv.data {
Some(VectorData::Vector(v)) => {
Expand Down Expand Up @@ -980,16 +979,15 @@ pub(crate) fn infer_expr_type<'a>(
Empty => (Type::Unknown, Some(GeneratedStatement::Empty)),
BM25Search(bm25_search) => {
// TODO: look into how best do type checking for type passed in
if let Some(ref ty) = bm25_search.type_arg {
if !ctx.node_set.contains(ty.as_str()) {
if let Some(ref ty) = bm25_search.type_arg
&& !ctx.node_set.contains(ty.as_str()) {
generate_error!(
ctx,
original_query,
bm25_search.loc.clone(),
E101,
ty.as_str()
);
}
}
let vec = match &bm25_search.data {
Some(ValueType::Literal { value, loc: _ }) => {
Expand Down
10 changes: 4 additions & 6 deletions helix-db/src/helixc/analyzer/methods/migration_validation.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,19 +130,17 @@ pub(crate) fn validate_migration(ctx: &mut Ctx, migration: &Migration) {
}

// check default value is valid for the new field type
if let Some(default) = &default {
if to_property_field.field_type != *default {
if let Some(default) = &default
&& to_property_field.field_type != *default {
// schema error - default value is not valid for the new field type
panic!("default value is not valid for the new field type");
}
}

// check the cast is valid for the new field type
if let Some(cast) = &cast {
if to_property_field.field_type != cast.cast_to {
if let Some(cast) = &cast
&& to_property_field.field_type != cast.cast_to {
// schema error - cast is not valid for the new field type
panic!("cast is not valid for the new field type");
}
}

// // warnings if name is same
Expand Down
5 changes: 2 additions & 3 deletions helix-db/src/helixc/analyzer/methods/query_validation.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,8 @@ pub(crate) fn validate_query<'a>(ctx: &mut Ctx<'a>, original_query: &'a Query) {
// Parameter validation
// -------------------------------------------------
for param in &original_query.parameters {
if let FieldType::Identifier(ref id) = param.param_type.1 {
if is_valid_identifier(ctx, original_query, param.param_type.0.clone(), id.as_str()) {
if let FieldType::Identifier(ref id) = param.param_type.1
&& is_valid_identifier(ctx, original_query, param.param_type.0.clone(), id.as_str()) {
// TODO: add support for edges
if !ctx.node_set.contains(id.as_str()) {
generate_error!(
Expand All @@ -51,7 +51,6 @@ pub(crate) fn validate_query<'a>(ctx: &mut Ctx<'a>, original_query: &'a Query) {
&param.name.1
);
}
}
}
// constructs parameters and sub‑parameters for generator
GeneratedParameter::unwrap_param(
Expand Down
54 changes: 23 additions & 31 deletions helix-db/src/helixc/analyzer/methods/traversal_validation.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,19 +89,17 @@ pub(crate) fn validate_traversal<'a>(
);
} else if let ValueType::Literal { ref value, ref loc } =
*value
{
if !field.field_type.eq(value) {
generate_error!(
ctx,
original_query,
loc.clone(),
E205,
&value.to_string(),
&field.field_type.to_string(),
"node",
node_type
);
}
&& !field.field_type.eq(value) {
generate_error!(
ctx,
original_query,
loc.clone(),
E205,
&value.to_string(),
&field.field_type.to_string(),
"node",
node_type
);
}
}
None => {
Expand Down Expand Up @@ -291,10 +289,9 @@ pub(crate) fn validate_traversal<'a>(
parent
}
StartNode::SearchVector(sv) => {
if let Some(ref ty) = sv.vector_type {
if !ctx.vector_set.contains(ty.as_str()) {
if let Some(ref ty) = sv.vector_type
&& !ctx.vector_set.contains(ty.as_str()) {
generate_error!(ctx, original_query, sv.loc.clone(), E103, ty.as_str());
}
}
let vec: VecData = match &sv.data {
Some(VectorData::Vector(v)) => {
Expand Down Expand Up @@ -1055,10 +1052,9 @@ pub(crate) fn validate_traversal<'a>(
}

StepType::AddEdge(add) => {
if let Some(ref ty) = add.edge_type {
if !ctx.edge_map.contains_key(ty.as_str()) {
if let Some(ref ty) = add.edge_type
&& !ctx.edge_map.contains_key(ty.as_str()) {
generate_error!(ctx, original_query, add.loc.clone(), E102, ty);
}
}
cur_ty = Type::Edges(add.edge_type.clone());
excluded.clear();
Expand All @@ -1077,8 +1073,8 @@ pub(crate) fn validate_traversal<'a>(
scope,
i.as_str(),
);
if let Some(ty) = ty {
if !ty.is_integer() {
if let Some(ty) = ty
&& !ty.is_integer() {
generate_error!(
ctx,
original_query,
Expand All @@ -1088,12 +1084,11 @@ pub(crate) fn validate_traversal<'a>(
[i.as_str()]
);
return cur_ty.clone(); // Not sure if this should be here
}
};
let ty =
type_in_scope(ctx, original_query, end.loc.clone(), scope, j.as_str());
if let Some(ty) = ty {
if !ty.is_integer() {
if let Some(ty) = ty
&& !ty.is_integer() {
generate_error!(
ctx,
original_query,
Expand All @@ -1103,7 +1098,6 @@ pub(crate) fn validate_traversal<'a>(
[j.as_str()]
);
return cur_ty.clone(); // Not sure if this should be here
}
}
(
gen_identifier_or_param(original_query, i.as_str(), false, true),
Expand All @@ -1124,8 +1118,8 @@ pub(crate) fn validate_traversal<'a>(
scope,
i.as_str(),
);
if let Some(ty) = ty {
if !ty.is_integer() {
if let Some(ty) = ty
&& !ty.is_integer() {
generate_error!(
ctx,
original_query,
Expand All @@ -1135,7 +1129,6 @@ pub(crate) fn validate_traversal<'a>(
[i.as_str()]
);
return cur_ty.clone(); // Not sure if this should be here
}
}

(
Expand All @@ -1147,8 +1140,8 @@ pub(crate) fn validate_traversal<'a>(
is_valid_identifier(ctx, original_query, end.loc.clone(), j.as_str());
let ty =
type_in_scope(ctx, original_query, end.loc.clone(), scope, j.as_str());
if let Some(ty) = ty {
if !ty.is_integer() {
if let Some(ty) = ty
&& !ty.is_integer() {
generate_error!(
ctx,
original_query,
Expand All @@ -1158,7 +1151,6 @@ pub(crate) fn validate_traversal<'a>(
[j.as_str()]
);
return cur_ty.clone();
}
}
(
GeneratedValue::Primitive(GenRef::Std(i.to_string())),
Expand Down
1 change: 1 addition & 0 deletions helix-db/src/helixc/generator/queries.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,3 +120,4 @@ impl Display for Parameter {
}
}
}

1 change: 1 addition & 0 deletions hql-tests/file14/file14.hx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,3 +6,4 @@ N::File14 {
QUERY file14() =>
res <- SearchBM25<File14>("John", 10)
RETURN res

Loading