Skip to content

HCD-181 AssertionError from CassandraStreamReader #1973

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 4 commits into
base: hcd-1.1-dev
Choose a base branch
from
Open

HCD-181 AssertionError from CassandraStreamReader #1973

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
e726210
HCD-181 AssertionError from CassandraStreamReader
Aug 29, 2025
88a1b57
Enhanced diags
Sep 3, 2025
a7f77fc
HCD-200 tracking in-flight compactions debug
Oct 8, 2025
9f5c223
HCD-200 extended diag
Oct 8, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions src/java/org/apache/cassandra/db/lifecycle/View.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import com.google.common.base.Predicate;
import com.google.common.collect.*;

import org.apache.commons.lang3.exception.ExceptionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down Expand Up @@ -105,6 +106,9 @@ public class View
this.sstablesByFilename = Maps.newHashMapWithExpectedSize(sstables.size());
for (SSTableReader sstable : this.sstables)
this.sstablesByFilename.put(sstable.getDataFile().name(), sstable);

if (!this.compacting.isEmpty())
logger.debug("New View {} at:\n{}", this, ExceptionUtils.getStackTrace(new Exception("HCD-200 debug - Ignore me")));
}

public Memtable getCurrentMemtable()
Expand Down Expand Up @@ -278,6 +282,9 @@ public static Function<View, Iterable<SSTableReader>> selectLive(AbstractBounds<
@VisibleForTesting
public static Function<View, View> updateCompacting(final Set<? extends SSTableReader> unmark, final Iterable<? extends SSTableReader> mark)
{
if (unmark != null && !unmark.isEmpty())
logger.debug("HCD-200 updateCompacting unmark {} at\n{}", unmark, ExceptionUtils.getStackTrace(new Exception("HCD-200 debug - Ignore me" )));

if (unmark.isEmpty() && Iterables.isEmpty(mark))
return Functions.identity();
return new Function<View, View>()
Expand Down
2 changes: 1 addition & 1 deletion src/java/org/apache/cassandra/db/streaming/CassandraStreamReader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ public CassandraStreamReader(StreamMessageHeader header, CassandraStreamHeader s
{
// we should only ever be streaming pending repair
// sstables if the session has a pending repair id
assert session.getPendingRepair().equals(header.pendingRepair);
assert session.getPendingRepair().equals(header.pendingRepair) : session.getPendingRepair() + " != " + header.pendingRepair;
}
this.session = session;
this.tableId = header.tableId;
Expand Down
6 changes: 5 additions & 1 deletion src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -223,9 +223,13 @@ else if (message.verb() == STATUS_RSP)
}
catch (Exception e)
{
logger.error("Got error processing {}, removing parent repair session", message.verb());
if (desc != null && desc.parentSessionId != null)
{
logger.error("Got error processing {}, removing parent repair session {}", message.verb(), desc.parentSessionId);
ActiveRepairService.instance.removeParentRepairSession(desc.parentSessionId);
}
else
logger.error("Got error processing {}, removing parent repair session", message.verb());
throw new RuntimeException(e);
}
}
Expand Down
3 changes: 2 additions & 1 deletion src/java/org/apache/cassandra/repair/consistent/CoordinatorSession.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.common.util.concurrent.SettableFuture;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.commons.lang3.time.DurationFormatUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand Down Expand Up @@ -274,7 +275,7 @@ public synchronized void fail()
logger.error("Can't transition endpoints {} to FAILED", cantFail, new RuntimeException());
return;
}
logger.info("Incremental repair session {} failed", sessionID);
logger.info("Incremental repair session {} failed \n {}", sessionID, ExceptionUtils.getStackTrace(new Exception("HCD-181 debug - Ignore me")));
sendFailureMessageToParticipants();
setAll(State.FAILED);

Expand Down
10 changes: 6 additions & 4 deletions src/java/org/apache/cassandra/repair/consistent/LocalSessions.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,8 @@
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;

import org.apache.commons.lang3.exception.ExceptionUtils;

import org.apache.cassandra.cql3.PageSize;
import org.apache.cassandra.db.compaction.AbstractCompactionTask;
import org.apache.cassandra.db.compaction.CleanupTask;
Expand Down Expand Up @@ -417,7 +419,7 @@ public boolean canCleanup(UUID sessionID)
*/
public void cancelSession(UUID sessionID, boolean force)
{
logger.info("Cancelling local repair session {}", sessionID);
logger.info("Cancelling local repair session {} \n {}", sessionID, ExceptionUtils.getStackTrace(new Exception("HCD-181 debug - Ignore me")));
LocalSession session = getSession(sessionID);
Preconditions.checkArgument(session != null, "Session {} does not exist", sessionID);
Preconditions.checkArgument(force || session.coordinator.equals(getBroadcastAddressAndPort()),
Expand Down Expand Up @@ -791,8 +793,8 @@ void setStateAndSave(LocalSession session, ConsistentSession.State state)
synchronized (session)
{
Preconditions.checkArgument(session.getState().canTransitionTo(state),
"Invalid state transition %s -> %s",
session.getState(), state);
"Invalid state transition %s -> %s for session %s",
session.getState(), state, session.sessionID);
logger.trace("Changing LocalSession state from {} -> {} for {}", session.getState(), state, session.sessionID);
boolean wasCompleted = session.isCompleted();
session.setState(state);
Expand Down Expand Up @@ -831,7 +833,7 @@ public void failSession(LocalSession session, boolean sendMessage)
}
else if (session.getState() != FAILED)
{
logger.info("Failing local repair session {}", session.sessionID);
logger.info("Failing local repair session {} \n {}", session.sessionID, ExceptionUtils.getStackTrace(new Exception("HCD-181 debug - Ignore me")));
setStateAndSave(session, FAILED);
}
}
Expand Down
3 changes: 3 additions & 0 deletions src/java/org/apache/cassandra/service/ActiveRepairService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,8 @@
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;

import org.apache.commons.lang3.exception.ExceptionUtils;

import org.apache.cassandra.concurrent.DebuggableThreadPoolExecutor;
import org.apache.cassandra.db.compaction.CompactionManager;
import org.apache.cassandra.locator.EndpointsByRange;
Expand Down Expand Up @@ -413,6 +415,7 @@ public void run()

public synchronized void terminateSessions()
{
logger.info("Terminating repair sessions \n {}", ExceptionUtils.getStackTrace(new Exception("HCD-181 debug - Ignore me")));
Throwable cause = new IOException("Terminate session is called");
for (RepairSession session : sessions.values())
{
Expand Down