Cassandra in error state due to MalformedInputException error
Applies to
StorageGRID 11
Issue
- After running postinstall-recovery.sh script,
/var/local/log/servermanager.logreports Cassandra failed to start and is in error state due to too many retries.
/var/local/log/cassandra/system.loghas the following exception:
ERROR [main] 2023-11-02 12:38:08,609 CassandraDaemon.java (line 737) Exception encountered during startup
java.lang.RuntimeException: java.nio.charset.MalformedInputException: Input length = 1
at org.apache.cassandra.io.util.FileUtils.readLines(FileUtils.java:613)
at org.apache.cassandra.db.lifecycle.LogReplica.readLines(LogReplica.java:74)
at java.util.stream.Collectors.lambda$toMap$58(Collectors.java:1321)
at java.util.stream.ReduceOps$3ReducingSink.accept(ReduceOps.java:169)
at java.util.Iterator.forEachRemaining(Iterator.java:116)
at java.util.Spliterators$IteratorSpliterator.forEachRemaining(Spliterators.java:1801)
at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:482)
at java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:472)
at java.util.stream.ReduceOps$ReduceOp.evaluateSequential(ReduceOps.java:708)
at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
at java.util.stream.ReferencePipeline.collect(ReferencePipeline.java:566)
at org.apache.cassandra.db.lifecycle.LogReplicaSet.readRecords(LogReplicaSet.java:111)
at org.apache.cassandra.db.lifecycle.LogFile.verify(LogFile.java:161)
at org.apache.cassandra.db.lifecycle.LogTransaction$LogFilesByName.removeUnfinishedLeftovers(LogTransaction.java:478)
at java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
at java.util.HashMap$EntrySpliterator.tryAdvance(HashMap.java:1744)
at java.util.stream.ReferencePipeline.forEachWithCancel(ReferencePipeline.java:126)
at java.util.stream.AbstractPipeline.copyIntoWithCancel(AbstractPipeline.java:499)
at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:486)
at java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:472)
at java.util.stream.MatchOps$MatchOp.evaluateSequential(MatchOps.java:230)
at java.util.stream.MatchOps$MatchOp.evaluateSequential(MatchOps.java:196)
at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
at java.util.stream.ReferencePipeline.allMatch(ReferencePipeline.java:521)
at org.apache.cassandra.db.lifecycle.LogTransaction$LogFilesByName.removeUnfinishedLeftovers(LogTransaction.java:471)
at org.apache.cassandra.db.lifecycle.LogTransaction.removeUnfinishedLeftovers(LogTransaction.java:438)
at org.apache.cassandra.db.lifecycle.LogTransaction.removeUnfinishedLeftovers(LogTransaction.java:430)
at org.apache.cassandra.db.lifecycle.LifecycleTransaction.removeUnfinishedLeftovers(LifecycleTransaction.java:544)
at org.apache.cassandra.db.ColumnFamilyStore.scrubDataDirectories(ColumnFamilyStore.java:656)
at org.apache.cassandra.service.CassandraDaemon.setup(CassandraDaemon.java:280)
at org.apache.cassandra.service.CassandraDaemon.activate(CassandraDaemon.java:631)
at org.apache.cassandra.service.CassandraDaemon.main(CassandraDaemon.java:720)
Caused by: java.nio.charset.MalformedInputException: Input length = 1
at java.nio.charset.CoderResult.throwException(CoderResult.java:281)
at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:339)
at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:178)
at java.io.InputStreamReader.read(InputStreamReader.java:184)
at java.io.BufferedReader.fill(BufferedReader.java:161)
at java.io.BufferedReader.readLine(BufferedReader.java:324)
at java.io.BufferedReader.readLine(BufferedReader.java:389)
at java.nio.file.Files.readAllLines(Files.java:3205)
at org.apache.cassandra.io.util.FileUtils.readLines(FileUtils.java:606)
... 31 common frames omitted