Sentry: db.go:866: log.Fatal: pebble: fatal commit error: read-only file system (1) attached stack trace -- stack trace: | github.com/cockroachdb/pebble.(*DB).applyInternal | github.com/cockroa...

[cockroach-sentry]

This issue was auto filed by Sentry. It represents a crash or reported error on a live cluster with telemetry enabled.

Sentry Link: https://cockroach-labs.sentry.io/issues/5995828461/?referrer=webhooks_plugin

Panic Message:

db.go:866: log.Fatal: pebble: fatal commit error: read-only file system
(1) attached stack trace
  -- stack trace:
  | github.com/cockroachdb/pebble.(*DB).applyInternal
  | 	github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:866
  | github.com/cockroachdb/pebble.(*DB).Apply
  | 	github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:792
  | github.com/cockroachdb/pebble.(*DB).Set
  | 	github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:629
  | github.com/cockroachdb/cockroach/pkg/storage.(*Pebble).put
  | 	github.com/cockroachdb/cockroach/pkg/storage/pebble.go:2010
  | github.com/cockroachdb/cockroach/pkg/storage.(*Pebble).PutUnversioned
  | 	github.com/cockroachdb/cockroach/pkg/storage/pebble.go:1995
  | github.com/cockroachdb/cockroach/pkg/storage.(*putBuffer).putInlineMeta
  | 	github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:1806
  | github.com/cockroachdb/cockroach/pkg/storage.mvccPutInternal
  | 	github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:2353
  | github.com/cockroachdb/cockroach/pkg/storage.mvccPutUsingIter
  | 	github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:2038
  | github.com/cockroachdb/cockroach/pkg/storage.MVCCPut
  | 	github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:1945
  | github.com/cockroachdb/cockroach/pkg/storage.MVCCPutProto
  | 	github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:1109
  | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).WriteLastUpTimestamp
  | 	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store.go:2782
  | github.com/cockroachdb/cockroach/pkg/server.NewServer.func5.1
  | 	github.com/cockroachdb/cockroach/pkg/server/server.go:539
  | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).VisitStores.func1
  | 	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:149
  | github.com/cockroachdb/cockroach/pkg/util/syncutil.(*Map[...]).Range
  | 	github.com/cockroachdb/cockroach/pkg/util/syncutil/map.go:383
  | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).VisitStores
  | 	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:148
  | github.com/cockroachdb/cockroach/pkg/server.NewServer.func5
  | 	github.com/cockroachdb/cockroach/pkg/server/server.go:538
  | github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness.(*NodeLiveness).Start.func1
  | 	github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness/liveness.go:671
  | github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2
  | 	github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:485
  | runtime.goexit
  | 	src/runtime/asm_arm64.s:1222
Wraps: (2) secondary error attachment
  | read-only file system
  | (1) read-only file system
  | Error types: (1) syscall.Errno
Wraps: (3) log.Fatal: pebble: fatal commit error: read-only file system
Error types: (1) *withstack.withStack (2) *secondary.withSecondaryError (3) *errutil.leafError
-- report composition:
*errutil.leafError: log.Fatal: pebble: fatal commit error: read-only file system
*secondary.withSecondaryError
db.go:866: *withstack.withStack (top exception)

Stacktrace (expand for inline code snippets):

src/runtime/asm_arm64.s#L1221-L1223

cockroach/pkg/util/stop/stopper.go

Lines 484 to 486 in 217b43e

	sp.UpdateGoroutineIDToCurrent()
	f(ctx)
	}()

cockroach/pkg/kv/kvserver/liveness/liveness.go

Lines 670 to 672 in 217b43e

	} else if nl.onSelfHeartbeat != nil {
	nl.onSelfHeartbeat(ctx)
	}

cockroach/pkg/server/server.go

Lines 537 to 539 in 217b43e

	now := clock.Now()
	if err := stores.VisitStores(func(s *kvserver.Store) error {
	return s.WriteLastUpTimestamp(ctx, now)

https://github.com/cockroachdb/cockroach/blob/217b43e69da61e9712d73e3e19fd5076fbdb26df/pkg/kv/kvserver/pkg/kv/kvserver/stores.go#L147-L149

cockroach/pkg/util/syncutil/map.go

Lines 382 to 384 in 217b43e

	}
	if !f(k, v) {
	break

https://github.com/cockroachdb/cockroach/blob/217b43e69da61e9712d73e3e19fd5076fbdb26df/pkg/kv/kvserver/pkg/kv/kvserver/stores.go#L148-L150

cockroach/pkg/server/server.go

Lines 538 to 540 in 217b43e

	if err := stores.VisitStores(func(s *kvserver.Store) error {
	return s.WriteLastUpTimestamp(ctx, now)
	}); err != nil {

https://github.com/cockroachdb/cockroach/blob/217b43e69da61e9712d73e3e19fd5076fbdb26df/pkg/kv/kvserver/pkg/kv/kvserver/store.go#L2781-L2783

cockroach/pkg/storage/mvcc.go

Lines 1108 to 1110 in 217b43e

	value.InitChecksum(key)
	_, err := MVCCPut(ctx, rw, key, timestamp, value, opts)
	return err

cockroach/pkg/storage/mvcc.go

Lines 1944 to 1946 in 217b43e

	}
	return mvccPutUsingIter(ctx, rw, iter, ltScanner, key, timestamp, value, nil, opts)
	}

cockroach/pkg/storage/mvcc.go

Lines 2037 to 2039 in 217b43e

	// do can call mvccPutInternal directly.
	_, acq, err := mvccPutInternal(
	ctx, writer, iter, ltScanner, key, timestamp, value, buf, valueFn, opts)

cockroach/pkg/storage/mvcc.go

Lines 2352 to 2354 in 217b43e

	buf.newMeta = enginepb.MVCCMetadata{RawBytes: value.RawBytes}
	metaKeySize, metaValSize, err = buf.putInlineMeta(writer, metaKey, &buf.newMeta)
	}

cockroach/pkg/storage/mvcc.go

Lines 1805 to 1807 in 217b43e

	}
	if err := writer.PutUnversioned(key.Key, bytes); err != nil {
	return 0, 0, err

cockroach/pkg/storage/pebble.go

Lines 1994 to 1996 in 217b43e

	func (p *Pebble) PutUnversioned(key roachpb.Key, value []byte) error {
	return p.put(MVCCKey{Key: key}, value)
	}

cockroach/pkg/storage/pebble.go

Lines 2009 to 2011 in 217b43e

	}
	return p.db.Set(EncodeMVCCKey(key), value, pebble.Sync)
	}

github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go#L628-L630
github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go#L791-L793
github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go#L865-L867

src/runtime/asm_arm64.s in runtime.goexit at line 1222
pkg/util/stop/stopper.go in pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2 at line 485
pkg/kv/kvserver/liveness/liveness.go in pkg/kv/kvserver/liveness.(*NodeLiveness).Start.func1 at line 671
pkg/server/server.go in pkg/server.NewServer.func5 at line 538
pkg/kv/kvserver/pkg/kv/kvserver/stores.go in pkg/kv/kvserver.(*Stores).VisitStores at line 148
pkg/util/syncutil/map.go in pkg/util/syncutil.(*Map[...]).Range at line 383
pkg/kv/kvserver/pkg/kv/kvserver/stores.go in pkg/kv/kvserver.(*Stores).VisitStores.func1 at line 149
pkg/server/server.go in pkg/server.NewServer.func5.1 at line 539
pkg/kv/kvserver/pkg/kv/kvserver/store.go in pkg/kv/kvserver.(*Store).WriteLastUpTimestamp at line 2782
pkg/storage/mvcc.go in pkg/storage.MVCCPutProto at line 1109
pkg/storage/mvcc.go in pkg/storage.MVCCPut at line 1945
pkg/storage/mvcc.go in pkg/storage.mvccPutUsingIter at line 2038
pkg/storage/mvcc.go in pkg/storage.mvccPutInternal at line 2353
pkg/storage/mvcc.go in pkg/storage.(*putBuffer).putInlineMeta at line 1806
pkg/storage/pebble.go in pkg/storage.(*Pebble).PutUnversioned at line 1995
pkg/storage/pebble.go in pkg/storage.(*Pebble).put at line 2010
github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go in github.com/cockroachdb/pebble.(*DB).Set at line 629
github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go in github.com/cockroachdb/pebble.(*DB).Apply at line 792
github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go in github.com/cockroachdb/pebble.(*DB).applyInternal at line 866

Tags

Tag	Value
Command	server
Environment	v24.2.3
Go Version	go1.22.5 X:nocoverageredesign
Platform	linux arm64
Distribution	CCL
Cockroach Release	v24.2.3
Cockroach SHA	217b43e
# of CPUs	2
# of Goroutines	177

Jira issue: CRDB-43257