Skip to content

Commit dc35072

Browse files
DracoLiDracoLi
authored
feat(x/blockdb): prevent multi-process access via file locks (#5420)
1 parent 15c98fb commit dc35072

6 files changed

Lines changed: 256 additions & 10 deletions

File tree

‎x/blockdb/BUILD.bazel‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ go_library(
88
"config.go",
99
"database.go",
1010
"errors.go",
11+
"lock.go",
1112
],
1213
importpath = "github.com/ava-labs/avalanchego/x/blockdb",
1314
visibility = ["//visibility:public"],
@@ -30,6 +31,7 @@ go_test(
3031
"database_test.go",
3132
"datasplit_test.go",
3233
"helpers_test.go",
34+
"lock_test.go",
3335
"readblock_test.go",
3436
"recovery_test.go",
3537
"writeblock_test.go",

‎x/blockdb/README.md‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,18 @@ On startup, BlockDB checks for signs of an unclean shutdown by comparing the dat
122122
3. Calculates the max block height
123123
4. Updates the index header with the updated max block height and next write offset
124124

125+
### Single-Process Access
126+
127+
BlockDB does not support concurrent access from multiple processes. It acquires an exclusive advisory file lock on a `LOCK` file at open time; a second process attempting to open the same database fails immediately. Within a single process, the database is safe to use from multiple goroutines.
128+
129+
`IndexDir` and `DataDir` must be dedicated to BlockDB. Sharing folders with other databases risks filename conflicts (e.g., the conventional `LOCK` file).
130+
131+
**Locked paths.** A `LOCK` file is created and locked in `IndexDir`, and (when distinct) in `DataDir`.
132+
133+
**Lifecycle.** The lock is acquired before any database files are opened or recovery is attempted, and released after all files are closed. If the process exits unexpectedly (panic, `SIGKILL`, OOM kill), the OS releases the lock automatically.
134+
135+
**Limits.** The lock is advisory and only coordinates between cooperating processes. It does not prevent non-cooperating tools (`rm`, backup utilities, editors) from modifying or deleting the database files. Deleting the `LOCK` file while a database is open breaks the protection.
136+
125137
## Usage
126138

127139
### Creating a Database

‎x/blockdb/database.go‎

Lines changed: 22 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -171,6 +171,7 @@ func (h *indexFileHeader) UnmarshalBinary(data []byte) error {
171171
// Database stores blockchain blocks on disk and provides methods to read and write blocks.
172172
type Database struct {
173173
indexFile *os.File
174+
locks *dbLocks
174175
config DatabaseConfig
175176
header indexFileHeader
176177
log logging.Logger
@@ -197,7 +198,7 @@ type Database struct {
197198
// Parameters:
198199
// - config: Configuration parameters
199200
// - log: Logger instance for structured logging
200-
func New(config DatabaseConfig, log logging.Logger) (database.HeightIndex, error) {
201+
func New(config DatabaseConfig, log logging.Logger) (_ database.HeightIndex, err error) {
201202
if err := config.Validate(); err != nil {
202203
return nil, err
203204
}
@@ -209,7 +210,6 @@ func New(config DatabaseConfig, log logging.Logger) (database.HeightIndex, error
209210

210211
// from benchmarks, zstd.BestSpeed is about 100% faster than the default
211212
// compression level while giving us ~5% better compression ratio than Snappy.
212-
var err error
213213
compressor, err := compression.NewZstdCompressorWithLevel(math.MaxUint32, zstd.BestSpeed)
214214
if err != nil {
215215
return nil, fmt.Errorf("failed to initialize compressor: %w", err)
@@ -234,6 +234,21 @@ func New(config DatabaseConfig, log logging.Logger) (database.HeightIndex, error
234234
zap.Uint16("blockCacheSize", config.BlockCacheSize),
235235
)
236236

237+
// Locks must be acquired before any database file is opened or recovery is
238+
// attempted; otherwise two processes can race on recovery and corrupt the index.
239+
s.locks, err = acquireDBLocks(config.IndexDir, config.DataDir)
240+
if err != nil {
241+
s.log.Error("Failed to acquire directory lock", zap.Error(err))
242+
return nil, err
243+
}
244+
defer func() {
245+
if err != nil {
246+
if rErr := s.locks.Release(); rErr != nil {
247+
s.log.Error("Failed to release directory lock after failed initialization", zap.Error(rErr))
248+
}
249+
}
250+
}()
251+
237252
if err := s.openAndInitializeIndex(); err != nil {
238253
s.log.Error("Failed to initialize database: failed to initialize index", zap.Error(err))
239254
return nil, err
@@ -275,11 +290,15 @@ func (s *Database) Close() error {
275290

276291
err := s.persistIndexHeader()
277292
if err != nil {
278-
s.log.Error("Failed to close database: failed to persist index header", zap.Error(err))
293+
s.log.Error("Failed to persist index header", zap.Error(err))
279294
}
280295

281296
s.closeFiles()
282297

298+
if rErr := s.locks.Release(); rErr != nil {
299+
s.log.Error("Failed to release directory lock", zap.Error(rErr))
300+
}
301+
283302
s.log.Info("Block database closed successfully")
284303
return err
285304
}
@@ -907,9 +926,6 @@ func (s *Database) listDataFiles() (map[int]string, int, error) {
907926

908927
func (s *Database) openAndInitializeIndex() error {
909928
indexPath := filepath.Join(s.config.IndexDir, indexFileName)
910-
if err := os.MkdirAll(s.config.IndexDir, 0o755); err != nil {
911-
return fmt.Errorf("failed to create index directory %s: %w", s.config.IndexDir, err)
912-
}
913929
openFlags := os.O_RDWR | os.O_CREATE
914930
var err error
915931
s.indexFile, err = os.OpenFile(indexPath, openFlags, defaultFilePermissions)
@@ -920,10 +936,6 @@ func (s *Database) openAndInitializeIndex() error {
920936
}
921937

922938
func (s *Database) initializeDataFiles() error {
923-
if err := os.MkdirAll(s.config.DataDir, 0o755); err != nil {
924-
return fmt.Errorf("failed to create data directory %s: %w", s.config.DataDir, err)
925-
}
926-
927939
// Pre-load the data file for the next write offset.
928940
nextOffset := s.nextDataWriteOffset.Load()
929941
if nextOffset > 0 {

‎x/blockdb/errors.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,4 +9,6 @@ var (
99
ErrInvalidBlockHeight = errors.New("blockdb: invalid block height")
1010
ErrCorrupted = errors.New("blockdb: unrecoverable corruption detected")
1111
ErrBlockTooLarge = errors.New("blockdb: block size too large")
12+
13+
errDatabaseInUse = errors.New("database directory is locked by another process")
1214
)

‎x/blockdb/lock.go‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
// Copyright (C) 2019, Ava Labs, Inc. All rights reserved.
2+
// See the file LICENSE for licensing terms.
3+
4+
package blockdb
5+
6+
import (
7+
"errors"
8+
"fmt"
9+
"os"
10+
"path/filepath"
11+
"syscall"
12+
)
13+
14+
const lockFileName = "LOCK"
15+
16+
type heldLock struct {
17+
path string
18+
file *os.File
19+
}
20+
21+
type dbLocks struct {
22+
locks []heldLock
23+
}
24+
25+
// acquireDBLocks creates indexDir and dataDir if needed and acquires exclusive
26+
// file locks in each. Returns errDatabaseInUse if another process holds a lock.
27+
func acquireDBLocks(indexDir, dataDir string) (_ *dbLocks, retErr error) {
28+
idx := filepath.Clean(indexDir)
29+
data := filepath.Clean(dataDir)
30+
31+
paths := []string{idx}
32+
// Lock both directories when they differ; the index and data files must
33+
// be used together as one database.
34+
if idx != data {
35+
paths = append(paths, data)
36+
}
37+
38+
l := &dbLocks{locks: make([]heldLock, 0, len(paths))}
39+
defer func() {
40+
if retErr != nil {
41+
_ = l.Release()
42+
}
43+
}()
44+
for _, dir := range paths {
45+
if err := os.MkdirAll(dir, 0o755); err != nil {
46+
return nil, fmt.Errorf("failed to create directory %s: %w", dir, err)
47+
}
48+
held, err := tryLockDir(dir)
49+
if err != nil {
50+
return nil, err
51+
}
52+
l.locks = append(l.locks, held)
53+
}
54+
return l, nil
55+
}
56+
57+
func tryLockDir(dir string) (heldLock, error) {
58+
path := filepath.Join(dir, lockFileName)
59+
f, err := os.OpenFile(path, os.O_CREATE|os.O_RDONLY, defaultFilePermissions)
60+
if err != nil {
61+
return heldLock{}, fmt.Errorf("failed to open lock file %s: %w", path, err)
62+
}
63+
if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil {
64+
_ = f.Close()
65+
if errors.Is(err, syscall.EWOULDBLOCK) {
66+
return heldLock{}, fmt.Errorf("%w: %s", errDatabaseInUse, dir)
67+
}
68+
return heldLock{}, fmt.Errorf("failed to acquire lock on %s: %w", dir, err)
69+
}
70+
return heldLock{path: path, file: f}, nil
71+
}
72+
73+
// Release unlocks every lock held by l.
74+
//
75+
// LOCK files are intentionally not removed. If removed, a process can
76+
// acquire the lock between release and removal, and after removal another
77+
// process can create a new LOCK file at the same path and lock it. Both
78+
// processes would then hold the database lock.
79+
func (l *dbLocks) Release() error {
80+
var errs []error
81+
for _, h := range l.locks {
82+
if err := syscall.Flock(int(h.file.Fd()), syscall.LOCK_UN); err != nil {
83+
errs = append(errs, fmt.Errorf("failed to release lock on %s: %w", h.path, err))
84+
}
85+
if err := h.file.Close(); err != nil {
86+
errs = append(errs, fmt.Errorf("failed to close lock file %s: %w", h.path, err))
87+
}
88+
}
89+
l.locks = nil
90+
return errors.Join(errs...)
91+
}

‎x/blockdb/lock_test.go‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
// Copyright (C) 2019, Ava Labs, Inc. All rights reserved.
2+
// See the file LICENSE for licensing terms.
3+
4+
package blockdb
5+
6+
import (
7+
"io"
8+
"os"
9+
"path/filepath"
10+
"testing"
11+
12+
"github.com/stretchr/testify/require"
13+
14+
"github.com/ava-labs/avalanchego/database"
15+
"github.com/ava-labs/avalanchego/utils/logging"
16+
)
17+
18+
func tryOpenAt(t *testing.T, indexDir, dataDir string) (database.HeightIndex, error) {
19+
t.Helper()
20+
21+
config := DefaultConfig().
22+
WithIndexDir(indexDir).
23+
WithDataDir(dataDir).
24+
WithBlockCacheSize(0)
25+
db, err := New(config, logging.NoLog{})
26+
if err != nil {
27+
return db, err
28+
}
29+
t.Cleanup(func() {
30+
_ = db.Close()
31+
})
32+
return db, nil
33+
}
34+
35+
func TestLock_DirectoryOverlap(t *testing.T) {
36+
type dbDirs struct {
37+
index string
38+
data string
39+
}
40+
41+
tests := []struct {
42+
name string
43+
first dbDirs
44+
second dbDirs
45+
wantErr error
46+
}{
47+
{
48+
name: "same",
49+
first: dbDirs{index: "same", data: "same"},
50+
second: dbDirs{index: "same", data: "same"},
51+
wantErr: errDatabaseInUse,
52+
},
53+
{
54+
name: "same_index",
55+
first: dbDirs{index: "index", data: "data1"},
56+
second: dbDirs{index: "index", data: "data2"},
57+
wantErr: errDatabaseInUse,
58+
},
59+
{
60+
name: "same_data",
61+
first: dbDirs{index: "index1", data: "data"},
62+
second: dbDirs{index: "index2", data: "data"},
63+
wantErr: errDatabaseInUse,
64+
},
65+
{
66+
name: "same_split_dirs",
67+
first: dbDirs{index: "index", data: "data"},
68+
second: dbDirs{index: "index", data: "data"},
69+
wantErr: errDatabaseInUse,
70+
},
71+
{
72+
name: "different_single_dirs",
73+
first: dbDirs{index: "db1", data: "db1"},
74+
second: dbDirs{index: "db2", data: "db2"},
75+
},
76+
{
77+
name: "different_split_dirs",
78+
first: dbDirs{index: "index1", data: "data1"},
79+
second: dbDirs{index: "index2", data: "data2"},
80+
},
81+
}
82+
83+
for _, test := range tests {
84+
t.Run(test.name, func(t *testing.T) {
85+
dir := t.TempDir()
86+
index1 := filepath.Join(dir, test.first.index)
87+
data1 := filepath.Join(dir, test.first.data)
88+
_, err := tryOpenAt(t, index1, data1)
89+
require.NoError(t, err)
90+
91+
index2 := filepath.Join(dir, test.second.index)
92+
data2 := filepath.Join(dir, test.second.data)
93+
_, err = tryOpenAt(t, index2, data2)
94+
require.ErrorIs(t, err, test.wantErr)
95+
})
96+
}
97+
}
98+
99+
func TestLock_AllowsReopenAfterClose(t *testing.T) {
100+
dir := t.TempDir()
101+
102+
db1, err := tryOpenAt(t, dir, dir)
103+
require.NoError(t, err)
104+
require.NoError(t, db1.Close())
105+
106+
_, err = tryOpenAt(t, dir, dir)
107+
require.NoError(t, err)
108+
}
109+
110+
func TestLock_AllowsOpenAfterError(t *testing.T) {
111+
dir := t.TempDir()
112+
113+
// Pre-write a corrupt index file so the first New() fails when loading the
114+
// index (an error unrelated to locking). The database should be openable
115+
// again once the corrupt file is removed.
116+
indexPath := filepath.Join(dir, indexFileName)
117+
require.NoError(t, os.WriteFile(indexPath, []byte("not a valid blockdb header"), 0o600))
118+
119+
db, err := tryOpenAt(t, dir, dir)
120+
require.Nil(t, db)
121+
require.ErrorIs(t, err, io.EOF)
122+
123+
require.NoError(t, os.Remove(indexPath))
124+
125+
_, err = tryOpenAt(t, dir, dir)
126+
require.NoError(t, err)
127+
}

0 commit comments

Comments
 (0)