From b2e86362273744be47ffec09fd4116a025c2a8a8 Mon Sep 17 00:00:00 2001 From: Nuno Cruces Date: Wed, 30 Oct 2024 12:20:21 +0000 Subject: [PATCH] Dot file locking. (#179) --- .github/workflows/test.yml | 8 ++ internal/util/mmap.go | 2 +- internal/util/mmap_other.go | 2 +- vfs/README.md | 19 ++--- vfs/file.go | 13 ++-- vfs/lock.go | 2 +- vfs/lock_other.go | 2 +- vfs/os_bsd.go | 2 +- vfs/os_dotlk.go | 143 ++++++++++++++++++++++++++++++++++++ vfs/os_ofd.go | 2 +- vfs/os_windows.go | 2 +- vfs/shm.go | 2 +- vfs/shm_bsd.go | 2 +- vfs/shm_other.go | 2 +- 14 files changed, 178 insertions(+), 25 deletions(-) create mode 100644 vfs/os_dotlk.go diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0a489b99..8a500390 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,6 +9,10 @@ on: - '**.wasm.bz2' pull_request: branches: [ "main" ] + paths: + - '**.go' + - '**.wasm' + - '**.wasm.bz2' workflow_dispatch: jobs: @@ -52,6 +56,10 @@ jobs: run: go test -v -tags sqlite3_flock ./... if: matrix.os == 'macos-latest' + - name: Test dot locks + run: go test -v -tags sqlite3_dotlk ./... + if: matrix.os == 'macos-latest' + - name: Test no shared memory run: go test -v -tags sqlite3_noshm ./... if: matrix.os == 'ubuntu-latest' diff --git a/internal/util/mmap.go b/internal/util/mmap.go index 5788eeb2..b17c6599 100644 --- a/internal/util/mmap.go +++ b/internal/util/mmap.go @@ -1,4 +1,4 @@ -//go:build unix && (386 || arm || amd64 || arm64 || riscv64 || ppc64le) && !(sqlite3_noshm || sqlite3_nosys) +//go:build unix && !(sqlite3_noshm || sqlite3_nosys) package util diff --git a/internal/util/mmap_other.go b/internal/util/mmap_other.go index a2fbf24d..7eb710b2 100644 --- a/internal/util/mmap_other.go +++ b/internal/util/mmap_other.go @@ -1,4 +1,4 @@ -//go:build !unix || !(386 || arm || amd64 || arm64 || riscv64 || ppc64le) || sqlite3_noshm || sqlite3_nosys +//go:build !unix || sqlite3_noshm || sqlite3_nosys package util diff --git a/vfs/README.md b/vfs/README.md index 77991486..aecae6d6 100644 --- a/vfs/README.md +++ b/vfs/README.md @@ -19,20 +19,19 @@ POSIX advisory locks, which SQLite uses on Unix, are On Linux and macOS, this package uses [OFD locks](https://www.gnu.org/software/libc/manual/html_node/Open-File-Description-Locks.html) to synchronize access to database files. -OFD locks are fully compatible with POSIX advisory locks. This package can also use [BSD locks](https://man.freebsd.org/cgi/man.cgi?query=flock&sektion=2), albeit with reduced concurrency (`BEGIN IMMEDIATE` behaves like `BEGIN EXCLUSIVE`). -On BSD, macOS, and illumos, BSD locks are fully compatible with POSIX advisory locks; -on Linux and z/OS, they are fully functional, but incompatible; -elsewhere, they are very likely broken. BSD locks are the default on BSD and illumos, but you can opt into them with the `sqlite3_flock` build tag. On Windows, this package uses `LockFileEx` and `UnlockFileEx`, like SQLite. +You can also opt into a cross platform locking implementation +with the `sqlite3_dotlk` build tag. + Otherwise, file locking is not supported, and you must use [`nolock=1`](https://sqlite.org/uri.html#urinolock) (or [`immutable=1`](https://sqlite.org/uri.html#uriimmutable)) @@ -86,8 +85,8 @@ The implementation is compatible with SQLite's ### Build Tags The VFS can be customized with a few build tags: -- `sqlite3_flock` forces the use of BSD locks; it can be used on z/OS to enable locking, - and elsewhere to test BSD locks. +- `sqlite3_flock` forces the use of BSD locks. +- `sqlite3_dotlk` forces the use of dot-file locks. - `sqlite3_nosys` prevents importing [`x/sys`](https://pkg.go.dev/golang.org/x/sys); disables locking _and_ shared memory on all platforms. - `sqlite3_noshm` disables shared memory on all platforms. @@ -96,17 +95,19 @@ The VFS can be customized with a few build tags: > The default configuration of this package is compatible with the standard > [Unix and Windows SQLite VFSes](https://sqlite.org/vfs.html#multiple_vfses); > `sqlite3_flock` builds are compatible with the -> [`unix-flock` VFS](https://sqlite.org/compile.html#enable_locking_style). +> [`unix-flock` VFS](https://sqlite.org/compile.html#enable_locking_style); +> `sqlite3_dotlk` builds are compatible with the +> [`unix-dotfile` VFS](https://sqlite.org/compile.html#enable_locking_style). > If incompatible file locking is used, accessing databases concurrently with > _other_ SQLite libraries will eventually corrupt data. ### Custom VFSes -- [`github.com/ncruces/go-sqlite3/vfs/adiantum`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/adiantum) - wraps a VFS to offer encryption at rest. - [`github.com/ncruces/go-sqlite3/vfs/memdb`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/memdb) implements an in-memory VFS. - [`github.com/ncruces/go-sqlite3/vfs/readervfs`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/readervfs) implements a VFS for immutable databases. +- [`github.com/ncruces/go-sqlite3/vfs/adiantum`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/adiantum) + wraps a VFS to offer encryption at rest. - [`github.com/ncruces/go-sqlite3/vfs/xts`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/xts) wraps a VFS to offer encryption at rest. \ No newline at end of file diff --git a/vfs/file.go b/vfs/file.go index ebd42e9a..ba70aa14 100644 --- a/vfs/file.go +++ b/vfs/file.go @@ -35,10 +35,10 @@ func testSymlinks(path string) error { func (vfsOS) Delete(path string, syncDir bool) error { err := os.Remove(path) + if errors.Is(err, fs.ErrNotExist) { + return _IOERR_DELETE_NOENT + } if err != nil { - if errors.Is(err, fs.ErrNotExist) { - return _IOERR_DELETE_NOENT - } return err } if runtime.GOOS != "windows" && syncDir { @@ -151,6 +151,7 @@ func (f *vfsFile) Close() error { if f.shm != nil { f.shm.Close() } + f.Unlock(LOCK_NONE) return f.File.Close() } @@ -206,10 +207,10 @@ func (f *vfsFile) HasMoved() (bool, error) { return false, err } pi, err := os.Stat(f.Name()) + if errors.Is(err, fs.ErrNotExist) { + return true, nil + } if err != nil { - if errors.Is(err, fs.ErrNotExist) { - return true, nil - } return false, err } return !os.SameFile(fi, pi), nil diff --git a/vfs/lock.go b/vfs/lock.go index 89068416..22e320a8 100644 --- a/vfs/lock.go +++ b/vfs/lock.go @@ -1,4 +1,4 @@ -//go:build (linux || darwin || windows || freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) && !sqlite3_nosys +//go:build ((linux || darwin || windows || freebsd || openbsd || netbsd || dragonfly || illumos) && !sqlite3_nosys) || sqlite3_flock || sqlite3_dotlk package vfs diff --git a/vfs/lock_other.go b/vfs/lock_other.go index c395f34a..81aacc62 100644 --- a/vfs/lock_other.go +++ b/vfs/lock_other.go @@ -1,4 +1,4 @@ -//go:build !(linux || darwin || windows || freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) || sqlite3_nosys +//go:build !(((linux || darwin || windows || freebsd || openbsd || netbsd || dragonfly || illumos) && !sqlite3_nosys) || sqlite3_flock || sqlite3_dotlk) package vfs diff --git a/vfs/os_bsd.go b/vfs/os_bsd.go index 1f54a692..56713e35 100644 --- a/vfs/os_bsd.go +++ b/vfs/os_bsd.go @@ -1,4 +1,4 @@ -//go:build (freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) && !sqlite3_nosys +//go:build ((freebsd || openbsd || netbsd || dragonfly || illumos) && !(sqlite3_dotlk || sqlite3_nosys)) || sqlite3_flock package vfs diff --git a/vfs/os_dotlk.go b/vfs/os_dotlk.go new file mode 100644 index 00000000..1c1a49c1 --- /dev/null +++ b/vfs/os_dotlk.go @@ -0,0 +1,143 @@ +//go:build sqlite3_dotlk + +package vfs + +import ( + "errors" + "io/fs" + "os" + "sync" +) + +var ( + // +checklocks:vfsDotLocksMtx + vfsDotLocks = map[string]*vfsDotLocker{} + vfsDotLocksMtx sync.Mutex +) + +type vfsDotLocker struct { + shared int // +checklocks:vfsDotLocksMtx + pending *os.File // +checklocks:vfsDotLocksMtx + reserved *os.File // +checklocks:vfsDotLocksMtx +} + +func osGetSharedLock(file *os.File) _ErrorCode { + vfsDotLocksMtx.Lock() + defer vfsDotLocksMtx.Unlock() + + name := file.Name() + locker := vfsDotLocks[name] + if locker == nil { + err := os.Mkdir(name+".lock", 0777) + if errors.Is(err, fs.ErrExist) { + return _BUSY // Another process has the lock. + } + if err != nil { + return _IOERR_LOCK + } + locker = &vfsDotLocker{} + vfsDotLocks[name] = locker + } + + if locker.pending != nil { + return _BUSY + } + locker.shared++ + return _OK +} + +func osGetReservedLock(file *os.File) _ErrorCode { + vfsDotLocksMtx.Lock() + defer vfsDotLocksMtx.Unlock() + + name := file.Name() + locker := vfsDotLocks[name] + if locker == nil { + return _IOERR_LOCK + } + + if locker.reserved != nil && locker.reserved != file { + return _BUSY + } + locker.reserved = file + return _OK +} + +func osGetExclusiveLock(file *os.File, _ *LockLevel) _ErrorCode { + vfsDotLocksMtx.Lock() + defer vfsDotLocksMtx.Unlock() + + name := file.Name() + locker := vfsDotLocks[name] + if locker == nil { + return _IOERR_LOCK + } + + if locker.pending != nil && locker.pending != file { + return _BUSY + } + locker.pending = file + if locker.shared > 1 { + return _BUSY + } + return _OK +} + +func osDowngradeLock(file *os.File, _ LockLevel) _ErrorCode { + vfsDotLocksMtx.Lock() + defer vfsDotLocksMtx.Unlock() + + name := file.Name() + locker := vfsDotLocks[name] + if locker == nil { + return _IOERR_UNLOCK + } + + if locker.reserved == file { + locker.reserved = nil + } + if locker.pending == file { + locker.pending = nil + } + return _OK +} + +func osReleaseLock(file *os.File, state LockLevel) _ErrorCode { + vfsDotLocksMtx.Lock() + defer vfsDotLocksMtx.Unlock() + + name := file.Name() + locker := vfsDotLocks[name] + if locker == nil { + return _IOERR_UNLOCK + } + + if locker.shared == 1 { + err := os.Remove(name + ".lock") + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return _IOERR_UNLOCK + } + delete(vfsDotLocks, name) + } + + if locker.reserved == file { + locker.reserved = nil + } + if locker.pending == file { + locker.pending = nil + } + locker.shared-- + return _OK +} + +func osCheckReservedLock(file *os.File) (bool, _ErrorCode) { + vfsDotLocksMtx.Lock() + defer vfsDotLocksMtx.Unlock() + + name := file.Name() + locker := vfsDotLocks[name] + if locker == nil { + return false, _OK + } + return locker.reserved != nil, _OK +} diff --git a/vfs/os_ofd.go b/vfs/os_ofd.go index 15730fe6..b4f570f4 100644 --- a/vfs/os_ofd.go +++ b/vfs/os_ofd.go @@ -1,4 +1,4 @@ -//go:build (linux || darwin) && !(sqlite3_flock || sqlite3_nosys) +//go:build (linux || darwin) && !(sqlite3_flock || sqlite3_dotlk || sqlite3_nosys) package vfs diff --git a/vfs/os_windows.go b/vfs/os_windows.go index 4f6149ba..b901f98a 100644 --- a/vfs/os_windows.go +++ b/vfs/os_windows.go @@ -1,4 +1,4 @@ -//go:build !sqlite3_nosys +//go:build !(sqlite3_dotlk || sqlite3_nosys) package vfs diff --git a/vfs/shm.go b/vfs/shm.go index 7ed87437..a63d4141 100644 --- a/vfs/shm.go +++ b/vfs/shm.go @@ -1,4 +1,4 @@ -//go:build (darwin || linux || freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) && (386 || arm || amd64 || arm64 || riscv64 || ppc64le) && !(sqlite3_noshm || sqlite3_nosys) +//go:build (darwin || linux || freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) && (386 || arm || amd64 || arm64 || riscv64 || ppc64le) && !(sqlite3_dotlk || sqlite3_noshm || sqlite3_nosys) package vfs diff --git a/vfs/shm_bsd.go b/vfs/shm_bsd.go index 17726eaf..c5a6aaf3 100644 --- a/vfs/shm_bsd.go +++ b/vfs/shm_bsd.go @@ -1,4 +1,4 @@ -//go:build (freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) && (386 || arm || amd64 || arm64 || riscv64 || ppc64le) && !(sqlite3_noshm || sqlite3_nosys) +//go:build (freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) && (386 || arm || amd64 || arm64 || riscv64 || ppc64le) && !(sqlite3_dotlk || sqlite3_noshm || sqlite3_nosys) package vfs diff --git a/vfs/shm_other.go b/vfs/shm_other.go index 12012033..31d71c4c 100644 --- a/vfs/shm_other.go +++ b/vfs/shm_other.go @@ -1,4 +1,4 @@ -//go:build !(darwin || linux || freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) || !(386 || arm || amd64 || arm64 || riscv64 || ppc64le) || sqlite3_noshm || sqlite3_nosys +//go:build !(darwin || linux || freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) || !(386 || arm || amd64 || arm64 || riscv64 || ppc64le) || sqlite3_dotlk || sqlite3_noshm || sqlite3_nosys package vfs