Files
filebrowser/backend/database/sql/sqlite.go
T
9d18a369c7 Beta/v1.1.2 (#1679)
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Kevin B. <kurami32_kbm@proton.me>
Co-authored-by: Swedish Wiking <121635232+Swedish-Wiking@users.noreply.github.com>
Co-authored-by: Pyro <pyrodd@gmail.com>
Co-authored-by: lps-rocks <10893911+lps-rocks@users.noreply.github.com>
Co-authored-by: contributor <foss.contributor@gmail.com>
Co-authored-by: Ironic <202444094+YoureIronic@users.noreply.github.com>
Co-authored-by: Sarabanga <96509626+Sarabanga@users.noreply.github.com>
Co-authored-by: ✿ corey <corey@x64.co>
2025-11-30 13:34:00 -05:00

254 lines
7.7 KiB
Go

package sql
import (
"database/sql"
"fmt"
"os"
"path/filepath"
"sync"
"time"
"github.com/gtsteffaniak/filebrowser/backend/adapters/fs/fileutils"
"github.com/gtsteffaniak/filebrowser/backend/common/settings"
"github.com/gtsteffaniak/go-logger/logger"
_ "modernc.org/sqlite"
)
// TempDB manages a temporary SQLite database for operations that need
// to stream large datasets without loading everything into memory.
// This can be used by any part of the codebase that needs temporary SQLite storage.
type TempDB struct {
db *sql.DB
path string
mu sync.Mutex
startTime time.Time
config *TempDBConfig
}
// TempDBConfig holds configuration options for temporary SQLite databases.
type TempDBConfig struct {
// CacheSizeKB is the page cache size in KB. Negative values are in pages.
// For one-time databases, a smaller cache (e.g., -2000 = ~8MB) is often sufficient.
// Default: -2000 (approximately 8MB)
CacheSizeKB int
// MmapSize is the memory-mapped I/O size in bytes. Set to 0 to disable mmap.
// For databases that fit in RAM, set this larger than the expected DB size.
// Default: 2GB (2147483648 bytes)
MmapSize int64
// Synchronous controls the synchronous mode. OFF is fastest but less safe.
// For temporary databases, OFF is acceptable.
// Default: OFF
Synchronous string
// TempStore controls where temporary tables and indices are stored.
// Valid values: "FILE" (default), "MEMORY", "DEFAULT"
// Default: FILE (temporary tables stored on disk)
TempStore string
// EnableLogging enables performance logging for debugging.
// Default: false
EnableLogging bool
}
// mergeConfig merges the provided config with defaults, returning a new config.
// If provided config is nil or empty, returns default config.
func mergeConfig(provided *TempDBConfig) *TempDBConfig {
defaults := &TempDBConfig{
CacheSizeKB: -2000, // ~8MB, appropriate for one-time databases
MmapSize: 2147483648, // 2GB
Synchronous: "OFF",
TempStore: "FILE", // Default to FILE, not MEMORY
EnableLogging: false,
}
if provided == nil {
return defaults
}
merged := *defaults // Copy defaults
// Override with provided values if they are non-zero/non-empty
if provided.CacheSizeKB != 0 {
merged.CacheSizeKB = provided.CacheSizeKB
}
if provided.MmapSize != 0 {
merged.MmapSize = provided.MmapSize
}
if provided.Synchronous != "" {
merged.Synchronous = provided.Synchronous
}
if provided.TempStore != "" {
merged.TempStore = provided.TempStore
}
merged.EnableLogging = provided.EnableLogging
return &merged
}
// NewTempDB creates a new temporary SQLite database.
// The database is created in the cache directory's sql/ subdirectory.
// The database will be cleaned up on Close().
//
// The database is optimized for bulk write-then-read operations with:
// - WAL journal mode for better concurrency
// - Configurable cache size (default: ~8MB for one-time DBs)
// - Memory-mapped I/O for faster access
// - OFF synchronous mode for maximum write performance
// - Configurable temp_store (default: FILE, can be set to MEMORY via config)
func NewTempDB(id string, config ...*TempDBConfig) (*TempDB, error) {
startTime := time.Now()
// Merge provided config with defaults
var providedConfig *TempDBConfig
if len(config) > 0 {
providedConfig = config[0]
}
cfg := mergeConfig(providedConfig)
// Create sql subdirectory in the cache directory
dbDir := filepath.Join(settings.Config.Server.CacheDir, "sql")
if err := os.MkdirAll(dbDir, fileutils.PermDir); err != nil {
return nil, fmt.Errorf("failed to create sql directory: %w", err)
}
// Create temporary file in the sql subdirectory
tmpFile, err := os.CreateTemp(dbDir, fmt.Sprintf("%s.db", id))
if err != nil {
return nil, fmt.Errorf("failed to create temp file: %w", err)
}
tmpPath := tmpFile.Name()
tmpFile.Close()
// Open SQLite database with basic connection string
// We'll set PRAGMAs after connection for better control and logging
db, err := sql.Open("sqlite", tmpPath)
if err != nil {
os.Remove(tmpPath)
return nil, fmt.Errorf("failed to open SQLite database: %w", err)
}
// Test the connection
if err := db.Ping(); err != nil {
db.Close()
os.Remove(tmpPath)
return nil, fmt.Errorf("failed to ping SQLite database: %w", err)
}
// Apply optimizations via PRAGMA statements
// Execute them individually for compatibility and better error reporting
pragmaStart := time.Now()
pragmas := []struct {
sql string
err string
}{
{"PRAGMA journal_mode = WAL;", "failed to set WAL mode"},
{fmt.Sprintf("PRAGMA cache_size = %d;", cfg.CacheSizeKB), "failed to set cache_size"},
{fmt.Sprintf("PRAGMA synchronous = %s;", cfg.Synchronous), "failed to set synchronous"},
{fmt.Sprintf("PRAGMA temp_store = %s;", cfg.TempStore), "failed to set temp_store"},
}
if cfg.MmapSize > 0 {
pragmas = append(pragmas, struct {
sql string
err string
}{fmt.Sprintf("PRAGMA mmap_size = %d;", cfg.MmapSize), "failed to set mmap_size"})
}
for _, pragma := range pragmas {
if _, err := db.Exec(pragma.sql); err != nil {
db.Close()
os.Remove(tmpPath)
return nil, fmt.Errorf("%s: %w", pragma.err, err)
}
}
pragmaDuration := time.Since(pragmaStart)
// Log configuration if enabled
if cfg.EnableLogging {
logger.Debugf("[TempDB:%s] Created with cache_size=%d KB, mmap_size=%d bytes, synchronous=%s, temp_store=%s (setup took %v)",
id, cfg.CacheSizeKB, cfg.MmapSize, cfg.Synchronous, cfg.TempStore, pragmaDuration)
}
return &TempDB{
db: db,
path: tmpPath,
startTime: startTime,
config: cfg,
}, nil
}
// DB returns the underlying *sql.DB connection.
// This allows callers to execute custom SQL queries if needed.
func (t *TempDB) DB() *sql.DB {
return t.db
}
// BeginTransaction starts a transaction for bulk operations.
// The caller must call Commit() or Rollback() on the returned transaction.
// The mutex is NOT held during the transaction - caller is responsible for coordination.
func (t *TempDB) BeginTransaction() (*sql.Tx, error) {
return t.db.Begin()
}
// Exec executes a SQL statement that doesn't return rows.
// This is a convenience method that handles locking.
func (t *TempDB) Exec(query string, args ...interface{}) (sql.Result, error) {
t.mu.Lock()
defer t.mu.Unlock()
return t.db.Exec(query, args...)
}
// Query executes a query that returns rows.
// This is a convenience method that handles locking.
func (t *TempDB) Query(query string, args ...interface{}) (*sql.Rows, error) {
t.mu.Lock()
defer t.mu.Unlock()
return t.db.Query(query, args...)
}
// QueryRow executes a query that is expected to return at most one row.
// This is a convenience method that handles locking.
func (t *TempDB) QueryRow(query string, args ...interface{}) *sql.Row {
t.mu.Lock()
defer t.mu.Unlock()
return t.db.QueryRow(query, args...)
}
// Close closes the database connection and removes the temporary file.
// This should always be called when done with the database, typically in a defer statement.
// If logging is enabled, it will log the total lifetime and file size for performance analysis.
func (t *TempDB) Close() error {
t.mu.Lock()
defer t.mu.Unlock()
if t.config != nil && t.config.EnableLogging {
totalDuration := time.Since(t.startTime)
fileInfo, err := os.Stat(t.path)
var fileSize int64
if err == nil {
fileSize = fileInfo.Size()
}
logger.Debugf("[TempDB] Closed after %v, final size: %d bytes (%.2f MB)",
totalDuration, fileSize, float64(fileSize)/(1024*1024))
}
if t.db != nil {
if err := t.db.Close(); err != nil {
os.Remove(t.path)
return err
}
}
return os.Remove(t.path)
}
// Path returns the path to the temporary database file.
// This is useful for debugging or if you need to inspect the database.
func (t *TempDB) Path() string {
return t.path
}