refactor(storage): new storage file structure

- new directory structure containing manifest.json and one json file per provider, by default.
- the manifest.json file can specify a filepath for each vpn provider
- each vpn provider json data file can contain the `"preferred": true` field to enforce it is used even if outdated, unless there is a version mismatch
- `STORAGE_SERVERS_DIRECTORY_PATH` replaces `STORAGE_FILEPATH` (which is now a migration source only). It sets the directory where server manifest and per-provider JSON files are stored (default: `/gluetun/servers/`).
- First-run migration: On startup, gluetun checks for the old /gluetun/servers.json file; if found and no new manifest exists, it automatically migrates all data to /gluetun/servers/ directory structure
- Silent fallback: If legacy file isn't found, uses the new directory path normally
- Legacy cleanup: After successful migration, attempts to remove the old fat JSON file (logs warning only if removal fails, e.g., read-only bind mounts)

Co-authored-by: Copilot <copilot@github.com>

internal/configuration/settings/settings.go

@@ -132,7 +132,7 @@ func (s *Settings) SetDefaults() {
 	s.IPv6.setDefaults()
 	s.PublicIP.setDefaults()
 	s.Shadowsocks.setDefaults()
-	s.Storage.setDefaults()
+	s.Storage.SetDefaults()
 	s.System.setDefaults()
 	s.Version.setDefaults()
 	s.VPN.setDefaults()
@@ -213,7 +213,7 @@ func (s *Settings) Read(r *reader.Reader, warner Warner) (err error) {
 			return s.PublicIP.read(r, warner)
 		},
 		"shadowsocks": s.Shadowsocks.read,
-		"storage":     s.Storage.read,
+		"storage":     s.Storage.Read,
 		"system":      s.System.read,
 		"updater":     s.Updater.read,
 		"version":     s.Version.read,

internal/configuration/settings/settings_test.go

@@ -90,7 +90,7 @@ func Test_Settings_String(t *testing.T) {
 |   ├── Logging: yes
 |   └── Authentication file path: /gluetun/auth/config.toml
 ├── Storage settings:
-|   └── Filepath: /gluetun/servers.json
+|   └── Servers directory path: /gluetun/servers/
 ├── OS Alpine settings:
 |   ├── Process UID: 1000
 |   └── Process GID: 1000

internal/configuration/settings/storage.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"path/filepath"
 
+	"github.com/qdm12/gluetun/internal/constants"
 	"github.com/qdm12/gosettings"
 	"github.com/qdm12/gosettings/reader"
 	"github.com/qdm12/gotree"
@@ -11,15 +12,25 @@ import (
 
 // Storage contains settings to configure the storage.
 type Storage struct {
-	// Filepath is the path to the servers.json file. An empty string disables on-disk storage.
-	Filepath *string
+	// ServersPath is the path to the servers files directory.
+	// An empty string disables on-disk storage.
+	ServersPath *string
+	// LegacyServersFilepath is the legacy "fat" JSON filepath to migrate from.
+	// TODO v4: remove
+	LegacyServersFilepath *string
 }
 
 func (s Storage) validate() (err error) {
-	if *s.Filepath != "" { // optional
-		_, err := filepath.Abs(*s.Filepath)
+	if *s.ServersPath != "" { // optional
+		_, err := filepath.Abs(*s.ServersPath)
 		if err != nil {
-			return fmt.Errorf("filepath is not valid: %w", err)
+			return fmt.Errorf("servers path is not valid: %w", err)
+		}
+	}
+	if *s.LegacyServersFilepath != "" {
+		_, err := filepath.Abs(*s.LegacyServersFilepath)
+		if err != nil {
+			return fmt.Errorf("legacy servers filepath is not valid: %w", err)
 		}
 	}
 	return nil
@@ -27,17 +38,20 @@ func (s Storage) validate() (err error) {
 
 func (s *Storage) copy() (copied Storage) {
 	return Storage{
-		Filepath: gosettings.CopyPointer(s.Filepath),
+		ServersPath:           gosettings.CopyPointer(s.ServersPath),
+		LegacyServersFilepath: gosettings.CopyPointer(s.LegacyServersFilepath),
 	}
 }
 
 func (s *Storage) overrideWith(other Storage) {
-	s.Filepath = gosettings.OverrideWithPointer(s.Filepath, other.Filepath)
+	s.ServersPath = gosettings.OverrideWithPointer(s.ServersPath, other.ServersPath)
+	s.LegacyServersFilepath = gosettings.OverrideWithPointer(s.LegacyServersFilepath, other.LegacyServersFilepath)
 }
 
-func (s *Storage) setDefaults() {
-	const defaultFilepath = "/gluetun/servers.json"
-	s.Filepath = gosettings.DefaultPointer(s.Filepath, defaultFilepath)
+func (s *Storage) SetDefaults() {
+	const defaultServersPath = "/gluetun/servers/"
+	s.ServersPath = gosettings.DefaultPointer(s.ServersPath, defaultServersPath)
+	s.LegacyServersFilepath = gosettings.DefaultPointer(s.LegacyServersFilepath, constants.ServersDataLegacy)
 }
 
 func (s Storage) String() string {
@@ -45,15 +59,29 @@ func (s Storage) String() string {
 }
 
 func (s Storage) toLinesNode() (node *gotree.Node) {
-	if *s.Filepath == "" {
+	if *s.ServersPath == "" {
 		return gotree.New("Storage settings: disabled")
 	}
 	node = gotree.New("Storage settings:")
-	node.Appendf("Filepath: %s", *s.Filepath)
+	node.Appendf("Servers directory path: %s", *s.ServersPath)
+	if *s.LegacyServersFilepath != constants.ServersDataLegacy {
+		node.Appendf("Legacy servers filepath: %s", *s.LegacyServersFilepath)
+	}
 	return node
 }
 
-func (s *Storage) read(r *reader.Reader) (err error) {
-	s.Filepath = r.Get("STORAGE_FILEPATH", reader.AcceptEmpty(true))
+func (s *Storage) Read(r *reader.Reader) (err error) {
+	// Retro-compatibility:
+	// TODO v4: remove support for STORAGE_FILEPATH
+	filePath := r.Get("STORAGE_FILEPATH", reader.AcceptEmpty(true), reader.IsRetro("STORAGE_SERVERS_DIRECTORY_PATH"))
+	if filePath != nil {
+		s.LegacyServersFilepath = filePath
+		if *filePath == "" {
+			s.ServersPath = ptrTo("") // skip disk operations
+		}
+	}
+	if s.ServersPath == nil {
+		s.ServersPath = r.Get("STORAGE_SERVERS_DIRECTORY_PATH", reader.AcceptEmpty(true))
+	}
 	return nil
 }