feat(storage): storage file structure changes (#3301)

- migrate persisted server data storage from `/gluetun/servers.json` to `/gluetun/servers/`
- add `STORAGE_SERVERS_ENABLED=on` to enable or disable on-disk server data storage
- add `STORAGE_SERVERS_DIRECTORY_PATH=/gluetun/servers` to configure where per-provider server files are stored
- keep backward compatibility with legacy `STORAGE_FILEPATH=/gluetun/servers.json`
- automatically read and migrate legacy `/gluetun/servers.json` into the new `/gluetun/servers/` layout when needed
- try to remove the legacy servers file after a successful migration to the new storage directory
- switch persisted server data from one large JSON file to a manifest plus per-provider JSON files
- add `UPDATER_PREFER_DIRECT_DOWNLOAD` to allow preferring direct download of provider server data
- keep deprecated updater flags `-enduser` and `-maintainer` as no-op warnings for backward compatibility
- preserve compatibility checks so persisted server data is discarded when its schema version no longer matches the built-in data
- allow preferred persisted provider data to override built-in data when versions match
- servers data now lives at https://github.com/qdm12/gluetun-servers/tree/main/pkg/servers

internal/configuration/settings/storage.go

@@ -11,15 +11,26 @@ import (
 
 // Storage contains settings to configure the storage.
 type Storage struct {
-	// Filepath is the path to the servers.json file. An empty string disables on-disk storage.
-	Filepath *string
+	// ServersEnabled is whether to enable storage of servers on disk.
+	// It defaults to true.
+	ServersEnabled *bool
+	// ServersPath is the path to the servers files directory, and cannot be
+	// the empty string.
+	ServersPath string
+	// LegacyServersFilepath is the legacy "fat" JSON filepath to migrate from.
+	// TODO v4: remove
+	LegacyServersFilepath string
 }
 
 func (s Storage) validate() (err error) {
-	if *s.Filepath != "" { // optional
-		_, err := filepath.Abs(*s.Filepath)
+	if *s.ServersEnabled {
+		_, err := filepath.Abs(s.ServersPath)
 		if err != nil {
-			return fmt.Errorf("filepath is not valid: %w", err)
+			return fmt.Errorf("servers path is not valid: %w", err)
+		}
+		_, err = filepath.Abs(s.LegacyServersFilepath)
+		if err != nil {
+			return fmt.Errorf("legacy servers filepath is not valid: %w", err)
 		}
 	}
 	return nil
@@ -27,17 +38,25 @@ func (s Storage) validate() (err error) {
 
 func (s *Storage) copy() (copied Storage) {
 	return Storage{
-		Filepath: gosettings.CopyPointer(s.Filepath),
+		ServersEnabled:        gosettings.CopyPointer(s.ServersEnabled),
+		ServersPath:           s.ServersPath,
+		LegacyServersFilepath: s.LegacyServersFilepath,
 	}
 }
 
 func (s *Storage) overrideWith(other Storage) {
-	s.Filepath = gosettings.OverrideWithPointer(s.Filepath, other.Filepath)
+	s.ServersEnabled = gosettings.OverrideWithPointer(s.ServersEnabled, other.ServersEnabled)
+	s.ServersPath = gosettings.OverrideWithComparable(s.ServersPath, other.ServersPath)
+	s.LegacyServersFilepath = gosettings.OverrideWithComparable(s.LegacyServersFilepath, other.LegacyServersFilepath)
 }
 
-func (s *Storage) setDefaults() {
-	const defaultFilepath = "/gluetun/servers.json"
-	s.Filepath = gosettings.DefaultPointer(s.Filepath, defaultFilepath)
+const defaultLegacyServersFilepath = "/gluetun/servers.json"
+
+func (s *Storage) SetDefaults() {
+	s.ServersEnabled = gosettings.DefaultPointer(s.ServersEnabled, true)
+	const defaultServersPath = "/gluetun/servers/"
+	s.ServersPath = gosettings.DefaultComparable(s.ServersPath, defaultServersPath)
+	s.LegacyServersFilepath = gosettings.DefaultComparable(s.LegacyServersFilepath, defaultLegacyServersFilepath)
 }
 
 func (s Storage) String() string {
@@ -45,15 +64,33 @@ func (s Storage) String() string {
 }
 
 func (s Storage) toLinesNode() (node *gotree.Node) {
-	if *s.Filepath == "" {
+	if !*s.ServersEnabled {
 		return gotree.New("Storage settings: disabled")
 	}
 	node = gotree.New("Storage settings:")
-	node.Appendf("Filepath: %s", *s.Filepath)
+	node.Appendf("Servers directory path: %s", s.ServersPath)
+	if s.LegacyServersFilepath != defaultLegacyServersFilepath {
+		node.Appendf("Legacy servers filepath: %s", s.LegacyServersFilepath)
+	}
 	return node
 }
 
-func (s *Storage) read(r *reader.Reader) (err error) {
-	s.Filepath = r.Get("STORAGE_FILEPATH", reader.AcceptEmpty(true))
+func (s *Storage) Read(r *reader.Reader) (err error) {
+	// Retro-compatibility:
+	// TODO v4: remove support for STORAGE_FILEPATH
+	filePath := r.Get("STORAGE_FILEPATH", reader.AcceptEmpty(true), reader.IsRetro("STORAGE_SERVERS_DIRECTORY_PATH"))
+	if filePath != nil {
+		if *filePath == "" {
+			s.ServersEnabled = ptrTo(false)
+		} else {
+			s.LegacyServersFilepath = *filePath
+		}
+	} else {
+		s.ServersEnabled, err = r.BoolPtr("STORAGE_SERVERS_ENABLED")
+		if err != nil {
+			return err
+		}
+		s.ServersPath = r.String("STORAGE_SERVERS_DIRECTORY_PATH")
+	}
 	return nil
 }