feat(storage): storage file structure changes (#3301)

- migrate persisted server data storage from `/gluetun/servers.json` to `/gluetun/servers/`
- add `STORAGE_SERVERS_ENABLED=on` to enable or disable on-disk server data storage
- add `STORAGE_SERVERS_DIRECTORY_PATH=/gluetun/servers` to configure where per-provider server files are stored
- keep backward compatibility with legacy `STORAGE_FILEPATH=/gluetun/servers.json`
- automatically read and migrate legacy `/gluetun/servers.json` into the new `/gluetun/servers/` layout when needed
- try to remove the legacy servers file after a successful migration to the new storage directory
- switch persisted server data from one large JSON file to a manifest plus per-provider JSON files
- add `UPDATER_PREFER_DIRECT_DOWNLOAD` to allow preferring direct download of provider server data
- keep deprecated updater flags `-enduser` and `-maintainer` as no-op warnings for backward compatibility
- preserve compatibility checks so persisted server data is discarded when its schema version no longer matches the built-in data
- allow preferred persisted provider data to override built-in data when versions match
- servers data now lives at https://github.com/qdm12/gluetun-servers/tree/main/pkg/servers

internal/storage/flush.go

@@ -2,6 +2,7 @@ package storage
 
 import (
 	"encoding/json"
+	"fmt"
 	"os"
 	"path/filepath"
 	"sort"
@@ -9,44 +10,76 @@ import (
 	"github.com/qdm12/gluetun/internal/models"
 )
 
-// FlushToFile flushes the merged servers data to the file
-// specified by path, as indented JSON.
-func (s *Storage) FlushToFile(path string) error {
-	s.mergedMutex.RLock()
-	defer s.mergedMutex.RUnlock()
+// flushToFile flushes the merged servers data to files
+// using the manifest file path given. It is not thread-safe.
+func (s *Storage) flushToFile(manifestPath string) error {
+	const (
+		filePermission = 0o644
+		dirPermission  = 0o755
+	)
 
-	return s.flushToFile(path)
-}
-
-// flushToFile flushes the merged servers data to the file
-// specified by path, as indented JSON. It is not thread-safe.
-func (s *Storage) flushToFile(path string) error {
-	if path == "" {
-		return nil // no file to write to
-	}
-	const permission = 0o644
-	dirPath := filepath.Dir(path)
-	if err := os.MkdirAll(dirPath, permission); err != nil {
-		return err
+	serversDirectoryPath := filepath.Dir(manifestPath)
+	if err := os.MkdirAll(serversDirectoryPath, dirPermission); err != nil {
+		return fmt.Errorf("creating directory: %w", err)
 	}
 
-	file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, permission)
+	for provider, providerServers := range s.mergedServers.ProviderToServers {
+		providerFilepath := providerServers.Filepath
+		if providerFilepath == "" {
+			providerFilepath = filepath.Join(serversDirectoryPath, provider+".json")
+		}
+
+		providerDirectoryPath := filepath.Dir(providerFilepath)
+		if err := os.MkdirAll(providerDirectoryPath, dirPermission); err != nil {
+			return fmt.Errorf("creating directory: %w", err)
+		}
+	}
+
+	metadata := map[string]any{"version": s.mergedServers.Version}
+
+	for provider, providerServers := range s.mergedServers.ProviderToServers {
+		sort.Sort(models.SortableServers(providerServers.Servers))
+
+		providerFilepath := providerServers.Filepath
+		if providerFilepath == "" {
+			providerFilepath = filepath.Join(serversDirectoryPath, provider+".json")
+		}
+
+		providerFile, err := os.OpenFile(providerFilepath,
+			os.O_CREATE|os.O_WRONLY|os.O_TRUNC, filePermission)
+		if err != nil {
+			return fmt.Errorf("opening servers data file for %s: %w", provider, err)
+		}
+
+		encoder := json.NewEncoder(providerFile)
+		encoder.SetIndent("", "  ")
+		err = encoder.Encode(providerServers)
+		if err != nil {
+			_ = providerFile.Close()
+			return fmt.Errorf("encoding servers data for %s: %w", provider, err)
+		}
+
+		err = providerFile.Close()
+		if err != nil {
+			return fmt.Errorf("closing servers data file for %s: %w", provider, err)
+		}
+
+		metadata[provider] = map[string]string{"filepath": providerFilepath}
+	}
+
+	serversFile, err := os.OpenFile(manifestPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, filePermission)
 	if err != nil {
 		return err
 	}
 
-	encoder := json.NewEncoder(file)
+	encoder := json.NewEncoder(serversFile)
 	encoder.SetIndent("", "  ")
 
-	for _, obj := range s.mergedServers.ProviderToServers {
-		sort.Sort(models.SortableServers(obj.Servers))
-	}
-
-	err = encoder.Encode(&s.mergedServers)
+	err = encoder.Encode(metadata)
 	if err != nil {
-		_ = file.Close()
+		_ = serversFile.Close()
 		return err
 	}
 
-	return file.Close()
+	return serversFile.Close()
 }