feat(storage): storage file structure changes (#3301)

- migrate persisted server data storage from `/gluetun/servers.json` to `/gluetun/servers/`
- add `STORAGE_SERVERS_ENABLED=on` to enable or disable on-disk server data storage
- add `STORAGE_SERVERS_DIRECTORY_PATH=/gluetun/servers` to configure where per-provider server files are stored
- keep backward compatibility with legacy `STORAGE_FILEPATH=/gluetun/servers.json`
- automatically read and migrate legacy `/gluetun/servers.json` into the new `/gluetun/servers/` layout when needed
- try to remove the legacy servers file after a successful migration to the new storage directory
- switch persisted server data from one large JSON file to a manifest plus per-provider JSON files
- add `UPDATER_PREFER_DIRECT_DOWNLOAD` to allow preferring direct download of provider server data
- keep deprecated updater flags `-enduser` and `-maintainer` as no-op warnings for backward compatibility
- preserve compatibility checks so persisted server data is discarded when its schema version no longer matches the built-in data
- allow preferred persisted provider data to override built-in data when versions match
- servers data now lives at https://github.com/qdm12/gluetun-servers/tree/main/pkg/servers

internal/storage/sync.go

@@ -2,6 +2,8 @@ package storage
 
 import (
 	"fmt"
+	"os"
+	"path/filepath"
 	"reflect"
 
 	"github.com/qdm12/gluetun/internal/models"
@@ -14,18 +16,31 @@ func countServers(allServers models.AllServers) (count int) {
 	return count
 }
 
-// syncServers merges the hardcoded servers with the ones from the file.
+// syncServers merges the hardcoded servers with the ones from on disk files.
+// It assumes s.directoryPath is set.
 func (s *Storage) syncServers() (err error) {
 	hardcodedVersions := make(map[string]uint16, len(s.hardcodedServers.ProviderToServers))
 	for provider, servers := range s.hardcodedServers.ProviderToServers {
 		hardcodedVersions[provider] = servers.Version
 	}
 
-	serversOnFile, err := s.readFromFile(s.filepath, hardcodedVersions)
+	sourceManifestPath := filepath.Join(s.directoryPath, manifestFilename)
+	destinationManifestPath := sourceManifestPath
+	serversOnFile, found, err := s.readFromFile(sourceManifestPath, hardcodedVersions)
 	if err != nil {
 		return fmt.Errorf("reading servers from file: %w", err)
 	}
 
+	hasLegacy := s.hasLegacy()
+	if !found && hasLegacy {
+		sourceManifestPath = s.legacyFilepath
+		s.logger.Infof("reading legacy servers file %s and migrating it to directory %s", sourceManifestPath, s.directoryPath)
+		serversOnFile, _, err = s.readFromFile(sourceManifestPath, hardcodedVersions)
+		if err != nil {
+			return fmt.Errorf("reading servers from file: %w", err)
+		}
+	}
+
 	hardcodedCount := countServers(s.hardcodedServers)
 	countOnFile := countServers(serversOnFile)
 
@@ -34,13 +49,13 @@ func (s *Storage) syncServers() (err error) {
 
 	if countOnFile == 0 {
 		s.logger.Info(fmt.Sprintf(
-			"creating %s with %d hardcoded servers",
-			s.filepath, hardcodedCount))
+			"writing servers data files to %s with %d hardcoded servers",
+			s.directoryPath, hardcodedCount))
 		s.mergedServers = s.hardcodedServers
 	} else {
 		s.logger.Info(fmt.Sprintf(
-			"merging by most recent %d hardcoded servers and %d servers read from %s",
-			hardcodedCount, countOnFile, s.filepath))
+			"merging by most recent %d hardcoded servers and %d servers read from manifest file %s",
+			hardcodedCount, countOnFile, sourceManifestPath))
 
 		s.mergedServers = s.mergeServers(s.hardcodedServers, serversOnFile)
 	}
@@ -50,9 +65,18 @@ func (s *Storage) syncServers() (err error) {
 		return nil
 	}
 
-	err = s.flushToFile(s.filepath)
+	err = s.flushToFile(destinationManifestPath)
 	if err != nil {
-		s.logger.Warn("failed writing servers to file: " + err.Error())
+		s.logger.Warn("failed writing servers to destination manifest: " + err.Error())
+		return nil
+	}
+
+	migratedFromLegacy := hasLegacy && sourceManifestPath == s.legacyFilepath
+	if migratedFromLegacy {
+		err = os.Remove(sourceManifestPath)
+		if err != nil && !os.IsNotExist(err) {
+			s.logger.Warn("failed removing legacy servers file " + sourceManifestPath + ": " + err.Error())
+		}
 	}
 	return nil
 }