refactor(storage): new storage file structure

- new directory structure containing manifest.json and one json file per provider, by default.
- the manifest.json file can specify a filepath for each vpn provider
- each vpn provider json data file can contain the `"preferred": true` field to enforce it is used even if outdated, unless there is a version mismatch
- `STORAGE_SERVERS_DIRECTORY_PATH` replaces `STORAGE_FILEPATH` (which is now a migration source only). It sets the directory where server manifest and per-provider JSON files are stored (default: `/gluetun/servers/`).
- First-run migration: On startup, gluetun checks for the old /gluetun/servers.json file; if found and no new manifest exists, it automatically migrates all data to /gluetun/servers/ directory structure
- Silent fallback: If legacy file isn't found, uses the new directory path normally
- Legacy cleanup: After successful migration, attempts to remove the old fat JSON file (logs warning only if removal fails, e.g., read-only bind mounts)

Co-authored-by: Copilot <copilot@github.com>

internal/storage/sync.go

@@ -2,6 +2,8 @@ package storage
 
 import (
 	"fmt"
+	"os"
+	"path/filepath"
 	"reflect"
 
 	"github.com/qdm12/gluetun/internal/models"
@@ -14,18 +16,31 @@ func countServers(allServers models.AllServers) (count int) {
 	return count
 }
 
-// syncServers merges the hardcoded servers with the ones from the file.
+// syncServers merges the hardcoded servers with the ones from on disk files.
+// It assumes s.directoryPath is set.
 func (s *Storage) syncServers() (err error) {
 	hardcodedVersions := make(map[string]uint16, len(s.hardcodedServers.ProviderToServers))
 	for provider, servers := range s.hardcodedServers.ProviderToServers {
 		hardcodedVersions[provider] = servers.Version
 	}
 
-	serversOnFile, err := s.readFromFile(s.filepath, hardcodedVersions)
+	sourceManifestPath := filepath.Join(s.directoryPath, manifestFilename)
+	destinationManifestPath := sourceManifestPath
+	serversOnFile, found, err := s.readFromFile(sourceManifestPath, hardcodedVersions)
 	if err != nil {
 		return fmt.Errorf("reading servers from file: %w", err)
 	}
 
+	hasLegacy := s.hasLegacy()
+	if !found && hasLegacy {
+		sourceManifestPath = s.legacyFilepath
+		s.logger.Infof("reading legacy servers file %s and migrating it to directory %s", sourceManifestPath, s.directoryPath)
+		serversOnFile, _, err = s.readFromFile(sourceManifestPath, hardcodedVersions)
+		if err != nil {
+			return fmt.Errorf("reading servers from file: %w", err)
+		}
+	}
+
 	hardcodedCount := countServers(s.hardcodedServers)
 	countOnFile := countServers(serversOnFile)
 
@@ -34,13 +49,13 @@ func (s *Storage) syncServers() (err error) {
 
 	if countOnFile == 0 {
 		s.logger.Info(fmt.Sprintf(
-			"creating %s with %d hardcoded servers",
-			s.filepath, hardcodedCount))
+			"writing servers data files to %s with %d hardcoded servers",
+			s.directoryPath, hardcodedCount))
 		s.mergedServers = s.hardcodedServers
 	} else {
 		s.logger.Info(fmt.Sprintf(
-			"merging by most recent %d hardcoded servers and %d servers read from %s",
-			hardcodedCount, countOnFile, s.filepath))
+			"merging by most recent %d hardcoded servers and %d servers read from manifest file %s",
+			hardcodedCount, countOnFile, sourceManifestPath))
 
 		s.mergedServers = s.mergeServers(s.hardcodedServers, serversOnFile)
 	}
@@ -50,9 +65,18 @@ func (s *Storage) syncServers() (err error) {
 		return nil
 	}
 
-	err = s.flushToFile(s.filepath)
+	err = s.flushToFile(destinationManifestPath)
 	if err != nil {
-		s.logger.Warn("failed writing servers to file: " + err.Error())
+		s.logger.Warn("failed writing servers to destination manifest: " + err.Error())
+		return nil
+	}
+
+	migratedFromLegacy := hasLegacy && sourceManifestPath == s.legacyFilepath
+	if migratedFromLegacy {
+		err = os.Remove(sourceManifestPath)
+		if err != nil && !os.IsNotExist(err) {
+			s.logger.Warn("failed removing legacy servers file " + sourceManifestPath + ": " + err.Error())
+		}
 	}
 	return nil
 }