refactor(storage): new storage file structure

- new directory structure containing manifest.json and one json file per provider, by default.
- the manifest.json file can specify a filepath for each vpn provider
- each vpn provider json data file can contain the `"preferred": true` field to enforce it is used even if outdated, unless there is a version mismatch
- `STORAGE_SERVERS_DIRECTORY_PATH` replaces `STORAGE_FILEPATH` (which is now a migration source only). It sets the directory where server manifest and per-provider JSON files are stored (default: `/gluetun/servers/`).
- First-run migration: On startup, gluetun checks for the old /gluetun/servers.json file; if found and no new manifest exists, it automatically migrates all data to /gluetun/servers/ directory structure
- Silent fallback: If legacy file isn't found, uses the new directory path normally
- Legacy cleanup: After successful migration, attempts to remove the old fat JSON file (logs warning only if removal fails, e.g., read-only bind mounts)

internal/storage/read.go

@@ -6,36 +6,37 @@ import (
 	"io"
 	"os"
 
+	umodels "github.com/qdm12/gluetun-servers/pkg/models"
 	"github.com/qdm12/gluetun/internal/constants/providers"
 	"github.com/qdm12/gluetun/internal/models"
-	umodels "github.com/qdm12/gluetun/pkg/updaters/models"
 	"golang.org/x/text/cases"
 	"golang.org/x/text/language"
 )
 
-// readFromFile reads the servers from server.json.
+// readFromFile reads the servers data starting from the given manifest file path.
 // It only reads servers that have the same version as the hardcoded servers version
 // to avoid JSON decoding errors.
-func (s *Storage) readFromFile(filepath string, hardcodedVersions map[string]uint16) (
-	servers models.AllServers, err error,
+func (s *Storage) readFromFile(manifestPath string, hardcodedVersions map[string]uint16) (
+	servers models.AllServers, found bool, err error,
 ) {
-	file, err := os.Open(filepath)
+	file, err := os.Open(manifestPath)
 	if os.IsNotExist(err) {
-		return servers, nil
+		return servers, false, nil
 	} else if err != nil {
-		return servers, err
+		return servers, false, err
 	}
 
 	b, err := io.ReadAll(file)
 	if err != nil {
-		return servers, err
+		return servers, true, err
 	}
 
 	if err := file.Close(); err != nil {
-		return servers, err
+		return servers, true, err
 	}
 
-	return s.extractServersFromBytes(b, hardcodedVersions)
+	servers, err = s.extractServersFromBytes(b, hardcodedVersions)
+	return servers, true, err
 }
 
 func (s *Storage) extractServersFromBytes(b []byte, hardcodedVersions map[string]uint16) (
@@ -47,6 +48,12 @@ func (s *Storage) extractServersFromBytes(b []byte, hardcodedVersions map[string
 	}
 
 	// Note schema version is at map key "version" as number
+	if rawVersion, ok := rawMessages["version"]; ok {
+		err := json.Unmarshal(rawVersion, &servers.Version)
+		if err != nil {
+			return servers, fmt.Errorf("decoding servers schema version: %w", err)
+		}
+	}
 
 	allProviders := providers.All()
 	servers.ProviderToServers = make(map[string]umodels.Servers, len(allProviders))
@@ -87,25 +94,20 @@ func (s *Storage) readServers(provider string, hardcodedVersion uint16,
 ) {
 	provider = titleCaser.String(provider)
 
-	var versionObject struct {
-		Version uint16 `json:"version"`
+	var metadata struct {
+		Version   uint16 `json:"version"`
+		Timestamp int64  `json:"timestamp"`
+		Filepath  string `json:"filepath"`
 	}
 
-	err = json.Unmarshal(rawMessage, &versionObject)
+	err = json.Unmarshal(rawMessage, &metadata)
 	if err != nil {
 		return servers, false, fmt.Errorf("decoding servers version for provider %s: %w",
 			provider, err)
 	}
 
-	persistedVersion := versionObject.Version
-
-	versionsMatch = hardcodedVersion == persistedVersion
-	if !versionsMatch {
-		s.logger.Info(fmt.Sprintf(
-			"%s servers from file discarded because they have "+
-				"version %d and hardcoded servers have version %d",
-			provider, persistedVersion, hardcodedVersion))
-		return servers, versionsMatch, nil
+	if metadata.Filepath != "" {
+		return s.readServersFromFilepath(provider, metadata.Filepath, hardcodedVersion)
 	}
 
 	err = json.Unmarshal(rawMessage, &servers)
@@ -114,5 +116,55 @@ func (s *Storage) readServers(provider string, hardcodedVersion uint16,
 			provider, err)
 	}
 
-	return servers, versionsMatch, nil
+	versionsMatch = servers.Version == hardcodedVersion
+	if !versionsMatch {
+		if servers.Preferred {
+			s.logger.Warn(fmt.Sprintf(
+				"%s preferred servers from file discarded because they have version %d and hardcoded servers have version %d",
+				provider, servers.Version, hardcodedVersion))
+		} else {
+			s.logger.Info(fmt.Sprintf(
+				"%s servers from file discarded because they have version %d and hardcoded servers have version %d",
+				provider, servers.Version, hardcodedVersion))
+		}
+		return servers, false, nil
+	}
+
+	return servers, true, nil
+}
+
+func (s *Storage) readServersFromFilepath(provider, filepath string, hardcodedVersion uint16) (
+	referencedServers umodels.Servers, versionsMatch bool, err error,
+) {
+	providerFile, err := os.Open(filepath)
+	if os.IsNotExist(err) {
+		return umodels.Servers{}, false, nil
+	} else if err != nil {
+		return umodels.Servers{}, false, fmt.Errorf("opening servers file %s for provider %s: %w",
+			filepath, provider, err)
+	}
+	defer providerFile.Close()
+
+	err = json.NewDecoder(providerFile).Decode(&referencedServers)
+	if err != nil {
+		return umodels.Servers{}, false, fmt.Errorf("decoding servers file %s for provider %s: %w",
+			filepath, provider, err)
+	}
+
+	versionsMatch = referencedServers.Version == hardcodedVersion
+	if !versionsMatch {
+		if referencedServers.Preferred {
+			s.logger.Warn(fmt.Sprintf(
+				"%s preferred servers from file %s discarded because they have version %d and hardcoded servers have version %d",
+				provider, filepath, referencedServers.Version, hardcodedVersion))
+		} else {
+			s.logger.Info(fmt.Sprintf(
+				"%s servers from file %s discarded because they have version %d and hardcoded servers have version %d",
+				provider, filepath, referencedServers.Version, hardcodedVersion))
+		}
+		return umodels.Servers{}, false, nil
+	}
+
+	referencedServers.Filepath = filepath
+	return referencedServers, true, nil
 }