refactor(storage): new storage file structure

- new directory structure containing manifest.json and one json file per provider, by default.
- the manifest.json file can specify a filepath for each vpn provider
- each vpn provider json data file can contain the `"preferred": true` field to enforce it is used even if outdated, unless there is a version mismatch
- `STORAGE_SERVERS_DIRECTORY_PATH` replaces `STORAGE_FILEPATH` (which is now a migration source only). It sets the directory where server manifest and per-provider JSON files are stored (default: `/gluetun/servers/`).
- First-run migration: On startup, gluetun checks for the old /gluetun/servers.json file; if found and no new manifest exists, it automatically migrates all data to /gluetun/servers/ directory structure
- Silent fallback: If legacy file isn't found, uses the new directory path normally
- Legacy cleanup: After successful migration, attempts to remove the old fat JSON file (logs warning only if removal fails, e.g., read-only bind mounts)

internal/storage/copy.go

@@ -3,7 +3,7 @@ package storage
 import (
 	"net/netip"
 
-	"github.com/qdm12/gluetun/pkg/updaters/models"
+	"github.com/qdm12/gluetun-servers/pkg/models"
 )
 
 func copyServer(server models.Server) (serverCopy models.Server) {

internal/storage/copy_test.go

@@ -4,7 +4,7 @@ import (
 	"net/netip"
 	"testing"
 
-	"github.com/qdm12/gluetun/pkg/updaters/models"
+	"github.com/qdm12/gluetun-servers/pkg/models"
 	"github.com/stretchr/testify/assert"
 )
 

internal/storage/filter.go

@@ -5,10 +5,10 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/qdm12/gluetun-servers/pkg/constants"
+	"github.com/qdm12/gluetun-servers/pkg/models"
 	"github.com/qdm12/gluetun/internal/configuration/settings"
 	"github.com/qdm12/gluetun/internal/constants/providers"
-	"github.com/qdm12/gluetun/pkg/updaters/constants"
-	"github.com/qdm12/gluetun/pkg/updaters/models"
 )
 
 // FilterServers filter servers for the given provider and according

internal/storage/flush.go

@@ -2,51 +2,84 @@ package storage
 
 import (
 	"encoding/json"
+	"fmt"
 	"os"
 	"path/filepath"
 	"sort"
 
-	"github.com/qdm12/gluetun/pkg/updaters/models"
+	"github.com/qdm12/gluetun-servers/pkg/models"
 )
 
-// FlushToFile flushes the merged servers data to the file
-// specified by path, as indented JSON.
-func (s *Storage) FlushToFile(path string) error {
-	s.mergedMutex.RLock()
-	defer s.mergedMutex.RUnlock()
+// flushToFile flushes the merged servers data to files
+// using the manifest file path given. It is not thread-safe.
+func (s *Storage) flushToFile(manifestPath string) error {
+	const (
+		filePermission = 0o644
+		dirPermission  = 0o755
+	)
 
-	return s.flushToFile(path)
-}
-
-// flushToFile flushes the merged servers data to the file
-// specified by path, as indented JSON. It is not thread-safe.
-func (s *Storage) flushToFile(path string) error {
-	if path == "" {
-		return nil // no file to write to
-	}
-	const permission = 0o644
-	dirPath := filepath.Dir(path)
-	if err := os.MkdirAll(dirPath, permission); err != nil {
-		return err
+	serversDirectoryPath := filepath.Dir(manifestPath)
+	if err := os.MkdirAll(serversDirectoryPath, dirPermission); err != nil {
+		return fmt.Errorf("creating directory: %w", err)
 	}
 
-	file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, permission)
+	for provider, providerServers := range s.mergedServers.ProviderToServers {
+		providerFilepath := providerServers.Filepath
+		if providerFilepath == "" {
+			providerFilepath = filepath.Join(serversDirectoryPath, provider+".json")
+		}
+
+		providerDirectoryPath := filepath.Dir(providerFilepath)
+		if err := os.MkdirAll(providerDirectoryPath, dirPermission); err != nil {
+			return fmt.Errorf("creating directory: %w", err)
+		}
+	}
+
+	metadata := map[string]any{"version": s.mergedServers.Version}
+
+	for provider, providerServers := range s.mergedServers.ProviderToServers {
+		sort.Sort(models.SortableServers(providerServers.Servers))
+
+		providerFilepath := providerServers.Filepath
+		if providerFilepath == "" {
+			providerFilepath = filepath.Join(serversDirectoryPath, provider+".json")
+		}
+
+		providerFile, err := os.OpenFile(providerFilepath,
+			os.O_CREATE|os.O_WRONLY|os.O_TRUNC, filePermission)
+		if err != nil {
+			return fmt.Errorf("opening servers data file for %s: %w", provider, err)
+		}
+
+		encoder := json.NewEncoder(providerFile)
+		encoder.SetIndent("", "  ")
+		err = encoder.Encode(providerServers)
+		if err != nil {
+			_ = providerFile.Close()
+			return fmt.Errorf("encoding servers data for %s: %w", provider, err)
+		}
+
+		err = providerFile.Close()
+		if err != nil {
+			return fmt.Errorf("closing servers data file for %s: %w", provider, err)
+		}
+
+		metadata[provider] = map[string]string{"filepath": providerFilepath}
+	}
+
+	serversFile, err := os.OpenFile(manifestPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, filePermission)
 	if err != nil {
 		return err
 	}
 
-	encoder := json.NewEncoder(file)
+	encoder := json.NewEncoder(serversFile)
 	encoder.SetIndent("", "  ")
 
-	for _, obj := range s.mergedServers.ProviderToServers {
-		sort.Sort(models.SortableServers(obj.Servers))
-	}
-
-	err = encoder.Encode(&s.mergedServers)
+	err = encoder.Encode(metadata)
 	if err != nil {
-		_ = file.Close()
+		_ = serversFile.Close()
 		return err
 	}
 
-	return file.Close()
+	return serversFile.Close()
 }

internal/storage/formatting.go

@@ -5,8 +5,8 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/qdm12/gluetun-servers/pkg/constants"
 	"github.com/qdm12/gluetun/internal/configuration/settings"
-	"github.com/qdm12/gluetun/pkg/updaters/constants"
 )
 
 func commaJoin(slice []string) string {

internal/storage/hardcoded.go

@@ -3,19 +3,51 @@ package storage
 import (
 	"embed"
 	"encoding/json"
+	"fmt"
+	"path"
 
+	umodels "github.com/qdm12/gluetun-servers/pkg/models"
+	serversmodule "github.com/qdm12/gluetun-servers/pkg/servers"
 	"github.com/qdm12/gluetun/internal/models"
 )
 
 //go:embed servers.json
 var allServersEmbedFS embed.FS
 
-func parseHardcodedServers() (allServers models.AllServers, err error) {
+func parseHardcodedServers() (allServers models.AllServers) {
 	f, err := allServersEmbedFS.Open("servers.json")
 	if err != nil {
-		return allServers, err
+		panic(err)
 	}
+	defer f.Close() // no-op
 	decoder := json.NewDecoder(f)
 	err = decoder.Decode(&allServers)
-	return allServers, err
+	if err != nil {
+		panic("decoding servers.json: " + err.Error())
+	}
+
+	for provider, metadata := range allServers.ProviderToServers {
+		filename := path.Base(metadata.Filepath)
+		providerFile, err := serversmodule.Files.Open(filename)
+		if err != nil {
+			panic(fmt.Sprintf("reading embedded provider file %s for %s: %s", filename, provider, err))
+		}
+		defer providerFile.Close() // no-op
+
+		var providerServers umodels.Servers
+		decoder := json.NewDecoder(providerFile)
+		err = decoder.Decode(&providerServers)
+		if err != nil {
+			panic(fmt.Sprintf("JSON decoding embedded provider file %s for %s: %s",
+				filename, provider, err))
+		} else if providerServers.Filepath != "" {
+			panic(fmt.Sprintf("embedded provider file %s for %s should not have filepath set",
+				filename, provider))
+		}
+
+		providerServers.Filepath = metadata.Filepath // inherit filepath from servers.json
+		allServers.ProviderToServers[provider] = providerServers
+	}
+
+	return allServers
 }

internal/storage/hardcoded_test.go

@@ -1,9 +1,13 @@
 package storage
 
 import (
+	"encoding/json"
+	"path"
 	"testing"
 
+	"github.com/qdm12/gluetun-servers/pkg/servers"
 	"github.com/qdm12/gluetun/internal/constants/providers"
+	"github.com/qdm12/gluetun/internal/models"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -11,9 +15,10 @@ import (
 func Test_parseHardcodedServers(t *testing.T) {
 	t.Parallel()
 
-	servers, err := parseHardcodedServers()
-
-	require.NoError(t, err)
+	var servers models.AllServers
+	assert.NotPanics(t, func() {
+		servers = parseHardcodedServers()
+	})
 
 	// all providers minus custom
 	allProviders := providers.All()
@@ -24,3 +29,35 @@ func Test_parseHardcodedServers(t *testing.T) {
 		assert.NotEmptyf(t, servers, "for provider %s", provider)
 	}
 }
+
+func Test_parseHardcodedServers_filepathsAndEmbeddedProviderFiles(t *testing.T) {
+	t.Parallel()
+
+	hardcodedServers := parseHardcodedServers()
+
+	allProviders := providers.All()
+	for _, provider := range allProviders {
+		providerServers, ok := hardcodedServers.ProviderToServers[provider]
+		require.Truef(t, ok, "for provider %s", provider)
+
+		require.NotEmptyf(t, providerServers.Filepath,
+			"embedded servers filepath should be set for provider %s", provider)
+
+		filename := path.Base(providerServers.Filepath)
+		file, err := servers.Files.Open(filename)
+		require.NoErrorf(t, err, "opening embedded provider file for %s", provider)
+
+		var fileServers struct {
+			Version   uint16            `json:"version"`
+			Timestamp int64             `json:"timestamp"`
+			Servers   []json.RawMessage `json:"servers"`
+		}
+		err = json.NewDecoder(file).Decode(&fileServers)
+		require.NoErrorf(t, err, "decoding embedded provider file for %s", provider)
+		require.NoError(t, file.Close())
+
+		assert.NotZerof(t, fileServers.Version, "for provider %s", provider)
+		assert.NotZerof(t, fileServers.Timestamp, "for provider %s", provider)
+		assert.NotEmptyf(t, fileServers.Servers, "for provider %s", provider)
+	}
+}

internal/storage/merge.go

@@ -5,10 +5,10 @@ import (
 	"sort"
 	"time"
 
+	umodels "github.com/qdm12/gluetun-servers/pkg/models"
 	"github.com/qdm12/gluetun/internal/constants/providers"
 	"github.com/qdm12/gluetun/internal/format"
 	"github.com/qdm12/gluetun/internal/models"
-	umodels "github.com/qdm12/gluetun/pkg/updaters/models"
 )
 
 func (s *Storage) mergeServers(hardcoded, persisted models.AllServers) models.AllServers {
@@ -31,6 +31,20 @@ func (s *Storage) mergeServers(hardcoded, persisted models.AllServers) models.Al
 func (s *Storage) mergeProviderServers(provider string,
 	hardcoded, persisted umodels.Servers,
 ) (merged umodels.Servers) {
+	if persisted.Preferred && persisted.Version != hardcoded.Version {
+		s.logger.Warn(fmt.Sprintf(
+			"persisted preferred %s servers are discarded because they have version %d and hardcoded servers have version %d",
+			provider, persisted.Version, hardcoded.Version))
+	}
+
+	// If persisted data is marked as preferred, use it regardless of timestamp
+	// (as long as versions match)
+	if persisted.Preferred && persisted.Version == hardcoded.Version && len(persisted.Servers) > 0 {
+		s.logger.Info(fmt.Sprintf(
+			"Using %s servers from file (marked as preferred)", provider))
+		return persisted
+	}
+
 	nowTimestamp := time.Now().Unix()
 	if persisted.Timestamp > nowTimestamp {
 		s.logger.Warn(fmt.Sprintf(

internal/storage/mocks_test.go

@@ -45,6 +45,23 @@ func (mr *MockLoggerMockRecorder) Info(arg0 interface{}) *gomock.Call {
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Info", reflect.TypeOf((*MockLogger)(nil).Info), arg0)
 }
 
+// Infof mocks base method.
+func (m *MockLogger) Infof(arg0 string, arg1 ...interface{}) {
+	m.ctrl.T.Helper()
+	varargs := []interface{}{arg0}
+	for _, a := range arg1 {
+		varargs = append(varargs, a)
+	}
+	m.ctrl.Call(m, "Infof", varargs...)
+}
+
+// Infof indicates an expected call of Infof.
+func (mr *MockLoggerMockRecorder) Infof(arg0 interface{}, arg1 ...interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	varargs := append([]interface{}{arg0}, arg1...)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Infof", reflect.TypeOf((*MockLogger)(nil).Infof), varargs...)
+}
+
 // Warn mocks base method.
 func (m *MockLogger) Warn(arg0 string) {
 	m.ctrl.T.Helper()

internal/storage/read.go

@@ -6,36 +6,37 @@ import (
 	"io"
 	"os"
 
+	umodels "github.com/qdm12/gluetun-servers/pkg/models"
 	"github.com/qdm12/gluetun/internal/constants/providers"
 	"github.com/qdm12/gluetun/internal/models"
-	umodels "github.com/qdm12/gluetun/pkg/updaters/models"
 	"golang.org/x/text/cases"
 	"golang.org/x/text/language"
 )
 
-// readFromFile reads the servers from server.json.
+// readFromFile reads the servers data starting from the given manifest file path.
 // It only reads servers that have the same version as the hardcoded servers version
 // to avoid JSON decoding errors.
-func (s *Storage) readFromFile(filepath string, hardcodedVersions map[string]uint16) (
-	servers models.AllServers, err error,
+func (s *Storage) readFromFile(manifestPath string, hardcodedVersions map[string]uint16) (
+	servers models.AllServers, found bool, err error,
 ) {
-	file, err := os.Open(filepath)
+	file, err := os.Open(manifestPath)
 	if os.IsNotExist(err) {
-		return servers, nil
+		return servers, false, nil
 	} else if err != nil {
-		return servers, err
+		return servers, false, err
 	}
 
 	b, err := io.ReadAll(file)
 	if err != nil {
-		return servers, err
+		return servers, true, err
 	}
 
 	if err := file.Close(); err != nil {
-		return servers, err
+		return servers, true, err
 	}
 
-	return s.extractServersFromBytes(b, hardcodedVersions)
+	servers, err = s.extractServersFromBytes(b, hardcodedVersions)
+	return servers, true, err
 }
 
 func (s *Storage) extractServersFromBytes(b []byte, hardcodedVersions map[string]uint16) (
@@ -47,6 +48,12 @@ func (s *Storage) extractServersFromBytes(b []byte, hardcodedVersions map[string
 	}
 
 	// Note schema version is at map key "version" as number
+	if rawVersion, ok := rawMessages["version"]; ok {
+		err := json.Unmarshal(rawVersion, &servers.Version)
+		if err != nil {
+			return servers, fmt.Errorf("decoding servers schema version: %w", err)
+		}
+	}
 
 	allProviders := providers.All()
 	servers.ProviderToServers = make(map[string]umodels.Servers, len(allProviders))
@@ -87,25 +94,20 @@ func (s *Storage) readServers(provider string, hardcodedVersion uint16,
 ) {
 	provider = titleCaser.String(provider)
 
-	var versionObject struct {
-		Version uint16 `json:"version"`
+	var metadata struct {
+		Version   uint16 `json:"version"`
+		Timestamp int64  `json:"timestamp"`
+		Filepath  string `json:"filepath"`
 	}
 
-	err = json.Unmarshal(rawMessage, &versionObject)
+	err = json.Unmarshal(rawMessage, &metadata)
 	if err != nil {
 		return servers, false, fmt.Errorf("decoding servers version for provider %s: %w",
 			provider, err)
 	}
 
-	persistedVersion := versionObject.Version
-
-	versionsMatch = hardcodedVersion == persistedVersion
-	if !versionsMatch {
-		s.logger.Info(fmt.Sprintf(
-			"%s servers from file discarded because they have "+
-				"version %d and hardcoded servers have version %d",
-			provider, persistedVersion, hardcodedVersion))
-		return servers, versionsMatch, nil
+	if metadata.Filepath != "" {
+		return s.readServersFromFilepath(provider, metadata.Filepath, hardcodedVersion)
 	}
 
 	err = json.Unmarshal(rawMessage, &servers)
@@ -114,5 +116,55 @@ func (s *Storage) readServers(provider string, hardcodedVersion uint16,
 			provider, err)
 	}
 
-	return servers, versionsMatch, nil
+	versionsMatch = servers.Version == hardcodedVersion
+	if !versionsMatch {
+		if servers.Preferred {
+			s.logger.Warn(fmt.Sprintf(
+				"%s preferred servers from file discarded because they have version %d and hardcoded servers have version %d",
+				provider, servers.Version, hardcodedVersion))
+		} else {
+			s.logger.Info(fmt.Sprintf(
+				"%s servers from file discarded because they have version %d and hardcoded servers have version %d",
+				provider, servers.Version, hardcodedVersion))
+		}
+		return servers, false, nil
+	}
+
+	return servers, true, nil
+}
+
+func (s *Storage) readServersFromFilepath(provider, filepath string, hardcodedVersion uint16) (
+	referencedServers umodels.Servers, versionsMatch bool, err error,
+) {
+	providerFile, err := os.Open(filepath)
+	if os.IsNotExist(err) {
+		return umodels.Servers{}, false, nil
+	} else if err != nil {
+		return umodels.Servers{}, false, fmt.Errorf("opening servers file %s for provider %s: %w",
+			filepath, provider, err)
+	}
+	defer providerFile.Close()
+
+	err = json.NewDecoder(providerFile).Decode(&referencedServers)
+	if err != nil {
+		return umodels.Servers{}, false, fmt.Errorf("decoding servers file %s for provider %s: %w",
+			filepath, provider, err)
+	}
+
+	versionsMatch = referencedServers.Version == hardcodedVersion
+	if !versionsMatch {
+		if referencedServers.Preferred {
+			s.logger.Warn(fmt.Sprintf(
+				"%s preferred servers from file %s discarded because they have version %d and hardcoded servers have version %d",
+				provider, filepath, referencedServers.Version, hardcodedVersion))
+		} else {
+			s.logger.Info(fmt.Sprintf(
+				"%s servers from file %s discarded because they have version %d and hardcoded servers have version %d",
+				provider, filepath, referencedServers.Version, hardcodedVersion))
+		}
+		return umodels.Servers{}, false, nil
+	}
+
+	referencedServers.Filepath = filepath
+	return referencedServers, true, nil
 }

internal/storage/read_test.go

@@ -5,9 +5,10 @@ import (
 	"testing"
 
 	"github.com/golang/mock/gomock"
+	umodels "github.com/qdm12/gluetun-servers/pkg/models"
 	"github.com/qdm12/gluetun/internal/constants/providers"
 	"github.com/qdm12/gluetun/internal/models"
-	umodels "github.com/qdm12/gluetun/pkg/updaters/models"
+	"github.com/qdm12/log"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -28,10 +29,15 @@ func populateProviderToVersion(providerToVersion map[string]uint16) map[string]u
 func Test_extractServersFromBytes(t *testing.T) {
 	t.Parallel()
 
+	type logLine struct {
+		level   log.Level
+		message string
+	}
+
 	testCases := map[string]struct {
 		b                 []byte
 		hardcodedVersions map[string]uint16
-		logged            []string
+		logged            []logLine
 		persisted         models.AllServers
 		errMessage        string
 	}{
@@ -43,7 +49,9 @@ func Test_extractServersFromBytes(t *testing.T) {
 			b:                 []byte(`{"cyberghost": "garbage"}`),
 			hardcodedVersions: populateProviderToVersion(map[string]uint16{}),
 			errMessage: "decoding servers version for provider Cyberghost: " +
-				"json: cannot unmarshal string into Go value of type struct { Version uint16 \"json:\\\"version\\\"\" }",
+				"json: cannot unmarshal string into Go value of type struct { Version uint16 \"json:\\\"version\\\"\"; " +
+				"Timestamp int64 \"json:\\\"timestamp\\\"\"; " +
+				"Filepath string \"json:\\\"filepath\\\"\" }",
 		},
 		"bad servers array JSON": {
 			b: []byte(`{"cyberghost": {"version": 1, "servers": "garbage"}}`),
@@ -82,13 +90,30 @@ func Test_extractServersFromBytes(t *testing.T) {
 			hardcodedVersions: populateProviderToVersion(map[string]uint16{
 				providers.Cyberghost: 2,
 			}),
-			logged: []string{
-				"Cyberghost servers from file discarded because they have version 1 and hardcoded servers have version 2",
+			logged: []logLine{
+				{level: log.LevelInfo, message: "Cyberghost servers from file discarded because they have version 1" +
+					" and hardcoded servers have version 2"},
 			},
 			persisted: models.AllServers{
 				ProviderToServers: map[string]umodels.Servers{},
 			},
 		},
+		"preferred_different_versions": {
+			b: []byte(`{
+				"cyberghost": {"version": 1, "timestamp": 1, "preferred": true}
+			}`),
+			hardcodedVersions: populateProviderToVersion(map[string]uint16{
+				providers.Cyberghost: 2,
+			}),
+			logged: []logLine{
+				{level: log.LevelWarn, message: "Cyberghost preferred servers from file discarded because they have version 1" +
+					" and hardcoded servers have version 2"},
+			},
+			persisted: models.AllServers{
+				ProviderToServers: map[string]umodels.Servers{},
+			},
+			errMessage: "",
+		},
 	}
 
 	for name, testCase := range testCases {
@@ -99,7 +124,15 @@ func Test_extractServersFromBytes(t *testing.T) {
 			logger := NewMockLogger(ctrl)
 			var previousLogCall *gomock.Call
 			for _, logged := range testCase.logged {
-				call := logger.EXPECT().Info(logged)
+				var call *gomock.Call
+				switch logged.level { //nolint:exhaustive
+				case log.LevelInfo:
+					call = logger.EXPECT().Info(logged.message)
+				case log.LevelWarn:
+					call = logger.EXPECT().Warn(logged.message)
+				default:
+					t.Fatalf("invalid log level %d in test case", logged.level)
+				}
 				if previousLogCall != nil {
 					call.After(previousLogCall)
 				}

internal/storage/servers.go

@@ -2,21 +2,23 @@ package storage
 
 import (
 	"fmt"
+	"os"
+	"path/filepath"
 	"time"
 
+	umodels "github.com/qdm12/gluetun-servers/pkg/models"
 	"github.com/qdm12/gluetun/internal/constants/providers"
 	"github.com/qdm12/gluetun/internal/models"
-	umodels "github.com/qdm12/gluetun/pkg/updaters/models"
 )
 
 // SetServers sets the given servers for the given provider
 // in the storage in-memory map and saves all the servers
-// to file.
+// to files.
 // Note the servers given are not copied so the caller must
 // NOT MUTATE them after calling this method.
 func (s *Storage) SetServers(provider string, servers []umodels.Server) (err error) {
 	if provider == providers.Custom {
-		return
+		return nil
 	}
 
 	s.mergedMutex.Lock()
@@ -27,10 +29,24 @@ func (s *Storage) SetServers(provider string, servers []umodels.Server) (err err
 	serversObject.Servers = servers
 	s.mergedServers.ProviderToServers[provider] = serversObject
 
-	err = s.flushToFile(s.filepath)
+	if s.directoryPath == "" {
+		return nil // no disk writing
+	}
+
+	manifestPath := filepath.Join(s.directoryPath, manifestFilename)
+	err = s.flushToFile(manifestPath)
 	if err != nil {
 		return fmt.Errorf("saving servers to file: %w", err)
 	}
+
+	if !s.hasLegacy() {
+		return nil
+	}
+	s.logger.Infof("removing legacy %s which is now migrated to %s", s.legacyFilepath, s.directoryPath)
+	err = os.Remove(s.legacyFilepath)
+	if err != nil && !os.IsNotExist(err) {
+		s.logger.Warn("failed removing legacy servers file " + s.legacyFilepath + ": " + err.Error())
+	}
 	return nil
 }
 

internal/storage/storage.go

@@ -1,6 +1,8 @@
 package storage
 
 import (
+	"os"
+	"path/filepath"
 	"sync"
 
 	"github.com/qdm12/gluetun/internal/models"
@@ -14,31 +16,36 @@ type Storage struct {
 	// SyncServers method.
 	hardcodedServers models.AllServers
 	logger           Logger
-	filepath         string
+	directoryPath    string
+	legacyFilepath   string
 }
 
+const manifestFilename = "manifest.json"
+
 type Logger interface {
 	Info(s string)
+	Infof(format string, args ...any)
 	Warn(s string)
 }
 
 // New creates a new storage and reads the servers from the
-// embedded servers file and the file on disk.
-// Passing an empty filepath disables the reading and writing of
+// embedded servers files and the files on disk.
+// Passing an empty directoryPath disables the reading and writing of
 // servers.
-func New(logger Logger, filepath string) (storage *Storage, err error) {
-	// A unit test prevents any error from being returned
+func New(logger Logger, directoryPath, legacyFilepath string) (storage *Storage, err error) {
+	// A unit test prevents [parseHardcodedServers] from ever failing,
 	// and ensures all providers are part of the servers returned.
-	hardcodedServers, _ := parseHardcodedServers()
+	hardcodedServers := parseHardcodedServers()
 
 	storage = &Storage{
 		hardcodedServers: hardcodedServers,
 		mergedServers:    hardcodedServers,
 		logger:           logger,
-		filepath:         filepath,
+		directoryPath:    directoryPath,
+		legacyFilepath:   legacyFilepath,
 	}
 
-	if filepath != "" {
+	if directoryPath != "" {
 		if err := storage.syncServers(); err != nil {
 			return nil, err
 		}
@@ -46,3 +53,18 @@ func New(logger Logger, filepath string) (storage *Storage, err error) {
 
 	return storage, nil
 }
+
+// hasLegacy returns true if the legacy file `legacyFilepath` exists AND is
+// different from the manifest file defined by `directoryPath`/[manifestFilename].
+// This is used to determine if the legacy file should be read and removed after flushing servers data.
+func (s *Storage) hasLegacy() bool {
+	if s.legacyFilepath == "" {
+		return false
+	}
+	if filepath.Clean(filepath.Join(s.directoryPath, manifestFilename)) ==
+		filepath.Clean(s.legacyFilepath) {
+		return false
+	}
+	stat, err := os.Stat(s.legacyFilepath)
+	return err == nil && !stat.IsDir()
+}

internal/storage/sync.go

@@ -2,6 +2,8 @@ package storage
 
 import (
 	"fmt"
+	"os"
+	"path/filepath"
 	"reflect"
 
 	"github.com/qdm12/gluetun/internal/models"
@@ -14,18 +16,31 @@ func countServers(allServers models.AllServers) (count int) {
 	return count
 }
 
-// syncServers merges the hardcoded servers with the ones from the file.
+// syncServers merges the hardcoded servers with the ones from on disk files.
+// It assumes s.directoryPath is set.
 func (s *Storage) syncServers() (err error) {
 	hardcodedVersions := make(map[string]uint16, len(s.hardcodedServers.ProviderToServers))
 	for provider, servers := range s.hardcodedServers.ProviderToServers {
 		hardcodedVersions[provider] = servers.Version
 	}
 
-	serversOnFile, err := s.readFromFile(s.filepath, hardcodedVersions)
+	sourceManifestPath := filepath.Join(s.directoryPath, manifestFilename)
+	destinationManifestPath := sourceManifestPath
+	serversOnFile, found, err := s.readFromFile(sourceManifestPath, hardcodedVersions)
 	if err != nil {
 		return fmt.Errorf("reading servers from file: %w", err)
 	}
 
+	hasLegacy := s.hasLegacy()
+	if !found && hasLegacy {
+		sourceManifestPath = s.legacyFilepath
+		s.logger.Infof("reading legacy servers file %s and migrating it to directory %s", sourceManifestPath, s.directoryPath)
+		serversOnFile, _, err = s.readFromFile(sourceManifestPath, hardcodedVersions)
+		if err != nil {
+			return fmt.Errorf("reading servers from file: %w", err)
+		}
+	}
+
 	hardcodedCount := countServers(s.hardcodedServers)
 	countOnFile := countServers(serversOnFile)
 
@@ -34,13 +49,13 @@ func (s *Storage) syncServers() (err error) {
 
 	if countOnFile == 0 {
 		s.logger.Info(fmt.Sprintf(
-			"creating %s with %d hardcoded servers",
-			s.filepath, hardcodedCount))
+			"writing servers data files to %s with %d hardcoded servers",
+			s.directoryPath, hardcodedCount))
 		s.mergedServers = s.hardcodedServers
 	} else {
 		s.logger.Info(fmt.Sprintf(
-			"merging by most recent %d hardcoded servers and %d servers read from %s",
-			hardcodedCount, countOnFile, s.filepath))
+			"merging by most recent %d hardcoded servers and %d servers read from manifest file %s",
+			hardcodedCount, countOnFile, sourceManifestPath))
 
 		s.mergedServers = s.mergeServers(s.hardcodedServers, serversOnFile)
 	}
@@ -50,9 +65,18 @@ func (s *Storage) syncServers() (err error) {
 		return nil
 	}
 
-	err = s.flushToFile(s.filepath)
+	err = s.flushToFile(destinationManifestPath)
 	if err != nil {
-		s.logger.Warn("failed writing servers to file: " + err.Error())
+		s.logger.Warn("failed writing servers to destination manifest: " + err.Error())
+		return nil
+	}
+
+	migratedFromLegacy := hasLegacy && sourceManifestPath == s.legacyFilepath
+	if migratedFromLegacy {
+		err = os.Remove(sourceManifestPath)
+		if err != nil && !os.IsNotExist(err) {
+			s.logger.Warn("failed removing legacy servers file " + sourceManifestPath + ": " + err.Error())
+		}
 	}
 	return nil
 }