reviewed

internal/storage/flush.go

@@ -2,6 +2,7 @@ package storage
 
 import (
 	"encoding/json"
+	"fmt"
 	"os"
 	"path/filepath"
 	"sort"
@@ -9,44 +10,76 @@ import (
 	"github.com/qdm12/gluetun/internal/models"
 )
 
-// FlushToFile flushes the merged servers data to the file
-// specified by path, as indented JSON.
-func (s *Storage) FlushToFile(path string) error {
-	s.mergedMutex.RLock()
-	defer s.mergedMutex.RUnlock()
+// flushToFile flushes the merged servers data to files
+// using the manifest file path given. It is not thread-safe.
+func (s *Storage) flushToFile(manifestPath string) error {
+	const (
+		filePermission = 0o644
+		dirPermission  = 0o755
+	)
 
-	return s.flushToFile(path)
-}
-
-// flushToFile flushes the merged servers data to the file
-// specified by path, as indented JSON. It is not thread-safe.
-func (s *Storage) flushToFile(path string) error {
-	if path == "" {
-		return nil // no file to write to
-	}
-	const permission = 0o644
-	dirPath := filepath.Dir(path)
-	if err := os.MkdirAll(dirPath, permission); err != nil {
-		return err
+	serversDirectoryPath := filepath.Dir(manifestPath)
+	if err := os.MkdirAll(serversDirectoryPath, dirPermission); err != nil {
+		return fmt.Errorf("creating directory: %w", err)
 	}
 
-	file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, permission)
+	for provider, providerServers := range s.mergedServers.ProviderToServers {
+		providerFilepath := providerServers.Filepath
+		if providerFilepath == "" {
+			providerFilepath = filepath.Join(serversDirectoryPath, provider+".json")
+		}
+
+		providerDirectoryPath := filepath.Dir(providerFilepath)
+		if err := os.MkdirAll(providerDirectoryPath, dirPermission); err != nil {
+			return fmt.Errorf("creating directory: %w", err)
+		}
+	}
+
+	metadata := map[string]any{"version": s.mergedServers.Version}
+
+	for provider, providerServers := range s.mergedServers.ProviderToServers {
+		sort.Sort(models.SortableServers(providerServers.Servers))
+
+		providerFilepath := providerServers.Filepath
+		if providerFilepath == "" {
+			providerFilepath = filepath.Join(serversDirectoryPath, provider+".json")
+		}
+
+		providerFile, err := os.OpenFile(providerFilepath,
+			os.O_CREATE|os.O_WRONLY|os.O_TRUNC, filePermission)
+		if err != nil {
+			return fmt.Errorf("opening servers data file for %s: %w", provider, err)
+		}
+
+		encoder := json.NewEncoder(providerFile)
+		encoder.SetIndent("", "  ")
+		err = encoder.Encode(providerServers)
+		if err != nil {
+			_ = providerFile.Close()
+			return fmt.Errorf("encoding servers data for %s: %w", provider, err)
+		}
+
+		err = providerFile.Close()
+		if err != nil {
+			return fmt.Errorf("closing servers data file for %s: %w", provider, err)
+		}
+
+		metadata[provider] = map[string]string{"filepath": providerFilepath}
+	}
+
+	serversFile, err := os.OpenFile(manifestPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, filePermission)
 	if err != nil {
 		return err
 	}
 
-	encoder := json.NewEncoder(file)
+	encoder := json.NewEncoder(serversFile)
 	encoder.SetIndent("", "  ")
 
-	for _, obj := range s.mergedServers.ProviderToServers {
-		sort.Sort(models.SortableServers(obj.Servers))
-	}
-
-	err = encoder.Encode(&s.mergedServers)
+	err = encoder.Encode(metadata)
 	if err != nil {
-		_ = file.Close()
+		_ = serversFile.Close()
 		return err
 	}
 
-	return file.Close()
+	return serversFile.Close()
 }

internal/storage/hardcoded.go

@@ -3,19 +3,50 @@ package storage
 import (
 	"embed"
 	"encoding/json"
+	"fmt"
+	"path"
 
+	serversmodule "github.com/qdm12/gluetun-servers/pkg/servers"
 	"github.com/qdm12/gluetun/internal/models"
 )
 
 //go:embed servers.json
 var allServersEmbedFS embed.FS
 
-func parseHardcodedServers() (allServers models.AllServers, err error) {
+func parseHardcodedServers() (allServers models.AllServers) {
 	f, err := allServersEmbedFS.Open("servers.json")
 	if err != nil {
-		return allServers, err
+		panic(err)
 	}
+	defer f.Close() // no-op
 	decoder := json.NewDecoder(f)
 	err = decoder.Decode(&allServers)
-	return allServers, err
+	if err != nil {
+		panic("decoding servers.json: " + err.Error())
+	}
+
+	for provider, metadata := range allServers.ProviderToServers {
+		filename := path.Base(metadata.Filepath)
+		providerFile, err := serversmodule.Files.Open(filename)
+		if err != nil {
+			panic(fmt.Sprintf("reading embedded provider file %s for %s: %s", filename, provider, err))
+		}
+		defer providerFile.Close() // no-op
+
+		var providerServers models.Servers
+		decoder := json.NewDecoder(providerFile)
+		err = decoder.Decode(&providerServers)
+		if err != nil {
+			panic(fmt.Sprintf("JSON decoding embedded provider file %s for %s: %s",
+				filename, provider, err))
+		} else if providerServers.Filepath != "" {
+			panic(fmt.Sprintf("embedded provider file %s for %s should not have filepath set",
+				filename, provider))
+		}
+
+		providerServers.Filepath = metadata.Filepath // inherit filepath from servers.json
+		allServers.ProviderToServers[provider] = providerServers
+	}
+
+	return allServers
 }

internal/storage/hardcoded_test.go

@@ -1,9 +1,13 @@
 package storage
 
 import (
+	"encoding/json"
+	"path"
 	"testing"
 
+	"github.com/qdm12/gluetun-servers/pkg/servers"
 	"github.com/qdm12/gluetun/internal/constants/providers"
+	"github.com/qdm12/gluetun/internal/models"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -11,9 +15,10 @@ import (
 func Test_parseHardcodedServers(t *testing.T) {
 	t.Parallel()
 
-	servers, err := parseHardcodedServers()
-
-	require.NoError(t, err)
+	var servers models.AllServers
+	assert.NotPanics(t, func() {
+		servers = parseHardcodedServers()
+	})
 
 	// all providers minus custom
 	allProviders := providers.All()
@@ -24,3 +29,35 @@ func Test_parseHardcodedServers(t *testing.T) {
 		assert.NotEmptyf(t, servers, "for provider %s", provider)
 	}
 }
+
+func Test_parseHardcodedServers_filepathsAndEmbeddedProviderFiles(t *testing.T) {
+	t.Parallel()
+
+	hardcodedServers := parseHardcodedServers()
+
+	allProviders := providers.All()
+	for _, provider := range allProviders {
+		providerServers, ok := hardcodedServers.ProviderToServers[provider]
+		require.Truef(t, ok, "for provider %s", provider)
+
+		require.NotEmptyf(t, providerServers.Filepath,
+			"embedded servers filepath should be set for provider %s", provider)
+
+		filename := path.Base(providerServers.Filepath)
+		file, err := servers.Files.Open(filename)
+		require.NoErrorf(t, err, "opening embedded provider file for %s", provider)
+
+		var fileServers struct {
+			Version   uint16            `json:"version"`
+			Timestamp int64             `json:"timestamp"`
+			Servers   []json.RawMessage `json:"servers"`
+		}
+		err = json.NewDecoder(file).Decode(&fileServers)
+		require.NoErrorf(t, err, "decoding embedded provider file for %s", provider)
+		require.NoError(t, file.Close())
+
+		assert.NotZerof(t, fileServers.Version, "for provider %s", provider)
+		assert.NotZerof(t, fileServers.Timestamp, "for provider %s", provider)
+		assert.NotEmptyf(t, fileServers.Servers, "for provider %s", provider)
+	}
+}

internal/storage/merge.go

@@ -30,6 +30,20 @@ func (s *Storage) mergeServers(hardcoded, persisted models.AllServers) models.Al
 func (s *Storage) mergeProviderServers(provider string,
 	hardcoded, persisted models.Servers,
 ) (merged models.Servers) {
+	if persisted.Preferred && persisted.Version != hardcoded.Version {
+		s.logger.Warn(fmt.Sprintf(
+			"persisted preferred %s servers are discarded because they have version %d and hardcoded servers have version %d",
+			provider, persisted.Version, hardcoded.Version))
+	}
+
+	// If persisted data is marked as preferred, use it regardless of timestamp
+	// (as long as versions match)
+	if persisted.Preferred && persisted.Version == hardcoded.Version && len(persisted.Servers) > 0 {
+		s.logger.Info(fmt.Sprintf(
+			"Using %s servers from file (marked as preferred)", provider))
+		return persisted
+	}
+
 	nowTimestamp := time.Now().Unix()
 	if persisted.Timestamp > nowTimestamp {
 		s.logger.Warn(fmt.Sprintf(

internal/storage/mocks_test.go

@@ -45,6 +45,23 @@ func (mr *MockLoggerMockRecorder) Info(arg0 interface{}) *gomock.Call {
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Info", reflect.TypeOf((*MockLogger)(nil).Info), arg0)
 }
 
+// Infof mocks base method.
+func (m *MockLogger) Infof(arg0 string, arg1 ...interface{}) {
+	m.ctrl.T.Helper()
+	varargs := []interface{}{arg0}
+	for _, a := range arg1 {
+		varargs = append(varargs, a)
+	}
+	m.ctrl.Call(m, "Infof", varargs...)
+}
+
+// Infof indicates an expected call of Infof.
+func (mr *MockLoggerMockRecorder) Infof(arg0 interface{}, arg1 ...interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	varargs := append([]interface{}{arg0}, arg1...)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Infof", reflect.TypeOf((*MockLogger)(nil).Infof), varargs...)
+}
+
 // Warn mocks base method.
 func (m *MockLogger) Warn(arg0 string) {
 	m.ctrl.T.Helper()