feat(storage): storage file structure changes (#3301)

- migrate persisted server data storage from `/gluetun/servers.json` to `/gluetun/servers/`
- add `STORAGE_SERVERS_ENABLED=on` to enable or disable on-disk server data storage
- add `STORAGE_SERVERS_DIRECTORY_PATH=/gluetun/servers` to configure where per-provider server files are stored
- keep backward compatibility with legacy `STORAGE_FILEPATH=/gluetun/servers.json`
- automatically read and migrate legacy `/gluetun/servers.json` into the new `/gluetun/servers/` layout when needed
- try to remove the legacy servers file after a successful migration to the new storage directory
- switch persisted server data from one large JSON file to a manifest plus per-provider JSON files
- add `UPDATER_PREFER_DIRECT_DOWNLOAD` to allow preferring direct download of provider server data
- keep deprecated updater flags `-enduser` and `-maintainer` as no-op warnings for backward compatibility
- preserve compatibility checks so persisted server data is discarded when its schema version no longer matches the built-in data
- allow preferred persisted provider data to override built-in data when versions match
- servers data now lives at https://github.com/qdm12/gluetun-servers/tree/main/pkg/servers

internal/storage/hardcoded_test.go

@@ -1,9 +1,13 @@
 package storage
 
 import (
+	"encoding/json"
+	"path"
 	"testing"
 
+	"github.com/qdm12/gluetun-servers/pkg/servers"
 	"github.com/qdm12/gluetun/internal/constants/providers"
+	"github.com/qdm12/gluetun/internal/models"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -11,9 +15,10 @@ import (
 func Test_parseHardcodedServers(t *testing.T) {
 	t.Parallel()
 
-	servers, err := parseHardcodedServers()
-
-	require.NoError(t, err)
+	var servers models.AllServers
+	assert.NotPanics(t, func() {
+		servers = parseHardcodedServers()
+	})
 
 	// all providers minus custom
 	allProviders := providers.All()
@@ -24,3 +29,35 @@ func Test_parseHardcodedServers(t *testing.T) {
 		assert.NotEmptyf(t, servers, "for provider %s", provider)
 	}
 }
+
+func Test_parseHardcodedServers_filepathsAndEmbeddedProviderFiles(t *testing.T) {
+	t.Parallel()
+
+	hardcodedServers := parseHardcodedServers()
+
+	allProviders := providers.All()
+	for _, provider := range allProviders {
+		providerServers, ok := hardcodedServers.ProviderToServers[provider]
+		require.Truef(t, ok, "for provider %s", provider)
+
+		require.NotEmptyf(t, providerServers.Filepath,
+			"embedded servers filepath should be set for provider %s", provider)
+
+		filename := path.Base(providerServers.Filepath)
+		file, err := servers.Files.Open(filename)
+		require.NoErrorf(t, err, "opening embedded provider file for %s", provider)
+
+		var fileServers struct {
+			Version   uint16            `json:"version"`
+			Timestamp int64             `json:"timestamp"`
+			Servers   []json.RawMessage `json:"servers"`
+		}
+		err = json.NewDecoder(file).Decode(&fileServers)
+		require.NoErrorf(t, err, "decoding embedded provider file for %s", provider)
+		require.NoError(t, file.Close())
+
+		assert.NotZerof(t, fileServers.Version, "for provider %s", provider)
+		assert.NotZerof(t, fileServers.Timestamp, "for provider %s", provider)
+		assert.NotEmptyf(t, fileServers.Servers, "for provider %s", provider)
+	}
+}