feat(storage): storage file structure changes (#3301)

- migrate persisted server data storage from `/gluetun/servers.json` to `/gluetun/servers/`
- add `STORAGE_SERVERS_ENABLED=on` to enable or disable on-disk server data storage
- add `STORAGE_SERVERS_DIRECTORY_PATH=/gluetun/servers` to configure where per-provider server files are stored
- keep backward compatibility with legacy `STORAGE_FILEPATH=/gluetun/servers.json`
- automatically read and migrate legacy `/gluetun/servers.json` into the new `/gluetun/servers/` layout when needed
- try to remove the legacy servers file after a successful migration to the new storage directory
- switch persisted server data from one large JSON file to a manifest plus per-provider JSON files
- add `UPDATER_PREFER_DIRECT_DOWNLOAD` to allow preferring direct download of provider server data
- keep deprecated updater flags `-enduser` and `-maintainer` as no-op warnings for backward compatibility
- preserve compatibility checks so persisted server data is discarded when its schema version no longer matches the built-in data
- allow preferred persisted provider data to override built-in data when versions match
- servers data now lives at https://github.com/qdm12/gluetun-servers/tree/main/pkg/servers

internal/updater/providers.go

@@ -5,6 +5,8 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"net/url"
+	"path"
 
 	"github.com/qdm12/gluetun/internal/models"
 	"github.com/qdm12/gluetun/internal/provider/common"
@@ -16,18 +18,37 @@ type Provider interface {
 }
 
 func (u *Updater) updateProvider(ctx context.Context, provider Provider,
-	minRatio float64,
+	manifest manifest, minRatio float64,
 ) (err error) {
 	providerName := provider.Name()
 	existingServersCount := u.storage.GetServersCount(providerName)
 	minServers := int(minRatio * float64(existingServersCount))
-	servers, err := provider.FetchServers(ctx, minServers)
-	if err != nil {
-		if errors.Is(err, common.ErrNotEnoughServers) {
+
+	var servers []models.Server
+	if manifest.providerToFilepath == nil {
+		servers, err = provider.FetchServers(ctx, minServers)
+		switch {
+		case errors.Is(err, common.ErrNotEnoughServers):
 			u.logger.Warn("note: if running the update manually, you can use the flag " +
 				"-minratio to allow the update to succeed with less servers found")
+			fallthrough
+		case err != nil:
+			return fmt.Errorf("getting %s servers: %w", providerName, err)
+		}
+	} else {
+		providerFilepath := manifest.providerToFilepath[providerName]
+		providerFileURL := buildProviderFileURL(providerName, providerFilepath)
+
+		var data models.Servers
+		err = u.fetchJSON(ctx, providerFileURL, &data)
+		if err != nil {
+			return fmt.Errorf("downloading provider file %s: %w", providerFileURL, err)
+		}
+		servers = data.Servers
+		if len(servers) < minServers {
+			return fmt.Errorf("provider %s has not enough servers from downloaded file: got %d and expected at least %d",
+				providerName, len(servers), minServers)
 		}
-		return fmt.Errorf("getting %s servers: %w", providerName, err)
 	}
 
 	for _, server := range servers {
@@ -55,3 +76,13 @@ func (u *Updater) updateProvider(ctx context.Context, provider Provider,
 	}
 	return nil
 }
+
+func buildProviderFileURL(providerName, filePath string) (providerFileURL string) {
+	filename := path.Base(filePath)
+	if filename == "." || filename == "/" || filename == "" {
+		filename = providerName + ".json"
+	}
+
+	const serversFilesBaseURL = "https://raw.githubusercontent.com/qdm12/gluetun-servers/main/pkg/servers/"
+	return serversFilesBaseURL + url.PathEscape(filename)
+}