feat(storage): storage file structure changes (#3301)

- migrate persisted server data storage from `/gluetun/servers.json` to `/gluetun/servers/`
- add `STORAGE_SERVERS_ENABLED=on` to enable or disable on-disk server data storage
- add `STORAGE_SERVERS_DIRECTORY_PATH=/gluetun/servers` to configure where per-provider server files are stored
- keep backward compatibility with legacy `STORAGE_FILEPATH=/gluetun/servers.json`
- automatically read and migrate legacy `/gluetun/servers.json` into the new `/gluetun/servers/` layout when needed
- try to remove the legacy servers file after a successful migration to the new storage directory
- switch persisted server data from one large JSON file to a manifest plus per-provider JSON files
- add `UPDATER_PREFER_DIRECT_DOWNLOAD` to allow preferring direct download of provider server data
- keep deprecated updater flags `-enduser` and `-maintainer` as no-op warnings for backward compatibility
- preserve compatibility checks so persisted server data is discarded when its schema version no longer matches the built-in data
- allow preferred persisted provider data to override built-in data when versions match
- servers data now lives at https://github.com/qdm12/gluetun-servers/tree/main/pkg/servers

internal/updater/loop/loop.go

@@ -50,7 +50,7 @@ func NewLoop(settings settings.Updater, providers updater.Providers,
 			status:   constants.Stopped,
 			settings: settings,
 		},
-		updater:      updater.New(client, storage, providers, logger),
+		updater:      updater.New(client, storage, providers, logger, *settings.PreferDirectDownload),
 		logger:       logger,
 		start:        make(chan struct{}),
 		running:      make(chan models.LoopStatus),

internal/updater/providers.go

@@ -5,6 +5,8 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"net/url"
+	"path"
 
 	"github.com/qdm12/gluetun/internal/models"
 	"github.com/qdm12/gluetun/internal/provider/common"
@@ -16,18 +18,37 @@ type Provider interface {
 }
 
 func (u *Updater) updateProvider(ctx context.Context, provider Provider,
-	minRatio float64,
+	manifest manifest, minRatio float64,
 ) (err error) {
 	providerName := provider.Name()
 	existingServersCount := u.storage.GetServersCount(providerName)
 	minServers := int(minRatio * float64(existingServersCount))
-	servers, err := provider.FetchServers(ctx, minServers)
-	if err != nil {
-		if errors.Is(err, common.ErrNotEnoughServers) {
+
+	var servers []models.Server
+	if manifest.providerToFilepath == nil {
+		servers, err = provider.FetchServers(ctx, minServers)
+		switch {
+		case errors.Is(err, common.ErrNotEnoughServers):
 			u.logger.Warn("note: if running the update manually, you can use the flag " +
 				"-minratio to allow the update to succeed with less servers found")
+			fallthrough
+		case err != nil:
+			return fmt.Errorf("getting %s servers: %w", providerName, err)
+		}
+	} else {
+		providerFilepath := manifest.providerToFilepath[providerName]
+		providerFileURL := buildProviderFileURL(providerName, providerFilepath)
+
+		var data models.Servers
+		err = u.fetchJSON(ctx, providerFileURL, &data)
+		if err != nil {
+			return fmt.Errorf("downloading provider file %s: %w", providerFileURL, err)
+		}
+		servers = data.Servers
+		if len(servers) < minServers {
+			return fmt.Errorf("provider %s has not enough servers from downloaded file: got %d and expected at least %d",
+				providerName, len(servers), minServers)
 		}
-		return fmt.Errorf("getting %s servers: %w", providerName, err)
 	}
 
 	for _, server := range servers {
@@ -55,3 +76,13 @@ func (u *Updater) updateProvider(ctx context.Context, provider Provider,
 	}
 	return nil
 }
+
+func buildProviderFileURL(providerName, filePath string) (providerFileURL string) {
+	filename := path.Base(filePath)
+	if filename == "." || filename == "/" || filename == "" {
+		filename = providerName + ".json"
+	}
+
+	const serversFilesBaseURL = "https://raw.githubusercontent.com/qdm12/gluetun-servers/main/pkg/servers/"
+	return serversFilesBaseURL + url.PathEscape(filename)
+}

internal/updater/updater.go

@@ -2,10 +2,15 @@ package updater
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
+	"fmt"
+	"io"
 	"net/http"
+	"strings"
 	"time"
 
+	"github.com/qdm12/gluetun/internal/constants/providers"
 	"github.com/qdm12/gluetun/internal/provider/common"
 	"github.com/qdm12/gluetun/internal/updater/unzip"
 	"golang.org/x/text/cases"
@@ -13,7 +18,8 @@ import (
 )
 
 type Updater struct {
-	providers Providers
+	providers            Providers
+	preferDirectDownload bool
 
 	// state
 	storage Storage
@@ -26,22 +32,31 @@ type Updater struct {
 }
 
 func New(httpClient *http.Client, storage Storage,
-	providers Providers, logger Logger,
+	providers Providers, logger Logger, preferDirectDownload bool,
 ) *Updater {
 	unzipper := unzip.New(httpClient)
 	return &Updater{
-		providers: providers,
-		storage:   storage,
-		logger:    logger,
-		timeNow:   time.Now,
-		client:    httpClient,
-		unzipper:  unzipper,
+		providers:            providers,
+		storage:              storage,
+		logger:               logger,
+		timeNow:              time.Now,
+		client:               httpClient,
+		unzipper:             unzipper,
+		preferDirectDownload: preferDirectDownload,
 	}
 }
 
 func (u *Updater) UpdateServers(ctx context.Context, providers []string,
 	minRatio float64,
 ) (err error) {
+	var manifest manifest
+	if u.preferDirectDownload {
+		manifest, err = u.fetchManifest(ctx)
+		if err != nil {
+			return fmt.Errorf("fetching remote manifest: %w", err)
+		}
+	}
+
 	caser := cases.Title(language.English)
 	for _, providerName := range providers {
 		u.logger.Info("updating " + caser.String(providerName) + " servers...")
@@ -49,7 +64,7 @@ func (u *Updater) UpdateServers(ctx context.Context, providers []string,
 		fetcher := u.providers.Get(providerName)
 		// TODO support servers offering only TCP or only UDP
 		// for NordVPN and PureVPN
-		err := u.updateProvider(ctx, fetcher, minRatio)
+		err := u.updateProvider(ctx, fetcher, manifest, minRatio)
 		switch {
 		case err == nil:
 			continue
@@ -70,3 +85,60 @@ func (u *Updater) UpdateServers(ctx context.Context, providers []string,
 
 	return nil
 }
+
+type manifest struct {
+	providerToFilepath map[string]string
+}
+
+func (u *Updater) fetchManifest(ctx context.Context) (m manifest, err error) {
+	const serversManifestURL = "https://raw.githubusercontent.com/qdm12/gluetun-servers/main/pkg/servers/manifest.json"
+	var raw map[string]json.RawMessage
+	err = u.fetchJSON(ctx, serversManifestURL, &raw)
+	if err != nil {
+		return m, err
+	}
+
+	providerNames := providers.All()
+	m.providerToFilepath = make(map[string]string, len(providerNames))
+	for _, name := range providerNames {
+		var metadata struct {
+			Filepath string `json:"filepath"`
+		}
+		err = json.Unmarshal(raw[name], &metadata)
+		if err != nil {
+			return m, fmt.Errorf("decoding manifest metadata for %s: %w", name, err)
+		} else if metadata.Filepath == "" {
+			return m, fmt.Errorf("manifest missing filepath for provider %s", name)
+		}
+		m.providerToFilepath[name] = metadata.Filepath
+	}
+
+	return m, nil
+}
+
+func (u *Updater) fetchJSON(ctx context.Context, rawURL string, dst any) (err error) {
+	request, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
+	if err != nil {
+		return fmt.Errorf("creating request: %w", err)
+	}
+
+	response, err := u.client.Do(request)
+	if err != nil {
+		return fmt.Errorf("doing request: %w", err)
+	}
+	defer response.Body.Close()
+
+	if response.StatusCode != http.StatusOK {
+		const limit = 10 * 1024 * 1024 // 10 MiB
+		body, _ := io.ReadAll(io.LimitReader(response.Body, limit))
+		return fmt.Errorf("HTTP status code %d for %s: %s",
+			response.StatusCode, rawURL, strings.TrimSpace(string(body)))
+	}
+
+	err = json.NewDecoder(response.Body).Decode(dst)
+	if err != nil {
+		return fmt.Errorf("decoding response body: %w", err)
+	}
+
+	return nil
+}