reviewed

internal/cli/cli.go

@@ -1,11 +1,7 @@
 package cli
 
-type CLI struct {
-	repoServersPath string
-}
+type CLI struct{}
 
 func New() *CLI {
-	return &CLI{
-		repoServersPath: "./internal/storage/servers.json",
-	}
+	return &CLI{}
 }

internal/cli/formatservers.go

@@ -9,9 +9,7 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/qdm12/gluetun/internal/constants"
 	"github.com/qdm12/gluetun/internal/constants/providers"
-	"github.com/qdm12/gluetun/internal/storage"
 	"golang.org/x/text/cases"
 	"golang.org/x/text/language"
 )
@@ -74,10 +72,9 @@ func (c *CLI) FormatServers(args []string) error {
 		}
 	}
 
-	logger := newNoopLogger()
-	storage, err := storage.New(logger, constants.ServersData)
+	storage, err := setupStorage(newNoopLogger())
 	if err != nil {
-		return fmt.Errorf("creating servers storage: %w", err)
+		return fmt.Errorf("setting up storage: %w", err)
 	}
 
 	formatted, err := storage.Format(providerToFormat, format)

internal/cli/helpers.go

@@ -0,0 +1,39 @@
+package cli
+
+import (
+	"fmt"
+
+	"github.com/qdm12/gluetun/internal/configuration/settings"
+	"github.com/qdm12/gluetun/internal/configuration/sources/files"
+	"github.com/qdm12/gluetun/internal/configuration/sources/secrets"
+	"github.com/qdm12/gluetun/internal/storage"
+	"github.com/qdm12/gosettings/reader"
+	"github.com/qdm12/gosettings/reader/sources/env"
+)
+
+type storageSetupLogger interface {
+	storage.Logger
+	files.Warner
+}
+
+func setupStorage(logger storageSetupLogger) (s *storage.Storage, err error) {
+	settingsReader := reader.New(reader.Settings{
+		Sources: []reader.Source{
+			secrets.New(logger),
+			files.New(logger),
+			env.New(env.Settings{}),
+		},
+	})
+	var settings settings.Storage
+	err = settings.Read(settingsReader)
+	if err != nil {
+		return nil, fmt.Errorf("reading storage settings: %w", err)
+	}
+	settings.SetDefaults()
+	storage, err := storage.New(logger, *settings.ServersPath,
+		*settings.LegacyServersFilepath)
+	if err != nil {
+		return nil, fmt.Errorf("creating storage: %w", err)
+	}
+	return storage, nil
+}

internal/cli/nooplogger.go

@@ -6,5 +6,7 @@ func newNoopLogger() *noopLogger {
 	return new(noopLogger)
 }
 
-func (l *noopLogger) Info(string) {}
-func (l *noopLogger) Warn(string) {}
+func (l *noopLogger) Info(string)          {}
+func (l *noopLogger) Infof(string, ...any) {}
+func (l *noopLogger) Warn(string)          {}
+func (l *noopLogger) Warnf(string, ...any) {}

internal/cli/openvpnconfig.go

@@ -9,12 +9,10 @@ import (
 	"time"
 
 	"github.com/qdm12/gluetun/internal/configuration/settings"
-	"github.com/qdm12/gluetun/internal/constants"
 	"github.com/qdm12/gluetun/internal/models"
 	"github.com/qdm12/gluetun/internal/netlink"
 	"github.com/qdm12/gluetun/internal/openvpn/extract"
 	"github.com/qdm12/gluetun/internal/provider"
-	"github.com/qdm12/gluetun/internal/storage"
 	"github.com/qdm12/gluetun/internal/updater/resolver"
 	"github.com/qdm12/gosettings/reader"
 )
@@ -49,9 +47,9 @@ type IPv6Checker interface {
 func (c *CLI) OpenvpnConfig(logger OpenvpnConfigLogger, reader *reader.Reader,
 	ipv6Checker IPv6Checker,
 ) error {
-	storage, err := storage.New(logger, constants.ServersData)
+	storage, err := setupStorage(newNoopLogger())
 	if err != nil {
-		return err
+		return fmt.Errorf("setting up storage: %w", err)
 	}
 
 	var allSettings settings.Settings

internal/cli/update.go

@@ -13,12 +13,10 @@ import (
 	"github.com/qdm12/dns/v2/pkg/doh"
 	dnsprovider "github.com/qdm12/dns/v2/pkg/provider"
 	"github.com/qdm12/gluetun/internal/configuration/settings"
-	"github.com/qdm12/gluetun/internal/constants"
 	"github.com/qdm12/gluetun/internal/constants/providers"
 	"github.com/qdm12/gluetun/internal/openvpn/extract"
 	"github.com/qdm12/gluetun/internal/provider"
 	"github.com/qdm12/gluetun/internal/publicip/api"
-	"github.com/qdm12/gluetun/internal/storage"
 	"github.com/qdm12/gluetun/internal/updater"
 	"github.com/qdm12/gluetun/internal/updater/resolver"
 	"github.com/qdm12/gluetun/internal/updater/unzip"
@@ -26,18 +24,19 @@ import (
 
 type UpdaterLogger interface {
 	Info(s string)
+	Infof(format string, args ...any)
 	Warn(s string)
+	Warnf(format string, args ...any)
 	Error(s string)
 }
 
 func (c *CLI) Update(ctx context.Context, args []string, logger UpdaterLogger) error {
 	options := settings.Updater{}
-	var endUserMode, maintainerMode, updateAll bool
+	// TODO v4: remove flags below already present in standard settings
+	var endUserMode, maintainerMode string
+	var updateAll bool
 	var dnsServer, csvProviders, ipToken, protonUsername, protonEmail, protonPassword string
 	flagSet := flag.NewFlagSet("update", flag.ExitOnError)
-	flagSet.BoolVar(&endUserMode, "enduser", false, "Write results to /gluetun/servers.json (for end users)")
-	flagSet.BoolVar(&maintainerMode, "maintainer", false,
-		"Write results to ./internal/storage/servers.json to modify the program (for maintainers)")
 	flagSet.StringVar(&dnsServer, "dns", "", "no longer used, your DNS will use DoH with Cloudflare and Google")
 	const defaultMinRatio = 0.8
 	flagSet.Float64Var(&options.MinRatio, "minratio", defaultMinRatio,
@@ -49,16 +48,19 @@ func (c *CLI) Update(ctx context.Context, args []string, logger UpdaterLogger) e
 		"(Retro-compatibility) Username to use to authenticate with Proton. Use -proton-email instead.") // v4 remove this
 	flagSet.StringVar(&protonEmail, "proton-email", "", "Email to use to authenticate with Proton")
 	flagSet.StringVar(&protonPassword, "proton-password", "", "Password to use to authenticate with Proton")
+	flagSet.StringVar(&endUserMode, "enduser", "", "deprecated")
+	flagSet.StringVar(&maintainerMode, "maintainer", "", "deprecated")
 	if err := flagSet.Parse(args); err != nil {
 		return err
 	}
 
-	if dnsServer != "" {
+	switch {
+	case dnsServer != "":
 		logger.Warn("The -dns flag is no longer used, your DNS will use DoH with Cloudflare and Google")
-	}
-
-	if !endUserMode && !maintainerMode {
-		return errors.New("at least one of -enduser or -maintainer must be specified")
+	case endUserMode != "":
+		logger.Warn("The -enduser flag is now unused")
+	case maintainerMode != "":
+		logger.Warn("The -maintainer flag is now unused")
 	}
 
 	if updateAll {
@@ -87,11 +89,7 @@ func (c *CLI) Update(ctx context.Context, args []string, logger UpdaterLogger) e
 		return fmt.Errorf("options validation failed: %w", err)
 	}
 
-	serversDataPath := constants.ServersData
-	if maintainerMode {
-		serversDataPath = ""
-	}
-	storage, err := storage.New(logger, serversDataPath)
+	storage, err := setupStorage(logger)
 	if err != nil {
 		return fmt.Errorf("creating servers storage: %w", err)
 	}
@@ -127,18 +125,11 @@ func (c *CLI) Update(ctx context.Context, args []string, logger UpdaterLogger) e
 	providers := provider.NewProviders(storage, time.Now, logger, httpClient,
 		unzipper, parallelResolver, ipFetcher, openvpnFileExtractor, options)
 
-	updater := updater.New(httpClient, storage, providers, logger)
+	updater := updater.New(httpClient, storage, providers, logger, *options.PreferDirectDownload)
 	err = updater.UpdateServers(ctx, options.Providers, options.MinRatio)
 	if err != nil {
 		return fmt.Errorf("updating server information: %w", err)
 	}
 
-	if maintainerMode {
-		err := storage.FlushToFile(c.repoServersPath)
-		if err != nil {
-			return fmt.Errorf("writing servers data to embedded JSON file: %w", err)
-		}
-	}
-
 	return nil
 }

internal/configuration/settings/settings.go

@@ -132,7 +132,7 @@ func (s *Settings) SetDefaults() {
 	s.IPv6.setDefaults()
 	s.PublicIP.setDefaults()
 	s.Shadowsocks.setDefaults()
-	s.Storage.setDefaults()
+	s.Storage.SetDefaults()
 	s.System.setDefaults()
 	s.Version.setDefaults()
 	s.VPN.setDefaults()
@@ -213,7 +213,7 @@ func (s *Settings) Read(r *reader.Reader, warner Warner) (err error) {
 			return s.PublicIP.read(r, warner)
 		},
 		"shadowsocks": s.Shadowsocks.read,
-		"storage":     s.Storage.read,
+		"storage":     s.Storage.Read,
 		"system":      s.System.read,
 		"updater":     s.Updater.read,
 		"version":     s.Version.read,

internal/configuration/settings/settings_test.go

@@ -90,7 +90,7 @@ func Test_Settings_String(t *testing.T) {
 |   ├── Logging: yes
 |   └── Authentication file path: /gluetun/auth/config.toml
 ├── Storage settings:
-|   └── Filepath: /gluetun/servers.json
+|   └── Servers directory path: /gluetun/servers/
 ├── OS Alpine settings:
 |   ├── Process UID: 1000
 |   └── Process GID: 1000

internal/configuration/settings/storage.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"path/filepath"
 
+	"github.com/qdm12/gluetun/internal/constants"
 	"github.com/qdm12/gosettings"
 	"github.com/qdm12/gosettings/reader"
 	"github.com/qdm12/gotree"
@@ -11,15 +12,25 @@ import (
 
 // Storage contains settings to configure the storage.
 type Storage struct {
-	// Filepath is the path to the servers.json file. An empty string disables on-disk storage.
-	Filepath *string
+	// ServersPath is the path to the servers files directory.
+	// An empty string disables on-disk storage.
+	ServersPath *string
+	// LegacyServersFilepath is the legacy "fat" JSON filepath to migrate from.
+	// TODO v4: remove
+	LegacyServersFilepath *string
 }
 
 func (s Storage) validate() (err error) {
-	if *s.Filepath != "" { // optional
-		_, err := filepath.Abs(*s.Filepath)
+	if *s.ServersPath != "" { // optional
+		_, err := filepath.Abs(*s.ServersPath)
 		if err != nil {
-			return fmt.Errorf("filepath is not valid: %w", err)
+			return fmt.Errorf("servers path is not valid: %w", err)
+		}
+	}
+	if *s.LegacyServersFilepath != "" {
+		_, err := filepath.Abs(*s.LegacyServersFilepath)
+		if err != nil {
+			return fmt.Errorf("legacy servers filepath is not valid: %w", err)
 		}
 	}
 	return nil
@@ -27,17 +38,20 @@ func (s Storage) validate() (err error) {
 
 func (s *Storage) copy() (copied Storage) {
 	return Storage{
-		Filepath: gosettings.CopyPointer(s.Filepath),
+		ServersPath:           gosettings.CopyPointer(s.ServersPath),
+		LegacyServersFilepath: gosettings.CopyPointer(s.LegacyServersFilepath),
 	}
 }
 
 func (s *Storage) overrideWith(other Storage) {
-	s.Filepath = gosettings.OverrideWithPointer(s.Filepath, other.Filepath)
+	s.ServersPath = gosettings.OverrideWithPointer(s.ServersPath, other.ServersPath)
+	s.LegacyServersFilepath = gosettings.OverrideWithPointer(s.LegacyServersFilepath, other.LegacyServersFilepath)
 }
 
-func (s *Storage) setDefaults() {
-	const defaultFilepath = "/gluetun/servers.json"
-	s.Filepath = gosettings.DefaultPointer(s.Filepath, defaultFilepath)
+func (s *Storage) SetDefaults() {
+	const defaultServersPath = "/gluetun/servers/"
+	s.ServersPath = gosettings.DefaultPointer(s.ServersPath, defaultServersPath)
+	s.LegacyServersFilepath = gosettings.DefaultPointer(s.LegacyServersFilepath, constants.ServersDataLegacy)
 }
 
 func (s Storage) String() string {
@@ -45,15 +59,29 @@ func (s Storage) String() string {
 }
 
 func (s Storage) toLinesNode() (node *gotree.Node) {
-	if *s.Filepath == "" {
+	if *s.ServersPath == "" {
 		return gotree.New("Storage settings: disabled")
 	}
 	node = gotree.New("Storage settings:")
-	node.Appendf("Filepath: %s", *s.Filepath)
+	node.Appendf("Servers directory path: %s", *s.ServersPath)
+	if *s.LegacyServersFilepath != constants.ServersDataLegacy {
+		node.Appendf("Legacy servers filepath: %s", *s.LegacyServersFilepath)
+	}
 	return node
 }
 
-func (s *Storage) read(r *reader.Reader) (err error) {
-	s.Filepath = r.Get("STORAGE_FILEPATH", reader.AcceptEmpty(true))
+func (s *Storage) Read(r *reader.Reader) (err error) {
+	// Retro-compatibility:
+	// TODO v4: remove support for STORAGE_FILEPATH
+	filePath := r.Get("STORAGE_FILEPATH", reader.AcceptEmpty(true), reader.IsRetro("STORAGE_SERVERS_DIRECTORY_PATH"))
+	if filePath != nil {
+		s.LegacyServersFilepath = filePath
+		if *filePath == "" {
+			s.ServersPath = ptrTo("") // skip disk operations
+		}
+	}
+	if s.ServersPath == nil {
+		s.ServersPath = r.Get("STORAGE_SERVERS_DIRECTORY_PATH", reader.AcceptEmpty(true))
+	}
 	return nil
 }

internal/configuration/settings/updater.go

@@ -29,6 +29,9 @@ type Updater struct {
 	// Providers is the list of VPN service providers
 	// to update server information for.
 	Providers []string
+	// PreferDirectDownload is whether to prefer direct download of
+	// server data from Github (recommended).
+	PreferDirectDownload *bool
 	// ProtonEmail is the email to authenticate with the Proton API.
 	ProtonEmail *string
 	// ProtonPassword is the password to authenticate with the Proton API.
@@ -72,11 +75,12 @@ func (u Updater) Validate() (err error) {
 
 func (u *Updater) copy() (copied Updater) {
 	return Updater{
-		Period:         gosettings.CopyPointer(u.Period),
-		MinRatio:       u.MinRatio,
-		Providers:      gosettings.CopySlice(u.Providers),
-		ProtonEmail:    gosettings.CopyPointer(u.ProtonEmail),
-		ProtonPassword: gosettings.CopyPointer(u.ProtonPassword),
+		Period:               gosettings.CopyPointer(u.Period),
+		MinRatio:             u.MinRatio,
+		Providers:            gosettings.CopySlice(u.Providers),
+		PreferDirectDownload: gosettings.CopyPointer(u.PreferDirectDownload),
+		ProtonEmail:          gosettings.CopyPointer(u.ProtonEmail),
+		ProtonPassword:       gosettings.CopyPointer(u.ProtonPassword),
 	}
 }
 
@@ -87,6 +91,7 @@ func (u *Updater) overrideWith(other Updater) {
 	u.Period = gosettings.OverrideWithPointer(u.Period, other.Period)
 	u.MinRatio = gosettings.OverrideWithComparable(u.MinRatio, other.MinRatio)
 	u.Providers = gosettings.OverrideWithSlice(u.Providers, other.Providers)
+	u.PreferDirectDownload = gosettings.OverrideWithPointer(u.PreferDirectDownload, other.PreferDirectDownload)
 	u.ProtonEmail = gosettings.OverrideWithPointer(u.ProtonEmail, other.ProtonEmail)
 	u.ProtonPassword = gosettings.OverrideWithPointer(u.ProtonPassword, other.ProtonPassword)
 }
@@ -104,6 +109,7 @@ func (u *Updater) SetDefaults(vpnProvider string) {
 	}
 
 	// Set these to empty strings to avoid nil pointer panics
+	u.PreferDirectDownload = gosettings.DefaultPointer(u.PreferDirectDownload, false)
 	u.ProtonEmail = gosettings.DefaultPointer(u.ProtonEmail, "")
 	u.ProtonPassword = gosettings.DefaultPointer(u.ProtonPassword, "")
 }
@@ -121,6 +127,7 @@ func (u Updater) toLinesNode() (node *gotree.Node) {
 	node.Appendf("Update period: %s", *u.Period)
 	node.Appendf("Minimum ratio: %.1f", u.MinRatio)
 	node.Appendf("Providers to update: %s", strings.Join(u.Providers, ", "))
+	node.Appendf("Prefer direct download: %s", gosettings.BoolToYesNo(u.PreferDirectDownload))
 	if slices.Contains(u.Providers, providers.Protonvpn) {
 		node.Appendf("Proton API email: %s", *u.ProtonEmail)
 		node.Appendf("Proton API password: %s", gosettings.ObfuscateKey(*u.ProtonPassword))
@@ -142,6 +149,11 @@ func (u *Updater) read(r *reader.Reader) (err error) {
 
 	u.Providers = r.CSV("UPDATER_VPN_SERVICE_PROVIDERS")
 
+	u.PreferDirectDownload, err = r.BoolPtr("UPDATER_PREFER_DIRECT_DOWNLOAD")
+	if err != nil {
+		return err
+	}
+
 	u.ProtonEmail = r.Get("UPDATER_PROTONVPN_EMAIL")
 	if u.ProtonEmail == nil {
 		protonUsername := r.String("UPDATER_PROTONVPN_USERNAME", reader.IsRetro("UPDATER_PROTONVPN_EMAIL"))

internal/constants/paths.go

@@ -1,6 +1,6 @@
 package constants
 
 const (
-	// ServersData is the server information filepath.
-	ServersData = "/gluetun/servers.json"
+	// ServersDataLegacy is the old server information filepath.
+	ServersDataLegacy = "/gluetun/servers.json"
 )

internal/models/servers.go

@@ -154,6 +154,8 @@ func (a *AllServers) Count() (count int) {
 type Servers struct {
 	Version   uint16   `json:"version"`
 	Timestamp int64    `json:"timestamp"`
+	Preferred bool     `json:"preferred,omitempty"`
+	Filepath  string   `json:"filepath,omitempty"`
 	Servers   []Server `json:"servers,omitempty"`
 }
 

internal/storage/flush.go

@@ -2,6 +2,7 @@ package storage
 
 import (
 	"encoding/json"
+	"fmt"
 	"os"
 	"path/filepath"
 	"sort"
@@ -9,44 +10,76 @@ import (
 	"github.com/qdm12/gluetun/internal/models"
 )
 
-// FlushToFile flushes the merged servers data to the file
-// specified by path, as indented JSON.
-func (s *Storage) FlushToFile(path string) error {
-	s.mergedMutex.RLock()
-	defer s.mergedMutex.RUnlock()
+// flushToFile flushes the merged servers data to files
+// using the manifest file path given. It is not thread-safe.
+func (s *Storage) flushToFile(manifestPath string) error {
+	const (
+		filePermission = 0o644
+		dirPermission  = 0o755
+	)
 
-	return s.flushToFile(path)
-}
-
-// flushToFile flushes the merged servers data to the file
-// specified by path, as indented JSON. It is not thread-safe.
-func (s *Storage) flushToFile(path string) error {
-	if path == "" {
-		return nil // no file to write to
-	}
-	const permission = 0o644
-	dirPath := filepath.Dir(path)
-	if err := os.MkdirAll(dirPath, permission); err != nil {
-		return err
+	serversDirectoryPath := filepath.Dir(manifestPath)
+	if err := os.MkdirAll(serversDirectoryPath, dirPermission); err != nil {
+		return fmt.Errorf("creating directory: %w", err)
 	}
 
-	file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, permission)
+	for provider, providerServers := range s.mergedServers.ProviderToServers {
+		providerFilepath := providerServers.Filepath
+		if providerFilepath == "" {
+			providerFilepath = filepath.Join(serversDirectoryPath, provider+".json")
+		}
+
+		providerDirectoryPath := filepath.Dir(providerFilepath)
+		if err := os.MkdirAll(providerDirectoryPath, dirPermission); err != nil {
+			return fmt.Errorf("creating directory: %w", err)
+		}
+	}
+
+	metadata := map[string]any{"version": s.mergedServers.Version}
+
+	for provider, providerServers := range s.mergedServers.ProviderToServers {
+		sort.Sort(models.SortableServers(providerServers.Servers))
+
+		providerFilepath := providerServers.Filepath
+		if providerFilepath == "" {
+			providerFilepath = filepath.Join(serversDirectoryPath, provider+".json")
+		}
+
+		providerFile, err := os.OpenFile(providerFilepath,
+			os.O_CREATE|os.O_WRONLY|os.O_TRUNC, filePermission)
+		if err != nil {
+			return fmt.Errorf("opening servers data file for %s: %w", provider, err)
+		}
+
+		encoder := json.NewEncoder(providerFile)
+		encoder.SetIndent("", "  ")
+		err = encoder.Encode(providerServers)
+		if err != nil {
+			_ = providerFile.Close()
+			return fmt.Errorf("encoding servers data for %s: %w", provider, err)
+		}
+
+		err = providerFile.Close()
+		if err != nil {
+			return fmt.Errorf("closing servers data file for %s: %w", provider, err)
+		}
+
+		metadata[provider] = map[string]string{"filepath": providerFilepath}
+	}
+
+	serversFile, err := os.OpenFile(manifestPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, filePermission)
 	if err != nil {
 		return err
 	}
 
-	encoder := json.NewEncoder(file)
+	encoder := json.NewEncoder(serversFile)
 	encoder.SetIndent("", "  ")
 
-	for _, obj := range s.mergedServers.ProviderToServers {
-		sort.Sort(models.SortableServers(obj.Servers))
-	}
-
-	err = encoder.Encode(&s.mergedServers)
+	err = encoder.Encode(metadata)
 	if err != nil {
-		_ = file.Close()
+		_ = serversFile.Close()
 		return err
 	}
 
-	return file.Close()
+	return serversFile.Close()
 }

internal/storage/hardcoded.go

@@ -3,19 +3,50 @@ package storage
 import (
 	"embed"
 	"encoding/json"
+	"fmt"
+	"path"
 
+	serversmodule "github.com/qdm12/gluetun-servers/pkg/servers"
 	"github.com/qdm12/gluetun/internal/models"
 )
 
 //go:embed servers.json
 var allServersEmbedFS embed.FS
 
-func parseHardcodedServers() (allServers models.AllServers, err error) {
+func parseHardcodedServers() (allServers models.AllServers) {
 	f, err := allServersEmbedFS.Open("servers.json")
 	if err != nil {
-		return allServers, err
+		panic(err)
 	}
+	defer f.Close() // no-op
 	decoder := json.NewDecoder(f)
 	err = decoder.Decode(&allServers)
-	return allServers, err
+	if err != nil {
+		panic("decoding servers.json: " + err.Error())
+	}
+
+	for provider, metadata := range allServers.ProviderToServers {
+		filename := path.Base(metadata.Filepath)
+		providerFile, err := serversmodule.Files.Open(filename)
+		if err != nil {
+			panic(fmt.Sprintf("reading embedded provider file %s for %s: %s", filename, provider, err))
+		}
+		defer providerFile.Close() // no-op
+
+		var providerServers models.Servers
+		decoder := json.NewDecoder(providerFile)
+		err = decoder.Decode(&providerServers)
+		if err != nil {
+			panic(fmt.Sprintf("JSON decoding embedded provider file %s for %s: %s",
+				filename, provider, err))
+		} else if providerServers.Filepath != "" {
+			panic(fmt.Sprintf("embedded provider file %s for %s should not have filepath set",
+				filename, provider))
+		}
+
+		providerServers.Filepath = metadata.Filepath // inherit filepath from servers.json
+		allServers.ProviderToServers[provider] = providerServers
+	}
+
+	return allServers
 }

internal/storage/hardcoded_test.go

@@ -1,9 +1,13 @@
 package storage
 
 import (
+	"encoding/json"
+	"path"
 	"testing"
 
+	"github.com/qdm12/gluetun-servers/pkg/servers"
 	"github.com/qdm12/gluetun/internal/constants/providers"
+	"github.com/qdm12/gluetun/internal/models"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -11,9 +15,10 @@ import (
 func Test_parseHardcodedServers(t *testing.T) {
 	t.Parallel()
 
-	servers, err := parseHardcodedServers()
-
-	require.NoError(t, err)
+	var servers models.AllServers
+	assert.NotPanics(t, func() {
+		servers = parseHardcodedServers()
+	})
 
 	// all providers minus custom
 	allProviders := providers.All()
@@ -24,3 +29,35 @@ func Test_parseHardcodedServers(t *testing.T) {
 		assert.NotEmptyf(t, servers, "for provider %s", provider)
 	}
 }
+
+func Test_parseHardcodedServers_filepathsAndEmbeddedProviderFiles(t *testing.T) {
+	t.Parallel()
+
+	hardcodedServers := parseHardcodedServers()
+
+	allProviders := providers.All()
+	for _, provider := range allProviders {
+		providerServers, ok := hardcodedServers.ProviderToServers[provider]
+		require.Truef(t, ok, "for provider %s", provider)
+
+		require.NotEmptyf(t, providerServers.Filepath,
+			"embedded servers filepath should be set for provider %s", provider)
+
+		filename := path.Base(providerServers.Filepath)
+		file, err := servers.Files.Open(filename)
+		require.NoErrorf(t, err, "opening embedded provider file for %s", provider)
+
+		var fileServers struct {
+			Version   uint16            `json:"version"`
+			Timestamp int64             `json:"timestamp"`
+			Servers   []json.RawMessage `json:"servers"`
+		}
+		err = json.NewDecoder(file).Decode(&fileServers)
+		require.NoErrorf(t, err, "decoding embedded provider file for %s", provider)
+		require.NoError(t, file.Close())
+
+		assert.NotZerof(t, fileServers.Version, "for provider %s", provider)
+		assert.NotZerof(t, fileServers.Timestamp, "for provider %s", provider)
+		assert.NotEmptyf(t, fileServers.Servers, "for provider %s", provider)
+	}
+}

internal/storage/merge.go

@@ -30,6 +30,20 @@ func (s *Storage) mergeServers(hardcoded, persisted models.AllServers) models.Al
 func (s *Storage) mergeProviderServers(provider string,
 	hardcoded, persisted models.Servers,
 ) (merged models.Servers) {
+	if persisted.Preferred && persisted.Version != hardcoded.Version {
+		s.logger.Warn(fmt.Sprintf(
+			"persisted preferred %s servers are discarded because they have version %d and hardcoded servers have version %d",
+			provider, persisted.Version, hardcoded.Version))
+	}
+
+	// If persisted data is marked as preferred, use it regardless of timestamp
+	// (as long as versions match)
+	if persisted.Preferred && persisted.Version == hardcoded.Version && len(persisted.Servers) > 0 {
+		s.logger.Info(fmt.Sprintf(
+			"Using %s servers from file (marked as preferred)", provider))
+		return persisted
+	}
+
 	nowTimestamp := time.Now().Unix()
 	if persisted.Timestamp > nowTimestamp {
 		s.logger.Warn(fmt.Sprintf(

internal/storage/mocks_test.go

@@ -45,6 +45,23 @@ func (mr *MockLoggerMockRecorder) Info(arg0 interface{}) *gomock.Call {
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Info", reflect.TypeOf((*MockLogger)(nil).Info), arg0)
 }
 
+// Infof mocks base method.
+func (m *MockLogger) Infof(arg0 string, arg1 ...interface{}) {
+	m.ctrl.T.Helper()
+	varargs := []interface{}{arg0}
+	for _, a := range arg1 {
+		varargs = append(varargs, a)
+	}
+	m.ctrl.Call(m, "Infof", varargs...)
+}
+
+// Infof indicates an expected call of Infof.
+func (mr *MockLoggerMockRecorder) Infof(arg0 interface{}, arg1 ...interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	varargs := append([]interface{}{arg0}, arg1...)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Infof", reflect.TypeOf((*MockLogger)(nil).Infof), varargs...)
+}
+
 // Warn mocks base method.
 func (m *MockLogger) Warn(arg0 string) {
 	m.ctrl.T.Helper()

internal/updater/interfaces.go

@@ -3,7 +3,6 @@ package updater
 import (
 	"context"
 
-	"github.com/qdm12/gluetun/internal/configuration/settings"
 	"github.com/qdm12/gluetun/internal/models"
 	"github.com/qdm12/gluetun/internal/provider"
 )
@@ -16,8 +15,6 @@ type Storage interface {
 	SetServers(provider string, servers []models.Server) (err error)
 	GetServersCount(provider string) (count int)
 	ServersAreEqual(provider string, servers []models.Server) (equal bool)
-	// Extra methods to match the provider.New storage interface
-	FilterServers(provider string, selection settings.ServerSelection) (filtered []models.Server, err error)
 }
 
 type Unzipper interface {

internal/updater/loop/loop.go

@@ -50,7 +50,7 @@ func NewLoop(settings settings.Updater, providers updater.Providers,
 			status:   constants.Stopped,
 			settings: settings,
 		},
-		updater:      updater.New(client, storage, providers, logger),
+		updater:      updater.New(client, storage, providers, logger, *settings.PreferDirectDownload),
 		logger:       logger,
 		start:        make(chan struct{}),
 		running:      make(chan models.LoopStatus),

internal/updater/providers.go

@@ -5,6 +5,8 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"net/url"
+	"path"
 
 	"github.com/qdm12/gluetun/internal/models"
 	"github.com/qdm12/gluetun/internal/provider/common"
@@ -16,18 +18,35 @@ type Provider interface {
 }
 
 func (u *Updater) updateProvider(ctx context.Context, provider Provider,
-	minRatio float64,
+	manifest manifest, minRatio float64,
 ) (err error) {
 	providerName := provider.Name()
 	existingServersCount := u.storage.GetServersCount(providerName)
 	minServers := int(minRatio * float64(existingServersCount))
-	servers, err := provider.FetchServers(ctx, minServers)
-	if err != nil {
-		if errors.Is(err, common.ErrNotEnoughServers) {
-			u.logger.Warn("note: if running the update manually, you can use the flag " +
-				"-minratio to allow the update to succeed with less servers found")
+
+	var servers []models.Server
+	if manifest.providerToFilepath == nil {
+		servers, err = provider.FetchServers(ctx, minServers)
+		if err != nil {
+			if errors.Is(err, common.ErrNotEnoughServers) {
+				u.logger.Warn("note: if running the update manually, you can use the flag " +
+					"-minratio to allow the update to succeed with less servers found")
+			}
+			return fmt.Errorf("getting %s servers: %w", providerName, err)
+		}
+	} else {
+		providerFilepath := manifest.providerToFilepath[providerName]
+		providerFileURL := buildProviderFileURL(providerName, providerFilepath)
+
+		var data models.Servers
+		err = u.fetchJSON(ctx, providerFileURL, &data)
+		if err != nil {
+			return fmt.Errorf("downloading provider file %s: %w", providerFileURL, err)
+		}
+		if len(servers) < minServers {
+			return fmt.Errorf("provider %s has not enough servers from downloaded file: got %d and expected at least %d",
+				providerName, len(servers), minServers)
 		}
-		return fmt.Errorf("getting %s servers: %w", providerName, err)
 	}
 
 	for _, server := range servers {
@@ -55,3 +74,13 @@ func (u *Updater) updateProvider(ctx context.Context, provider Provider,
 	}
 	return nil
 }
+
+func buildProviderFileURL(providerName, filePath string) (providerFileURL string) {
+	filename := path.Base(filePath)
+	if filename == "." || filename == "/" || filename == "" {
+		filename = providerName + ".json"
+	}
+
+	const serversFilesBaseURL = "https://raw.githubusercontent.com/qdm12/gluetun-servers/main/pkg/servers/"
+	return serversFilesBaseURL + url.PathEscape(filename)
+}

internal/updater/updater.go

@@ -2,10 +2,15 @@ package updater
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
+	"fmt"
+	"io"
 	"net/http"
+	"strings"
 	"time"
 
+	"github.com/qdm12/gluetun/internal/constants/providers"
 	"github.com/qdm12/gluetun/internal/provider/common"
 	"github.com/qdm12/gluetun/internal/updater/unzip"
 	"golang.org/x/text/cases"
@@ -13,7 +18,8 @@ import (
 )
 
 type Updater struct {
-	providers Providers
+	providers            Providers
+	preferDirectDownload bool
 
 	// state
 	storage Storage
@@ -26,22 +32,31 @@ type Updater struct {
 }
 
 func New(httpClient *http.Client, storage Storage,
-	providers Providers, logger Logger,
+	providers Providers, logger Logger, preferDirectDownload bool,
 ) *Updater {
 	unzipper := unzip.New(httpClient)
 	return &Updater{
-		providers: providers,
-		storage:   storage,
-		logger:    logger,
-		timeNow:   time.Now,
-		client:    httpClient,
-		unzipper:  unzipper,
+		providers:            providers,
+		storage:              storage,
+		logger:               logger,
+		timeNow:              time.Now,
+		client:               httpClient,
+		unzipper:             unzipper,
+		preferDirectDownload: preferDirectDownload,
 	}
 }
 
 func (u *Updater) UpdateServers(ctx context.Context, providers []string,
 	minRatio float64,
 ) (err error) {
+	var manifest manifest
+	if u.preferDirectDownload {
+		manifest, err = u.fetchManifest(ctx)
+		if err != nil {
+			return fmt.Errorf("fetching remote manifest: %w", err)
+		}
+	}
+
 	caser := cases.Title(language.English)
 	for _, providerName := range providers {
 		u.logger.Info("updating " + caser.String(providerName) + " servers...")
@@ -49,7 +64,7 @@ func (u *Updater) UpdateServers(ctx context.Context, providers []string,
 		fetcher := u.providers.Get(providerName)
 		// TODO support servers offering only TCP or only UDP
 		// for NordVPN and PureVPN
-		err := u.updateProvider(ctx, fetcher, minRatio)
+		err := u.updateProvider(ctx, fetcher, manifest, minRatio)
 		switch {
 		case err == nil:
 			continue
@@ -70,3 +85,60 @@ func (u *Updater) UpdateServers(ctx context.Context, providers []string,
 
 	return nil
 }
+
+type manifest struct {
+	providerToFilepath map[string]string
+}
+
+func (u *Updater) fetchManifest(ctx context.Context) (m manifest, err error) {
+	const serversManifestURL = "https://raw.githubusercontent.com/qdm12/gluetun-servers/main/pkg/servers/manifest.json"
+	var raw map[string]json.RawMessage
+	err = u.fetchJSON(ctx, serversManifestURL, &raw)
+	if err != nil {
+		return m, err
+	}
+
+	providerNames := providers.All()
+	m.providerToFilepath = make(map[string]string, len(providerNames))
+	for _, name := range providerNames {
+		var metadata struct {
+			Filepath string `json:"filepath"`
+		}
+		err = json.Unmarshal(raw[name], &metadata)
+		if err != nil {
+			return m, fmt.Errorf("decoding manifest metadata for %s: %w", name, err)
+		} else if metadata.Filepath == "" {
+			return m, fmt.Errorf("manifest missing filepath for provider %s", name)
+		}
+		m.providerToFilepath[name] = metadata.Filepath
+	}
+
+	return m, nil
+}
+
+func (u *Updater) fetchJSON(ctx context.Context, rawURL string, dst any) (err error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
+	if err != nil {
+		return fmt.Errorf("creating request: %w", err)
+	}
+
+	response, err := u.client.Do(req)
+	if err != nil {
+		return fmt.Errorf("doing request: %w", err)
+	}
+	defer response.Body.Close()
+
+	if response.StatusCode != http.StatusOK {
+		const limit = 10 * 1024 * 1024 // 10 MiB
+		body, _ := io.ReadAll(io.LimitReader(response.Body, limit))
+		return fmt.Errorf("HTTP status code %d for %s: %s",
+			response.StatusCode, rawURL, strings.TrimSpace(string(body)))
+	}
+
+	err = json.NewDecoder(response.Body).Decode(dst)
+	if err != nil {
+		return fmt.Errorf("decoding response body: %w", err)
+	}
+
+	return nil
+}