gluetun updater only downloads files from gluetun-servers

.vscode/tasks.json

@@ -1,19 +1,6 @@
 {
     "version": "2.0.0",
     "tasks": [
-        {
-            "label": "Update a VPN provider servers data",
-            "type": "shell",
-            "command": "go",
-            "args": [
-                "run",
-                "./cmd/gluetun/main.go",
-                "update",
-                "${input:updateMode}",
-                "-providers",
-                "${input:provider}"
-            ],
-        },
         {
             "label": "Add a Gluetun Github Git remote",
             "type": "shell",
@@ -36,21 +23,6 @@
         }
     ],
     "inputs": [
-        {
-            "id": "provider",
-            "type": "promptString",
-            "description": "Please enter a provider (or comma separated list of providers)",
-        },
-        {
-            "id": "updateMode",
-            "type": "pickString",
-            "description": "Update mode to use",
-            "options": [
-                "-maintainer",
-                "-enduser"
-            ],
-            "default": "-maintainer"
-        },
         {
             "id": "githubRemoteUsername",
             "type": "promptString",

cmd/gluetun/main.go

@@ -464,7 +464,7 @@ func _main(ctx context.Context, buildInfo models.BuildInformation,
 	go vpnLooper.Run(vpnCtx, vpnDone)
 
 	updaterLooper := updater.NewLoop(allSettings.Updater,
-		providers, storage, httpClient, updaterLogger)
+		storage, httpClient, updaterLogger)
 	updaterHandler, updaterCtx, updaterDone := goshutdown.NewGoRoutineHandler(
 		"updater", goroutine.OptionTimeout(defaultShutdownTimeout))
 	// wait for updaterLooper.Restart() or its ticket launched with RunRestartTicker

go.mod

@@ -3,7 +3,6 @@ module github.com/qdm12/gluetun
 go 1.25.0
 
 require (
-	github.com/ProtonMail/go-srp v0.0.7
 	github.com/amnezia-vpn/amneziawg-go v0.2.16
 	github.com/breml/rootcerts v0.3.4
 	github.com/fatih/color v1.18.0
@@ -37,6 +36,7 @@ require (
 require (
 	github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf // indirect
 	github.com/ProtonMail/go-crypto v1.3.0-proton // indirect
+	github.com/ProtonMail/go-srp v0.0.7 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/cloudflare/circl v1.6.1 // indirect

internal/cli/update.go

@@ -10,15 +10,8 @@ import (
 	"strings"
 	"time"
 
-	"github.com/qdm12/dns/v2/pkg/doh"
-	dnsprovider "github.com/qdm12/dns/v2/pkg/provider"
-	"github.com/qdm12/gluetun-servers/pkg/publicip/api"
-	"github.com/qdm12/gluetun-servers/pkg/updaters/resolver"
-	"github.com/qdm12/gluetun-servers/pkg/updaters/unzip"
 	"github.com/qdm12/gluetun/internal/configuration/settings"
 	"github.com/qdm12/gluetun/internal/constants/providers"
-	"github.com/qdm12/gluetun/internal/openvpn/extract"
-	"github.com/qdm12/gluetun/internal/provider"
 	"github.com/qdm12/gluetun/internal/updater"
 )
 
@@ -52,8 +45,19 @@ func (c *CLI) Update(ctx context.Context, args []string, logger UpdaterLogger) e
 		return err
 	}
 
-	if dnsServer != "" {
+	switch {
+	case dnsServer != "":
 		logger.Warn("The -dns flag is no longer used, your DNS will use DoH with Cloudflare and Google")
+	case endUserMode:
+		logger.Warn("The -enduser flag is no longer used and has no effect")
+	case ipToken != "":
+		logger.Warn("The -ip-token flag is no longer used and has no effect")
+	case protonUsername != "":
+		logger.Warn("The -proton-username flag is no longer used and has no effect. Use -proton-email instead.")
+	case protonEmail != "":
+		logger.Warn("The -proton-email flag is no longer used and has no effect")
+	case protonPassword != "":
+		logger.Warn("The -proton-password flag is no longer used and has no effect")
 	}
 
 	if updateAll {
@@ -87,38 +91,9 @@ func (c *CLI) Update(ctx context.Context, args []string, logger UpdaterLogger) e
 		return fmt.Errorf("creating servers storage: %w", err)
 	}
 
-	dohSettings := doh.Settings{
-		UpstreamResolvers: []dnsprovider.Provider{
-			dnsprovider.Cloudflare(),
-			dnsprovider.Google(),
-		},
-	}
-	dnsDialer, err := doh.New(dohSettings)
-	if err != nil {
-		return fmt.Errorf("creating DoH dialer: %w", err)
-	}
-
 	const clientTimeout = 10 * time.Second
 	httpClient := &http.Client{Timeout: clientTimeout}
-	unzipper := unzip.New(httpClient)
+	updater := updater.New(httpClient, storage, logger)
-	parallelResolver := resolver.NewParallelResolver(dnsDialer)
-	nameTokenPairs := []api.NameToken{
-		{Name: string(api.IPInfo), Token: ipToken},
-		{Name: string(api.IP2Location)},
-		{Name: string(api.IfConfigCo)},
-	}
-	fetchers, err := api.New(nameTokenPairs, httpClient)
-	if err != nil {
-		return fmt.Errorf("creating public IP fetchers: %w", err)
-	}
-	ipFetcher := api.NewResilient(fetchers, logger)
-
-	openvpnFileExtractor := extract.New()
-
-	providers := provider.NewProviders(storage, time.Now, logger, httpClient,
-		unzipper, parallelResolver, ipFetcher, openvpnFileExtractor, options)
-
-	updater := updater.New(httpClient, storage, providers, logger)
 	err = updater.UpdateServers(ctx, options.Providers, options.MinRatio)
 	if err != nil {
 		return fmt.Errorf("updating server information: %w", err)

internal/updater/interfaces.go

@@ -1,28 +1,13 @@
 package updater
 
 import (
-	"context"
-
 	"github.com/qdm12/gluetun-servers/pkg/models"
-	"github.com/qdm12/gluetun/internal/configuration/settings"
-	"github.com/qdm12/gluetun/internal/provider"
 )
 
-type Providers interface {
-	Get(providerName string) provider.Provider
-}
-
 type Storage interface {
 	SetServers(provider string, servers []models.Server) (err error)
 	GetServersCount(provider string) (count int)
 	ServersAreEqual(provider string, servers []models.Server) (equal bool)
-	// Extra methods to match the provider.New storage interface
-	FilterServers(provider string, selection settings.ServerSelection) (filtered []models.Server, err error)
-}
-
-type Unzipper interface {
-	FetchAndExtract(ctx context.Context, url string) (
-		contents map[string][]byte, err error)
 }
 
 type Logger interface {

internal/updater/loop/loop.go

@@ -42,15 +42,15 @@ type Logger interface {
 	Error(s string)
 }
 
-func NewLoop(settings settings.Updater, providers updater.Providers,
+func NewLoop(settings settings.Updater, storage updater.Storage,
-	storage updater.Storage, client *http.Client, logger Logger,
+	client *http.Client, logger Logger,
 ) *Loop {
 	return &Loop{
 		state: state{
 			status:   constants.Stopped,
 			settings: settings,
 		},
-		updater:      updater.New(client, storage, providers, logger),
+		updater:      updater.New(client, storage, logger),
 		logger:       logger,
 		start:        make(chan struct{}),
 		running:      make(chan models.LoopStatus),

internal/updater/providers.go

@@ -3,31 +3,29 @@ package updater
 import (
 	"context"
 	"encoding/json"
-	"errors"
 	"fmt"
 
 	"github.com/qdm12/gluetun-servers/pkg/models"
-	"github.com/qdm12/gluetun-servers/pkg/updaters/common"
 )
 
-type Provider interface {
+func (u *Updater) updateProvider(ctx context.Context, providerName string,
-	Name() string
+	manifest manifest, minRatio float64,
-	FetchServers(ctx context.Context, minServers int) (servers []models.Server, err error)
-}
-
-func (u *Updater) updateProvider(ctx context.Context, provider Provider,
-	minRatio float64,
 ) (err error) {
-	providerName := provider.Name()
+	providerFilepath := manifest.providerToFilepath[providerName]
+	providerFileURL := buildProviderFileURL(providerName, providerFilepath)
+
+	var data models.Servers
+	err = u.fetchJSON(ctx, providerFileURL, &data)
+	if err != nil {
+		return fmt.Errorf("downloading provider file %s: %w", providerFileURL, err)
+	}
+
 	existingServersCount := u.storage.GetServersCount(providerName)
 	minServers := int(minRatio * float64(existingServersCount))
-	servers, err := provider.FetchServers(ctx, minServers)
+	servers := data.Servers
-	if err != nil {
+	if len(servers) < minServers {
-		if errors.Is(err, common.ErrNotEnoughServers) {
+		return fmt.Errorf("provider %s has not enough servers from downloaded file: got %d and expected at least %d",
-			u.logger.Warn("note: if running the update manually, you can use the flag " +
+			providerName, len(servers), minServers)
-				"-minratio to allow the update to succeed with less servers found")
-		}
-		return fmt.Errorf("getting %s servers: %w", providerName, err)
 	}
 
 	for _, server := range servers {

internal/updater/updater.go

@@ -2,60 +2,59 @@ package updater
 
 import (
 	"context"
-	"errors"
+	"encoding/json"
+	"fmt"
+	"io"
 	"net/http"
+	"net/url"
+	"path"
+	"strings"
 	"time"
 
-	"github.com/qdm12/gluetun-servers/pkg/updaters/common"
-	"github.com/qdm12/gluetun-servers/pkg/updaters/unzip"
 	"golang.org/x/text/cases"
 	"golang.org/x/text/language"
 )
 
 type Updater struct {
-	providers Providers
-
 	// state
 	storage Storage
 
 	// Functions for tests
-	logger   Logger
+	logger  Logger
-	timeNow  func() time.Time
+	timeNow func() time.Time
-	client   *http.Client
+	client  *http.Client
-	unzipper Unzipper
 }
 
-func New(httpClient *http.Client, storage Storage,
+func New(httpClient *http.Client, storage Storage, logger Logger) *Updater {
-	providers Providers, logger Logger,
-) *Updater {
-	unzipper := unzip.New(httpClient)
 	return &Updater{
-		providers: providers,
+		storage: storage,
-		storage:   storage,
+		logger:  logger,
-		logger:    logger,
+		timeNow: time.Now,
-		timeNow:   time.Now,
+		client:  httpClient,
-		client:    httpClient,
-		unzipper:  unzipper,
 	}
 }
 
-func (u *Updater) UpdateServers(ctx context.Context, providers []string,
+const (
-	minRatio float64,
+	serversManifestURL  = "https://raw.githubusercontent.com/qdm12/gluetun-servers/main/pkg/servers/manifest.json"
-) (err error) {
+	serversFilesBaseURL = "https://raw.githubusercontent.com/qdm12/gluetun-servers/main/pkg/servers/"
+)
+
+func (u *Updater) UpdateServers(ctx context.Context, providers []string, minRatio float64) (err error) {
+	manifest, err := u.fetchManifest(ctx)
+	if err != nil {
+		return fmt.Errorf("fetching remote manifest: %w", err)
+	}
+
 	caser := cases.Title(language.English)
 	for _, providerName := range providers {
 		u.logger.Info("updating " + caser.String(providerName) + " servers...")
 
-		fetcher := u.providers.Get(providerName)
 		// TODO support servers offering only TCP or only UDP
 		// for NordVPN and PureVPN
-		err := u.updateProvider(ctx, fetcher, minRatio)
+		err := u.updateProvider(ctx, providerName, manifest, minRatio)
 		switch {
 		case err == nil:
 			continue
-		case errors.Is(err, common.ErrCredentialsMissing):
-			u.logger.Warn(err.Error() + " - skipping update for " + providerName)
-			continue
 		case len(providers) == 1:
 			// return the only error for the single provider.
 			return err
@@ -70,3 +69,69 @@ func (u *Updater) UpdateServers(ctx context.Context, providers []string,
 
 	return nil
 }
+
+type manifest struct {
+	providerToFilepath map[string]string
+}
+
+func (u *Updater) fetchManifest(ctx context.Context) (m manifest, err error) {
+	var raw map[string]json.RawMessage
+	err = u.fetchJSON(ctx, serversManifestURL, &raw)
+	if err != nil {
+		return m, err
+	}
+
+	m.providerToFilepath = make(map[string]string, len(raw))
+	for provider, rawMessage := range raw {
+		if provider == "version" {
+			continue
+		}
+
+		var metadata struct {
+			Filepath string `json:"filepath"`
+		}
+		err = json.Unmarshal(rawMessage, &metadata)
+		if err != nil {
+			return m, fmt.Errorf("decoding manifest metadata for %s: %w", provider, err)
+		}
+		m.providerToFilepath[provider] = metadata.Filepath
+	}
+
+	return m, nil
+}
+
+func (u *Updater) fetchJSON(ctx context.Context, rawURL string, dst any) (err error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
+	if err != nil {
+		return fmt.Errorf("creating request: %w", err)
+	}
+
+	response, err := u.client.Do(req)
+	if err != nil {
+		return fmt.Errorf("doing request: %w", err)
+	}
+	defer response.Body.Close()
+
+	if response.StatusCode != http.StatusOK {
+		const limit = 10 * 1024 * 1024 // 10 MiB
+		body, _ := io.ReadAll(io.LimitReader(response.Body, limit))
+		return fmt.Errorf("HTTP status code %d for %s: %s",
+			response.StatusCode, rawURL, strings.TrimSpace(string(body)))
+	}
+
+	err = json.NewDecoder(response.Body).Decode(dst)
+	if err != nil {
+		return fmt.Errorf("decoding response body: %w", err)
+	}
+
+	return nil
+}
+
+func buildProviderFileURL(providerName, filePath string) (providerFileURL string) {
+	filename := path.Base(filePath)
+	if filename == "." || filename == "/" || filename == "" {
+		filename = providerName + ".json"
+	}
+
+	return serversFilesBaseURL + url.PathEscape(filename)
+}