From 4d2ec68e66a1b02bef7dcd4a99a5c7174a3ac778 Mon Sep 17 00:00:00 2001 From: Quentin McGaw Date: Mon, 11 May 2026 03:34:54 +0000 Subject: [PATCH] gluetun updater only downloads files from gluetun-servers --- .vscode/tasks.json | 28 -------- cmd/gluetun/main.go | 2 +- go.mod | 2 +- internal/cli/update.go | 51 ++++---------- internal/updater/interfaces.go | 15 ----- internal/updater/loop/loop.go | 6 +- internal/updater/providers.go | 32 +++++---- internal/updater/updater.go | 119 +++++++++++++++++++++++++-------- 8 files changed, 125 insertions(+), 130 deletions(-) diff --git a/.vscode/tasks.json b/.vscode/tasks.json index d359045d..a356c119 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -1,19 +1,6 @@ { "version": "2.0.0", "tasks": [ - { - "label": "Update a VPN provider servers data", - "type": "shell", - "command": "go", - "args": [ - "run", - "./cmd/gluetun/main.go", - "update", - "${input:updateMode}", - "-providers", - "${input:provider}" - ], - }, { "label": "Add a Gluetun Github Git remote", "type": "shell", @@ -36,21 +23,6 @@ } ], "inputs": [ - { - "id": "provider", - "type": "promptString", - "description": "Please enter a provider (or comma separated list of providers)", - }, - { - "id": "updateMode", - "type": "pickString", - "description": "Update mode to use", - "options": [ - "-maintainer", - "-enduser" - ], - "default": "-maintainer" - }, { "id": "githubRemoteUsername", "type": "promptString", diff --git a/cmd/gluetun/main.go b/cmd/gluetun/main.go index 80d124ee..80932a76 100644 --- a/cmd/gluetun/main.go +++ b/cmd/gluetun/main.go @@ -464,7 +464,7 @@ func _main(ctx context.Context, buildInfo models.BuildInformation, go vpnLooper.Run(vpnCtx, vpnDone) updaterLooper := updater.NewLoop(allSettings.Updater, - providers, storage, httpClient, updaterLogger) + storage, httpClient, updaterLogger) updaterHandler, updaterCtx, updaterDone := goshutdown.NewGoRoutineHandler( "updater", goroutine.OptionTimeout(defaultShutdownTimeout)) // wait for updaterLooper.Restart() or its ticket launched with RunRestartTicker diff --git a/go.mod b/go.mod index f03cfa50..50fe7fc1 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,6 @@ module github.com/qdm12/gluetun go 1.25.0 require ( - github.com/ProtonMail/go-srp v0.0.7 github.com/amnezia-vpn/amneziawg-go v0.2.16 github.com/breml/rootcerts v0.3.4 github.com/fatih/color v1.18.0 @@ -37,6 +36,7 @@ require ( require ( github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf // indirect github.com/ProtonMail/go-crypto v1.3.0-proton // indirect + github.com/ProtonMail/go-srp v0.0.7 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudflare/circl v1.6.1 // indirect diff --git a/internal/cli/update.go b/internal/cli/update.go index 318d014e..e82b65e8 100644 --- a/internal/cli/update.go +++ b/internal/cli/update.go @@ -10,15 +10,8 @@ import ( "strings" "time" - "github.com/qdm12/dns/v2/pkg/doh" - dnsprovider "github.com/qdm12/dns/v2/pkg/provider" - "github.com/qdm12/gluetun-servers/pkg/publicip/api" - "github.com/qdm12/gluetun-servers/pkg/updaters/resolver" - "github.com/qdm12/gluetun-servers/pkg/updaters/unzip" "github.com/qdm12/gluetun/internal/configuration/settings" "github.com/qdm12/gluetun/internal/constants/providers" - "github.com/qdm12/gluetun/internal/openvpn/extract" - "github.com/qdm12/gluetun/internal/provider" "github.com/qdm12/gluetun/internal/updater" ) @@ -52,8 +45,19 @@ func (c *CLI) Update(ctx context.Context, args []string, logger UpdaterLogger) e return err } - if dnsServer != "" { + switch { + case dnsServer != "": logger.Warn("The -dns flag is no longer used, your DNS will use DoH with Cloudflare and Google") + case endUserMode: + logger.Warn("The -enduser flag is no longer used and has no effect") + case ipToken != "": + logger.Warn("The -ip-token flag is no longer used and has no effect") + case protonUsername != "": + logger.Warn("The -proton-username flag is no longer used and has no effect. Use -proton-email instead.") + case protonEmail != "": + logger.Warn("The -proton-email flag is no longer used and has no effect") + case protonPassword != "": + logger.Warn("The -proton-password flag is no longer used and has no effect") } if updateAll { @@ -87,38 +91,9 @@ func (c *CLI) Update(ctx context.Context, args []string, logger UpdaterLogger) e return fmt.Errorf("creating servers storage: %w", err) } - dohSettings := doh.Settings{ - UpstreamResolvers: []dnsprovider.Provider{ - dnsprovider.Cloudflare(), - dnsprovider.Google(), - }, - } - dnsDialer, err := doh.New(dohSettings) - if err != nil { - return fmt.Errorf("creating DoH dialer: %w", err) - } - const clientTimeout = 10 * time.Second httpClient := &http.Client{Timeout: clientTimeout} - unzipper := unzip.New(httpClient) - parallelResolver := resolver.NewParallelResolver(dnsDialer) - nameTokenPairs := []api.NameToken{ - {Name: string(api.IPInfo), Token: ipToken}, - {Name: string(api.IP2Location)}, - {Name: string(api.IfConfigCo)}, - } - fetchers, err := api.New(nameTokenPairs, httpClient) - if err != nil { - return fmt.Errorf("creating public IP fetchers: %w", err) - } - ipFetcher := api.NewResilient(fetchers, logger) - - openvpnFileExtractor := extract.New() - - providers := provider.NewProviders(storage, time.Now, logger, httpClient, - unzipper, parallelResolver, ipFetcher, openvpnFileExtractor, options) - - updater := updater.New(httpClient, storage, providers, logger) + updater := updater.New(httpClient, storage, logger) err = updater.UpdateServers(ctx, options.Providers, options.MinRatio) if err != nil { return fmt.Errorf("updating server information: %w", err) diff --git a/internal/updater/interfaces.go b/internal/updater/interfaces.go index 1e66784c..442ebe4b 100644 --- a/internal/updater/interfaces.go +++ b/internal/updater/interfaces.go @@ -1,28 +1,13 @@ package updater import ( - "context" - "github.com/qdm12/gluetun-servers/pkg/models" - "github.com/qdm12/gluetun/internal/configuration/settings" - "github.com/qdm12/gluetun/internal/provider" ) -type Providers interface { - Get(providerName string) provider.Provider -} - type Storage interface { SetServers(provider string, servers []models.Server) (err error) GetServersCount(provider string) (count int) ServersAreEqual(provider string, servers []models.Server) (equal bool) - // Extra methods to match the provider.New storage interface - FilterServers(provider string, selection settings.ServerSelection) (filtered []models.Server, err error) -} - -type Unzipper interface { - FetchAndExtract(ctx context.Context, url string) ( - contents map[string][]byte, err error) } type Logger interface { diff --git a/internal/updater/loop/loop.go b/internal/updater/loop/loop.go index 432218b9..da1c130e 100644 --- a/internal/updater/loop/loop.go +++ b/internal/updater/loop/loop.go @@ -42,15 +42,15 @@ type Logger interface { Error(s string) } -func NewLoop(settings settings.Updater, providers updater.Providers, - storage updater.Storage, client *http.Client, logger Logger, +func NewLoop(settings settings.Updater, storage updater.Storage, + client *http.Client, logger Logger, ) *Loop { return &Loop{ state: state{ status: constants.Stopped, settings: settings, }, - updater: updater.New(client, storage, providers, logger), + updater: updater.New(client, storage, logger), logger: logger, start: make(chan struct{}), running: make(chan models.LoopStatus), diff --git a/internal/updater/providers.go b/internal/updater/providers.go index ef41cd0e..86448f5b 100644 --- a/internal/updater/providers.go +++ b/internal/updater/providers.go @@ -3,31 +3,29 @@ package updater import ( "context" "encoding/json" - "errors" "fmt" "github.com/qdm12/gluetun-servers/pkg/models" - "github.com/qdm12/gluetun-servers/pkg/updaters/common" ) -type Provider interface { - Name() string - FetchServers(ctx context.Context, minServers int) (servers []models.Server, err error) -} - -func (u *Updater) updateProvider(ctx context.Context, provider Provider, - minRatio float64, +func (u *Updater) updateProvider(ctx context.Context, providerName string, + manifest manifest, minRatio float64, ) (err error) { - providerName := provider.Name() + providerFilepath := manifest.providerToFilepath[providerName] + providerFileURL := buildProviderFileURL(providerName, providerFilepath) + + var data models.Servers + err = u.fetchJSON(ctx, providerFileURL, &data) + if err != nil { + return fmt.Errorf("downloading provider file %s: %w", providerFileURL, err) + } + existingServersCount := u.storage.GetServersCount(providerName) minServers := int(minRatio * float64(existingServersCount)) - servers, err := provider.FetchServers(ctx, minServers) - if err != nil { - if errors.Is(err, common.ErrNotEnoughServers) { - u.logger.Warn("note: if running the update manually, you can use the flag " + - "-minratio to allow the update to succeed with less servers found") - } - return fmt.Errorf("getting %s servers: %w", providerName, err) + servers := data.Servers + if len(servers) < minServers { + return fmt.Errorf("provider %s has not enough servers from downloaded file: got %d and expected at least %d", + providerName, len(servers), minServers) } for _, server := range servers { diff --git a/internal/updater/updater.go b/internal/updater/updater.go index 5618e2bd..9abece77 100644 --- a/internal/updater/updater.go +++ b/internal/updater/updater.go @@ -2,60 +2,59 @@ package updater import ( "context" - "errors" + "encoding/json" + "fmt" + "io" "net/http" + "net/url" + "path" + "strings" "time" - "github.com/qdm12/gluetun-servers/pkg/updaters/common" - "github.com/qdm12/gluetun-servers/pkg/updaters/unzip" "golang.org/x/text/cases" "golang.org/x/text/language" ) type Updater struct { - providers Providers - // state storage Storage // Functions for tests - logger Logger - timeNow func() time.Time - client *http.Client - unzipper Unzipper + logger Logger + timeNow func() time.Time + client *http.Client } -func New(httpClient *http.Client, storage Storage, - providers Providers, logger Logger, -) *Updater { - unzipper := unzip.New(httpClient) +func New(httpClient *http.Client, storage Storage, logger Logger) *Updater { return &Updater{ - providers: providers, - storage: storage, - logger: logger, - timeNow: time.Now, - client: httpClient, - unzipper: unzipper, + storage: storage, + logger: logger, + timeNow: time.Now, + client: httpClient, } } -func (u *Updater) UpdateServers(ctx context.Context, providers []string, - minRatio float64, -) (err error) { +const ( + serversManifestURL = "https://raw.githubusercontent.com/qdm12/gluetun-servers/main/pkg/servers/manifest.json" + serversFilesBaseURL = "https://raw.githubusercontent.com/qdm12/gluetun-servers/main/pkg/servers/" +) + +func (u *Updater) UpdateServers(ctx context.Context, providers []string, minRatio float64) (err error) { + manifest, err := u.fetchManifest(ctx) + if err != nil { + return fmt.Errorf("fetching remote manifest: %w", err) + } + caser := cases.Title(language.English) for _, providerName := range providers { u.logger.Info("updating " + caser.String(providerName) + " servers...") - fetcher := u.providers.Get(providerName) // TODO support servers offering only TCP or only UDP // for NordVPN and PureVPN - err := u.updateProvider(ctx, fetcher, minRatio) + err := u.updateProvider(ctx, providerName, manifest, minRatio) switch { case err == nil: continue - case errors.Is(err, common.ErrCredentialsMissing): - u.logger.Warn(err.Error() + " - skipping update for " + providerName) - continue case len(providers) == 1: // return the only error for the single provider. return err @@ -70,3 +69,69 @@ func (u *Updater) UpdateServers(ctx context.Context, providers []string, return nil } + +type manifest struct { + providerToFilepath map[string]string +} + +func (u *Updater) fetchManifest(ctx context.Context) (m manifest, err error) { + var raw map[string]json.RawMessage + err = u.fetchJSON(ctx, serversManifestURL, &raw) + if err != nil { + return m, err + } + + m.providerToFilepath = make(map[string]string, len(raw)) + for provider, rawMessage := range raw { + if provider == "version" { + continue + } + + var metadata struct { + Filepath string `json:"filepath"` + } + err = json.Unmarshal(rawMessage, &metadata) + if err != nil { + return m, fmt.Errorf("decoding manifest metadata for %s: %w", provider, err) + } + m.providerToFilepath[provider] = metadata.Filepath + } + + return m, nil +} + +func (u *Updater) fetchJSON(ctx context.Context, rawURL string, dst any) (err error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) + if err != nil { + return fmt.Errorf("creating request: %w", err) + } + + response, err := u.client.Do(req) + if err != nil { + return fmt.Errorf("doing request: %w", err) + } + defer response.Body.Close() + + if response.StatusCode != http.StatusOK { + const limit = 10 * 1024 * 1024 // 10 MiB + body, _ := io.ReadAll(io.LimitReader(response.Body, limit)) + return fmt.Errorf("HTTP status code %d for %s: %s", + response.StatusCode, rawURL, strings.TrimSpace(string(body))) + } + + err = json.NewDecoder(response.Body).Decode(dst) + if err != nil { + return fmt.Errorf("decoding response body: %w", err) + } + + return nil +} + +func buildProviderFileURL(providerName, filePath string) (providerFileURL string) { + filename := path.Base(filePath) + if filename == "." || filename == "/" || filename == "" { + filename = providerName + ".json" + } + + return serversFilesBaseURL + url.PathEscape(filename) +}