vanja/gluetun
1
0
Fork 0
mirror of https://github.com/qdm12/gluetun.git synced 2026-06-25 05:17:36 +02:00
Code Issues Packages Projects Releases Wiki Activity

feat(storage): storage file structure changes (#3301)

Browse Source 
- migrate persisted server data storage from `/gluetun/servers.json` to `/gluetun/servers/`
- add `STORAGE_SERVERS_ENABLED=on` to enable or disable on-disk server data storage
- add `STORAGE_SERVERS_DIRECTORY_PATH=/gluetun/servers` to configure where per-provider server files are stored
- keep backward compatibility with legacy `STORAGE_FILEPATH=/gluetun/servers.json`
- automatically read and migrate legacy `/gluetun/servers.json` into the new `/gluetun/servers/` layout when needed
- try to remove the legacy servers file after a successful migration to the new storage directory
- switch persisted server data from one large JSON file to a manifest plus per-provider JSON files
- add `UPDATER_PREFER_DIRECT_DOWNLOAD` to allow preferring direct download of provider server data
- keep deprecated updater flags `-enduser` and `-maintainer` as no-op warnings for backward compatibility
- preserve compatibility checks so persisted server data is discarded when its schema version no longer matches the built-in data
- allow preferred persisted provider data to override built-in data when versions match
- servers data now lives at https://github.com/qdm12/gluetun-servers/tree/main/pkg/servers
This commit is contained in:
Quentin McGaw
2026-05-18 22:28:25 -04:00
committed by GitHub
parent cd19093d1d
commit 8f82376996
31 changed files with 654 additions and 304041 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Dockerfile
+3 -1
View File
@@ -249,6 +249,7 @@ ENV VPN_SERVICE_PROVIDER=pia \
UPDATER_PERIOD=0 \
UPDATER_MIN_RATIO=0.8 \
UPDATER_VPN_SERVICE_PROVIDERS= \
UPDATER_PREFER_DIRECT_DOWNLOAD=no \
UPDATER_PROTONVPN_EMAIL= \
UPDATER_PROTONVPN_PASSWORD= \
# Public IP
@@ -257,7 +258,8 @@ ENV VPN_SERVICE_PROVIDER=pia \
PUBLICIP_API=ipinfo,ifconfigco,ip2location,cloudflare \
PUBLICIP_API_TOKEN= \
# Storage
STORAGE_FILEPATH=/gluetun/servers.json \
STORAGE_SERVERS_ENABLED=on \
STORAGE_SERVERS_DIRECTORY_PATH=/gluetun/servers/ \
# Pprof
PPROF_ENABLED=no \
PPROF_BLOCK_PROFILE_RATE=0 \
cmd/gluetun/main.go
+4 -5
View File
@@ -169,7 +169,7 @@ func _main(ctx context.Context, buildInfo models.BuildInformation,
defer fmt.Println(gluetunLogo)
announcementExp, err := time.Parse(time.RFC3339, "2026-04-30T00:00:00Z")
announcementExp, err := time.Parse(time.RFC3339, "2026-06-30T00:00:00Z")
if err != nil {
return err
}
@@ -180,9 +180,7 @@ func _main(ctx context.Context, buildInfo models.BuildInformation,
Version: buildInfo.Version,
Commit: buildInfo.Commit,
Created: buildInfo.Created,
Announcement: "the repository will be migrated to https://github.com/passteque/gluetun on 2026-05-21, " +
"which is a Github organization under my sole control, so don't get alarmed if you get redirected " +
"in the coming days (reason: personal paperwork ugh)",
Announcement: "Your servers data files are now migrated to /gluetun/servers/",
AnnounceExp: announcementExp,
// Sponsor information
PaypalUser: "qmcgaw",
@@ -245,7 +243,8 @@ func _main(ctx context.Context, buildInfo models.BuildInformation,
// TODO run this in a loop or in openvpn to reload from file without restarting
storageLogger := logger.New(log.SetComponent("storage"))
storage, err := storage.New(storageLogger, *allSettings.Storage.Filepath)
storage, err := storage.New(storageLogger, *allSettings.Storage.ServersEnabled,
allSettings.Storage.ServersPath, allSettings.Storage.LegacyServersFilepath)
if err != nil {
return err
}
go.mod
+1
View File
@@ -15,6 +15,7 @@ require (
github.com/mdlayher/netlink v1.9.0
github.com/pelletier/go-toml/v2 v2.2.4
github.com/qdm12/dns/v2 v2.0.0-rc9.0.20260421173011-9de8e7fdbe3a
github.com/qdm12/gluetun-servers v0.1.0
github.com/qdm12/gosettings v0.4.4
github.com/qdm12/goshutdown v0.3.0
github.com/qdm12/gosplash v0.2.1-0.20260305164749-b713de4fee6c
go.sum
+2
View File
@@ -76,6 +76,8 @@ github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0leargg
github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
github.com/qdm12/dns/v2 v2.0.0-rc9.0.20260421173011-9de8e7fdbe3a h1:TE157yPQmAbVruH0MWCQzs0vTT/6t96DkoWUXd6PVuc=
github.com/qdm12/dns/v2 v2.0.0-rc9.0.20260421173011-9de8e7fdbe3a/go.mod h1:98foWgXJZ+g8gJIuO+fdO+oWpFei5WShMFTeN4Im2lE=
github.com/qdm12/gluetun-servers v0.1.0 h1:w9JLghKZwI0Gzpp9p5rNANgEYUUZ1dxdxsG6NKIojaY=
github.com/qdm12/gluetun-servers v0.1.0/go.mod h1:acttuyHyoFDu6GTbf3kAV+QXeiX8oJeh0MBic67/9z8=
github.com/qdm12/goservices v0.1.1-0.20251104135713-6bee97bd4978 h1:TRGpCU1l0lNwtogEUSs5U+RFceYxkAJUmrGabno7J5c=
github.com/qdm12/goservices v0.1.1-0.20251104135713-6bee97bd4978/go.mod h1:D1Po4CRQLYjccnAR2JsVlN1sBMgQrcNLONbvyuzcdTg=
github.com/qdm12/gosettings v0.4.4 h1:SM6tOZDf6k8qbjWU8KWyBF4mWIixfsKCfh9DGRLHlj4=
internal/cli/cli.go
+2 -6
View File
@@ -1,11 +1,7 @@
package cli
type CLI struct {
repoServersPath string
}
type CLI struct{}
func New() *CLI {
return &CLI{
repoServersPath: "./internal/storage/servers.json",
}
return &CLI{}
}
internal/cli/formatservers.go
+2 -5
View File
@@ -9,9 +9,7 @@ import (
"path/filepath"
"strings"
"github.com/qdm12/gluetun/internal/constants"
"github.com/qdm12/gluetun/internal/constants/providers"
"github.com/qdm12/gluetun/internal/storage"
"golang.org/x/text/cases"
"golang.org/x/text/language"
)
@@ -74,10 +72,9 @@ func (c *CLI) FormatServers(args []string) error {
}
}
logger := newNoopLogger()
storage, err := storage.New(logger, constants.ServersData)
storage, err := setupStorage(newNoopLogger())
if err != nil {
return fmt.Errorf("creating servers storage: %w", err)
return fmt.Errorf("setting up storage: %w", err)
}
formatted, err := storage.Format(providerToFormat, format)
internal/cli/helpers.go
+39
View File
@@ -0,0 +1,39 @@
package cli
import (
"fmt"
"github.com/qdm12/gluetun/internal/configuration/settings"
"github.com/qdm12/gluetun/internal/configuration/sources/files"
"github.com/qdm12/gluetun/internal/configuration/sources/secrets"
"github.com/qdm12/gluetun/internal/storage"
"github.com/qdm12/gosettings/reader"
"github.com/qdm12/gosettings/reader/sources/env"
)
type storageSetupLogger interface {
storage.Logger
files.Warner
}
func setupStorage(logger storageSetupLogger) (s *storage.Storage, err error) {
settingsReader := reader.New(reader.Settings{
Sources: []reader.Source{
secrets.New(logger),
files.New(logger),
env.New(env.Settings{}),
},
})
var settings settings.Storage
err = settings.Read(settingsReader)
if err != nil {
return nil, fmt.Errorf("reading storage settings: %w", err)
}
settings.SetDefaults()
storage, err := storage.New(logger, *settings.ServersEnabled, settings.ServersPath,
settings.LegacyServersFilepath)
if err != nil {
return nil, fmt.Errorf("creating storage: %w", err)
}
return storage, nil
}
internal/cli/nooplogger.go
+2
View File
@@ -7,4 +7,6 @@ func newNoopLogger() *noopLogger {
}
func (l *noopLogger) Info(string) {}
func (l *noopLogger) Infof(string, ...any) {}
func (l *noopLogger) Warn(string) {}
func (l *noopLogger) Warnf(string, ...any) {}
internal/cli/openvpnconfig.go
+2 -4
View File
@@ -9,12 +9,10 @@ import (
"time"
"github.com/qdm12/gluetun/internal/configuration/settings"
"github.com/qdm12/gluetun/internal/constants"
"github.com/qdm12/gluetun/internal/models"
"github.com/qdm12/gluetun/internal/netlink"
"github.com/qdm12/gluetun/internal/openvpn/extract"
"github.com/qdm12/gluetun/internal/provider"
"github.com/qdm12/gluetun/internal/storage"
"github.com/qdm12/gluetun/internal/updater/resolver"
"github.com/qdm12/gosettings/reader"
)
@@ -49,9 +47,9 @@ type IPv6Checker interface {
func (c *CLI) OpenvpnConfig(logger OpenvpnConfigLogger, reader *reader.Reader,
ipv6Checker IPv6Checker,
) error {
storage, err := storage.New(logger, constants.ServersData)
storage, err := setupStorage(newNoopLogger())
if err != nil {
return err
return fmt.Errorf("setting up storage: %w", err)
}
var allSettings settings.Settings
internal/cli/update.go
+15 -24
View File
@@ -13,12 +13,10 @@ import (
"github.com/qdm12/dns/v2/pkg/doh"
dnsprovider "github.com/qdm12/dns/v2/pkg/provider"
"github.com/qdm12/gluetun/internal/configuration/settings"
"github.com/qdm12/gluetun/internal/constants"
"github.com/qdm12/gluetun/internal/constants/providers"
"github.com/qdm12/gluetun/internal/openvpn/extract"
"github.com/qdm12/gluetun/internal/provider"
"github.com/qdm12/gluetun/internal/publicip/api"
"github.com/qdm12/gluetun/internal/storage"
"github.com/qdm12/gluetun/internal/updater"
"github.com/qdm12/gluetun/internal/updater/resolver"
"github.com/qdm12/gluetun/internal/updater/unzip"
@@ -26,18 +24,19 @@ import (
type UpdaterLogger interface {
Info(s string)
Infof(format string, args ...any)
Warn(s string)
Warnf(format string, args ...any)
Error(s string)
}
func (c *CLI) Update(ctx context.Context, args []string, logger UpdaterLogger) error {
options := settings.Updater{}
var endUserMode, maintainerMode, updateAll bool
// TODO v4: remove flags below already present in standard settings
var endUserMode, maintainerMode bool
var updateAll bool
var dnsServer, csvProviders, ipToken, protonUsername, protonEmail, protonPassword string
flagSet := flag.NewFlagSet("update", flag.ExitOnError)
flagSet.BoolVar(&endUserMode, "enduser", false, "Write results to /gluetun/servers.json (for end users)")
flagSet.BoolVar(&maintainerMode, "maintainer", false,
"Write results to ./internal/storage/servers.json to modify the program (for maintainers)")
flagSet.StringVar(&dnsServer, "dns", "", "no longer used, your DNS will use DoH with Cloudflare and Google")
const defaultMinRatio = 0.8
flagSet.Float64Var(&options.MinRatio, "minratio", defaultMinRatio,
@@ -49,16 +48,19 @@ func (c *CLI) Update(ctx context.Context, args []string, logger UpdaterLogger) e
"(Retro-compatibility) Username to use to authenticate with Proton. Use -proton-email instead.") // v4 remove this
flagSet.StringVar(&protonEmail, "proton-email", "", "Email to use to authenticate with Proton")
flagSet.StringVar(&protonPassword, "proton-password", "", "Password to use to authenticate with Proton")
flagSet.BoolVar(&endUserMode, "enduser", false, "deprecated")
flagSet.BoolVar(&maintainerMode, "maintainer", false, "deprecated")
if err := flagSet.Parse(args); err != nil {
return err
}
if dnsServer != "" {
switch {
case dnsServer != "":
logger.Warn("The -dns flag is no longer used, your DNS will use DoH with Cloudflare and Google")
}
if !endUserMode && !maintainerMode {
return errors.New("at least one of -enduser or -maintainer must be specified")
case endUserMode:
logger.Warn("The -enduser flag is now unused")
case maintainerMode:
logger.Warn("The -maintainer flag is now unused")
}
if updateAll {
@@ -87,11 +89,7 @@ func (c *CLI) Update(ctx context.Context, args []string, logger UpdaterLogger) e
return fmt.Errorf("options validation failed: %w", err)
}
serversDataPath := constants.ServersData
if maintainerMode {
serversDataPath = ""
}
storage, err := storage.New(logger, serversDataPath)
storage, err := setupStorage(logger)
if err != nil {
return fmt.Errorf("creating servers storage: %w", err)
}
@@ -127,18 +125,11 @@ func (c *CLI) Update(ctx context.Context, args []string, logger UpdaterLogger) e
providers := provider.NewProviders(storage, time.Now, logger, httpClient,
unzipper, parallelResolver, ipFetcher, openvpnFileExtractor, options)
updater := updater.New(httpClient, storage, providers, logger)
updater := updater.New(httpClient, storage, providers, logger, *options.PreferDirectDownload)
err = updater.UpdateServers(ctx, options.Providers, options.MinRatio)
if err != nil {
return fmt.Errorf("updating server information: %w", err)
}
if maintainerMode {
err := storage.FlushToFile(c.repoServersPath)
if err != nil {
return fmt.Errorf("writing servers data to embedded JSON file: %w", err)
}
}
return nil
}
internal/configuration/settings/settings.go
+2 -2
View File
@@ -132,7 +132,7 @@ func (s *Settings) SetDefaults() {
s.IPv6.setDefaults()
s.PublicIP.setDefaults()
s.Shadowsocks.setDefaults()
s.Storage.setDefaults()
s.Storage.SetDefaults()
s.System.setDefaults()
s.Version.setDefaults()
s.VPN.setDefaults()
@@ -213,7 +213,7 @@ func (s *Settings) Read(r *reader.Reader, warner Warner) (err error) {
return s.PublicIP.read(r, warner)
},
"shadowsocks": s.Shadowsocks.read,
"storage": s.Storage.read,
"storage": s.Storage.Read,
"system": s.System.read,
"updater": s.Updater.read,
"version": s.Version.read,
internal/configuration/settings/settings_test.go
+1 -1
View File
@@ -90,7 +90,7 @@ func Test_Settings_String(t *testing.T) {
| ├── Logging: yes
| └── Authentication file path: /gluetun/auth/config.toml
├── Storage settings:
| └── Filepath: /gluetun/servers.json
| └── Servers directory path: /gluetun/servers/
├── OS Alpine settings:
| ├── Process UID: 1000
| └── Process GID: 1000
internal/configuration/settings/storage.go
+51 -14
View File
@@ -11,15 +11,26 @@ import (
// Storage contains settings to configure the storage.
type Storage struct {
// Filepath is the path to the servers.json file. An empty string disables on-disk storage.
Filepath *string
// ServersEnabled is whether to enable storage of servers on disk.
// It defaults to true.
ServersEnabled *bool
// ServersPath is the path to the servers files directory, and cannot be
// the empty string.
ServersPath string
// LegacyServersFilepath is the legacy "fat" JSON filepath to migrate from.
// TODO v4: remove
LegacyServersFilepath string
}
func (s Storage) validate() (err error) {
if *s.Filepath != "" { // optional
_, err := filepath.Abs(*s.Filepath)
if *s.ServersEnabled {
_, err := filepath.Abs(s.ServersPath)
if err != nil {
return fmt.Errorf("filepath is not valid: %w", err)
return fmt.Errorf("servers path is not valid: %w", err)
}
_, err = filepath.Abs(s.LegacyServersFilepath)
if err != nil {
return fmt.Errorf("legacy servers filepath is not valid: %w", err)
}
}
return nil
@@ -27,17 +38,25 @@ func (s Storage) validate() (err error) {
func (s *Storage) copy() (copied Storage) {
return Storage{
Filepath: gosettings.CopyPointer(s.Filepath),
ServersEnabled: gosettings.CopyPointer(s.ServersEnabled),
ServersPath: s.ServersPath,
LegacyServersFilepath: s.LegacyServersFilepath,
}
}
func (s *Storage) overrideWith(other Storage) {
s.Filepath = gosettings.OverrideWithPointer(s.Filepath, other.Filepath)
s.ServersEnabled = gosettings.OverrideWithPointer(s.ServersEnabled, other.ServersEnabled)
s.ServersPath = gosettings.OverrideWithComparable(s.ServersPath, other.ServersPath)
s.LegacyServersFilepath = gosettings.OverrideWithComparable(s.LegacyServersFilepath, other.LegacyServersFilepath)
}
func (s *Storage) setDefaults() {
const defaultFilepath = "/gluetun/servers.json"
s.Filepath = gosettings.DefaultPointer(s.Filepath, defaultFilepath)
const defaultLegacyServersFilepath = "/gluetun/servers.json"
func (s *Storage) SetDefaults() {
s.ServersEnabled = gosettings.DefaultPointer(s.ServersEnabled, true)
const defaultServersPath = "/gluetun/servers/"
s.ServersPath = gosettings.DefaultComparable(s.ServersPath, defaultServersPath)
s.LegacyServersFilepath = gosettings.DefaultComparable(s.LegacyServersFilepath, defaultLegacyServersFilepath)
}
func (s Storage) String() string {
@@ -45,15 +64,33 @@ func (s Storage) String() string {
}
func (s Storage) toLinesNode() (node *gotree.Node) {
if *s.Filepath == "" {
if !*s.ServersEnabled {
return gotree.New("Storage settings: disabled")
}
node = gotree.New("Storage settings:")
node.Appendf("Filepath: %s", *s.Filepath)
node.Appendf("Servers directory path: %s", s.ServersPath)
if s.LegacyServersFilepath != defaultLegacyServersFilepath {
node.Appendf("Legacy servers filepath: %s", s.LegacyServersFilepath)
}
return node
}
func (s *Storage) read(r *reader.Reader) (err error) {
s.Filepath = r.Get("STORAGE_FILEPATH", reader.AcceptEmpty(true))
func (s *Storage) Read(r *reader.Reader) (err error) {
// Retro-compatibility:
// TODO v4: remove support for STORAGE_FILEPATH
filePath := r.Get("STORAGE_FILEPATH", reader.AcceptEmpty(true), reader.IsRetro("STORAGE_SERVERS_DIRECTORY_PATH"))
if filePath != nil {
if *filePath == "" {
s.ServersEnabled = ptrTo(false)
} else {
s.LegacyServersFilepath = *filePath
}
} else {
s.ServersEnabled, err = r.BoolPtr("STORAGE_SERVERS_ENABLED")
if err != nil {
return err
}
s.ServersPath = r.String("STORAGE_SERVERS_DIRECTORY_PATH")
}
return nil
}
internal/configuration/settings/updater.go
+12
View File
@@ -29,6 +29,9 @@ type Updater struct {
// Providers is the list of VPN service providers
// to update server information for.
Providers []string
// PreferDirectDownload is whether to prefer direct download of
// server data from Github (recommended).
PreferDirectDownload *bool
// ProtonEmail is the email to authenticate with the Proton API.
ProtonEmail *string
// ProtonPassword is the password to authenticate with the Proton API.
@@ -75,6 +78,7 @@ func (u *Updater) copy() (copied Updater) {
Period: gosettings.CopyPointer(u.Period),
MinRatio: u.MinRatio,
Providers: gosettings.CopySlice(u.Providers),
PreferDirectDownload: gosettings.CopyPointer(u.PreferDirectDownload),
ProtonEmail: gosettings.CopyPointer(u.ProtonEmail),
ProtonPassword: gosettings.CopyPointer(u.ProtonPassword),
}
@@ -87,6 +91,7 @@ func (u *Updater) overrideWith(other Updater) {
u.Period = gosettings.OverrideWithPointer(u.Period, other.Period)
u.MinRatio = gosettings.OverrideWithComparable(u.MinRatio, other.MinRatio)
u.Providers = gosettings.OverrideWithSlice(u.Providers, other.Providers)
u.PreferDirectDownload = gosettings.OverrideWithPointer(u.PreferDirectDownload, other.PreferDirectDownload)
u.ProtonEmail = gosettings.OverrideWithPointer(u.ProtonEmail, other.ProtonEmail)
u.ProtonPassword = gosettings.OverrideWithPointer(u.ProtonPassword, other.ProtonPassword)
}
@@ -104,6 +109,7 @@ func (u *Updater) SetDefaults(vpnProvider string) {
}
// Set these to empty strings to avoid nil pointer panics
u.PreferDirectDownload = gosettings.DefaultPointer(u.PreferDirectDownload, false)
u.ProtonEmail = gosettings.DefaultPointer(u.ProtonEmail, "")
u.ProtonPassword = gosettings.DefaultPointer(u.ProtonPassword, "")
}
@@ -121,6 +127,7 @@ func (u Updater) toLinesNode() (node *gotree.Node) {
node.Appendf("Update period: %s", *u.Period)
node.Appendf("Minimum ratio: %.1f", u.MinRatio)
node.Appendf("Providers to update: %s", strings.Join(u.Providers, ", "))
node.Appendf("Prefer direct download: %s", gosettings.BoolToYesNo(u.PreferDirectDownload))
if slices.Contains(u.Providers, providers.Protonvpn) {
node.Appendf("Proton API email: %s", *u.ProtonEmail)
node.Appendf("Proton API password: %s", gosettings.ObfuscateKey(*u.ProtonPassword))
@@ -142,6 +149,11 @@ func (u *Updater) read(r *reader.Reader) (err error) {
u.Providers = r.CSV("UPDATER_VPN_SERVICE_PROVIDERS")
u.PreferDirectDownload, err = r.BoolPtr("UPDATER_PREFER_DIRECT_DOWNLOAD")
if err != nil {
return err
}
u.ProtonEmail = r.Get("UPDATER_PROTONVPN_EMAIL")
if u.ProtonEmail == nil {
protonUsername := r.String("UPDATER_PROTONVPN_USERNAME", reader.IsRetro("UPDATER_PROTONVPN_EMAIL"))
internal/constants/paths.go
-6
View File
@@ -1,6 +0,0 @@
package constants
const (
// ServersData is the server information filepath.
ServersData = "/gluetun/servers.json"
)
internal/models/servers.go
+2
View File
@@ -154,6 +154,8 @@ func (a *AllServers) Count() (count int) {
type Servers struct {
Version uint16 `json:"version"`
Timestamp int64 `json:"timestamp"`
Preferred bool `json:"preferred,omitempty"`
Filepath string `json:"filepath,omitempty"`
Servers []Server `json:"servers,omitempty"`
}
internal/storage/flush.go
+58 -25
View File
@@ -2,6 +2,7 @@ package storage
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"sort"
@@ -9,44 +10,76 @@ import (
"github.com/qdm12/gluetun/internal/models"
)
// FlushToFile flushes the merged servers data to the file
// specified by path, as indented JSON.
func (s *Storage) FlushToFile(path string) error {
s.mergedMutex.RLock()
defer s.mergedMutex.RUnlock()
// flushToFile flushes the merged servers data to files
// using the manifest file path given. It is not thread-safe.
func (s *Storage) flushToFile(manifestPath string) error {
const (
filePermission = 0o644
dirPermission = 0o755
)
return s.flushToFile(path)
serversDirectoryPath := filepath.Dir(manifestPath)
if err := os.MkdirAll(serversDirectoryPath, dirPermission); err != nil {
return fmt.Errorf("creating directory: %w", err)
}
// flushToFile flushes the merged servers data to the file
// specified by path, as indented JSON. It is not thread-safe.
func (s *Storage) flushToFile(path string) error {
if path == "" {
return nil // no file to write to
}
const permission = 0o644
dirPath := filepath.Dir(path)
if err := os.MkdirAll(dirPath, permission); err != nil {
return err
for provider, providerServers := range s.mergedServers.ProviderToServers {
providerFilepath := providerServers.Filepath
if providerFilepath == "" {
providerFilepath = filepath.Join(serversDirectoryPath, provider+".json")
}
file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, permission)
providerDirectoryPath := filepath.Dir(providerFilepath)
if err := os.MkdirAll(providerDirectoryPath, dirPermission); err != nil {
return fmt.Errorf("creating directory: %w", err)
}
}
metadata := map[string]any{"version": s.mergedServers.Version}
for provider, providerServers := range s.mergedServers.ProviderToServers {
sort.Sort(models.SortableServers(providerServers.Servers))
providerFilepath := providerServers.Filepath
if providerFilepath == "" {
providerFilepath = filepath.Join(serversDirectoryPath, provider+".json")
}
providerFile, err := os.OpenFile(providerFilepath,
os.O_CREATE|os.O_WRONLY|os.O_TRUNC, filePermission)
if err != nil {
return fmt.Errorf("opening servers data file for %s: %w", provider, err)
}
encoder := json.NewEncoder(providerFile)
encoder.SetIndent("", " ")
err = encoder.Encode(providerServers)
if err != nil {
_ = providerFile.Close()
return fmt.Errorf("encoding servers data for %s: %w", provider, err)
}
err = providerFile.Close()
if err != nil {
return fmt.Errorf("closing servers data file for %s: %w", provider, err)
}
metadata[provider] = map[string]string{"filepath": providerFilepath}
}
serversFile, err := os.OpenFile(manifestPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, filePermission)
if err != nil {
return err
}
encoder := json.NewEncoder(file)
encoder := json.NewEncoder(serversFile)
encoder.SetIndent("", " ")
for _, obj := range s.mergedServers.ProviderToServers {
sort.Sort(models.SortableServers(obj.Servers))
}
err = encoder.Encode(&s.mergedServers)
err = encoder.Encode(metadata)
if err != nil {
_ = file.Close()
_ = serversFile.Close()
return err
}
return file.Close()
return serversFile.Close()
}
internal/storage/hardcoded.go
+34 -3
View File
@@ -3,19 +3,50 @@ package storage
import (
"embed"
"encoding/json"
"fmt"
"path"
serversmodule "github.com/qdm12/gluetun-servers/pkg/servers"
"github.com/qdm12/gluetun/internal/models"
)
//go:embed servers.json
var allServersEmbedFS embed.FS
func parseHardcodedServers() (allServers models.AllServers, err error) {
func parseHardcodedServers() (allServers models.AllServers) {
f, err := allServersEmbedFS.Open("servers.json")
if err != nil {
return allServers, err
panic(err)
}
defer f.Close() // no-op
decoder := json.NewDecoder(f)
err = decoder.Decode(&allServers)
return allServers, err
if err != nil {
panic("decoding servers.json: " + err.Error())
}
for provider, metadata := range allServers.ProviderToServers {
filename := path.Base(metadata.Filepath)
providerFile, err := serversmodule.Files.Open(filename)
if err != nil {
panic(fmt.Sprintf("reading embedded provider file %s for %s: %s", filename, provider, err))
}
defer providerFile.Close() // no-op
var providerServers models.Servers
decoder := json.NewDecoder(providerFile)
err = decoder.Decode(&providerServers)
if err != nil {
panic(fmt.Sprintf("JSON decoding embedded provider file %s for %s: %s",
filename, provider, err))
} else if providerServers.Filepath != "" {
panic(fmt.Sprintf("embedded provider file %s for %s should not have filepath set",
filename, provider))
}
providerServers.Filepath = metadata.Filepath // inherit filepath from servers.json
allServers.ProviderToServers[provider] = providerServers
}
return allServers
}
internal/storage/hardcoded_test.go
+40 -3
View File
@@ -1,9 +1,13 @@
package storage
import (
"encoding/json"
"path"
"testing"
"github.com/qdm12/gluetun-servers/pkg/servers"
"github.com/qdm12/gluetun/internal/constants/providers"
"github.com/qdm12/gluetun/internal/models"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -11,9 +15,10 @@ import (
func Test_parseHardcodedServers(t *testing.T) {
t.Parallel()
servers, err := parseHardcodedServers()
require.NoError(t, err)
var servers models.AllServers
assert.NotPanics(t, func() {
servers = parseHardcodedServers()
})
// all providers minus custom
allProviders := providers.All()
@@ -24,3 +29,35 @@ func Test_parseHardcodedServers(t *testing.T) {
assert.NotEmptyf(t, servers, "for provider %s", provider)
}
}
func Test_parseHardcodedServers_filepathsAndEmbeddedProviderFiles(t *testing.T) {
t.Parallel()
hardcodedServers := parseHardcodedServers()
allProviders := providers.All()
for _, provider := range allProviders {
providerServers, ok := hardcodedServers.ProviderToServers[provider]
require.Truef(t, ok, "for provider %s", provider)
require.NotEmptyf(t, providerServers.Filepath,
"embedded servers filepath should be set for provider %s", provider)
filename := path.Base(providerServers.Filepath)
file, err := servers.Files.Open(filename)
require.NoErrorf(t, err, "opening embedded provider file for %s", provider)
var fileServers struct {
Version uint16 `json:"version"`
Timestamp int64 `json:"timestamp"`
Servers []json.RawMessage `json:"servers"`
}
err = json.NewDecoder(file).Decode(&fileServers)
require.NoErrorf(t, err, "decoding embedded provider file for %s", provider)
require.NoError(t, file.Close())
assert.NotZerof(t, fileServers.Version, "for provider %s", provider)
assert.NotZerof(t, fileServers.Timestamp, "for provider %s", provider)
assert.NotEmptyf(t, fileServers.Servers, "for provider %s", provider)
}
}
internal/storage/merge.go
+14
View File
@@ -30,6 +30,20 @@ func (s *Storage) mergeServers(hardcoded, persisted models.AllServers) models.Al
func (s *Storage) mergeProviderServers(provider string,
hardcoded, persisted models.Servers,
) (merged models.Servers) {
if persisted.Preferred && persisted.Version != hardcoded.Version {
s.logger.Warn(fmt.Sprintf(
"persisted preferred %s servers are discarded because they have version %d and hardcoded servers have version %d",
provider, persisted.Version, hardcoded.Version))
}
// If persisted data is marked as preferred, use it regardless of timestamp
// (as long as versions match)
if persisted.Preferred && persisted.Version == hardcoded.Version && len(persisted.Servers) > 0 {
s.logger.Info(fmt.Sprintf(
"Using %s servers from file (marked as preferred)", provider))
return persisted
}
nowTimestamp := time.Now().Unix()
if persisted.Timestamp > nowTimestamp {
s.logger.Warn(fmt.Sprintf(
internal/storage/mocks_test.go
+17
View File
@@ -45,6 +45,23 @@ func (mr *MockLoggerMockRecorder) Info(arg0 interface{}) *gomock.Call {
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Info", reflect.TypeOf((*MockLogger)(nil).Info), arg0)
}
// Infof mocks base method.
func (m *MockLogger) Infof(arg0 string, arg1 ...interface{}) {
m.ctrl.T.Helper()
varargs := []interface{}{arg0}
for _, a := range arg1 {
varargs = append(varargs, a)
}
m.ctrl.Call(m, "Infof", varargs...)
}
// Infof indicates an expected call of Infof.
func (mr *MockLoggerMockRecorder) Infof(arg0 interface{}, arg1 ...interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
varargs := append([]interface{}{arg0}, arg1...)
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Infof", reflect.TypeOf((*MockLogger)(nil).Infof), varargs...)
}
// Warn mocks base method.
func (m *MockLogger) Warn(arg0 string) {
m.ctrl.T.Helper()
internal/storage/read.go
+78 -21
View File
@@ -12,29 +12,30 @@ import (
"golang.org/x/text/language"
)
// readFromFile reads the servers from server.json.
// readFromFile reads the servers data starting from the given manifest file path.
// It only reads servers that have the same version as the hardcoded servers version
// to avoid JSON decoding errors.
func (s *Storage) readFromFile(filepath string, hardcodedVersions map[string]uint16) (
servers models.AllServers, err error,
func (s *Storage) readFromFile(manifestPath string, hardcodedVersions map[string]uint16) (
servers models.AllServers, found bool, err error,
) {
file, err := os.Open(filepath)
file, err := os.Open(manifestPath)
if os.IsNotExist(err) {
return servers, nil
return servers, false, nil
} else if err != nil {
return servers, err
return servers, false, err
}
b, err := io.ReadAll(file)
if err != nil {
return servers, err
return servers, true, err
}
if err := file.Close(); err != nil {
return servers, err
return servers, true, err
}
return s.extractServersFromBytes(b, hardcodedVersions)
servers, err = s.extractServersFromBytes(b, hardcodedVersions)
return servers, true, err
}
func (s *Storage) extractServersFromBytes(b []byte, hardcodedVersions map[string]uint16) (
@@ -46,6 +47,12 @@ func (s *Storage) extractServersFromBytes(b []byte, hardcodedVersions map[string
}
// Note schema version is at map key "version" as number
if rawVersion, ok := rawMessages["version"]; ok {
err := json.Unmarshal(rawVersion, &servers.Version)
if err != nil {
return servers, fmt.Errorf("decoding servers schema version: %w", err)
}
}
allProviders := providers.All()
servers.ProviderToServers = make(map[string]models.Servers, len(allProviders))
@@ -86,25 +93,20 @@ func (s *Storage) readServers(provider string, hardcodedVersion uint16,
) {
provider = titleCaser.String(provider)
var versionObject struct {
var metadata struct {
Version uint16 `json:"version"`
Timestamp uint64 `json:"timestamp"`
Filepath string `json:"filepath"`
}
err = json.Unmarshal(rawMessage, &versionObject)
err = json.Unmarshal(rawMessage, &metadata)
if err != nil {
return servers, false, fmt.Errorf("decoding servers version for provider %s: %w",
provider, err)
}
persistedVersion := versionObject.Version
versionsMatch = hardcodedVersion == persistedVersion
if !versionsMatch {
s.logger.Info(fmt.Sprintf(
"%s servers from file discarded because they have "+
"version %d and hardcoded servers have version %d",
provider, persistedVersion, hardcodedVersion))
return servers, versionsMatch, nil
if metadata.Filepath != "" {
return s.readServersFromFilepath(provider, metadata.Filepath, hardcodedVersion)
}
err = json.Unmarshal(rawMessage, &servers)
@@ -113,5 +115,60 @@ func (s *Storage) readServers(provider string, hardcodedVersion uint16,
provider, err)
}
return servers, versionsMatch, nil
const sourcePath = ""
if !checkVersions(hardcodedVersion, servers.Version, provider, sourcePath,
servers.Preferred, s.logger) {
return models.Servers{}, false, nil
}
return servers, true, nil
}
func (s *Storage) readServersFromFilepath(provider, filepath string, hardcodedVersion uint16) (
referencedServers models.Servers, versionsMatch bool, err error,
) {
providerFile, err := os.Open(filepath)
if os.IsNotExist(err) {
return models.Servers{}, false, nil
} else if err != nil {
return models.Servers{}, false, fmt.Errorf("opening servers file %s for provider %s: %w",
filepath, provider, err)
}
defer providerFile.Close()
err = json.NewDecoder(providerFile).Decode(&referencedServers)
if err != nil {
return models.Servers{}, false, fmt.Errorf("decoding servers file %s for provider %s: %w",
filepath, provider, err)
}
if !checkVersions(hardcodedVersion, referencedServers.Version, provider, filepath,
referencedServers.Preferred, s.logger) {
return models.Servers{}, false, nil
}
referencedServers.Filepath = filepath
return referencedServers, true, nil
}
func checkVersions(builtinVersion, version uint16, provider, sourcePath string,
preferred bool, logger Logger,
) (match bool) {
if version == builtinVersion {
return true
}
name := provider
log := logger.Info
if preferred {
name += " preferred"
log = logger.Warn
}
name += " servers"
if sourcePath != "" {
name += " from file " + sourcePath
}
log(fmt.Sprintf(
"%s discarded because they have version %d and hardcoded servers have version %d",
name, version, builtinVersion))
return false
}
internal/storage/read_test.go
+32 -14
View File
@@ -30,25 +30,30 @@ func Test_extractServersFromBytes(t *testing.T) {
testCases := map[string]struct {
b []byte
hardcodedVersions map[string]uint16
logged []string
makeLogger func(ctrl *gomock.Controller) *MockLogger
persisted models.AllServers
errMessage string
}{
"bad JSON": {
b: []byte("garbage"),
makeLogger: func(_ *gomock.Controller) *MockLogger { return nil },
errMessage: "decoding servers: invalid character 'g' looking for beginning of value",
},
"bad provider JSON": {
b: []byte(`{"cyberghost": "garbage"}`),
hardcodedVersions: populateProviderToVersion(map[string]uint16{}),
makeLogger: func(_ *gomock.Controller) *MockLogger { return nil },
errMessage: "decoding servers version for provider Cyberghost: " +
"json: cannot unmarshal string into Go value of type struct { Version uint16 \"json:\\\"version\\\"\" }",
"json: cannot unmarshal string into Go value of type struct { Version uint16 \"json:\\\"version\\\"\"; " +
"Timestamp uint64 \"json:\\\"timestamp\\\"\"; " +
"Filepath string \"json:\\\"filepath\\\"\" }",
},
"bad servers array JSON": {
b: []byte(`{"cyberghost": {"version": 1, "servers": "garbage"}}`),
hardcodedVersions: populateProviderToVersion(map[string]uint16{
providers.Cyberghost: 1,
}),
makeLogger: func(_ *gomock.Controller) *MockLogger { return nil },
errMessage: "decoding servers for provider Cyberghost: " +
"json: cannot unmarshal string into Go struct field Servers.servers of type []models.Server",
},
@@ -57,6 +62,7 @@ func Test_extractServersFromBytes(t *testing.T) {
hardcodedVersions: populateProviderToVersion(map[string]uint16{
providers.Cyberghost: 1,
}),
makeLogger: func(_ *gomock.Controller) *MockLogger { return nil },
persisted: models.AllServers{
ProviderToServers: map[string]models.Servers{},
},
@@ -68,6 +74,7 @@ func Test_extractServersFromBytes(t *testing.T) {
hardcodedVersions: populateProviderToVersion(map[string]uint16{
providers.Cyberghost: 1,
}),
makeLogger: func(_ *gomock.Controller) *MockLogger { return nil },
persisted: models.AllServers{
ProviderToServers: map[string]models.Servers{
providers.Cyberghost: {Version: 1},
@@ -81,8 +88,28 @@ func Test_extractServersFromBytes(t *testing.T) {
hardcodedVersions: populateProviderToVersion(map[string]uint16{
providers.Cyberghost: 2,
}),
logged: []string{
"Cyberghost servers from file discarded because they have version 1 and hardcoded servers have version 2",
makeLogger: func(ctrl *gomock.Controller) *MockLogger {
logger := NewMockLogger(ctrl)
logger.EXPECT().Info("Cyberghost servers discarded because " +
"they have version 1 and hardcoded servers have version 2")
return logger
},
persisted: models.AllServers{
ProviderToServers: map[string]models.Servers{},
},
},
"preferred_different_versions": {
b: []byte(`{
"cyberghost": {"version": 1, "timestamp": 1, "preferred": true}
}`),
hardcodedVersions: populateProviderToVersion(map[string]uint16{
providers.Cyberghost: 2,
}),
makeLogger: func(ctrl *gomock.Controller) *MockLogger {
logger := NewMockLogger(ctrl)
logger.EXPECT().Warn("Cyberghost preferred servers discarded because " +
"they have version 1 and hardcoded servers have version 2")
return logger
},
persisted: models.AllServers{
ProviderToServers: map[string]models.Servers{},
@@ -95,16 +122,7 @@ func Test_extractServersFromBytes(t *testing.T) {
t.Parallel()
ctrl := gomock.NewController(t)
logger := NewMockLogger(ctrl)
var previousLogCall *gomock.Call
for _, logged := range testCase.logged {
call := logger.EXPECT().Info(logged)
if previousLogCall != nil {
call.After(previousLogCall)
}
previousLogCall = call
}
logger := testCase.makeLogger(ctrl)
s := &Storage{
logger: logger,
}
internal/storage/servers.go
+19 -3
View File
@@ -2,6 +2,8 @@ package storage
import (
"fmt"
"os"
"path/filepath"
"time"
"github.com/qdm12/gluetun/internal/constants/providers"
@@ -10,12 +12,12 @@ import (
// SetServers sets the given servers for the given provider
// in the storage in-memory map and saves all the servers
// to file.
// to files.
// Note the servers given are not copied so the caller must
// NOT MUTATE them after calling this method.
func (s *Storage) SetServers(provider string, servers []models.Server) (err error) {
if provider == providers.Custom {
return
return nil
}
s.mergedMutex.Lock()
@@ -26,10 +28,24 @@ func (s *Storage) SetServers(provider string, servers []models.Server) (err erro
serversObject.Servers = servers
s.mergedServers.ProviderToServers[provider] = serversObject
err = s.flushToFile(s.filepath)
if !s.disk {
return nil
}
manifestPath := filepath.Join(s.directoryPath, manifestFilename)
err = s.flushToFile(manifestPath)
if err != nil {
return fmt.Errorf("saving servers to file: %w", err)
}
if !s.hasLegacy() {
return nil
}
s.logger.Infof("removing legacy %s which is now migrated to %s", s.legacyFilepath, s.directoryPath)
err = os.Remove(s.legacyFilepath)
if err != nil {
s.logger.Warn("failed removing legacy servers file " + s.legacyFilepath + ": " + err.Error())
}
return nil
}
internal/storage/servers.json
+23 -303854
View File
File diff suppressed because it is too large Load Diff
internal/storage/storage.go
+32 -8
View File
@@ -1,6 +1,8 @@
package storage
import (
"os"
"path/filepath"
"sync"
"github.com/qdm12/gluetun/internal/models"
@@ -14,31 +16,38 @@ type Storage struct {
// SyncServers method.
hardcodedServers models.AllServers
logger Logger
filepath string
disk bool
directoryPath string
legacyFilepath string
}
const manifestFilename = "manifest.json"
type Logger interface {
Info(s string)
Infof(format string, args ...any)
Warn(s string)
}
// New creates a new storage and reads the servers from the
// embedded servers file and the file on disk.
// Passing an empty filepath disables the reading and writing of
// embedded servers files and the files on disk.
// Passing an empty directoryPath disables the reading and writing of
// servers.
func New(logger Logger, filepath string) (storage *Storage, err error) {
// A unit test prevents any error from being returned
func New(logger Logger, disk bool, directoryPath, legacyFilepath string) (storage *Storage, err error) {
// A unit test prevents [parseHardcodedServers] from ever failing,
// and ensures all providers are part of the servers returned.
hardcodedServers, _ := parseHardcodedServers()
hardcodedServers := parseHardcodedServers()
storage = &Storage{
hardcodedServers: hardcodedServers,
mergedServers: hardcodedServers,
logger: logger,
filepath: filepath,
disk: disk,
directoryPath: directoryPath,
legacyFilepath: legacyFilepath,
}
if filepath != "" {
if disk {
if err := storage.syncServers(); err != nil {
return nil, err
}
@@ -46,3 +55,18 @@ func New(logger Logger, filepath string) (storage *Storage, err error) {
return storage, nil
}
// hasLegacy returns true if the legacy file `legacyFilepath` exists AND is
// different from the manifest file defined by `directoryPath`/[manifestFilename].
// This is used to determine if the legacy file should be read and removed after flushing servers data.
func (s *Storage) hasLegacy() bool {
if !s.disk {
return false
}
if filepath.Clean(filepath.Join(s.directoryPath, manifestFilename)) ==
filepath.Clean(s.legacyFilepath) {
return false
}
stat, err := os.Stat(s.legacyFilepath)
return err == nil && !stat.IsDir()
}
internal/storage/sync.go
+32 -8
View File
@@ -2,6 +2,8 @@ package storage
import (
"fmt"
"os"
"path/filepath"
"reflect"
"github.com/qdm12/gluetun/internal/models"
@@ -14,18 +16,31 @@ func countServers(allServers models.AllServers) (count int) {
return count
}
// syncServers merges the hardcoded servers with the ones from the file.
// syncServers merges the hardcoded servers with the ones from on disk files.
// It assumes s.directoryPath is set.
func (s *Storage) syncServers() (err error) {
hardcodedVersions := make(map[string]uint16, len(s.hardcodedServers.ProviderToServers))
for provider, servers := range s.hardcodedServers.ProviderToServers {
hardcodedVersions[provider] = servers.Version
}
serversOnFile, err := s.readFromFile(s.filepath, hardcodedVersions)
sourceManifestPath := filepath.Join(s.directoryPath, manifestFilename)
destinationManifestPath := sourceManifestPath
serversOnFile, found, err := s.readFromFile(sourceManifestPath, hardcodedVersions)
if err != nil {
return fmt.Errorf("reading servers from file: %w", err)
}
hasLegacy := s.hasLegacy()
if !found && hasLegacy {
sourceManifestPath = s.legacyFilepath
s.logger.Infof("reading legacy servers file %s and migrating it to directory %s", sourceManifestPath, s.directoryPath)
serversOnFile, _, err = s.readFromFile(sourceManifestPath, hardcodedVersions)
if err != nil {
return fmt.Errorf("reading servers from file: %w", err)
}
}
hardcodedCount := countServers(s.hardcodedServers)
countOnFile := countServers(serversOnFile)
@@ -34,13 +49,13 @@ func (s *Storage) syncServers() (err error) {
if countOnFile == 0 {
s.logger.Info(fmt.Sprintf(
"creating %s with %d hardcoded servers",
s.filepath, hardcodedCount))
"writing servers data files to %s with %d hardcoded servers",
s.directoryPath, hardcodedCount))
s.mergedServers = s.hardcodedServers
} else {
s.logger.Info(fmt.Sprintf(
"merging by most recent %d hardcoded servers and %d servers read from %s",
hardcodedCount, countOnFile, s.filepath))
"merging by most recent %d hardcoded servers and %d servers read from manifest file %s",
hardcodedCount, countOnFile, sourceManifestPath))
s.mergedServers = s.mergeServers(s.hardcodedServers, serversOnFile)
}
@@ -50,9 +65,18 @@ func (s *Storage) syncServers() (err error) {
return nil
}
err = s.flushToFile(s.filepath)
err = s.flushToFile(destinationManifestPath)
if err != nil {
s.logger.Warn("failed writing servers to file: " + err.Error())
s.logger.Warn("failed writing servers to destination manifest: " + err.Error())
return nil
}
migratedFromLegacy := hasLegacy && sourceManifestPath == s.legacyFilepath
if migratedFromLegacy {
err = os.Remove(sourceManifestPath)
if err != nil && !os.IsNotExist(err) {
s.logger.Warn("failed removing legacy servers file " + sourceManifestPath + ": " + err.Error())
}
}
return nil
}
internal/updater/loop/loop.go
+1 -1
View File
@@ -50,7 +50,7 @@ func NewLoop(settings settings.Updater, providers updater.Providers,
status: constants.Stopped,
settings: settings,
},
updater: updater.New(client, storage, providers, logger),
updater: updater.New(client, storage, providers, logger, *settings.PreferDirectDownload),
logger: logger,
start: make(chan struct{}),
running: make(chan models.LoopStatus),
internal/updater/providers.go
+36 -5
View File
@@ -5,6 +5,8 @@ import (
"encoding/json"
"errors"
"fmt"
"net/url"
"path"
"github.com/qdm12/gluetun/internal/models"
"github.com/qdm12/gluetun/internal/provider/common"
@@ -16,19 +18,38 @@ type Provider interface {
}
func (u *Updater) updateProvider(ctx context.Context, provider Provider,
minRatio float64,
manifest manifest, minRatio float64,
) (err error) {
providerName := provider.Name()
existingServersCount := u.storage.GetServersCount(providerName)
minServers := int(minRatio * float64(existingServersCount))
servers, err := provider.FetchServers(ctx, minServers)
if err != nil {
if errors.Is(err, common.ErrNotEnoughServers) {
var servers []models.Server
if manifest.providerToFilepath == nil {
servers, err = provider.FetchServers(ctx, minServers)
switch {
case errors.Is(err, common.ErrNotEnoughServers):
u.logger.Warn("note: if running the update manually, you can use the flag " +
"-minratio to allow the update to succeed with less servers found")
}
fallthrough
case err != nil:
return fmt.Errorf("getting %s servers: %w", providerName, err)
}
} else {
providerFilepath := manifest.providerToFilepath[providerName]
providerFileURL := buildProviderFileURL(providerName, providerFilepath)
var data models.Servers
err = u.fetchJSON(ctx, providerFileURL, &data)
if err != nil {
return fmt.Errorf("downloading provider file %s: %w", providerFileURL, err)
}
servers = data.Servers
if len(servers) < minServers {
return fmt.Errorf("provider %s has not enough servers from downloaded file: got %d and expected at least %d",
providerName, len(servers), minServers)
}
}
for _, server := range servers {
err := server.HasMinimumInformation()
@@ -55,3 +76,13 @@ func (u *Updater) updateProvider(ctx context.Context, provider Provider,
}
return nil
}
func buildProviderFileURL(providerName, filePath string) (providerFileURL string) {
filename := path.Base(filePath)
if filename == "." || filename == "/" || filename == "" {
filename = providerName + ".json"
}
const serversFilesBaseURL = "https://raw.githubusercontent.com/qdm12/gluetun-servers/main/pkg/servers/"
return serversFilesBaseURL + url.PathEscape(filename)
}
internal/updater/updater.go
+74 -2
View File
@@ -2,10 +2,15 @@ package updater
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"strings"
"time"
"github.com/qdm12/gluetun/internal/constants/providers"
"github.com/qdm12/gluetun/internal/provider/common"
"github.com/qdm12/gluetun/internal/updater/unzip"
"golang.org/x/text/cases"
@@ -14,6 +19,7 @@ import (
type Updater struct {
providers Providers
preferDirectDownload bool
// state
storage Storage
@@ -26,7 +32,7 @@ type Updater struct {
}
func New(httpClient *http.Client, storage Storage,
providers Providers, logger Logger,
providers Providers, logger Logger, preferDirectDownload bool,
) *Updater {
unzipper := unzip.New(httpClient)
return &Updater{
@@ -36,12 +42,21 @@ func New(httpClient *http.Client, storage Storage,
timeNow: time.Now,
client: httpClient,
unzipper: unzipper,
preferDirectDownload: preferDirectDownload,
}
}
func (u *Updater) UpdateServers(ctx context.Context, providers []string,
minRatio float64,
) (err error) {
var manifest manifest
if u.preferDirectDownload {
manifest, err = u.fetchManifest(ctx)
if err != nil {
return fmt.Errorf("fetching remote manifest: %w", err)
}
}
caser := cases.Title(language.English)
for _, providerName := range providers {
u.logger.Info("updating " + caser.String(providerName) + " servers...")
@@ -49,7 +64,7 @@ func (u *Updater) UpdateServers(ctx context.Context, providers []string,
fetcher := u.providers.Get(providerName)
// TODO support servers offering only TCP or only UDP
// for NordVPN and PureVPN
err := u.updateProvider(ctx, fetcher, minRatio)
err := u.updateProvider(ctx, fetcher, manifest, minRatio)
switch {
case err == nil:
continue
@@ -70,3 +85,60 @@ func (u *Updater) UpdateServers(ctx context.Context, providers []string,
return nil
}
type manifest struct {
providerToFilepath map[string]string
}
func (u *Updater) fetchManifest(ctx context.Context) (m manifest, err error) {
const serversManifestURL = "https://raw.githubusercontent.com/qdm12/gluetun-servers/main/pkg/servers/manifest.json"
var raw map[string]json.RawMessage
err = u.fetchJSON(ctx, serversManifestURL, &raw)
if err != nil {
return m, err
}
providerNames := providers.All()
m.providerToFilepath = make(map[string]string, len(providerNames))
for _, name := range providerNames {
var metadata struct {
Filepath string `json:"filepath"`
}
err = json.Unmarshal(raw[name], &metadata)
if err != nil {
return m, fmt.Errorf("decoding manifest metadata for %s: %w", name, err)
} else if metadata.Filepath == "" {
return m, fmt.Errorf("manifest missing filepath for provider %s", name)
}
m.providerToFilepath[name] = metadata.Filepath
}
return m, nil
}
func (u *Updater) fetchJSON(ctx context.Context, rawURL string, dst any) (err error) {
request, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
if err != nil {
return fmt.Errorf("creating request: %w", err)
}
response, err := u.client.Do(request)
if err != nil {
return fmt.Errorf("doing request: %w", err)
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
const limit = 10 * 1024 * 1024 // 10 MiB
body, _ := io.ReadAll(io.LimitReader(response.Body, limit))
return fmt.Errorf("HTTP status code %d for %s: %s",
response.StatusCode, rawURL, strings.TrimSpace(string(body)))
}
err = json.NewDecoder(response.Body).Decode(dst)
if err != nil {
return fmt.Errorf("decoding response body: %w", err)
}
return nil
}
internal/vpn/interfaces.go
-2
View File
@@ -45,8 +45,6 @@ type Provider interface {
GetConnection(selection settings.ServerSelection, ipv6Supported bool) (connection models.Connection, err error)
OpenVPNConfig(connection models.Connection, settings settings.OpenVPN, ipv6Supported bool) (lines []string)
Name() string
FetchServers(ctx context.Context, minServers int) (
servers []models.Server, err error)
}
type PortForwarder interface {