Compare commits

...

4 Commits

Author SHA1 Message Date
Quentin McGaw b119325241 hotfix(storage): do not write filepath field for non-manifest files 2026-05-19 03:03:30 +00:00
Quentin McGaw 7720b1fad4 fix(storage): ignore empty manifest servers file
- Fix #3318
2026-05-19 02:53:45 +00:00
Quentin McGaw 854bf5811d fix(wireguard): skip tun device checks when using kernelspace 2026-05-19 02:46:40 +00:00
Quentin McGaw 8f82376996 feat(storage): storage file structure changes (#3301)
- migrate persisted server data storage from `/gluetun/servers.json` to `/gluetun/servers/`
- add `STORAGE_SERVERS_ENABLED=on` to enable or disable on-disk server data storage
- add `STORAGE_SERVERS_DIRECTORY_PATH=/gluetun/servers` to configure where per-provider server files are stored
- keep backward compatibility with legacy `STORAGE_FILEPATH=/gluetun/servers.json`
- automatically read and migrate legacy `/gluetun/servers.json` into the new `/gluetun/servers/` layout when needed
- try to remove the legacy servers file after a successful migration to the new storage directory
- switch persisted server data from one large JSON file to a manifest plus per-provider JSON files
- add `UPDATER_PREFER_DIRECT_DOWNLOAD` to allow preferring direct download of provider server data
- keep deprecated updater flags `-enduser` and `-maintainer` as no-op warnings for backward compatibility
- preserve compatibility checks so persisted server data is discarded when its schema version no longer matches the built-in data
- allow preferred persisted provider data to override built-in data when versions match
- servers data now lives at https://github.com/qdm12/gluetun-servers/tree/main/pkg/servers
2026-05-19 04:28:25 +02:00
42 changed files with 867 additions and 304181 deletions
+3 -1
View File
@@ -249,6 +249,7 @@ ENV VPN_SERVICE_PROVIDER=pia \
UPDATER_PERIOD=0 \
UPDATER_MIN_RATIO=0.8 \
UPDATER_VPN_SERVICE_PROVIDERS= \
UPDATER_PREFER_DIRECT_DOWNLOAD=no \
UPDATER_PROTONVPN_EMAIL= \
UPDATER_PROTONVPN_PASSWORD= \
# Public IP
@@ -257,7 +258,8 @@ ENV VPN_SERVICE_PROVIDER=pia \
PUBLICIP_API=ipinfo,ifconfigco,ip2location,cloudflare \
PUBLICIP_API_TOKEN= \
# Storage
STORAGE_FILEPATH=/gluetun/servers.json \
STORAGE_SERVERS_ENABLED=on \
STORAGE_SERVERS_DIRECTORY_PATH=/gluetun/servers/ \
# Pprof
PPROF_ENABLED=no \
PPROF_BLOCK_PROFILE_RATE=0 \
+13 -35
View File
@@ -2,7 +2,6 @@ package main
import (
"context"
"errors"
"fmt"
"io/fs"
"net/http"
@@ -43,7 +42,6 @@ import (
"github.com/qdm12/gluetun/internal/server"
"github.com/qdm12/gluetun/internal/shadowsocks"
"github.com/qdm12/gluetun/internal/storage"
"github.com/qdm12/gluetun/internal/tun"
updater "github.com/qdm12/gluetun/internal/updater/loop"
"github.com/qdm12/gluetun/internal/updater/resolver"
"github.com/qdm12/gluetun/internal/updater/unzip"
@@ -80,7 +78,6 @@ func main() {
logger := log.New(log.SetLevel(log.LevelInfo))
args := os.Args
tun := tun.New()
netLinkDebugLogger := logger.New(log.SetComponent("netlink"))
netLinker := netlink.New(netLinkDebugLogger)
cli := cli.New()
@@ -100,7 +97,7 @@ func main() {
errorCh := make(chan error)
go func() {
errorCh <- _main(ctx, buildInfo, args, logger, reader, tun, netLinker, cmder, cli)
errorCh <- _main(ctx, buildInfo, args, logger, reader, netLinker, cmder, cli)
}()
// Wait for OS signal or run error
@@ -145,7 +142,7 @@ func main() {
//nolint:gocognit,gocyclo,maintidx
func _main(ctx context.Context, buildInfo models.BuildInformation,
args []string, logger log.LoggerInterface, reader *reader.Reader,
tun Tun, netLinker netLinker, cmder RunStarter,
netLinker netLinker, cmder RunStarter,
cli clier,
) error {
if len(args) > 1 { // cli operation
@@ -169,21 +166,19 @@ func _main(ctx context.Context, buildInfo models.BuildInformation,
defer fmt.Println(gluetunLogo)
announcementExp, err := time.Parse(time.RFC3339, "2026-04-30T00:00:00Z")
announcementExp, err := time.Parse(time.RFC3339, "2026-06-30T00:00:00Z")
if err != nil {
return err
}
splashSettings := gosplash.Settings{
User: "qdm12",
Repository: "gluetun",
Emails: []string{"quentin.mcgaw@gmail.com"},
Version: buildInfo.Version,
Commit: buildInfo.Commit,
Created: buildInfo.Created,
Announcement: "the repository will be migrated to https://github.com/passteque/gluetun on 2026-05-21, " +
"which is a Github organization under my sole control, so don't get alarmed if you get redirected " +
"in the coming days (reason: personal paperwork ugh)",
AnnounceExp: announcementExp,
User: "qdm12",
Repository: "gluetun",
Emails: []string{"quentin.mcgaw@gmail.com"},
Version: buildInfo.Version,
Commit: buildInfo.Commit,
Created: buildInfo.Created,
Announcement: "Your servers data files are now migrated to /gluetun/servers/",
AnnounceExp: announcementExp,
// Sponsor information
PaypalUser: "qmcgaw",
GithubSponsor: "qdm12",
@@ -245,7 +240,8 @@ func _main(ctx context.Context, buildInfo models.BuildInformation,
// TODO run this in a loop or in openvpn to reload from file without restarting
storageLogger := logger.New(log.SetComponent("storage"))
storage, err := storage.New(storageLogger, *allSettings.Storage.Filepath)
storage, err := storage.New(storageLogger, *allSettings.Storage.ServersEnabled,
allSettings.Storage.ServersPath, allSettings.Storage.LegacyServersFilepath)
if err != nil {
return err
}
@@ -343,19 +339,6 @@ func _main(ctx context.Context, buildInfo models.BuildInformation,
return fmt.Errorf("adding local rules: %w", err)
}
const tunDevice = "/dev/net/tun"
err = tun.Check(tunDevice)
if err != nil {
if !errors.Is(err, os.ErrNotExist) {
return fmt.Errorf("checking TUN device: %w (see the Wiki errors/tun page)", err)
}
logger.Info(err.Error() + "; creating it...")
err = tun.Create(tunDevice)
if err != nil {
return fmt.Errorf("creating tun device: %w", err)
}
}
for _, port := range allSettings.Firewall.InputPorts {
for _, defaultRoute := range defaultRoutes {
err = firewallConf.SetAllowedPort(ctx, port, defaultRoute.NetInterface)
@@ -627,11 +610,6 @@ type clier interface {
GenKey(args []string) error
}
type Tun interface {
Check(tunDevice string) error
Create(tunDevice string) error
}
type RunStarter interface {
Run(cmd *exec.Cmd) (output string, err error)
Start(cmd *exec.Cmd) (stdoutLines, stderrLines <-chan string,
+1
View File
@@ -15,6 +15,7 @@ require (
github.com/mdlayher/netlink v1.9.0
github.com/pelletier/go-toml/v2 v2.2.4
github.com/qdm12/dns/v2 v2.0.0-rc9.0.20260421173011-9de8e7fdbe3a
github.com/qdm12/gluetun-servers v0.1.0
github.com/qdm12/gosettings v0.4.4
github.com/qdm12/goshutdown v0.3.0
github.com/qdm12/gosplash v0.2.1-0.20260305164749-b713de4fee6c
+2
View File
@@ -76,6 +76,8 @@ github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0leargg
github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
github.com/qdm12/dns/v2 v2.0.0-rc9.0.20260421173011-9de8e7fdbe3a h1:TE157yPQmAbVruH0MWCQzs0vTT/6t96DkoWUXd6PVuc=
github.com/qdm12/dns/v2 v2.0.0-rc9.0.20260421173011-9de8e7fdbe3a/go.mod h1:98foWgXJZ+g8gJIuO+fdO+oWpFei5WShMFTeN4Im2lE=
github.com/qdm12/gluetun-servers v0.1.0 h1:w9JLghKZwI0Gzpp9p5rNANgEYUUZ1dxdxsG6NKIojaY=
github.com/qdm12/gluetun-servers v0.1.0/go.mod h1:acttuyHyoFDu6GTbf3kAV+QXeiX8oJeh0MBic67/9z8=
github.com/qdm12/goservices v0.1.1-0.20251104135713-6bee97bd4978 h1:TRGpCU1l0lNwtogEUSs5U+RFceYxkAJUmrGabno7J5c=
github.com/qdm12/goservices v0.1.1-0.20251104135713-6bee97bd4978/go.mod h1:D1Po4CRQLYjccnAR2JsVlN1sBMgQrcNLONbvyuzcdTg=
github.com/qdm12/gosettings v0.4.4 h1:SM6tOZDf6k8qbjWU8KWyBF4mWIixfsKCfh9DGRLHlj4=
+2 -6
View File
@@ -1,11 +1,7 @@
package cli
type CLI struct {
repoServersPath string
}
type CLI struct{}
func New() *CLI {
return &CLI{
repoServersPath: "./internal/storage/servers.json",
}
return &CLI{}
}
+2 -5
View File
@@ -9,9 +9,7 @@ import (
"path/filepath"
"strings"
"github.com/qdm12/gluetun/internal/constants"
"github.com/qdm12/gluetun/internal/constants/providers"
"github.com/qdm12/gluetun/internal/storage"
"golang.org/x/text/cases"
"golang.org/x/text/language"
)
@@ -74,10 +72,9 @@ func (c *CLI) FormatServers(args []string) error {
}
}
logger := newNoopLogger()
storage, err := storage.New(logger, constants.ServersData)
storage, err := setupStorage(newNoopLogger())
if err != nil {
return fmt.Errorf("creating servers storage: %w", err)
return fmt.Errorf("setting up storage: %w", err)
}
formatted, err := storage.Format(providerToFormat, format)
+39
View File
@@ -0,0 +1,39 @@
package cli
import (
"fmt"
"github.com/qdm12/gluetun/internal/configuration/settings"
"github.com/qdm12/gluetun/internal/configuration/sources/files"
"github.com/qdm12/gluetun/internal/configuration/sources/secrets"
"github.com/qdm12/gluetun/internal/storage"
"github.com/qdm12/gosettings/reader"
"github.com/qdm12/gosettings/reader/sources/env"
)
type storageSetupLogger interface {
storage.Logger
files.Warner
}
func setupStorage(logger storageSetupLogger) (s *storage.Storage, err error) {
settingsReader := reader.New(reader.Settings{
Sources: []reader.Source{
secrets.New(logger),
files.New(logger),
env.New(env.Settings{}),
},
})
var settings settings.Storage
err = settings.Read(settingsReader)
if err != nil {
return nil, fmt.Errorf("reading storage settings: %w", err)
}
settings.SetDefaults()
storage, err := storage.New(logger, *settings.ServersEnabled, settings.ServersPath,
settings.LegacyServersFilepath)
if err != nil {
return nil, fmt.Errorf("creating storage: %w", err)
}
return storage, nil
}
+4 -2
View File
@@ -6,5 +6,7 @@ func newNoopLogger() *noopLogger {
return new(noopLogger)
}
func (l *noopLogger) Info(string) {}
func (l *noopLogger) Warn(string) {}
func (l *noopLogger) Info(string) {}
func (l *noopLogger) Infof(string, ...any) {}
func (l *noopLogger) Warn(string) {}
func (l *noopLogger) Warnf(string, ...any) {}
+2 -4
View File
@@ -9,12 +9,10 @@ import (
"time"
"github.com/qdm12/gluetun/internal/configuration/settings"
"github.com/qdm12/gluetun/internal/constants"
"github.com/qdm12/gluetun/internal/models"
"github.com/qdm12/gluetun/internal/netlink"
"github.com/qdm12/gluetun/internal/openvpn/extract"
"github.com/qdm12/gluetun/internal/provider"
"github.com/qdm12/gluetun/internal/storage"
"github.com/qdm12/gluetun/internal/updater/resolver"
"github.com/qdm12/gosettings/reader"
)
@@ -49,9 +47,9 @@ type IPv6Checker interface {
func (c *CLI) OpenvpnConfig(logger OpenvpnConfigLogger, reader *reader.Reader,
ipv6Checker IPv6Checker,
) error {
storage, err := storage.New(logger, constants.ServersData)
storage, err := setupStorage(newNoopLogger())
if err != nil {
return err
return fmt.Errorf("setting up storage: %w", err)
}
var allSettings settings.Settings
+15 -24
View File
@@ -13,12 +13,10 @@ import (
"github.com/qdm12/dns/v2/pkg/doh"
dnsprovider "github.com/qdm12/dns/v2/pkg/provider"
"github.com/qdm12/gluetun/internal/configuration/settings"
"github.com/qdm12/gluetun/internal/constants"
"github.com/qdm12/gluetun/internal/constants/providers"
"github.com/qdm12/gluetun/internal/openvpn/extract"
"github.com/qdm12/gluetun/internal/provider"
"github.com/qdm12/gluetun/internal/publicip/api"
"github.com/qdm12/gluetun/internal/storage"
"github.com/qdm12/gluetun/internal/updater"
"github.com/qdm12/gluetun/internal/updater/resolver"
"github.com/qdm12/gluetun/internal/updater/unzip"
@@ -26,18 +24,19 @@ import (
type UpdaterLogger interface {
Info(s string)
Infof(format string, args ...any)
Warn(s string)
Warnf(format string, args ...any)
Error(s string)
}
func (c *CLI) Update(ctx context.Context, args []string, logger UpdaterLogger) error {
options := settings.Updater{}
var endUserMode, maintainerMode, updateAll bool
// TODO v4: remove flags below already present in standard settings
var endUserMode, maintainerMode bool
var updateAll bool
var dnsServer, csvProviders, ipToken, protonUsername, protonEmail, protonPassword string
flagSet := flag.NewFlagSet("update", flag.ExitOnError)
flagSet.BoolVar(&endUserMode, "enduser", false, "Write results to /gluetun/servers.json (for end users)")
flagSet.BoolVar(&maintainerMode, "maintainer", false,
"Write results to ./internal/storage/servers.json to modify the program (for maintainers)")
flagSet.StringVar(&dnsServer, "dns", "", "no longer used, your DNS will use DoH with Cloudflare and Google")
const defaultMinRatio = 0.8
flagSet.Float64Var(&options.MinRatio, "minratio", defaultMinRatio,
@@ -49,16 +48,19 @@ func (c *CLI) Update(ctx context.Context, args []string, logger UpdaterLogger) e
"(Retro-compatibility) Username to use to authenticate with Proton. Use -proton-email instead.") // v4 remove this
flagSet.StringVar(&protonEmail, "proton-email", "", "Email to use to authenticate with Proton")
flagSet.StringVar(&protonPassword, "proton-password", "", "Password to use to authenticate with Proton")
flagSet.BoolVar(&endUserMode, "enduser", false, "deprecated")
flagSet.BoolVar(&maintainerMode, "maintainer", false, "deprecated")
if err := flagSet.Parse(args); err != nil {
return err
}
if dnsServer != "" {
switch {
case dnsServer != "":
logger.Warn("The -dns flag is no longer used, your DNS will use DoH with Cloudflare and Google")
}
if !endUserMode && !maintainerMode {
return errors.New("at least one of -enduser or -maintainer must be specified")
case endUserMode:
logger.Warn("The -enduser flag is now unused")
case maintainerMode:
logger.Warn("The -maintainer flag is now unused")
}
if updateAll {
@@ -87,11 +89,7 @@ func (c *CLI) Update(ctx context.Context, args []string, logger UpdaterLogger) e
return fmt.Errorf("options validation failed: %w", err)
}
serversDataPath := constants.ServersData
if maintainerMode {
serversDataPath = ""
}
storage, err := storage.New(logger, serversDataPath)
storage, err := setupStorage(logger)
if err != nil {
return fmt.Errorf("creating servers storage: %w", err)
}
@@ -127,18 +125,11 @@ func (c *CLI) Update(ctx context.Context, args []string, logger UpdaterLogger) e
providers := provider.NewProviders(storage, time.Now, logger, httpClient,
unzipper, parallelResolver, ipFetcher, openvpnFileExtractor, options)
updater := updater.New(httpClient, storage, providers, logger)
updater := updater.New(httpClient, storage, providers, logger, *options.PreferDirectDownload)
err = updater.UpdateServers(ctx, options.Providers, options.MinRatio)
if err != nil {
return fmt.Errorf("updating server information: %w", err)
}
if maintainerMode {
err := storage.FlushToFile(c.repoServersPath)
if err != nil {
return fmt.Errorf("writing servers data to embedded JSON file: %w", err)
}
}
return nil
}
+2 -2
View File
@@ -132,7 +132,7 @@ func (s *Settings) SetDefaults() {
s.IPv6.setDefaults()
s.PublicIP.setDefaults()
s.Shadowsocks.setDefaults()
s.Storage.setDefaults()
s.Storage.SetDefaults()
s.System.setDefaults()
s.Version.setDefaults()
s.VPN.setDefaults()
@@ -213,7 +213,7 @@ func (s *Settings) Read(r *reader.Reader, warner Warner) (err error) {
return s.PublicIP.read(r, warner)
},
"shadowsocks": s.Shadowsocks.read,
"storage": s.Storage.read,
"storage": s.Storage.Read,
"system": s.System.read,
"updater": s.Updater.read,
"version": s.Version.read,
@@ -90,7 +90,7 @@ func Test_Settings_String(t *testing.T) {
| ├── Logging: yes
| └── Authentication file path: /gluetun/auth/config.toml
├── Storage settings:
| └── Filepath: /gluetun/servers.json
| └── Servers directory path: /gluetun/servers/
├── OS Alpine settings:
| ├── Process UID: 1000
| └── Process GID: 1000
+51 -14
View File
@@ -11,15 +11,26 @@ import (
// Storage contains settings to configure the storage.
type Storage struct {
// Filepath is the path to the servers.json file. An empty string disables on-disk storage.
Filepath *string
// ServersEnabled is whether to enable storage of servers on disk.
// It defaults to true.
ServersEnabled *bool
// ServersPath is the path to the servers files directory, and cannot be
// the empty string.
ServersPath string
// LegacyServersFilepath is the legacy "fat" JSON filepath to migrate from.
// TODO v4: remove
LegacyServersFilepath string
}
func (s Storage) validate() (err error) {
if *s.Filepath != "" { // optional
_, err := filepath.Abs(*s.Filepath)
if *s.ServersEnabled {
_, err := filepath.Abs(s.ServersPath)
if err != nil {
return fmt.Errorf("filepath is not valid: %w", err)
return fmt.Errorf("servers path is not valid: %w", err)
}
_, err = filepath.Abs(s.LegacyServersFilepath)
if err != nil {
return fmt.Errorf("legacy servers filepath is not valid: %w", err)
}
}
return nil
@@ -27,17 +38,25 @@ func (s Storage) validate() (err error) {
func (s *Storage) copy() (copied Storage) {
return Storage{
Filepath: gosettings.CopyPointer(s.Filepath),
ServersEnabled: gosettings.CopyPointer(s.ServersEnabled),
ServersPath: s.ServersPath,
LegacyServersFilepath: s.LegacyServersFilepath,
}
}
func (s *Storage) overrideWith(other Storage) {
s.Filepath = gosettings.OverrideWithPointer(s.Filepath, other.Filepath)
s.ServersEnabled = gosettings.OverrideWithPointer(s.ServersEnabled, other.ServersEnabled)
s.ServersPath = gosettings.OverrideWithComparable(s.ServersPath, other.ServersPath)
s.LegacyServersFilepath = gosettings.OverrideWithComparable(s.LegacyServersFilepath, other.LegacyServersFilepath)
}
func (s *Storage) setDefaults() {
const defaultFilepath = "/gluetun/servers.json"
s.Filepath = gosettings.DefaultPointer(s.Filepath, defaultFilepath)
const defaultLegacyServersFilepath = "/gluetun/servers.json"
func (s *Storage) SetDefaults() {
s.ServersEnabled = gosettings.DefaultPointer(s.ServersEnabled, true)
const defaultServersPath = "/gluetun/servers/"
s.ServersPath = gosettings.DefaultComparable(s.ServersPath, defaultServersPath)
s.LegacyServersFilepath = gosettings.DefaultComparable(s.LegacyServersFilepath, defaultLegacyServersFilepath)
}
func (s Storage) String() string {
@@ -45,15 +64,33 @@ func (s Storage) String() string {
}
func (s Storage) toLinesNode() (node *gotree.Node) {
if *s.Filepath == "" {
if !*s.ServersEnabled {
return gotree.New("Storage settings: disabled")
}
node = gotree.New("Storage settings:")
node.Appendf("Filepath: %s", *s.Filepath)
node.Appendf("Servers directory path: %s", s.ServersPath)
if s.LegacyServersFilepath != defaultLegacyServersFilepath {
node.Appendf("Legacy servers filepath: %s", s.LegacyServersFilepath)
}
return node
}
func (s *Storage) read(r *reader.Reader) (err error) {
s.Filepath = r.Get("STORAGE_FILEPATH", reader.AcceptEmpty(true))
func (s *Storage) Read(r *reader.Reader) (err error) {
// Retro-compatibility:
// TODO v4: remove support for STORAGE_FILEPATH
filePath := r.Get("STORAGE_FILEPATH", reader.AcceptEmpty(true), reader.IsRetro("STORAGE_SERVERS_DIRECTORY_PATH"))
if filePath != nil {
if *filePath == "" {
s.ServersEnabled = ptrTo(false)
} else {
s.LegacyServersFilepath = *filePath
}
} else {
s.ServersEnabled, err = r.BoolPtr("STORAGE_SERVERS_ENABLED")
if err != nil {
return err
}
s.ServersPath = r.String("STORAGE_SERVERS_DIRECTORY_PATH")
}
return nil
}
+17 -5
View File
@@ -29,6 +29,9 @@ type Updater struct {
// Providers is the list of VPN service providers
// to update server information for.
Providers []string
// PreferDirectDownload is whether to prefer direct download of
// server data from Github (recommended).
PreferDirectDownload *bool
// ProtonEmail is the email to authenticate with the Proton API.
ProtonEmail *string
// ProtonPassword is the password to authenticate with the Proton API.
@@ -72,11 +75,12 @@ func (u Updater) Validate() (err error) {
func (u *Updater) copy() (copied Updater) {
return Updater{
Period: gosettings.CopyPointer(u.Period),
MinRatio: u.MinRatio,
Providers: gosettings.CopySlice(u.Providers),
ProtonEmail: gosettings.CopyPointer(u.ProtonEmail),
ProtonPassword: gosettings.CopyPointer(u.ProtonPassword),
Period: gosettings.CopyPointer(u.Period),
MinRatio: u.MinRatio,
Providers: gosettings.CopySlice(u.Providers),
PreferDirectDownload: gosettings.CopyPointer(u.PreferDirectDownload),
ProtonEmail: gosettings.CopyPointer(u.ProtonEmail),
ProtonPassword: gosettings.CopyPointer(u.ProtonPassword),
}
}
@@ -87,6 +91,7 @@ func (u *Updater) overrideWith(other Updater) {
u.Period = gosettings.OverrideWithPointer(u.Period, other.Period)
u.MinRatio = gosettings.OverrideWithComparable(u.MinRatio, other.MinRatio)
u.Providers = gosettings.OverrideWithSlice(u.Providers, other.Providers)
u.PreferDirectDownload = gosettings.OverrideWithPointer(u.PreferDirectDownload, other.PreferDirectDownload)
u.ProtonEmail = gosettings.OverrideWithPointer(u.ProtonEmail, other.ProtonEmail)
u.ProtonPassword = gosettings.OverrideWithPointer(u.ProtonPassword, other.ProtonPassword)
}
@@ -104,6 +109,7 @@ func (u *Updater) SetDefaults(vpnProvider string) {
}
// Set these to empty strings to avoid nil pointer panics
u.PreferDirectDownload = gosettings.DefaultPointer(u.PreferDirectDownload, false)
u.ProtonEmail = gosettings.DefaultPointer(u.ProtonEmail, "")
u.ProtonPassword = gosettings.DefaultPointer(u.ProtonPassword, "")
}
@@ -121,6 +127,7 @@ func (u Updater) toLinesNode() (node *gotree.Node) {
node.Appendf("Update period: %s", *u.Period)
node.Appendf("Minimum ratio: %.1f", u.MinRatio)
node.Appendf("Providers to update: %s", strings.Join(u.Providers, ", "))
node.Appendf("Prefer direct download: %s", gosettings.BoolToYesNo(u.PreferDirectDownload))
if slices.Contains(u.Providers, providers.Protonvpn) {
node.Appendf("Proton API email: %s", *u.ProtonEmail)
node.Appendf("Proton API password: %s", gosettings.ObfuscateKey(*u.ProtonPassword))
@@ -142,6 +149,11 @@ func (u *Updater) read(r *reader.Reader) (err error) {
u.Providers = r.CSV("UPDATER_VPN_SERVICE_PROVIDERS")
u.PreferDirectDownload, err = r.BoolPtr("UPDATER_PREFER_DIRECT_DOWNLOAD")
if err != nil {
return err
}
u.ProtonEmail = r.Get("UPDATER_PROTONVPN_EMAIL")
if u.ProtonEmail == nil {
protonUsername := r.String("UPDATER_PROTONVPN_USERNAME", reader.IsRetro("UPDATER_PROTONVPN_EMAIL"))
-6
View File
@@ -1,6 +0,0 @@
package constants
const (
// ServersData is the server information filepath.
ServersData = "/gluetun/servers.json"
)
+2
View File
@@ -154,6 +154,8 @@ func (a *AllServers) Count() (count int) {
type Servers struct {
Version uint16 `json:"version"`
Timestamp int64 `json:"timestamp"`
Preferred bool `json:"preferred,omitempty"`
Filepath string `json:"filepath,omitempty"`
Servers []Server `json:"servers,omitempty"`
}
+63 -27
View File
@@ -2,6 +2,7 @@ package storage
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"sort"
@@ -9,44 +10,79 @@ import (
"github.com/qdm12/gluetun/internal/models"
)
// FlushToFile flushes the merged servers data to the file
// specified by path, as indented JSON.
func (s *Storage) FlushToFile(path string) error {
s.mergedMutex.RLock()
defer s.mergedMutex.RUnlock()
// flushToFile flushes the merged servers data to files
// using the manifest file path given. It is not thread-safe.
func (s *Storage) flushToFile(manifestPath string) error {
const (
filePermission = 0o644
dirPermission = 0o755
)
return s.flushToFile(path)
}
// flushToFile flushes the merged servers data to the file
// specified by path, as indented JSON. It is not thread-safe.
func (s *Storage) flushToFile(path string) error {
if path == "" {
return nil // no file to write to
}
const permission = 0o644
dirPath := filepath.Dir(path)
if err := os.MkdirAll(dirPath, permission); err != nil {
return err
serversDirectoryPath := filepath.Dir(manifestPath)
if err := os.MkdirAll(serversDirectoryPath, dirPermission); err != nil {
return fmt.Errorf("creating directory: %w", err)
}
file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, permission)
for provider, providerServers := range s.mergedServers.ProviderToServers {
providerFilepath := providerServers.Filepath
if providerFilepath == "" {
providerFilepath = filepath.Join(serversDirectoryPath, provider+".json")
}
providerDirectoryPath := filepath.Dir(providerFilepath)
if err := os.MkdirAll(providerDirectoryPath, dirPermission); err != nil {
return fmt.Errorf("creating directory: %w", err)
}
}
metadata := map[string]any{"version": s.mergedServers.Version}
for provider, providerServers := range s.mergedServers.ProviderToServers {
sort.Sort(models.SortableServers(providerServers.Servers))
providerFilepath := providerServers.Filepath
if providerFilepath == "" {
providerFilepath = filepath.Join(serversDirectoryPath, provider+".json")
}
providerFile, err := os.OpenFile(providerFilepath,
os.O_CREATE|os.O_WRONLY|os.O_TRUNC, filePermission)
if err != nil {
return fmt.Errorf("opening servers data file for %s: %w", provider, err)
}
encodedProviderServers := providerServers
encodedProviderServers.Filepath = ""
encoder := json.NewEncoder(providerFile)
encoder.SetIndent("", " ")
err = encoder.Encode(encodedProviderServers)
if err != nil {
_ = providerFile.Close()
return fmt.Errorf("encoding servers data for %s: %w", provider, err)
}
err = providerFile.Close()
if err != nil {
return fmt.Errorf("closing servers data file for %s: %w", provider, err)
}
metadata[provider] = map[string]string{"filepath": providerFilepath}
}
serversFile, err := os.OpenFile(manifestPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, filePermission)
if err != nil {
return err
}
encoder := json.NewEncoder(file)
encoder := json.NewEncoder(serversFile)
encoder.SetIndent("", " ")
for _, obj := range s.mergedServers.ProviderToServers {
sort.Sort(models.SortableServers(obj.Servers))
}
err = encoder.Encode(&s.mergedServers)
err = encoder.Encode(metadata)
if err != nil {
_ = file.Close()
_ = serversFile.Close()
return err
}
return file.Close()
return serversFile.Close()
}
+64
View File
@@ -0,0 +1,64 @@
package storage
import (
"encoding/json"
"os"
"path/filepath"
"testing"
"github.com/qdm12/gluetun/internal/models"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func Test_flushToFile_filepathOnlyInManifest(t *testing.T) {
t.Parallel()
tempPath := t.TempDir()
providerFilepath := filepath.Join(tempPath, "provider.json")
manifestPath := filepath.Join(tempPath, "manifest.json")
storage := &Storage{
mergedServers: models.AllServers{
Version: 1,
ProviderToServers: map[string]models.Servers{
"provider": {
Version: 1,
Timestamp: 1,
Filepath: providerFilepath,
},
},
},
}
err := storage.flushToFile(manifestPath)
require.NoError(t, err)
providerFile, err := os.Open(providerFilepath)
require.NoError(t, err)
defer providerFile.Close()
providerContent := make(map[string]json.RawMessage)
err = json.NewDecoder(providerFile).Decode(&providerContent)
require.NoError(t, err)
_, hasFilepath := providerContent["filepath"]
assert.False(t, hasFilepath)
manifestFile, err := os.Open(manifestPath)
require.NoError(t, err)
defer manifestFile.Close()
manifestContent := make(map[string]json.RawMessage)
err = json.NewDecoder(manifestFile).Decode(&manifestContent)
require.NoError(t, err)
providerMetadataRaw, ok := manifestContent["provider"]
require.True(t, ok)
var providerMetadata struct {
Filepath string `json:"filepath"`
}
err = json.Unmarshal(providerMetadataRaw, &providerMetadata)
require.NoError(t, err)
assert.Equal(t, providerFilepath, providerMetadata.Filepath)
}
+34 -3
View File
@@ -3,19 +3,50 @@ package storage
import (
"embed"
"encoding/json"
"fmt"
"path"
serversmodule "github.com/qdm12/gluetun-servers/pkg/servers"
"github.com/qdm12/gluetun/internal/models"
)
//go:embed servers.json
var allServersEmbedFS embed.FS
func parseHardcodedServers() (allServers models.AllServers, err error) {
func parseHardcodedServers() (allServers models.AllServers) {
f, err := allServersEmbedFS.Open("servers.json")
if err != nil {
return allServers, err
panic(err)
}
defer f.Close() // no-op
decoder := json.NewDecoder(f)
err = decoder.Decode(&allServers)
return allServers, err
if err != nil {
panic("decoding servers.json: " + err.Error())
}
for provider, metadata := range allServers.ProviderToServers {
filename := path.Base(metadata.Filepath)
providerFile, err := serversmodule.Files.Open(filename)
if err != nil {
panic(fmt.Sprintf("reading embedded provider file %s for %s: %s", filename, provider, err))
}
defer providerFile.Close() // no-op
var providerServers models.Servers
decoder := json.NewDecoder(providerFile)
err = decoder.Decode(&providerServers)
if err != nil {
panic(fmt.Sprintf("JSON decoding embedded provider file %s for %s: %s",
filename, provider, err))
} else if providerServers.Filepath != "" {
panic(fmt.Sprintf("embedded provider file %s for %s should not have filepath set",
filename, provider))
}
providerServers.Filepath = metadata.Filepath // inherit filepath from servers.json
allServers.ProviderToServers[provider] = providerServers
}
return allServers
}
+40 -3
View File
@@ -1,9 +1,13 @@
package storage
import (
"encoding/json"
"path"
"testing"
"github.com/qdm12/gluetun-servers/pkg/servers"
"github.com/qdm12/gluetun/internal/constants/providers"
"github.com/qdm12/gluetun/internal/models"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -11,9 +15,10 @@ import (
func Test_parseHardcodedServers(t *testing.T) {
t.Parallel()
servers, err := parseHardcodedServers()
require.NoError(t, err)
var servers models.AllServers
assert.NotPanics(t, func() {
servers = parseHardcodedServers()
})
// all providers minus custom
allProviders := providers.All()
@@ -24,3 +29,35 @@ func Test_parseHardcodedServers(t *testing.T) {
assert.NotEmptyf(t, servers, "for provider %s", provider)
}
}
func Test_parseHardcodedServers_filepathsAndEmbeddedProviderFiles(t *testing.T) {
t.Parallel()
hardcodedServers := parseHardcodedServers()
allProviders := providers.All()
for _, provider := range allProviders {
providerServers, ok := hardcodedServers.ProviderToServers[provider]
require.Truef(t, ok, "for provider %s", provider)
require.NotEmptyf(t, providerServers.Filepath,
"embedded servers filepath should be set for provider %s", provider)
filename := path.Base(providerServers.Filepath)
file, err := servers.Files.Open(filename)
require.NoErrorf(t, err, "opening embedded provider file for %s", provider)
var fileServers struct {
Version uint16 `json:"version"`
Timestamp int64 `json:"timestamp"`
Servers []json.RawMessage `json:"servers"`
}
err = json.NewDecoder(file).Decode(&fileServers)
require.NoErrorf(t, err, "decoding embedded provider file for %s", provider)
require.NoError(t, file.Close())
assert.NotZerof(t, fileServers.Version, "for provider %s", provider)
assert.NotZerof(t, fileServers.Timestamp, "for provider %s", provider)
assert.NotEmptyf(t, fileServers.Servers, "for provider %s", provider)
}
}
+14
View File
@@ -30,6 +30,20 @@ func (s *Storage) mergeServers(hardcoded, persisted models.AllServers) models.Al
func (s *Storage) mergeProviderServers(provider string,
hardcoded, persisted models.Servers,
) (merged models.Servers) {
if persisted.Preferred && persisted.Version != hardcoded.Version {
s.logger.Warn(fmt.Sprintf(
"persisted preferred %s servers are discarded because they have version %d and hardcoded servers have version %d",
provider, persisted.Version, hardcoded.Version))
}
// If persisted data is marked as preferred, use it regardless of timestamp
// (as long as versions match)
if persisted.Preferred && persisted.Version == hardcoded.Version && len(persisted.Servers) > 0 {
s.logger.Info(fmt.Sprintf(
"Using %s servers from file (marked as preferred)", provider))
return persisted
}
nowTimestamp := time.Now().Unix()
if persisted.Timestamp > nowTimestamp {
s.logger.Warn(fmt.Sprintf(
+17
View File
@@ -45,6 +45,23 @@ func (mr *MockLoggerMockRecorder) Info(arg0 interface{}) *gomock.Call {
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Info", reflect.TypeOf((*MockLogger)(nil).Info), arg0)
}
// Infof mocks base method.
func (m *MockLogger) Infof(arg0 string, arg1 ...interface{}) {
m.ctrl.T.Helper()
varargs := []interface{}{arg0}
for _, a := range arg1 {
varargs = append(varargs, a)
}
m.ctrl.Call(m, "Infof", varargs...)
}
// Infof indicates an expected call of Infof.
func (mr *MockLoggerMockRecorder) Infof(arg0 interface{}, arg1 ...interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
varargs := append([]interface{}{arg0}, arg1...)
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Infof", reflect.TypeOf((*MockLogger)(nil).Infof), varargs...)
}
// Warn mocks base method.
func (m *MockLogger) Warn(arg0 string) {
m.ctrl.T.Helper()
+85 -22
View File
@@ -12,29 +12,36 @@ import (
"golang.org/x/text/language"
)
// readFromFile reads the servers from server.json.
// readFromFile reads the servers data starting from the given manifest file path.
// It only reads servers that have the same version as the hardcoded servers version
// to avoid JSON decoding errors.
func (s *Storage) readFromFile(filepath string, hardcodedVersions map[string]uint16) (
servers models.AllServers, err error,
func (s *Storage) readFromFile(manifestPath string, hardcodedVersions map[string]uint16) (
servers models.AllServers, found bool, err error,
) {
file, err := os.Open(filepath)
file, err := os.Open(manifestPath)
if os.IsNotExist(err) {
return servers, nil
return servers, false, nil
} else if err != nil {
return servers, err
return servers, false, err
}
b, err := io.ReadAll(file)
if err != nil {
return servers, err
return servers, true, err
}
if err := file.Close(); err != nil {
return servers, err
return servers, true, err
}
return s.extractServersFromBytes(b, hardcodedVersions)
if len(b) == 0 {
// To satisfy https://github.com/qdm12/gluetun/issues/3318
// not too sure why but eh I'm feeling generous adding 3 code lines for it.
return servers, false, nil
}
servers, err = s.extractServersFromBytes(b, hardcodedVersions)
return servers, true, err
}
func (s *Storage) extractServersFromBytes(b []byte, hardcodedVersions map[string]uint16) (
@@ -46,6 +53,12 @@ func (s *Storage) extractServersFromBytes(b []byte, hardcodedVersions map[string
}
// Note schema version is at map key "version" as number
if rawVersion, ok := rawMessages["version"]; ok {
err := json.Unmarshal(rawVersion, &servers.Version)
if err != nil {
return servers, fmt.Errorf("decoding servers schema version: %w", err)
}
}
allProviders := providers.All()
servers.ProviderToServers = make(map[string]models.Servers, len(allProviders))
@@ -86,25 +99,20 @@ func (s *Storage) readServers(provider string, hardcodedVersion uint16,
) {
provider = titleCaser.String(provider)
var versionObject struct {
Version uint16 `json:"version"`
var metadata struct {
Version uint16 `json:"version"`
Timestamp uint64 `json:"timestamp"`
Filepath string `json:"filepath"`
}
err = json.Unmarshal(rawMessage, &versionObject)
err = json.Unmarshal(rawMessage, &metadata)
if err != nil {
return servers, false, fmt.Errorf("decoding servers version for provider %s: %w",
provider, err)
}
persistedVersion := versionObject.Version
versionsMatch = hardcodedVersion == persistedVersion
if !versionsMatch {
s.logger.Info(fmt.Sprintf(
"%s servers from file discarded because they have "+
"version %d and hardcoded servers have version %d",
provider, persistedVersion, hardcodedVersion))
return servers, versionsMatch, nil
if metadata.Filepath != "" {
return s.readServersFromFilepath(provider, metadata.Filepath, hardcodedVersion)
}
err = json.Unmarshal(rawMessage, &servers)
@@ -113,5 +121,60 @@ func (s *Storage) readServers(provider string, hardcodedVersion uint16,
provider, err)
}
return servers, versionsMatch, nil
const sourcePath = ""
if !checkVersions(hardcodedVersion, servers.Version, provider, sourcePath,
servers.Preferred, s.logger) {
return models.Servers{}, false, nil
}
return servers, true, nil
}
func (s *Storage) readServersFromFilepath(provider, filepath string, hardcodedVersion uint16) (
referencedServers models.Servers, versionsMatch bool, err error,
) {
providerFile, err := os.Open(filepath)
if os.IsNotExist(err) {
return models.Servers{}, false, nil
} else if err != nil {
return models.Servers{}, false, fmt.Errorf("opening servers file %s for provider %s: %w",
filepath, provider, err)
}
defer providerFile.Close()
err = json.NewDecoder(providerFile).Decode(&referencedServers)
if err != nil {
return models.Servers{}, false, fmt.Errorf("decoding servers file %s for provider %s: %w",
filepath, provider, err)
}
if !checkVersions(hardcodedVersion, referencedServers.Version, provider, filepath,
referencedServers.Preferred, s.logger) {
return models.Servers{}, false, nil
}
referencedServers.Filepath = filepath
return referencedServers, true, nil
}
func checkVersions(builtinVersion, version uint16, provider, sourcePath string,
preferred bool, logger Logger,
) (match bool) {
if version == builtinVersion {
return true
}
name := provider
log := logger.Info
if preferred {
name += " preferred"
log = logger.Warn
}
name += " servers"
if sourcePath != "" {
name += " from file " + sourcePath
}
log(fmt.Sprintf(
"%s discarded because they have version %d and hardcoded servers have version %d",
name, version, builtinVersion))
return false
}
+32 -14
View File
@@ -30,25 +30,30 @@ func Test_extractServersFromBytes(t *testing.T) {
testCases := map[string]struct {
b []byte
hardcodedVersions map[string]uint16
logged []string
makeLogger func(ctrl *gomock.Controller) *MockLogger
persisted models.AllServers
errMessage string
}{
"bad JSON": {
b: []byte("garbage"),
makeLogger: func(_ *gomock.Controller) *MockLogger { return nil },
errMessage: "decoding servers: invalid character 'g' looking for beginning of value",
},
"bad provider JSON": {
b: []byte(`{"cyberghost": "garbage"}`),
hardcodedVersions: populateProviderToVersion(map[string]uint16{}),
makeLogger: func(_ *gomock.Controller) *MockLogger { return nil },
errMessage: "decoding servers version for provider Cyberghost: " +
"json: cannot unmarshal string into Go value of type struct { Version uint16 \"json:\\\"version\\\"\" }",
"json: cannot unmarshal string into Go value of type struct { Version uint16 \"json:\\\"version\\\"\"; " +
"Timestamp uint64 \"json:\\\"timestamp\\\"\"; " +
"Filepath string \"json:\\\"filepath\\\"\" }",
},
"bad servers array JSON": {
b: []byte(`{"cyberghost": {"version": 1, "servers": "garbage"}}`),
hardcodedVersions: populateProviderToVersion(map[string]uint16{
providers.Cyberghost: 1,
}),
makeLogger: func(_ *gomock.Controller) *MockLogger { return nil },
errMessage: "decoding servers for provider Cyberghost: " +
"json: cannot unmarshal string into Go struct field Servers.servers of type []models.Server",
},
@@ -57,6 +62,7 @@ func Test_extractServersFromBytes(t *testing.T) {
hardcodedVersions: populateProviderToVersion(map[string]uint16{
providers.Cyberghost: 1,
}),
makeLogger: func(_ *gomock.Controller) *MockLogger { return nil },
persisted: models.AllServers{
ProviderToServers: map[string]models.Servers{},
},
@@ -68,6 +74,7 @@ func Test_extractServersFromBytes(t *testing.T) {
hardcodedVersions: populateProviderToVersion(map[string]uint16{
providers.Cyberghost: 1,
}),
makeLogger: func(_ *gomock.Controller) *MockLogger { return nil },
persisted: models.AllServers{
ProviderToServers: map[string]models.Servers{
providers.Cyberghost: {Version: 1},
@@ -81,8 +88,28 @@ func Test_extractServersFromBytes(t *testing.T) {
hardcodedVersions: populateProviderToVersion(map[string]uint16{
providers.Cyberghost: 2,
}),
logged: []string{
"Cyberghost servers from file discarded because they have version 1 and hardcoded servers have version 2",
makeLogger: func(ctrl *gomock.Controller) *MockLogger {
logger := NewMockLogger(ctrl)
logger.EXPECT().Info("Cyberghost servers discarded because " +
"they have version 1 and hardcoded servers have version 2")
return logger
},
persisted: models.AllServers{
ProviderToServers: map[string]models.Servers{},
},
},
"preferred_different_versions": {
b: []byte(`{
"cyberghost": {"version": 1, "timestamp": 1, "preferred": true}
}`),
hardcodedVersions: populateProviderToVersion(map[string]uint16{
providers.Cyberghost: 2,
}),
makeLogger: func(ctrl *gomock.Controller) *MockLogger {
logger := NewMockLogger(ctrl)
logger.EXPECT().Warn("Cyberghost preferred servers discarded because " +
"they have version 1 and hardcoded servers have version 2")
return logger
},
persisted: models.AllServers{
ProviderToServers: map[string]models.Servers{},
@@ -95,16 +122,7 @@ func Test_extractServersFromBytes(t *testing.T) {
t.Parallel()
ctrl := gomock.NewController(t)
logger := NewMockLogger(ctrl)
var previousLogCall *gomock.Call
for _, logged := range testCase.logged {
call := logger.EXPECT().Info(logged)
if previousLogCall != nil {
call.After(previousLogCall)
}
previousLogCall = call
}
logger := testCase.makeLogger(ctrl)
s := &Storage{
logger: logger,
}
+19 -3
View File
@@ -2,6 +2,8 @@ package storage
import (
"fmt"
"os"
"path/filepath"
"time"
"github.com/qdm12/gluetun/internal/constants/providers"
@@ -10,12 +12,12 @@ import (
// SetServers sets the given servers for the given provider
// in the storage in-memory map and saves all the servers
// to file.
// to files.
// Note the servers given are not copied so the caller must
// NOT MUTATE them after calling this method.
func (s *Storage) SetServers(provider string, servers []models.Server) (err error) {
if provider == providers.Custom {
return
return nil
}
s.mergedMutex.Lock()
@@ -26,10 +28,24 @@ func (s *Storage) SetServers(provider string, servers []models.Server) (err erro
serversObject.Servers = servers
s.mergedServers.ProviderToServers[provider] = serversObject
err = s.flushToFile(s.filepath)
if !s.disk {
return nil
}
manifestPath := filepath.Join(s.directoryPath, manifestFilename)
err = s.flushToFile(manifestPath)
if err != nil {
return fmt.Errorf("saving servers to file: %w", err)
}
if !s.hasLegacy() {
return nil
}
s.logger.Infof("removing legacy %s which is now migrated to %s", s.legacyFilepath, s.directoryPath)
err = os.Remove(s.legacyFilepath)
if err != nil {
s.logger.Warn("failed removing legacy servers file " + s.legacyFilepath + ": " + err.Error())
}
return nil
}
+23 -303854
View File
File diff suppressed because it is too large Load Diff
+32 -8
View File
@@ -1,6 +1,8 @@
package storage
import (
"os"
"path/filepath"
"sync"
"github.com/qdm12/gluetun/internal/models"
@@ -14,31 +16,38 @@ type Storage struct {
// SyncServers method.
hardcodedServers models.AllServers
logger Logger
filepath string
disk bool
directoryPath string
legacyFilepath string
}
const manifestFilename = "manifest.json"
type Logger interface {
Info(s string)
Infof(format string, args ...any)
Warn(s string)
}
// New creates a new storage and reads the servers from the
// embedded servers file and the file on disk.
// Passing an empty filepath disables the reading and writing of
// embedded servers files and the files on disk.
// Passing an empty directoryPath disables the reading and writing of
// servers.
func New(logger Logger, filepath string) (storage *Storage, err error) {
// A unit test prevents any error from being returned
func New(logger Logger, disk bool, directoryPath, legacyFilepath string) (storage *Storage, err error) {
// A unit test prevents [parseHardcodedServers] from ever failing,
// and ensures all providers are part of the servers returned.
hardcodedServers, _ := parseHardcodedServers()
hardcodedServers := parseHardcodedServers()
storage = &Storage{
hardcodedServers: hardcodedServers,
mergedServers: hardcodedServers,
logger: logger,
filepath: filepath,
disk: disk,
directoryPath: directoryPath,
legacyFilepath: legacyFilepath,
}
if filepath != "" {
if disk {
if err := storage.syncServers(); err != nil {
return nil, err
}
@@ -46,3 +55,18 @@ func New(logger Logger, filepath string) (storage *Storage, err error) {
return storage, nil
}
// hasLegacy returns true if the legacy file `legacyFilepath` exists AND is
// different from the manifest file defined by `directoryPath`/[manifestFilename].
// This is used to determine if the legacy file should be read and removed after flushing servers data.
func (s *Storage) hasLegacy() bool {
if !s.disk {
return false
}
if filepath.Clean(filepath.Join(s.directoryPath, manifestFilename)) ==
filepath.Clean(s.legacyFilepath) {
return false
}
stat, err := os.Stat(s.legacyFilepath)
return err == nil && !stat.IsDir()
}
+32 -8
View File
@@ -2,6 +2,8 @@ package storage
import (
"fmt"
"os"
"path/filepath"
"reflect"
"github.com/qdm12/gluetun/internal/models"
@@ -14,18 +16,31 @@ func countServers(allServers models.AllServers) (count int) {
return count
}
// syncServers merges the hardcoded servers with the ones from the file.
// syncServers merges the hardcoded servers with the ones from on disk files.
// It assumes s.directoryPath is set.
func (s *Storage) syncServers() (err error) {
hardcodedVersions := make(map[string]uint16, len(s.hardcodedServers.ProviderToServers))
for provider, servers := range s.hardcodedServers.ProviderToServers {
hardcodedVersions[provider] = servers.Version
}
serversOnFile, err := s.readFromFile(s.filepath, hardcodedVersions)
sourceManifestPath := filepath.Join(s.directoryPath, manifestFilename)
destinationManifestPath := sourceManifestPath
serversOnFile, found, err := s.readFromFile(sourceManifestPath, hardcodedVersions)
if err != nil {
return fmt.Errorf("reading servers from file: %w", err)
}
hasLegacy := s.hasLegacy()
if !found && hasLegacy {
sourceManifestPath = s.legacyFilepath
s.logger.Infof("reading legacy servers file %s and migrating it to directory %s", sourceManifestPath, s.directoryPath)
serversOnFile, _, err = s.readFromFile(sourceManifestPath, hardcodedVersions)
if err != nil {
return fmt.Errorf("reading servers from file: %w", err)
}
}
hardcodedCount := countServers(s.hardcodedServers)
countOnFile := countServers(serversOnFile)
@@ -34,13 +49,13 @@ func (s *Storage) syncServers() (err error) {
if countOnFile == 0 {
s.logger.Info(fmt.Sprintf(
"creating %s with %d hardcoded servers",
s.filepath, hardcodedCount))
"writing servers data files to %s with %d hardcoded servers",
s.directoryPath, hardcodedCount))
s.mergedServers = s.hardcodedServers
} else {
s.logger.Info(fmt.Sprintf(
"merging by most recent %d hardcoded servers and %d servers read from %s",
hardcodedCount, countOnFile, s.filepath))
"merging by most recent %d hardcoded servers and %d servers read from manifest file %s",
hardcodedCount, countOnFile, sourceManifestPath))
s.mergedServers = s.mergeServers(s.hardcodedServers, serversOnFile)
}
@@ -50,9 +65,18 @@ func (s *Storage) syncServers() (err error) {
return nil
}
err = s.flushToFile(s.filepath)
err = s.flushToFile(destinationManifestPath)
if err != nil {
s.logger.Warn("failed writing servers to file: " + err.Error())
s.logger.Warn("failed writing servers to destination manifest: " + err.Error())
return nil
}
migratedFromLegacy := hasLegacy && sourceManifestPath == s.legacyFilepath
if migratedFromLegacy {
err = os.Remove(sourceManifestPath)
if err != nil && !os.IsNotExist(err) {
s.logger.Warn("failed removing legacy servers file " + sourceManifestPath + ": " + err.Error())
}
}
return nil
}
-41
View File
@@ -1,41 +0,0 @@
//go:build linux || darwin
package tun
import (
"errors"
"fmt"
"os"
"syscall"
)
// Check checks the tunnel device specified by path is present and accessible.
func (t *Tun) Check(path string) error {
f, err := os.OpenFile(path, os.O_RDWR, 0)
if err != nil {
return fmt.Errorf("TUN device is not available: %w", err)
}
defer f.Close()
info, err := f.Stat()
if err != nil {
return fmt.Errorf("getting stat information for TUN file: %w", err)
}
sys, ok := info.Sys().(*syscall.Stat_t)
if !ok {
return errors.New("cannot get syscall stat info of TUN file")
}
const expectedRdev = 2760 // corresponds to major 10 and minor 200
if sys.Rdev != expectedRdev {
return fmt.Errorf("TUN file has an unexpected rdev: %d instead of expected %d",
sys.Rdev, expectedRdev)
}
if err := f.Close(); err != nil {
return fmt.Errorf("closing TUN device: %w", err)
}
return nil
}
-7
View File
@@ -1,7 +0,0 @@
//go:build !linux && !darwin
package tun
func (t *Tun) Check(path string) error {
panic("not implemented")
}
-50
View File
@@ -1,50 +0,0 @@
//go:build linux || darwin
package tun
import (
"fmt"
"math"
"os"
"path/filepath"
"golang.org/x/sys/unix"
)
// Create creates a TUN device at the path specified.
func (t *Tun) Create(path string) (err error) {
parentDir := filepath.Dir(path)
err = os.MkdirAll(parentDir, 0o751) //nolint:mnd
if err != nil {
return err
}
const (
major = 10
minor = 200
)
dev := unix.Mkdev(major, minor)
if dev > math.MaxInt {
panic("dev is too high")
}
err = unix.Mknod(path, unix.S_IFCHR, int(dev))
if err != nil {
return fmt.Errorf("creating TUN device file node: %w", err)
}
fd, err := unix.Open(path, 0, 0)
if err != nil {
if err.Error() == "operation not permitted" {
err = fmt.Errorf("%w (did you specify --device /dev/net/tun to your container command?)", err)
}
return fmt.Errorf("unix opening TUN device file: %w", err)
}
const nonBlocking = true
err = unix.SetNonblock(fd, nonBlocking)
if err != nil {
return fmt.Errorf("setting non block to TUN device file descriptor: %w", err)
}
return nil
}
-8
View File
@@ -1,8 +0,0 @@
//go:build !linux && !darwin
package tun
// Create creates a TUN device at the path specified.
func (t *Tun) Create(path string) error {
panic("not implemented")
}
+96 -3
View File
@@ -1,7 +1,100 @@
//go:build linux || darwin
package tun
type Tun struct{}
import (
"errors"
"fmt"
"math"
"os"
"path/filepath"
"syscall"
func New() *Tun {
return &Tun{}
"golang.org/x/sys/unix"
)
func Setup() error {
const tunDevice = "/dev/net/tun"
err := check(tunDevice)
switch {
case err == nil:
return nil
case errors.Is(err, os.ErrNotExist):
err = create(tunDevice)
if err != nil {
return fmt.Errorf("creating TUN device: %w", err)
}
return nil
default:
return fmt.Errorf("checking TUN device: %w (see the Wiki errors/tun page)", err)
}
}
// check checks the tunnel device specified by path is present and accessible.
func check(path string) error {
f, err := os.OpenFile(path, os.O_RDWR, 0)
if err != nil {
return fmt.Errorf("TUN device is not available: %w", err)
}
defer f.Close()
info, err := f.Stat()
if err != nil {
return fmt.Errorf("getting stat information for TUN file: %w", err)
}
sys, ok := info.Sys().(*syscall.Stat_t)
if !ok {
return errors.New("cannot get syscall stat info of TUN file")
}
const expectedRdev = 2760 // corresponds to major 10 and minor 200
if sys.Rdev != expectedRdev {
return fmt.Errorf("TUN file has an unexpected rdev: %d instead of expected %d",
sys.Rdev, expectedRdev)
}
if err := f.Close(); err != nil {
return fmt.Errorf("closing TUN device: %w", err)
}
return nil
}
// create creates a TUN device at the path specified.
func create(path string) (err error) {
parentDir := filepath.Dir(path)
err = os.MkdirAll(parentDir, 0o751) //nolint:mnd
if err != nil {
return err
}
const (
major = 10
minor = 200
)
dev := unix.Mkdev(major, minor)
if dev > math.MaxInt {
panic("dev is too high")
}
err = unix.Mknod(path, unix.S_IFCHR, int(dev))
if err != nil {
return fmt.Errorf("creating TUN device file node: %w", err)
}
fd, err := unix.Open(path, 0, 0)
if err != nil {
if err.Error() == "operation not permitted" {
err = fmt.Errorf("%w (did you specify --device /dev/net/tun to your container command?)", err)
}
return fmt.Errorf("unix opening TUN device file: %w", err)
}
const nonBlocking = true
err = unix.SetNonblock(fd, nonBlocking)
if err != nil {
return fmt.Errorf("setting non block to TUN device file descriptor: %w", err)
}
return nil
}
+6 -8
View File
@@ -10,20 +10,18 @@ import (
"github.com/stretchr/testify/require"
)
func Test_Tun(t *testing.T) {
func Test_Setup(t *testing.T) {
t.Parallel()
path := getTempPath(t)
tun := New()
defer func() {
err := os.RemoveAll(path)
require.NoError(t, err)
}()
// No file check fail
err := tun.Check(path)
err := check(path)
require.Error(t, err)
expectedMessage := "TUN device is not available: open " + path + ": no such file or directory"
require.Equal(t, expectedMessage, err.Error())
@@ -35,13 +33,13 @@ func Test_Tun(t *testing.T) {
require.NoError(t, err)
// Simple file check fail
err = tun.Check(path)
err = check(path)
require.Error(t, err)
expectedMessage = "TUN file has an unexpected rdev: 0 instead of expected 2760"
require.Equal(t, expectedMessage, err.Error())
// Create TUN device fail as file exists
err = tun.Create(path)
err = create(path)
require.Error(t, err)
require.EqualError(t, err, "creating TUN device file node: file exists")
@@ -50,7 +48,7 @@ func Test_Tun(t *testing.T) {
require.NoError(t, err)
// Create TUN device success
err = tun.Create(path)
err = create(path)
if err != nil && strings.HasSuffix(err.Error(), "operation not permitted") {
t.Skip("You do not have root privileges to create a TUN device, skipping test")
return
@@ -58,7 +56,7 @@ func Test_Tun(t *testing.T) {
require.NoError(t, err)
// Check TUN device success
err = tun.Check(path)
err = check(path)
require.NoError(t, err)
}
+12
View File
@@ -0,0 +1,12 @@
//go:build !linux && !darwin
package tun
import (
"fmt"
"runtime"
)
func Setup() error {
return fmt.Errorf("not implemented for %s", runtime.GOOS)
}
+1 -1
View File
@@ -50,7 +50,7 @@ func NewLoop(settings settings.Updater, providers updater.Providers,
status: constants.Stopped,
settings: settings,
},
updater: updater.New(client, storage, providers, logger),
updater: updater.New(client, storage, providers, logger, *settings.PreferDirectDownload),
logger: logger,
start: make(chan struct{}),
running: make(chan models.LoopStatus),
+36 -5
View File
@@ -5,6 +5,8 @@ import (
"encoding/json"
"errors"
"fmt"
"net/url"
"path"
"github.com/qdm12/gluetun/internal/models"
"github.com/qdm12/gluetun/internal/provider/common"
@@ -16,18 +18,37 @@ type Provider interface {
}
func (u *Updater) updateProvider(ctx context.Context, provider Provider,
minRatio float64,
manifest manifest, minRatio float64,
) (err error) {
providerName := provider.Name()
existingServersCount := u.storage.GetServersCount(providerName)
minServers := int(minRatio * float64(existingServersCount))
servers, err := provider.FetchServers(ctx, minServers)
if err != nil {
if errors.Is(err, common.ErrNotEnoughServers) {
var servers []models.Server
if manifest.providerToFilepath == nil {
servers, err = provider.FetchServers(ctx, minServers)
switch {
case errors.Is(err, common.ErrNotEnoughServers):
u.logger.Warn("note: if running the update manually, you can use the flag " +
"-minratio to allow the update to succeed with less servers found")
fallthrough
case err != nil:
return fmt.Errorf("getting %s servers: %w", providerName, err)
}
} else {
providerFilepath := manifest.providerToFilepath[providerName]
providerFileURL := buildProviderFileURL(providerName, providerFilepath)
var data models.Servers
err = u.fetchJSON(ctx, providerFileURL, &data)
if err != nil {
return fmt.Errorf("downloading provider file %s: %w", providerFileURL, err)
}
servers = data.Servers
if len(servers) < minServers {
return fmt.Errorf("provider %s has not enough servers from downloaded file: got %d and expected at least %d",
providerName, len(servers), minServers)
}
return fmt.Errorf("getting %s servers: %w", providerName, err)
}
for _, server := range servers {
@@ -55,3 +76,13 @@ func (u *Updater) updateProvider(ctx context.Context, provider Provider,
}
return nil
}
func buildProviderFileURL(providerName, filePath string) (providerFileURL string) {
filename := path.Base(filePath)
if filename == "." || filename == "/" || filename == "" {
filename = providerName + ".json"
}
const serversFilesBaseURL = "https://raw.githubusercontent.com/qdm12/gluetun-servers/main/pkg/servers/"
return serversFilesBaseURL + url.PathEscape(filename)
}
+81 -9
View File
@@ -2,10 +2,15 @@ package updater
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"strings"
"time"
"github.com/qdm12/gluetun/internal/constants/providers"
"github.com/qdm12/gluetun/internal/provider/common"
"github.com/qdm12/gluetun/internal/updater/unzip"
"golang.org/x/text/cases"
@@ -13,7 +18,8 @@ import (
)
type Updater struct {
providers Providers
providers Providers
preferDirectDownload bool
// state
storage Storage
@@ -26,22 +32,31 @@ type Updater struct {
}
func New(httpClient *http.Client, storage Storage,
providers Providers, logger Logger,
providers Providers, logger Logger, preferDirectDownload bool,
) *Updater {
unzipper := unzip.New(httpClient)
return &Updater{
providers: providers,
storage: storage,
logger: logger,
timeNow: time.Now,
client: httpClient,
unzipper: unzipper,
providers: providers,
storage: storage,
logger: logger,
timeNow: time.Now,
client: httpClient,
unzipper: unzipper,
preferDirectDownload: preferDirectDownload,
}
}
func (u *Updater) UpdateServers(ctx context.Context, providers []string,
minRatio float64,
) (err error) {
var manifest manifest
if u.preferDirectDownload {
manifest, err = u.fetchManifest(ctx)
if err != nil {
return fmt.Errorf("fetching remote manifest: %w", err)
}
}
caser := cases.Title(language.English)
for _, providerName := range providers {
u.logger.Info("updating " + caser.String(providerName) + " servers...")
@@ -49,7 +64,7 @@ func (u *Updater) UpdateServers(ctx context.Context, providers []string,
fetcher := u.providers.Get(providerName)
// TODO support servers offering only TCP or only UDP
// for NordVPN and PureVPN
err := u.updateProvider(ctx, fetcher, minRatio)
err := u.updateProvider(ctx, fetcher, manifest, minRatio)
switch {
case err == nil:
continue
@@ -70,3 +85,60 @@ func (u *Updater) UpdateServers(ctx context.Context, providers []string,
return nil
}
type manifest struct {
providerToFilepath map[string]string
}
func (u *Updater) fetchManifest(ctx context.Context) (m manifest, err error) {
const serversManifestURL = "https://raw.githubusercontent.com/qdm12/gluetun-servers/main/pkg/servers/manifest.json"
var raw map[string]json.RawMessage
err = u.fetchJSON(ctx, serversManifestURL, &raw)
if err != nil {
return m, err
}
providerNames := providers.All()
m.providerToFilepath = make(map[string]string, len(providerNames))
for _, name := range providerNames {
var metadata struct {
Filepath string `json:"filepath"`
}
err = json.Unmarshal(raw[name], &metadata)
if err != nil {
return m, fmt.Errorf("decoding manifest metadata for %s: %w", name, err)
} else if metadata.Filepath == "" {
return m, fmt.Errorf("manifest missing filepath for provider %s", name)
}
m.providerToFilepath[name] = metadata.Filepath
}
return m, nil
}
func (u *Updater) fetchJSON(ctx context.Context, rawURL string, dst any) (err error) {
request, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
if err != nil {
return fmt.Errorf("creating request: %w", err)
}
response, err := u.client.Do(request)
if err != nil {
return fmt.Errorf("doing request: %w", err)
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
const limit = 10 * 1024 * 1024 // 10 MiB
body, _ := io.ReadAll(io.LimitReader(response.Body, limit))
return fmt.Errorf("HTTP status code %d for %s: %s",
response.StatusCode, rawURL, strings.TrimSpace(string(body)))
}
err = json.NewDecoder(response.Body).Decode(dst)
if err != nil {
return fmt.Errorf("decoding response body: %w", err)
}
return nil
}
+6
View File
@@ -9,6 +9,7 @@ import (
"github.com/qdm12/gluetun/internal/models"
"github.com/qdm12/gluetun/internal/netlink"
"github.com/qdm12/gluetun/internal/provider"
"github.com/qdm12/gluetun/internal/tun"
"github.com/qdm12/gluetun/internal/wireguard"
"github.com/qdm12/gosettings"
)
@@ -19,6 +20,11 @@ func setupAmneziaWg(ctx context.Context, netlinker NetLinker,
settings settings.VPN, ipv6SupportLevel netlink.IPv6SupportLevel, logger wireguard.Logger) (
amneziawger *amneziawg.Amneziawg, connection models.Connection, err error,
) {
err = tun.Setup()
if err != nil {
return nil, models.Connection{}, fmt.Errorf("setting up tun device: %w", err)
}
ipv6Internet := ipv6SupportLevel == netlink.IPv6Internet
connection, err = providerConf.GetConnection(settings.Provider.ServerSelection, ipv6Internet)
if err != nil {
-2
View File
@@ -45,8 +45,6 @@ type Provider interface {
GetConnection(selection settings.ServerSelection, ipv6Supported bool) (connection models.Connection, err error)
OpenVPNConfig(connection models.Connection, settings settings.OpenVPN, ipv6Supported bool) (lines []string)
Name() string
FetchServers(ctx context.Context, minServers int) (
servers []models.Server, err error)
}
type PortForwarder interface {
+6
View File
@@ -9,6 +9,7 @@ import (
"github.com/qdm12/gluetun/internal/netlink"
"github.com/qdm12/gluetun/internal/openvpn"
"github.com/qdm12/gluetun/internal/provider"
"github.com/qdm12/gluetun/internal/tun"
)
// setupOpenVPN sets OpenVPN up using the configurators and settings given.
@@ -18,6 +19,11 @@ func setupOpenVPN(ctx context.Context, fw Firewall,
settings settings.VPN, ipv6SupportLevel netlink.IPv6SupportLevel, starter Cmder,
logger openvpn.Logger) (runner *openvpn.Runner, connection models.Connection, err error,
) {
err = tun.Setup()
if err != nil {
return nil, models.Connection{}, fmt.Errorf("setting up tun device: %w", err)
}
ipv6Internet := ipv6SupportLevel == netlink.IPv6Internet
connection, err = providerConf.GetConnection(settings.Provider.ServerSelection, ipv6Internet)
if err != nil {
+12
View File
@@ -8,6 +8,7 @@ import (
"github.com/qdm12/gluetun/internal/cleanup"
"github.com/qdm12/gluetun/internal/netlink"
gtun "github.com/qdm12/gluetun/internal/tun"
"golang.zx2c4.com/wireguard/conn"
"golang.zx2c4.com/wireguard/device"
"golang.zx2c4.com/wireguard/tun"
@@ -27,15 +28,18 @@ func (w *Wireguard) Run(ctx context.Context, waitError chan<- error, ready chan<
}
setupFunction := setupUserSpace
userspace := false
switch w.settings.Implementation {
case "auto": //nolint:goconst
if !kernelSupported {
w.logger.Info("Using userspace implementation since Kernel support does not exist")
userspace = true
break
}
w.logger.Info("Using available kernelspace implementation")
setupFunction = setupKernelSpace
case "userspace":
userspace = true
case "kernelspace":
if !kernelSupported {
waitError <- errors.New("kernel does not support Wireguard")
@@ -46,6 +50,14 @@ func (w *Wireguard) Run(ctx context.Context, waitError chan<- error, ready chan<
panic(fmt.Sprintf("unknown implementation %q", w.settings.Implementation))
}
if userspace {
err = gtun.Setup()
if err != nil {
waitError <- fmt.Errorf("setting up userspace tun device: %w", err)
return
}
}
setup := func(ctx context.Context, cleanups *cleanup.Cleanups) (
linkIndex uint32, waitAndCleanup func() error, err error,
) {