refactor(storage): new storage file structure

- new directory structure containing manifest.json and one json file per provider, by default.
- the manifest.json file can specify a filepath for each vpn provider
- each vpn provider json data file can contain the `"preferred": true` field to enforce it is used even if outdated, unless there is a version mismatch
- `STORAGE_SERVERS_DIRECTORY_PATH` replaces `STORAGE_FILEPATH` (which is now a migration source only). It sets the directory where server manifest and per-provider JSON files are stored (default: `/gluetun/servers/`).
- First-run migration: On startup, gluetun checks for the old /gluetun/servers.json file; if found and no new manifest exists, it automatically migrates all data to /gluetun/servers/ directory structure
- Silent fallback: If legacy file isn't found, uses the new directory path normally
- Legacy cleanup: After successful migration, attempts to remove the old fat JSON file (logs warning only if removal fails, e.g., read-only bind mounts)

gluetun-servers/pkg/updaters/resolver/interfaces.go

@@ -0,0 +1,10 @@
+package resolver
+
+import (
+	"context"
+	"net"
+)
+
+type Dialer interface {
+	Dial(ctx context.Context, network, address string) (net.Conn, error)
+}

gluetun-servers/pkg/updaters/resolver/ips.go

@@ -0,0 +1,20 @@
+package resolver
+
+import (
+	"net/netip"
+)
+
+func uniqueIPsToSlice(uniqueIPs map[string]struct{}) (ips []netip.Addr) {
+	ips = make([]netip.Addr, 0, len(uniqueIPs))
+	for key := range uniqueIPs {
+		ip, err := netip.ParseAddr(key)
+		if err != nil {
+			panic(err)
+		}
+		if ip.Is4In6() {
+			ip = netip.AddrFrom4(ip.As4())
+		}
+		ips = append(ips, ip)
+	}
+	return ips
+}

gluetun-servers/pkg/updaters/resolver/ips_test.go

@@ -0,0 +1,40 @@
+package resolver
+
+import (
+	"net/netip"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_uniqueIPsToSlice(t *testing.T) {
+	t.Parallel()
+	testCases := map[string]struct {
+		inputIPs  map[string]struct{}
+		outputIPs []netip.Addr
+	}{
+		"nil": {
+			inputIPs:  nil,
+			outputIPs: []netip.Addr{},
+		},
+		"empty": {
+			inputIPs:  map[string]struct{}{},
+			outputIPs: []netip.Addr{},
+		},
+		"single IPv4": {
+			inputIPs:  map[string]struct{}{"1.1.1.1": {}},
+			outputIPs: []netip.Addr{netip.AddrFrom4([4]byte{1, 1, 1, 1})},
+		},
+		"two IPv4s": {
+			inputIPs:  map[string]struct{}{"1.1.1.1": {}, "1.1.2.1": {}},
+			outputIPs: []netip.Addr{netip.AddrFrom4([4]byte{1, 1, 1, 1}), netip.AddrFrom4([4]byte{1, 1, 2, 1})},
+		},
+	}
+	for name, testCase := range testCases {
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+			outputIPs := uniqueIPsToSlice(testCase.inputIPs)
+			assert.ElementsMatch(t, testCase.outputIPs, outputIPs)
+		})
+	}
+}

gluetun-servers/pkg/updaters/resolver/net.go

@@ -0,0 +1,12 @@
+package resolver
+
+import (
+	"net"
+)
+
+func newResolver(d Dialer) *net.Resolver {
+	return &net.Resolver{
+		PreferGo: true,
+		Dial:     d.Dial,
+	}
+}

gluetun-servers/pkg/updaters/resolver/parallel.go

@@ -0,0 +1,105 @@
+package resolver
+
+import (
+	"context"
+	"fmt"
+	"net/netip"
+)
+
+type Parallel struct {
+	repeatResolver *Repeat
+}
+
+func NewParallelResolver(dialer Dialer) *Parallel {
+	return &Parallel{
+		repeatResolver: NewRepeat(dialer),
+	}
+}
+
+type ParallelSettings struct {
+	// Hosts to resolve in parallel.
+	Hosts     []string
+	Repeat    RepeatSettings
+	FailEarly bool
+	// Maximum ratio of the hosts failing DNS resolution
+	// divided by the total number of hosts requested.
+	// This value is between 0 and 1. Note this is only
+	// applicable if FailEarly is not set to true.
+	MaxFailRatio float64
+}
+
+type parallelResult struct {
+	host string
+	IPs  []netip.Addr
+}
+
+func (pr *Parallel) Resolve(ctx context.Context, settings ParallelSettings) (
+	hostToIPs map[string][]netip.Addr, warnings []string, err error,
+) {
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	results := make(chan parallelResult)
+	defer close(results)
+	errors := make(chan error)
+	defer close(errors)
+
+	for _, host := range settings.Hosts {
+		go pr.resolveAsync(ctx, host, settings.Repeat, results, errors)
+	}
+
+	hostToIPs = make(map[string][]netip.Addr, len(settings.Hosts))
+	maxFails := int(settings.MaxFailRatio * float64(len(settings.Hosts)))
+
+	for range settings.Hosts {
+		select {
+		case newErr := <-errors:
+			if settings.FailEarly {
+				if err == nil {
+					// only set the error to the first error encountered
+					// and not the context canceled errors coming after.
+					err = newErr
+					cancel()
+				}
+				break
+			}
+
+			// do not add warnings coming from the call to cancel()
+			if len(warnings) < maxFails {
+				warnings = append(warnings, newErr.Error())
+			}
+
+			if len(warnings) == maxFails {
+				cancel() // cancel only once when we reach maxFails
+			}
+		case result := <-results:
+			hostToIPs[result.host] = result.IPs
+		}
+	}
+
+	if err != nil { // fail early
+		return nil, warnings, err
+	}
+
+	failureRatio := float64(len(warnings)) / float64(len(settings.Hosts))
+	if failureRatio > settings.MaxFailRatio {
+		return hostToIPs, warnings,
+			fmt.Errorf("maximum failure ratio reached: %.2f failure ratio reached", failureRatio)
+	}
+
+	return hostToIPs, warnings, nil
+}
+
+func (pr *Parallel) resolveAsync(ctx context.Context, host string,
+	settings RepeatSettings, results chan<- parallelResult, errors chan<- error,
+) {
+	IPs, err := pr.repeatResolver.Resolve(ctx, host, settings)
+	if err != nil {
+		errors <- err
+		return
+	}
+	results <- parallelResult{
+		host: host,
+		IPs:  IPs,
+	}
+}

gluetun-servers/pkg/updaters/resolver/repeat.go

@@ -0,0 +1,135 @@
+package resolver
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"net/netip"
+	"sort"
+	"time"
+)
+
+type Repeat struct {
+	resolver *net.Resolver
+}
+
+func NewRepeat(dialer Dialer) *Repeat {
+	return &Repeat{
+		resolver: newResolver(dialer),
+	}
+}
+
+type RepeatSettings struct {
+	Address         string
+	MaxDuration     time.Duration
+	BetweenDuration time.Duration
+	MaxNoNew        int
+	// Maximum consecutive DNS resolution failures
+	MaxFails int
+	SortIPs  bool
+}
+
+func (r *Repeat) Resolve(ctx context.Context, host string, settings RepeatSettings) (
+	ips []netip.Addr, err error,
+) {
+	timedCtx, cancel := context.WithTimeout(ctx, settings.MaxDuration)
+	defer cancel()
+
+	noNewCounter := 0
+	failCounter := 0
+	uniqueIPs := make(map[string]struct{})
+
+	for err == nil {
+		// TODO
+		// - one resolving every 100ms for round robin DNS responses
+		// - one every second for time based DNS cycling responses
+		noNewCounter, failCounter, err = r.resolveOnce(ctx, timedCtx, host, settings, uniqueIPs, noNewCounter, failCounter)
+	}
+
+	if len(uniqueIPs) == 0 {
+		return nil, err
+	}
+
+	ips = uniqueIPsToSlice(uniqueIPs)
+
+	if settings.SortIPs {
+		sort.Slice(ips, func(i, j int) bool {
+			return ips[i].Compare(ips[j]) < 1
+		})
+	}
+
+	return ips, nil
+}
+
+func (r *Repeat) resolveOnce(ctx, timedCtx context.Context, host string,
+	settings RepeatSettings, uniqueIPs map[string]struct{}, noNewCounter, failCounter int) (
+	newNoNewCounter, newFailCounter int, err error,
+) {
+	IPs, err := r.lookupIPs(timedCtx, host)
+	if err != nil {
+		failCounter++
+		if settings.MaxFails > 0 && failCounter == settings.MaxFails {
+			return noNewCounter, failCounter, fmt.Errorf("reached the maximum number of consecutive failures: "+
+				"%d failed attempts resolving %s: %s", settings.MaxFails, host, err)
+		}
+		// it's fine to fail some of the resolutions
+		return noNewCounter, failCounter, nil
+	}
+	failCounter = 0 // reset the counter if we had no error
+
+	anyNew := false
+	for _, IP := range IPs {
+		key := IP.String()
+		if _, ok := uniqueIPs[key]; !ok {
+			anyNew = true
+			uniqueIPs[key] = struct{}{}
+		}
+	}
+
+	if !anyNew {
+		noNewCounter++
+	}
+
+	if settings.MaxNoNew > 0 && noNewCounter == settings.MaxNoNew {
+		// we reached the maximum number of resolutions without
+		// finding any new IP address to our unique IP addresses set.
+		return noNewCounter, failCounter,
+			fmt.Errorf("reached the maximum number of no new update: "+
+				"%d times no updated for %d IP addresses found",
+				noNewCounter, len(uniqueIPs))
+	}
+
+	timer := time.NewTimer(settings.BetweenDuration)
+	select {
+	case <-timer.C:
+		return noNewCounter, failCounter, nil
+	case <-ctx.Done():
+		if !timer.Stop() {
+			<-timer.C
+		}
+		return noNewCounter, failCounter, ctx.Err()
+	case <-timedCtx.Done():
+		if err := ctx.Err(); err != nil {
+			// timedCtx was canceled from its parent context
+			return noNewCounter, failCounter, err
+		}
+		return noNewCounter, failCounter,
+			fmt.Errorf("reached the timeout: %w", timedCtx.Err())
+	}
+}
+
+func (r *Repeat) lookupIPs(ctx context.Context, host string) (ips []netip.Addr, err error) {
+	addresses, err := r.resolver.LookupIPAddr(ctx, host)
+	if err != nil {
+		return nil, err
+	}
+	ips = make([]netip.Addr, 0, len(addresses))
+	for i := range addresses {
+		ip, ok := netip.AddrFromSlice(addresses[i].IP)
+		if !ok {
+			continue
+		}
+		ips = append(ips, ip)
+	}
+	return ips, nil
+}