refactor(storage): new storage file structure

- new directory structure containing manifest.json and one json file per provider, by default.
- the manifest.json file can specify a filepath for each vpn provider
- each vpn provider json data file can contain the `"preferred": true` field to enforce it is used even if outdated, unless there is a version mismatch
- `STORAGE_SERVERS_DIRECTORY_PATH` replaces `STORAGE_FILEPATH` (which is now a migration source only). It sets the directory where server manifest and per-provider JSON files are stored (default: `/gluetun/servers/`).
- First-run migration: On startup, gluetun checks for the old /gluetun/servers.json file; if found and no new manifest exists, it automatically migrates all data to /gluetun/servers/ directory structure
- Silent fallback: If legacy file isn't found, uses the new directory path normally
- Legacy cleanup: After successful migration, attempts to remove the old fat JSON file (logs warning only if removal fails, e.g., read-only bind mounts)

gluetun-servers/pkg/updaters/openvpn/extract.go

@@ -0,0 +1,93 @@
+package openvpn
+
+import (
+	"errors"
+	"fmt"
+	"net/netip"
+	"sort"
+	"strings"
+)
+
+func ExtractProto(b []byte) (tcp, udp bool, err error) {
+	lines := strings.Split(string(b), "\n")
+	const protoPrefix = "proto "
+	for _, line := range lines {
+		if !strings.HasPrefix(line, protoPrefix) {
+			continue
+		}
+		s := strings.TrimPrefix(line, protoPrefix)
+		s = strings.TrimSpace(s)
+		s = strings.ToLower(s)
+		switch s {
+		case "tcp", "tcp4", "tcp6", "tcp-client":
+			return true, false, nil
+		case "udp", "udp4", "udp6":
+			return false, true, nil
+		default:
+			return false, false, fmt.Errorf("unknown protocol: %s", s)
+		}
+	}
+
+	// default is UDP if unspecified in openvpn configuration
+	return false, true, nil
+}
+
+func ExtractHost(b []byte) (host, warning string, err error) {
+	const (
+		rejectIP     = true
+		rejectDomain = false
+	)
+	hosts := extractRemoteHosts(b, rejectIP, rejectDomain)
+	if len(hosts) == 0 {
+		return "", "", errors.New("remote host not found")
+	} else if len(hosts) > 1 {
+		warning = fmt.Sprintf(
+			"only using the first host %q and discarding %d other hosts",
+			hosts[0], len(hosts)-1)
+	}
+	return hosts[0], warning, nil
+}
+
+func ExtractIPs(b []byte) (ips []netip.Addr, err error) {
+	const rejectIP, rejectDomain = false, true
+	ipStrings := extractRemoteHosts(b, rejectIP, rejectDomain)
+	if len(ipStrings) == 0 {
+		return nil, errors.New("remote IP not found")
+	}
+
+	sort.Slice(ipStrings, func(i, j int) bool {
+		return ipStrings[i] < ipStrings[j]
+	})
+
+	ips = make([]netip.Addr, len(ipStrings))
+	for i := range ipStrings {
+		ips[i], err = netip.ParseAddr(ipStrings[i])
+		if err != nil {
+			return nil, fmt.Errorf("parsing IP address: %w", err)
+		}
+	}
+
+	return ips, nil
+}
+
+func extractRemoteHosts(content []byte, rejectIP, rejectDomain bool) (hosts []string) {
+	lines := strings.Split(string(content), "\n")
+	for _, line := range lines {
+		line = strings.TrimSpace(line)
+		if !strings.HasPrefix(line, "remote ") {
+			continue
+		}
+		fields := strings.Fields(line)
+		if len(fields) == 1 || fields[1] == "" {
+			continue
+		}
+		host := fields[1]
+		_, err := netip.ParseAddr(host)
+		if (rejectIP && err == nil) ||
+			(rejectDomain && err != nil) {
+			continue
+		}
+		hosts = append(hosts, host)
+	}
+	return hosts
+}

gluetun-servers/pkg/updaters/openvpn/fetch.go

@@ -0,0 +1,43 @@
+package openvpn
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+)
+
+func FetchFile(ctx context.Context, client *http.Client, url string) (
+	host string, err error,
+) {
+	b, err := fetchData(ctx, client, url)
+	if err != nil {
+		return "", err
+	}
+
+	const rejectIP = true
+	const rejectDomain = false
+	hosts := extractRemoteHosts(b, rejectIP, rejectDomain)
+	if len(hosts) == 0 {
+		return "", fmt.Errorf("remote host not found for url %s", url)
+	}
+
+	return hosts[0], nil
+}
+
+func fetchData(ctx context.Context, client *http.Client, url string) (
+	b []byte, err error,
+) {
+	request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	response, err := client.Do(request)
+	if err != nil {
+		return nil, err
+	}
+	defer response.Body.Close()
+
+	return io.ReadAll(response.Body)
+}

gluetun-servers/pkg/updaters/openvpn/multifetch.go

@@ -0,0 +1,67 @@
+package openvpn
+
+import (
+	"context"
+	"net/http"
+)
+
+// FetchMultiFiles fetches multiple Openvpn files in parallel and
+// parses them to extract each of their host. A mapping from host to
+// URL is returned.
+func FetchMultiFiles(ctx context.Context, client *http.Client, urls []string,
+	failEarly bool,
+) (hostToURL map[string]string, errors []error) {
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	hostToURL = make(map[string]string, len(urls))
+
+	type Result struct {
+		url  string
+		host string
+	}
+
+	results := make(chan Result)
+	defer close(results)
+	errorsCh := make(chan error)
+	defer close(errorsCh)
+
+	for _, url := range urls {
+		go func(url string) {
+			host, err := FetchFile(ctx, client, url)
+			if err != nil {
+				errorsCh <- err
+				return
+			}
+			results <- Result{
+				url:  url,
+				host: host,
+			}
+		}(url)
+	}
+
+	for range urls {
+		select {
+		case result := <-results:
+			hostToURL[result.host] = result.url
+		case err := <-errorsCh:
+			if !failEarly {
+				errors = append(errors, err)
+				break
+			}
+
+			if len(errors) == 0 {
+				errors = []error{err} // keep only the first error
+				// stop other operations, this will trigger other errors we ignore
+				cancel()
+			}
+		}
+	}
+
+	if len(errors) > 0 && failEarly {
+		// we don't care about the result found
+		return nil, errors
+	}
+
+	return hostToURL, errors
+}