refactor(storage): new storage file structure

- new directory structure containing manifest.json and one json file per provider, by default. - the manifest.json file can specify a filepath for each vpn provider - each vpn provider json data file can contain the `"preferred": true` field to enforce it is used even if outdated, unless there is a version mismatch - `STORAGE_SERVERS_DIRECTORY_PATH` replaces `STORAGE_FILEPATH` (which is now a migration source only). It sets the directory where server manifest and per-provider JSON files are stored (default: `/gluetun/servers/`). - First-run migration: On startup, gluetun checks for the old /gluetun/servers.json file; if found and no new manifest exists, it automatically migrates all data to /gluetun/servers/ directory structure - Silent fallback: If legacy file isn't found, uses the new directory path normally - Legacy cleanup: After successful migration, attempts to remove the old fat JSON file (logs warning only if removal fails, e.g., read-only bind mounts)
2026-07-25 20:06:25 +02:00 · 2026-04-27 02:47:30 +00:00
parent 13503b0ae0
commit d9cc7dcffb
303 changed files with 304957 additions and 304344 deletions
@@ -0,0 +1,107 @@
+package privatevpn
+
+import (
+	"context"
+	"fmt"
+	"sort"
+	"strings"
+
+	"github.com/qdm12/gluetun-servers/pkg/constants"
+	"github.com/qdm12/gluetun-servers/pkg/models"
+	"github.com/qdm12/gluetun-servers/pkg/updaters/common"
+	"github.com/qdm12/gluetun-servers/pkg/updaters/openvpn"
+)
+
+func (u *Updater) FetchServers(ctx context.Context, minServers int) (
+	servers []models.Server, err error,
+) {
+	const url = "https://privatevpn.com/client/PrivateVPN-TUN.zip"
+	contents, err := u.unzipper.FetchAndExtract(ctx, url)
+	if err != nil {
+		return nil, err
+	} else if len(contents) < minServers {
+		return nil, fmt.Errorf("%w: %d and expected at least %d",
+			common.ErrNotEnoughServers, len(contents), minServers)
+	}
+
+	countryCodes := constants.CountryCodes()
+
+	hts := make(hostToServer)
+	noHostnameServers := make([]models.Server, 0, 1) // there is only one for now
+
+	for fileName, content := range contents {
+		if !strings.HasSuffix(fileName, ".ovpn") {
+			continue // not an OpenVPN file
+		}
+
+		countryCode, city, err := parseFilename(fileName)
+		if err != nil {
+			// treat error as warning and go to next file
+			u.warner.Warn(err.Error() + " in " + fileName)
+			continue
+		}
+
+		country, warning := codeToCountry(countryCode, countryCodes)
+		if warning != "" {
+			u.warner.Warn(warning)
+		}
+
+		host, warning, err := openvpn.ExtractHost(content)
+		if warning != "" {
+			u.warner.Warn(warning)
+		}
+		if err == nil { // found host
+			hts.add(host, country, city)
+			continue
+		}
+
+		ips, extractIPErr := openvpn.ExtractIPs(content)
+		if warning != "" {
+			u.warner.Warn(warning)
+		}
+		if extractIPErr != nil {
+			// treat extract host error as warning and go to next file
+			u.warner.Warn(extractIPErr.Error() + " in " + fileName)
+			continue
+		}
+		server := models.Server{
+			VPN:     constants.OpenVPN,
+			Country: country,
+			City:    city,
+			IPs:     ips,
+			UDP:     true,
+			TCP:     true,
+		}
+		noHostnameServers = append(noHostnameServers, server)
+	}
+
+	if len(noHostnameServers)+len(hts) < minServers {
+		return nil, fmt.Errorf("%w: %d and expected at least %d",
+			common.ErrNotEnoughServers, len(servers)+len(hts), minServers)
+	}
+
+	hosts := hts.toHostsSlice()
+
+	resolveSettings := parallelResolverSettings(hosts)
+	hostToIPs, warnings, err := u.parallelResolver.Resolve(ctx, resolveSettings)
+	for _, warning := range warnings {
+		u.warner.Warn(warning)
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	if len(noHostnameServers)+len(hostToIPs) < minServers {
+		return nil, fmt.Errorf("%w: %d and expected at least %d",
+			common.ErrNotEnoughServers, len(servers), minServers)
+	}
+
+	hts.adaptWithIPs(hostToIPs)
+
+	servers = hts.toServersSlice()
+	servers = append(servers, noHostnameServers...)
+
+	sort.Sort(models.SortableServers(servers))
+
+	return servers, nil
+}