refactor(storage): new storage file structure

- new directory structure containing manifest.json and one json file per provider, by default.
- the manifest.json file can specify a filepath for each vpn provider
- each vpn provider json data file can contain the `"preferred": true` field to enforce it is used even if outdated, unless there is a version mismatch
- `STORAGE_SERVERS_DIRECTORY_PATH` replaces `STORAGE_FILEPATH` (which is now a migration source only). It sets the directory where server manifest and per-provider JSON files are stored (default: `/gluetun/servers/`).
- First-run migration: On startup, gluetun checks for the old /gluetun/servers.json file; if found and no new manifest exists, it automatically migrates all data to /gluetun/servers/ directory structure
- Silent fallback: If legacy file isn't found, uses the new directory path normally
- Legacy cleanup: After successful migration, attempts to remove the old fat JSON file (logs warning only if removal fails, e.g., read-only bind mounts)
This commit is contained in:
Quentin McGaw
2026-04-27 02:47:30 +00:00
parent 13503b0ae0
commit d9cc7dcffb
303 changed files with 304957 additions and 304344 deletions
@@ -0,0 +1,47 @@
package perfectprivacy
import (
"net/netip"
"github.com/qdm12/gluetun-servers/pkg/constants"
"github.com/qdm12/gluetun-servers/pkg/models"
)
type cityToServer map[string]models.Server
func (cts cityToServer) add(city string, ips []netip.Addr) {
server, ok := cts[city]
if !ok {
server.VPN = constants.OpenVPN
server.City = city
server.IPs = ips
server.TCP = true
server.UDP = true
} else {
// Do not insert duplicate IP addresses
existingIPs := make(map[string]struct{}, len(server.IPs))
for _, ip := range server.IPs {
existingIPs[ip.String()] = struct{}{}
}
for _, ip := range ips {
ipString := ip.String()
_, ok := existingIPs[ipString]
if ok {
continue
}
existingIPs[ipString] = struct{}{}
server.IPs = append(server.IPs, ip)
}
}
cts[city] = server
}
func (cts cityToServer) toServersSlice() (servers []models.Server) {
servers = make([]models.Server, 0, len(cts))
for _, server := range cts {
servers = append(servers, server)
}
return servers
}
@@ -0,0 +1,20 @@
package perfectprivacy
import (
"strings"
"unicode"
)
func parseFilename(fileName string) (city string) {
const suffix = ".conf"
s := strings.TrimSuffix(fileName, suffix)
for i, r := range s {
if unicode.IsDigit(r) {
s = s[:i]
break
}
}
return s
}
@@ -0,0 +1,73 @@
package perfectprivacy
import (
"context"
"fmt"
"net/url"
"sort"
"strings"
"github.com/qdm12/gluetun-servers/pkg/models"
"github.com/qdm12/gluetun-servers/pkg/updaters/common"
"github.com/qdm12/gluetun-servers/pkg/updaters/openvpn"
)
func (u *Updater) FetchServers(ctx context.Context, minServers int) (
servers []models.Server, err error,
) {
zipURL := url.URL{
Scheme: "https",
Host: "www.perfect-privacy.com",
Path: "/downloads/openvpn/get",
}
values := make(url.Values)
values.Set("system", "linux")
values.Set("scope", "server")
values.Set("filetype", "zip")
values.Set("protocol", "udp") // all support both TCP and UDP
zipURL.RawQuery = values.Encode()
contents, err := u.unzipper.FetchAndExtract(ctx, zipURL.String())
if err != nil {
return nil, err
}
cts := make(cityToServer)
for fileName, content := range contents {
err := addServerFromOvpn(cts, fileName, content)
if err != nil {
u.warner.Warn(err.Error() + " in " + fileName)
}
}
if len(cts) < minServers {
return nil, fmt.Errorf("%w: %d and expected at least %d",
common.ErrNotEnoughServers, len(cts), minServers)
}
servers = cts.toServersSlice()
sort.Sort(models.SortableServers(servers))
return servers, nil
}
func addServerFromOvpn(cts cityToServer,
fileName string, content []byte,
) (err error) {
if !strings.HasSuffix(fileName, ".conf") {
return nil // not an OpenVPN file
}
ips, err := openvpn.ExtractIPs(content)
if err != nil {
return err
}
city := parseFilename(fileName)
cts.add(city, ips)
return nil
}
@@ -0,0 +1,25 @@
package perfectprivacy
import (
"github.com/qdm12/gluetun-servers/pkg/updaters/common"
)
type Updater struct {
unzipper common.Unzipper
warner common.Warner
}
func New(unzipper common.Unzipper, warner common.Warner) *Updater {
return &Updater{
unzipper: unzipper,
warner: warner,
}
}
func (u *Updater) Version() uint16 {
return 1
}
func (u *Updater) Name() string {
return "perfect privacy"
}