mirror of
https://github.com/qdm12/gluetun.git
synced 2026-07-24 11:26:25 +02:00
refactor(storage): new storage file structure
- new directory structure containing manifest.json and one json file per provider, by default. - the manifest.json file can specify a filepath for each vpn provider - each vpn provider json data file can contain the `"preferred": true` field to enforce it is used even if outdated, unless there is a version mismatch - `STORAGE_SERVERS_DIRECTORY_PATH` replaces `STORAGE_FILEPATH` (which is now a migration source only). It sets the directory where server manifest and per-provider JSON files are stored (default: `/gluetun/servers/`). - First-run migration: On startup, gluetun checks for the old /gluetun/servers.json file; if found and no new manifest exists, it automatically migrates all data to /gluetun/servers/ directory structure - Silent fallback: If legacy file isn't found, uses the new directory path normally - Legacy cleanup: After successful migration, attempts to remove the old fat JSON file (logs warning only if removal fails, e.g., read-only bind mounts)
This commit is contained in:
Quentin McGaw
@@ -0,0 +1,51 @@
|
||||
package mullvad
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
type serverData struct {
|
||||
Hostname string `json:"hostname"`
|
||||
Country string `json:"country_name"`
|
||||
City string `json:"city_name"`
|
||||
Active bool `json:"active"`
|
||||
Owned bool `json:"owned"`
|
||||
Provider string `json:"provider"`
|
||||
IPv4 string `json:"ipv4_addr_in"`
|
||||
IPv6 string `json:"ipv6_addr_in"`
|
||||
Type string `json:"type"`
|
||||
PubKey string `json:"pubkey"` // Wireguard public key
|
||||
}
|
||||
|
||||
func fetchAPI(ctx context.Context, client *http.Client) (data []serverData, err error) {
|
||||
const url = "https://api.mullvad.net/www/relays/all/"
|
||||
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
response, err := client.Do(request)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer response.Body.Close()
|
||||
|
||||
if response.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("HTTP status code not OK: %d %s", response.StatusCode, response.Status)
|
||||
}
|
||||
|
||||
decoder := json.NewDecoder(response.Body)
|
||||
if err := decoder.Decode(&data); err != nil {
|
||||
return nil, fmt.Errorf("failed decoding response body: %s", err)
|
||||
}
|
||||
|
||||
if err := response.Body.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return data, nil
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
package mullvad
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/netip"
|
||||
"strings"
|
||||
|
||||
"github.com/qdm12/gluetun-servers/pkg/constants"
|
||||
"github.com/qdm12/gluetun-servers/pkg/models"
|
||||
)
|
||||
|
||||
type hostToServer map[string]models.Server
|
||||
|
||||
func (hts hostToServer) add(data serverData) (err error) {
|
||||
if !data.Active {
|
||||
return nil
|
||||
}
|
||||
|
||||
if data.IPv4 == "" && data.IPv6 == "" {
|
||||
return errors.New("no IP address for VPN server")
|
||||
}
|
||||
|
||||
server, ok := hts[data.Hostname]
|
||||
if ok { // API returns a server per hostname at most
|
||||
return nil
|
||||
}
|
||||
|
||||
switch data.Type {
|
||||
case "wireguard":
|
||||
server.VPN = constants.Wireguard
|
||||
case "bridge":
|
||||
// ignore bridge servers
|
||||
return nil
|
||||
default:
|
||||
return fmt.Errorf("VPN type not supported: %s", data.Type)
|
||||
}
|
||||
|
||||
if data.IPv4 != "" {
|
||||
ipv4, err := netip.ParseAddr(data.IPv4)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parsing IPv4 address: %w", err)
|
||||
} else if !ipv4.Is4() {
|
||||
return fmt.Errorf("IP address is not IPv4: %s", data.IPv4)
|
||||
}
|
||||
server.IPs = append(server.IPs, ipv4)
|
||||
}
|
||||
|
||||
if data.IPv6 != "" {
|
||||
ipv6, err := netip.ParseAddr(data.IPv6)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parsing IPv6 address: %w", err)
|
||||
} else if !ipv6.Is6() {
|
||||
return fmt.Errorf("IP address is not IPv6: %s", data.IPv6)
|
||||
}
|
||||
server.IPs = append(server.IPs, ipv6)
|
||||
}
|
||||
|
||||
server.Country = data.Country
|
||||
server.City = strings.ReplaceAll(data.City, ",", "")
|
||||
server.Hostname = data.Hostname
|
||||
server.ISP = data.Provider
|
||||
server.Owned = data.Owned
|
||||
server.WgPubKey = data.PubKey
|
||||
|
||||
hts[data.Hostname] = server
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (hts hostToServer) toServersSlice() (servers []models.Server) {
|
||||
servers = make([]models.Server, 0, len(hts))
|
||||
for _, server := range hts {
|
||||
server.IPs = uniqueSortedIPs(server.IPs)
|
||||
servers = append(servers, server)
|
||||
}
|
||||
return servers
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package mullvad
|
||||
|
||||
import (
|
||||
"net/netip"
|
||||
"sort"
|
||||
)
|
||||
|
||||
func uniqueSortedIPs(ips []netip.Addr) []netip.Addr {
|
||||
uniqueIPs := make(map[string]struct{}, len(ips))
|
||||
for _, ip := range ips {
|
||||
key := ip.String()
|
||||
uniqueIPs[key] = struct{}{}
|
||||
}
|
||||
|
||||
ips = make([]netip.Addr, 0, len(uniqueIPs))
|
||||
for key := range uniqueIPs {
|
||||
ip, err := netip.ParseAddr(key)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if ip.Is4In6() {
|
||||
ip = netip.AddrFrom4(ip.As4())
|
||||
}
|
||||
ips = append(ips, ip)
|
||||
}
|
||||
|
||||
sort.Slice(ips, func(i, j int) bool {
|
||||
return ips[i].Compare(ips[j]) < 0
|
||||
})
|
||||
|
||||
return ips
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package mullvad
|
||||
|
||||
import (
|
||||
"net/netip"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func Test_uniqueSortedIPs(t *testing.T) {
|
||||
t.Parallel()
|
||||
testCases := map[string]struct {
|
||||
inputIPs []netip.Addr
|
||||
outputIPs []netip.Addr
|
||||
}{
|
||||
"nil": {
|
||||
inputIPs: nil,
|
||||
outputIPs: []netip.Addr{},
|
||||
},
|
||||
"empty": {
|
||||
inputIPs: []netip.Addr{},
|
||||
outputIPs: []netip.Addr{},
|
||||
},
|
||||
"single IPv4": {
|
||||
inputIPs: []netip.Addr{netip.AddrFrom4([4]byte{1, 1, 1, 1})},
|
||||
outputIPs: []netip.Addr{netip.AddrFrom4([4]byte{1, 1, 1, 1})},
|
||||
},
|
||||
"two IPv4s": {
|
||||
inputIPs: []netip.Addr{netip.AddrFrom4([4]byte{1, 1, 2, 1}), netip.AddrFrom4([4]byte{1, 1, 1, 1})},
|
||||
outputIPs: []netip.Addr{netip.AddrFrom4([4]byte{1, 1, 1, 1}), netip.AddrFrom4([4]byte{1, 1, 2, 1})},
|
||||
},
|
||||
}
|
||||
for name, testCase := range testCases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
outputIPs := uniqueSortedIPs(testCase.inputIPs)
|
||||
assert.Equal(t, testCase.outputIPs, outputIPs)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
package mullvad
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
"github.com/qdm12/gluetun-servers/pkg/models"
|
||||
"github.com/qdm12/gluetun-servers/pkg/updaters/common"
|
||||
)
|
||||
|
||||
func (u *Updater) FetchServers(ctx context.Context, minServers int) (
|
||||
servers []models.Server, err error,
|
||||
) {
|
||||
data, err := fetchAPI(ctx, u.client)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
hts := make(hostToServer)
|
||||
for _, serverData := range data {
|
||||
if err := hts.add(serverData); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
if len(hts) < minServers {
|
||||
return nil, fmt.Errorf("%w: %d and expected at least %d",
|
||||
common.ErrNotEnoughServers, len(hts), minServers)
|
||||
}
|
||||
|
||||
servers = hts.toServersSlice()
|
||||
|
||||
sort.Sort(models.SortableServers(servers))
|
||||
|
||||
return servers, nil
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
package mullvad
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
)
|
||||
|
||||
type Updater struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
func New(client *http.Client) *Updater {
|
||||
return &Updater{
|
||||
client: client,
|
||||
}
|
||||
}
|
||||
|
||||
func (u *Updater) Version() uint16 {
|
||||
return 4 //nolint:mnd
|
||||
}
|
||||
|
||||
func (u *Updater) Name() string {
|
||||
return "mullvad"
|
||||
}
|
||||
Reference in New Issue
Block a user