refactor(storage): new storage file structure

- new directory structure containing manifest.json and one json file per provider, by default.
- the manifest.json file can specify a filepath for each vpn provider
- each vpn provider json data file can contain the `"preferred": true` field to enforce it is used even if outdated, unless there is a version mismatch
- `STORAGE_SERVERS_DIRECTORY_PATH` replaces `STORAGE_FILEPATH` (which is now a migration source only). It sets the directory where server manifest and per-provider JSON files are stored (default: `/gluetun/servers/`).
- First-run migration: On startup, gluetun checks for the old /gluetun/servers.json file; if found and no new manifest exists, it automatically migrates all data to /gluetun/servers/ directory structure
- Silent fallback: If legacy file isn't found, uses the new directory path normally
- Legacy cleanup: After successful migration, attempts to remove the old fat JSON file (logs warning only if removal fails, e.g., read-only bind mounts)
This commit is contained in:
Quentin McGaw
2026-04-27 02:47:30 +00:00
parent 13503b0ae0
commit d9cc7dcffb
303 changed files with 304957 additions and 304344 deletions
@@ -0,0 +1,86 @@
package surfshark
import (
"context"
"encoding/json"
"fmt"
"net/http"
"github.com/qdm12/gluetun-servers/pkg/updaters/providers/surfshark/servers"
)
func addServersFromAPI(ctx context.Context, client *http.Client,
hts hostToServers,
) (err error) {
data, err := fetchAPI(ctx, client)
if err != nil {
return err
}
locationData := servers.LocationData()
hostToLocation := hostToLocation(locationData)
for _, serverData := range data {
locationData := hostToLocation[serverData.Host] // TODO remove in v4
retroLoc := locationData.RetroLoc // empty string if the host has no retro-compatible region
tcp, udp := true, true // OpenVPN servers from API supports both TCP and UDP
hts.addOpenVPN(serverData.Host, serverData.Region, serverData.Country,
serverData.Location, retroLoc, tcp, udp)
if serverData.PubKey != "" {
hts.addWireguard(serverData.Host, serverData.Region, serverData.Country,
serverData.Location, retroLoc, serverData.PubKey)
}
}
return nil
}
type serverData struct {
Host string `json:"connectionName"`
Region string `json:"region"`
Country string `json:"country"`
Location string `json:"location"`
PubKey string `json:"pubKey"`
}
func fetchAPI(ctx context.Context, client *http.Client) (
servers []serverData, err error,
) {
const url = "https://api.surfshark.com/v4/server/clusters"
for _, clustersType := range [...]string{"generic", "double", "static", "obfuscated"} {
request, err := http.NewRequestWithContext(ctx, http.MethodGet, url+"/"+clustersType, nil)
if err != nil {
return nil, err
}
response, err := client.Do(request)
if err != nil {
return nil, err
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
return nil, fmt.Errorf("HTTP status code not OK: %d %s",
response.StatusCode, response.Status)
}
decoder := json.NewDecoder(response.Body)
var newServers []serverData
err = decoder.Decode(&newServers)
if err != nil {
return nil, fmt.Errorf("decoding response body: %w", err)
}
err = response.Body.Close()
if err != nil {
return nil, err
}
servers = append(servers, newServers...)
}
return servers, nil
}
@@ -0,0 +1,240 @@
package surfshark
import (
"context"
"errors"
"io"
"net/http"
"strings"
"testing"
"github.com/qdm12/gluetun-servers/pkg/constants"
"github.com/qdm12/gluetun-servers/pkg/models"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
type httpExchange struct {
requestURL string
responseStatus int
responseBody io.ReadCloser
}
func Test_addServersFromAPI(t *testing.T) {
t.Parallel()
testCases := map[string]struct {
hts hostToServers
exchanges []httpExchange
expected hostToServers
err error
}{
"fetch API error": {
exchanges: []httpExchange{{
requestURL: "https://api.surfshark.com/v4/server/clusters/generic",
responseStatus: http.StatusNoContent,
}},
err: errors.New("HTTP status code not OK: 204 No Content"),
},
"success": {
hts: hostToServers{
"existinghost": []models.Server{{Hostname: "existinghost"}},
},
exchanges: []httpExchange{{
requestURL: "https://api.surfshark.com/v4/server/clusters/generic",
responseStatus: http.StatusOK,
responseBody: io.NopCloser(strings.NewReader(`[
{"connectionName":"host1","region":"region1","country":"country1","location":"location1"},
{"connectionName":"host1","region":"region1","country":"country1","location":"location1","pubkey":"pubKeyValue"},
{"connectionName":"host2","region":"region2","country":"country1","location":"location2"}
]`)),
}, {
requestURL: "https://api.surfshark.com/v4/server/clusters/double",
responseStatus: http.StatusOK,
responseBody: io.NopCloser(strings.NewReader(`[]`)),
}, {
requestURL: "https://api.surfshark.com/v4/server/clusters/static",
responseStatus: http.StatusOK,
responseBody: io.NopCloser(strings.NewReader(`[]`)),
}, {
requestURL: "https://api.surfshark.com/v4/server/clusters/obfuscated",
responseStatus: http.StatusOK,
responseBody: io.NopCloser(strings.NewReader(`[]`)),
}},
expected: map[string][]models.Server{
"existinghost": {{Hostname: "existinghost"}},
"host1": {{
VPN: constants.OpenVPN,
Region: "region1",
Country: "country1",
City: "location1",
Hostname: "host1",
TCP: true,
UDP: true,
}, {
VPN: constants.Wireguard,
Region: "region1",
Country: "country1",
City: "location1",
Hostname: "host1",
WgPubKey: "pubKeyValue",
}},
"host2": {{
VPN: constants.OpenVPN,
Region: "region2",
Country: "country1",
City: "location2",
Hostname: "host2",
TCP: true,
UDP: true,
}},
},
},
}
for name, testCase := range testCases {
t.Run(name, func(t *testing.T) {
t.Parallel()
ctx := context.Background()
currentExchangeIndex := 0
client := &http.Client{
Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) {
assert.Equal(t, http.MethodGet, r.Method)
exchange := testCase.exchanges[currentExchangeIndex]
currentExchangeIndex++
assert.Equal(t, exchange.requestURL, r.URL.String())
return &http.Response{
StatusCode: exchange.responseStatus,
Status: http.StatusText(exchange.responseStatus),
Body: exchange.responseBody,
}, nil
}),
}
err := addServersFromAPI(ctx, client, testCase.hts)
assert.Equal(t, testCase.expected, testCase.hts)
if testCase.err != nil {
require.Error(t, err)
assert.Equal(t, testCase.err.Error(), err.Error())
} else {
assert.NoError(t, err)
}
})
}
}
func Test_fetchAPI(t *testing.T) {
t.Parallel()
testCases := map[string]struct {
exchanges []httpExchange
data []serverData
err error
}{
"http response status not ok": {
exchanges: []httpExchange{{
requestURL: "https://api.surfshark.com/v4/server/clusters/generic",
responseStatus: http.StatusNoContent,
}},
err: errors.New("HTTP status code not OK: 204 No Content"),
},
"nil body": {
exchanges: []httpExchange{{
requestURL: "https://api.surfshark.com/v4/server/clusters/generic",
responseStatus: http.StatusOK,
}},
err: errors.New("decoding response body: EOF"),
},
"no server": {
exchanges: []httpExchange{{
requestURL: "https://api.surfshark.com/v4/server/clusters/generic",
responseStatus: http.StatusOK,
responseBody: io.NopCloser(strings.NewReader(`[]`)),
}, {
requestURL: "https://api.surfshark.com/v4/server/clusters/double",
responseStatus: http.StatusOK,
responseBody: io.NopCloser(strings.NewReader(`[]`)),
}, {
requestURL: "https://api.surfshark.com/v4/server/clusters/static",
responseStatus: http.StatusOK,
responseBody: io.NopCloser(strings.NewReader(`[]`)),
}, {
requestURL: "https://api.surfshark.com/v4/server/clusters/obfuscated",
responseStatus: http.StatusOK,
responseBody: io.NopCloser(strings.NewReader(`[]`)),
}},
},
"success": {
exchanges: []httpExchange{{
requestURL: "https://api.surfshark.com/v4/server/clusters/generic",
responseStatus: http.StatusOK,
responseBody: io.NopCloser(strings.NewReader(`[
{"connectionName":"host1","region":"region1","country":"country1","location":"location1"},
{"connectionName":"host2","region":"region2","country":"country1","location":"location2"}
]`)),
}, {
requestURL: "https://api.surfshark.com/v4/server/clusters/double",
responseStatus: http.StatusOK,
responseBody: io.NopCloser(strings.NewReader(`[]`)),
}, {
requestURL: "https://api.surfshark.com/v4/server/clusters/static",
responseStatus: http.StatusOK,
responseBody: io.NopCloser(strings.NewReader(`[]`)),
}, {
requestURL: "https://api.surfshark.com/v4/server/clusters/obfuscated",
responseStatus: http.StatusOK,
responseBody: io.NopCloser(strings.NewReader(`[]`)),
}},
data: []serverData{
{
Region: "region1",
Country: "country1",
Location: "location1",
Host: "host1",
},
{
Region: "region2",
Country: "country1",
Location: "location2",
Host: "host2",
},
},
},
}
for name, testCase := range testCases {
t.Run(name, func(t *testing.T) {
t.Parallel()
ctx := context.Background()
currentExchangeIndex := 0
client := &http.Client{
Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) {
assert.Equal(t, http.MethodGet, r.Method)
exchange := testCase.exchanges[currentExchangeIndex]
currentExchangeIndex++
assert.Equal(t, exchange.requestURL, r.URL.String())
return &http.Response{
StatusCode: exchange.responseStatus,
Status: http.StatusText(exchange.responseStatus),
Body: exchange.responseBody,
}, nil
}),
}
data, err := fetchAPI(ctx, client)
assert.Equal(t, testCase.data, data)
if testCase.err != nil {
require.Error(t, err)
assert.Equal(t, testCase.err.Error(), err.Error())
} else {
assert.NoError(t, err)
}
})
}
}
@@ -0,0 +1,102 @@
package surfshark
import (
"net/netip"
"github.com/qdm12/gluetun-servers/pkg/constants"
"github.com/qdm12/gluetun-servers/pkg/models"
)
type hostToServers map[string][]models.Server
func (hts hostToServers) addOpenVPN(host, region, country, city,
retroLoc string, tcp, udp bool,
) {
// Check for existing server for this host and OpenVPN.
servers := hts[host]
for i, existingServer := range servers {
if existingServer.Hostname != host ||
existingServer.VPN != constants.OpenVPN {
continue
}
// Update OpenVPN supported protocols and return
if !existingServer.TCP {
servers[i].TCP = tcp
}
if !existingServer.UDP {
servers[i].UDP = udp
}
return
}
server := models.Server{
VPN: constants.OpenVPN,
Region: region,
Country: country,
City: city,
RetroLoc: retroLoc,
Hostname: host,
TCP: tcp,
UDP: udp,
}
hts[host] = append(servers, server)
}
func (hts hostToServers) addWireguard(host, region, country, city, retroLoc,
wgPubKey string,
) {
// Check for existing server for this host and Wireguard.
servers := hts[host]
for _, existingServer := range servers {
if existingServer.Hostname == host &&
existingServer.VPN == constants.Wireguard {
// No update necessary for Wireguard
return
}
}
server := models.Server{
VPN: constants.Wireguard,
Region: region,
Country: country,
City: city,
RetroLoc: retroLoc,
Hostname: host,
WgPubKey: wgPubKey,
}
hts[host] = append(servers, server)
}
func (hts hostToServers) toHostsSlice() (hosts []string) {
const vpnServerTypes = 2 // OpenVPN + Wireguard
hosts = make([]string, 0, vpnServerTypes*len(hts))
for host := range hts {
hosts = append(hosts, host)
}
return hosts
}
func (hts hostToServers) adaptWithIPs(hostToIPs map[string][]netip.Addr) {
for host, IPs := range hostToIPs {
servers := hts[host]
for i := range servers {
servers[i].IPs = IPs
}
hts[host] = servers
}
for host, servers := range hts {
if len(servers[0].IPs) == 0 {
delete(hts, host)
}
}
}
func (hts hostToServers) toServersSlice() (servers []models.Server) {
const vpnServerTypes = 2 // OpenVPN + Wireguard
servers = make([]models.Server, 0, vpnServerTypes*len(hts))
for _, serversForHost := range hts {
servers = append(servers, serversForHost...)
}
return servers
}
@@ -0,0 +1,28 @@
package surfshark
import (
"fmt"
"github.com/qdm12/gluetun-servers/pkg/updaters/providers/surfshark/servers"
)
func getHostInformation(host string, hostnameToLocation map[string]servers.ServerLocation) (
data servers.ServerLocation, err error,
) {
locationData, ok := hostnameToLocation[host]
if !ok {
return locationData, fmt.Errorf("hostname %s not found in hostname to location mapping", host)
}
return locationData, nil
}
func hostToLocation(locationData []servers.ServerLocation) (
hostToLocation map[string]servers.ServerLocation,
) {
hostToLocation = make(map[string]servers.ServerLocation, len(locationData))
for _, data := range locationData {
hostToLocation[data.Hostname] = data
}
return hostToLocation
}
@@ -0,0 +1,21 @@
package surfshark
import (
"github.com/qdm12/gluetun-servers/pkg/updaters/providers/surfshark/servers"
)
// getRemainingServers finds extra servers not found in the API or in the ZIP file.
func getRemainingServers(hts hostToServers) {
locationData := servers.LocationData()
hostnameToLocationLeft := hostToLocation(locationData)
for _, hostnameDone := range hts.toHostsSlice() {
delete(hostnameToLocationLeft, hostnameDone)
}
for hostname, locationData := range hostnameToLocationLeft {
// we assume the OpenVPN server supports both TCP and UDP
const tcp, udp = true, true
hts.addOpenVPN(hostname, locationData.Region, locationData.Country,
locationData.City, locationData.RetroLoc, tcp, udp)
}
}
@@ -0,0 +1,28 @@
package surfshark
import (
"time"
"github.com/qdm12/gluetun-servers/pkg/updaters/resolver"
)
func parallelResolverSettings(hosts []string) (settings resolver.ParallelSettings) {
const (
maxFailRatio = 0.1
maxDuration = 20 * time.Second
betweenDuration = time.Second
maxNoNew = 2
maxFails = 2
)
return resolver.ParallelSettings{
Hosts: hosts,
MaxFailRatio: maxFailRatio,
Repeat: resolver.RepeatSettings{
MaxDuration: maxDuration,
BetweenDuration: betweenDuration,
MaxNoNew: maxNoNew,
MaxFails: maxFails,
SortIPs: true,
},
}
}
@@ -0,0 +1,9 @@
package surfshark
import "net/http"
type roundTripFunc func(r *http.Request) (*http.Response, error)
func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) {
return f(r)
}
@@ -0,0 +1,53 @@
package surfshark
import (
"context"
"fmt"
"sort"
"github.com/qdm12/gluetun-servers/pkg/models"
"github.com/qdm12/gluetun-servers/pkg/updaters/common"
)
func (u *Updater) FetchServers(ctx context.Context, minServers int) (
servers []models.Server, err error,
) {
hts := make(hostToServers)
err = addServersFromAPI(ctx, u.client, hts)
if err != nil {
return nil, fmt.Errorf("fetching server information from API: %w", err)
}
warnings, err := addOpenVPNServersFromZip(ctx, u.unzipper, hts)
for _, warning := range warnings {
u.warner.Warn(warning)
}
if err != nil {
return nil, fmt.Errorf("getting OpenVPN ZIP file: %w", err)
}
getRemainingServers(hts)
hosts := hts.toHostsSlice()
resolveSettings := parallelResolverSettings(hosts)
hostToIPs, warnings, err := u.parallelResolver.Resolve(ctx, resolveSettings)
for _, warning := range warnings {
u.warner.Warn(warning)
}
if err != nil {
return nil, err
}
hts.adaptWithIPs(hostToIPs)
if len(hts) < minServers {
return nil, fmt.Errorf("%w: %d and expected at least %d",
common.ErrNotEnoughServers, len(hts), minServers)
}
servers = hts.toServersSlice()
sort.Sort(models.SortableServers(servers))
return servers, nil
}
@@ -0,0 +1,158 @@
package servers
// LocationData is required to keep location data on Surfshark
// servers that are not obtained through their API.
type ServerLocation struct {
Region string
Country string
City string
RetroLoc string // TODO remove in v4
Hostname string
MultiHop bool
}
// TODO remove retroRegion and servers from API in v4.
func LocationData() (data []ServerLocation) {
//nolint:lll
return []ServerLocation{
{Region: "Asia Pacific", Country: "Australia", City: "Adelaide", RetroLoc: "Australia Adelaide", Hostname: "au-adl.prod.surfshark.com", MultiHop: false},
{Region: "Asia Pacific", Country: "Australia", City: "Brisbane", RetroLoc: "Australia Brisbane", Hostname: "au-bne.prod.surfshark.com", MultiHop: false},
{Region: "Asia Pacific", Country: "Australia", City: "Melbourne", RetroLoc: "Australia Melbourne", Hostname: "au-mel.prod.surfshark.com", MultiHop: false},
{Region: "Asia Pacific", Country: "Australia", City: "Perth", RetroLoc: "Australia Perth", Hostname: "au-per.prod.surfshark.com", MultiHop: false},
{Region: "Asia Pacific", Country: "Australia", City: "Sydney", RetroLoc: "Australia Sydney", Hostname: "au-syd.prod.surfshark.com", MultiHop: false},
{Region: "Asia Pacific", Country: "Azerbaijan", City: "Baku", RetroLoc: "Azerbaijan", Hostname: "az-bak.prod.surfshark.com", MultiHop: false},
{Region: "Asia Pacific", Country: "Hong Kong", City: "Hong Kong", RetroLoc: "Hong Kong", Hostname: "hk-hkg.prod.surfshark.com", MultiHop: false},
{Region: "Asia Pacific", Country: "Hong Kong", City: "Hong Kong", RetroLoc: "Hong Kong", Hostname: "lk-cmb.prod.surfshark.com", MultiHop: false},
{Region: "Asia Pacific", Country: "Hong Kong", City: "Hong Kong", RetroLoc: "Hong Kong", Hostname: "mn-uln.prod.surfshark.com", MultiHop: false},
{Region: "Asia Pacific", Country: "Indonesia", City: "Jakarta", RetroLoc: "Indonesia", Hostname: "id-jak.prod.surfshark.com", MultiHop: false},
{Region: "Asia Pacific", Country: "Japan", City: "Tokyo", Hostname: "jp-tok-st014.prod.surfshark.com"},
{Region: "Asia Pacific", Country: "Japan", City: "Tokyo", Hostname: "jp-tok-st015.prod.surfshark.com"},
{Region: "Asia Pacific", Country: "Japan", City: "Tokyo", Hostname: "jp-tok-st016.prod.surfshark.com"},
{Region: "Asia Pacific", Country: "Japan", City: "Tokyo", Hostname: "jp-tok-st017.prod.surfshark.com"},
{Region: "Asia Pacific", Country: "Japan", City: "Tokyo", Hostname: "jp-tok-st018.prod.surfshark.com"},
{Region: "Asia Pacific", Country: "Japan", City: "Tokyo", Hostname: "jp-tok-st019.prod.surfshark.com"},
{Region: "Asia Pacific", Country: "Japan", City: "Tokyo", Hostname: "jp-tok-st020.prod.surfshark.com"},
{Region: "Asia Pacific", Country: "Japan", City: "Tokyo", Hostname: "jp-tok-st021.prod.surfshark.com"},
{Region: "Asia Pacific", Country: "Japan", City: "Tokyo", Hostname: "jp-tok-st022.prod.surfshark.com"},
{Region: "Asia Pacific", Country: "Japan", City: "Tokyo", Hostname: "jp-tok-st023.prod.surfshark.com"},
{Region: "Asia Pacific", Country: "Japan", City: "Tokyo", Hostname: "jp-tok-st024.prod.surfshark.com"},
{Region: "Asia Pacific", Country: "Japan", City: "Tokyo", Hostname: "jp-tok-st025.prod.surfshark.com"},
{Region: "Asia Pacific", Country: "Japan", City: "Tokyo", RetroLoc: "Japan Tokyo", Hostname: "jp-tok.prod.surfshark.com", MultiHop: false},
{Region: "Asia Pacific", Country: "Malaysia", City: "Kuala Lumpur", RetroLoc: "Malaysia", Hostname: "my-kul.prod.surfshark.com", MultiHop: false},
{Region: "Asia Pacific", Country: "New Zealand", City: "Auckland", RetroLoc: "New Zealand", Hostname: "nz-akl.prod.surfshark.com", MultiHop: false},
{Region: "Asia Pacific", Country: "Philippines", City: "Manila", RetroLoc: "Philippines", Hostname: "ph-mnl.prod.surfshark.com", MultiHop: false},
{Region: "Asia Pacific", Country: "Singapore", City: "Singapore", RetroLoc: "Singapore mp001", Hostname: "sg-sng-mp001.prod.surfshark.com", MultiHop: false},
{Region: "Asia Pacific", Country: "Singapore", City: "Singapore", Hostname: "sg-sng-st005.prod.surfshark.com"},
{Region: "Asia Pacific", Country: "Singapore", City: "Singapore", Hostname: "sg-sng-st006.prod.surfshark.com"},
{Region: "Asia Pacific", Country: "Singapore", City: "Singapore", Hostname: "sg-sng-st007.prod.surfshark.com"},
{Region: "Asia Pacific", Country: "Singapore", City: "Singapore", Hostname: "sg-sng-st008.prod.surfshark.com"},
{Region: "Asia Pacific", Country: "Singapore", City: "Singapore", Hostname: "sg-sng-st009.prod.surfshark.com"},
{Region: "Asia Pacific", Country: "Singapore", City: "Singapore", Hostname: "sg-sng-st010.prod.surfshark.com"},
{Region: "Asia Pacific", Country: "Singapore", City: "Singapore", RetroLoc: "Singapore", Hostname: "sg-sng.prod.surfshark.com", MultiHop: false},
{Region: "Asia Pacific", Country: "South Korea", City: "Seoul", RetroLoc: "Korea", Hostname: "kr-seo.prod.surfshark.com", MultiHop: false},
{Region: "Asia Pacific", Country: "Taiwan", City: "Taichung City", RetroLoc: "Taiwan", Hostname: "tw-tai.prod.surfshark.com", MultiHop: false},
{Region: "Asia Pacific", Country: "Thailand", City: "Bangkok", RetroLoc: "Thailand", Hostname: "th-bkk.prod.surfshark.com", MultiHop: false},
{Region: "Europe", Country: "Albania", City: "Tirana", RetroLoc: "Albania", Hostname: "al-tia.prod.surfshark.com", MultiHop: false},
{Region: "Europe", Country: "Austria", City: "Vienna", RetroLoc: "Austria", Hostname: "at-vie.prod.surfshark.com", MultiHop: false},
{Region: "Europe", Country: "Belgium", City: "Brussels", RetroLoc: "Belgium", Hostname: "be-bru.prod.surfshark.com", MultiHop: false},
{Region: "Europe", Country: "Bosnia and Herzegovina", City: "Sarajevo", RetroLoc: "Bosnia and Herzegovina", Hostname: "ba-sjj.prod.surfshark.com", MultiHop: false},
{Region: "Europe", Country: "Bulgaria", City: "Sofia", RetroLoc: "Bulgaria", Hostname: "bg-sof.prod.surfshark.com", MultiHop: false},
{Region: "Europe", Country: "Croatia", City: "Zagreb", RetroLoc: "Croatia", Hostname: "hr-zag.prod.surfshark.com", MultiHop: false},
{Region: "Europe", Country: "Cyprus", City: "Nicosia", RetroLoc: "Cyprus", Hostname: "cy-nic.prod.surfshark.com", MultiHop: false},
{Region: "Europe", Country: "Czech Republic", City: "Prague", RetroLoc: "Czech Republic", Hostname: "cz-prg.prod.surfshark.com", MultiHop: false},
{Region: "Europe", Country: "Denmark", City: "Copenhagen", RetroLoc: "Denmark", Hostname: "dk-cph.prod.surfshark.com", MultiHop: false},
{Region: "Europe", Country: "Estonia", City: "Tallinn", RetroLoc: "Estonia", Hostname: "ee-tll.prod.surfshark.com", MultiHop: false},
{Region: "Europe", Country: "Finland", City: "Helsinki", RetroLoc: "Finland", Hostname: "fi-hel.prod.surfshark.com", MultiHop: false},
{Region: "Europe", Country: "France", City: "Bordeaux", RetroLoc: "France Bordeaux", Hostname: "fr-bod.prod.surfshark.com", MultiHop: false},
{Region: "Europe", Country: "France", City: "Marseille", RetroLoc: "France Marseilles", Hostname: "fr-mrs.prod.surfshark.com", MultiHop: false},
{Region: "Europe", Country: "France", City: "Paris", RetroLoc: "France Paris", Hostname: "fr-par.prod.surfshark.com", MultiHop: false},
{Region: "Europe", Country: "Germany", City: "Berlin", RetroLoc: "Germany Berlin", Hostname: "de-ber.prod.surfshark.com", MultiHop: false},
{Region: "Europe", Country: "Germany", City: "Frankfurt am Main", RetroLoc: "Germany Frankfurt am Main st001", Hostname: "de-fra-st001.prod.surfshark.com", MultiHop: false},
{Region: "Europe", Country: "Germany", City: "Frankfurt am Main", RetroLoc: "Germany Frankfurt am Main st002", Hostname: "de-fra-st002.prod.surfshark.com", MultiHop: false},
{Region: "Europe", Country: "Germany", City: "Frankfurt am Main", RetroLoc: "Germany Frankfurt am Main st003", Hostname: "de-fra-st003.prod.surfshark.com", MultiHop: false},
{Region: "Europe", Country: "Germany", City: "Frankfurt am Main", RetroLoc: "Germany Frankfurt am Main st004", Hostname: "de-fra-st004.prod.surfshark.com", MultiHop: false},
{Region: "Europe", Country: "Germany", City: "Frankfurt am Main", RetroLoc: "Germany Frankfurt am Main st005", Hostname: "de-fra-st005.prod.surfshark.com", MultiHop: false},
{Region: "Europe", Country: "Germany", City: "Frankfurt am Main", RetroLoc: "Germany Frankfurt am Main st006", Hostname: "de-fra-st006.prod.surfshark.com", MultiHop: false},
{Region: "Europe", Country: "Germany", City: "Frankfurt am Main", RetroLoc: "Germany Frankfurt am Main st007", Hostname: "de-fra-st007.prod.surfshark.com", MultiHop: false},
{Region: "Europe", Country: "Germany", City: "Frankfurt am Main", RetroLoc: "Germany Frankfurt am Main", Hostname: "de-fra.prod.surfshark.com", MultiHop: false},
{Region: "Europe", Country: "Germany", City: "Frankfurt am Main", RetroLoc: "Germany Frankfurt mp001", Hostname: "de-fra-mp001.prod.surfshark.com", MultiHop: false},
{Region: "Europe", Country: "Greece", City: "Athens", RetroLoc: "Greece", Hostname: "gr-ath.prod.surfshark.com", MultiHop: false},
{Region: "Europe", Country: "Hungary", City: "Budapest", RetroLoc: "Hungary", Hostname: "hu-bud.prod.surfshark.com", MultiHop: false},
{Region: "Europe", Country: "Iceland", City: "Reykjavik", RetroLoc: "Iceland", Hostname: "is-rkv.prod.surfshark.com", MultiHop: false},
{Region: "Europe", Country: "Ireland", City: "Dublin", RetroLoc: "Ireland", Hostname: "ie-dub.prod.surfshark.com", MultiHop: false},
{Region: "Europe", Country: "Italy", City: "Milan", RetroLoc: "Italy Milan", Hostname: "it-mil.prod.surfshark.com", MultiHop: false},
{Region: "Europe", Country: "Italy", City: "Rome", RetroLoc: "Italy Rome", Hostname: "it-rom.prod.surfshark.com", MultiHop: false},
{Region: "Europe", Country: "Latvia", City: "Riga", RetroLoc: "Latvia", Hostname: "lv-rig.prod.surfshark.com", MultiHop: false},
{Region: "Europe", Country: "Luxembourg", City: "Luxembourg", RetroLoc: "Luxembourg", Hostname: "lu-ste.prod.surfshark.com", MultiHop: false},
{Region: "Europe", Country: "Moldova", City: "Chisinau", RetroLoc: "Moldova", Hostname: "md-chi.prod.surfshark.com", MultiHop: false},
{Region: "Europe", Country: "Netherlands", City: "Amsterdam", RetroLoc: "Netherlands Amsterdam mp001", Hostname: "nl-ams-mp001.prod.surfshark.com", MultiHop: false},
{Region: "Europe", Country: "Netherlands", City: "Amsterdam", RetroLoc: "Netherlands Amsterdam st001", Hostname: "nl-ams-st001.prod.surfshark.com", MultiHop: false},
{Region: "Europe", Country: "Netherlands", City: "Amsterdam", RetroLoc: "Netherlands Amsterdam", Hostname: "nl-ams.prod.surfshark.com", MultiHop: false},
{Region: "Europe", Country: "North Macedonia", City: "Skopje", RetroLoc: "North Macedonia", Hostname: "mk-skp.prod.surfshark.com", MultiHop: false},
{Region: "Europe", Country: "Norway", City: "Oslo", RetroLoc: "Norway", Hostname: "no-osl.prod.surfshark.com", MultiHop: false},
{Region: "Europe", Country: "Poland", City: "Gdansk", RetroLoc: "Poland Gdansk", Hostname: "pl-gdn.prod.surfshark.com", MultiHop: false},
{Region: "Europe", Country: "Poland", City: "Warsaw", RetroLoc: "Poland Warsaw", Hostname: "pl-waw.prod.surfshark.com", MultiHop: false},
{Region: "Europe", Country: "Portugal", City: "Lisbon", RetroLoc: "Portugal Lisbon", Hostname: "pt-lis.prod.surfshark.com", MultiHop: false},
{Region: "Europe", Country: "Portugal", City: "Porto", RetroLoc: "Portugal Porto", Hostname: "pt-opo.prod.surfshark.com", MultiHop: false},
{Region: "Europe", Country: "Romania", City: "Bucharest", RetroLoc: "Romania", Hostname: "ro-buc.prod.surfshark.com", MultiHop: false},
{Region: "Europe", Country: "Serbia", City: "Belgrade", RetroLoc: "Serbia", Hostname: "rs-beg.prod.surfshark.com", MultiHop: false},
{Region: "Europe", Country: "Slovakia", City: "Bratislava", RetroLoc: "Slovekia", Hostname: "sk-bts.prod.surfshark.com", MultiHop: false},
{Region: "Europe", Country: "Slovenia", City: "Ljubljana", RetroLoc: "Slovenia", Hostname: "si-lju.prod.surfshark.com", MultiHop: false},
{Region: "Europe", Country: "Spain", City: "Barcelona", RetroLoc: "Spain Barcelona", Hostname: "es-bcn.prod.surfshark.com", MultiHop: false},
{Region: "Europe", Country: "Spain", City: "Madrid", RetroLoc: "Spain Madrid", Hostname: "es-mad.prod.surfshark.com", MultiHop: false},
{Region: "Europe", Country: "Spain", City: "Valencia", RetroLoc: "Spain Valencia", Hostname: "es-vlc.prod.surfshark.com", MultiHop: false},
{Region: "Europe", Country: "Sweden", City: "Stockholm", RetroLoc: "Sweden", Hostname: "se-sto.prod.surfshark.com", MultiHop: false},
{Region: "Europe", Country: "Switzerland", City: "Zurich", RetroLoc: "Switzerland", Hostname: "ch-zur.prod.surfshark.com", MultiHop: false},
{Region: "Europe", Country: "Turkey", City: "Istanbul", RetroLoc: "Turkey Istanbul", Hostname: "tr-ist.prod.surfshark.com", MultiHop: false},
{Region: "Europe", Country: "Ukraine", City: "Kyiv", RetroLoc: "Ukraine", Hostname: "ua-iev.prod.surfshark.com", MultiHop: false},
{Region: "Europe", Country: "United Kingdom", City: "Glasgow", RetroLoc: "UK Glasgow", Hostname: "uk-gla.prod.surfshark.com", MultiHop: false},
{Region: "Europe", Country: "United Kingdom", City: "London", RetroLoc: "UK London mp001", Hostname: "uk-lon-mp001.prod.surfshark.com", MultiHop: false},
{Region: "Europe", Country: "United Kingdom", City: "London", RetroLoc: "UK London st001", Hostname: "uk-lon-st001.prod.surfshark.com", MultiHop: false},
{Region: "Europe", Country: "United Kingdom", City: "London", RetroLoc: "UK London st002", Hostname: "uk-lon-st002.prod.surfshark.com", MultiHop: false},
{Region: "Europe", Country: "United Kingdom", City: "London", RetroLoc: "UK London st003", Hostname: "uk-lon-st003.prod.surfshark.com", MultiHop: false},
{Region: "Europe", Country: "United Kingdom", City: "London", RetroLoc: "UK London st004", Hostname: "uk-lon-st004.prod.surfshark.com", MultiHop: false},
{Region: "Europe", Country: "United Kingdom", City: "London", RetroLoc: "UK London st005", Hostname: "uk-lon-st005.prod.surfshark.com", MultiHop: false},
{Region: "Europe", Country: "United Kingdom", City: "London", RetroLoc: "UK London", Hostname: "uk-lon.prod.surfshark.com", MultiHop: false},
{Region: "Europe", Country: "United Kingdom", City: "Manchester", RetroLoc: "UK Manchester", Hostname: "uk-man.prod.surfshark.com", MultiHop: false},
{Region: "Middle East and Africa", Country: "Israel", City: "Tel Aviv", RetroLoc: "Israel", Hostname: "il-tlv.prod.surfshark.com", MultiHop: false},
{Region: "Middle East and Africa", Country: "Nigeria", City: "Lagos", RetroLoc: "Nigeria", Hostname: "ng-lag.prod.surfshark.com", MultiHop: false},
{Region: "Middle East and Africa", Country: "South Africa", City: "Johannesburg", RetroLoc: "South Africa", Hostname: "za-jnb.prod.surfshark.com", MultiHop: false},
{Region: "Middle East and Africa", Country: "United Arab Emirates", City: "Dubai", RetroLoc: "United Arab Emirates", Hostname: "ae-dub.prod.surfshark.com", MultiHop: false},
{Region: "The Americas", Country: "Argentina", City: "Buenos Aires", RetroLoc: "Argentina Buenos Aires", Hostname: "ar-bua.prod.surfshark.com", MultiHop: false},
{Region: "The Americas", Country: "Brazil", City: "Sao Paulo", RetroLoc: "Brazil", Hostname: "br-sao.prod.surfshark.com", MultiHop: false},
{Region: "The Americas", Country: "Canada", City: "Montreal", RetroLoc: "Canada Montreal", Hostname: "ca-mon.prod.surfshark.com", MultiHop: false},
{Region: "The Americas", Country: "Canada", City: "Toronto", RetroLoc: "Canada Toronto mp001", Hostname: "ca-tor-mp001.prod.surfshark.com", MultiHop: false},
{Region: "The Americas", Country: "Canada", City: "Toronto", RetroLoc: "Canada Toronto", Hostname: "ca-tor.prod.surfshark.com", MultiHop: false},
{Region: "The Americas", Country: "Canada", City: "Vancouver", RetroLoc: "Canada Vancouver", Hostname: "ca-van.prod.surfshark.com", MultiHop: false},
{Region: "The Americas", Country: "Chile", City: "Santiago", RetroLoc: "Chile", Hostname: "cl-san.prod.surfshark.com", MultiHop: false},
{Region: "The Americas", Country: "Colombia", City: "Bogota", RetroLoc: "Colombia", Hostname: "co-bog.prod.surfshark.com", MultiHop: false},
{Region: "The Americas", Country: "Costa Rica", City: "San Jose", RetroLoc: "Costa Rica", Hostname: "cr-sjn.prod.surfshark.com", MultiHop: false},
{Region: "The Americas", Country: "United States", City: "Atlanta", RetroLoc: "US Atlanta", Hostname: "us-atl.prod.surfshark.com", MultiHop: false},
{Region: "The Americas", Country: "United States", City: "Bend", RetroLoc: "US Bend", Hostname: "us-bdn.prod.surfshark.com", MultiHop: false},
{Region: "The Americas", Country: "United States", City: "Boston", RetroLoc: "US Boston", Hostname: "us-bos.prod.surfshark.com", MultiHop: false},
{Region: "The Americas", Country: "United States", City: "Buffalo", RetroLoc: "US Buffalo", Hostname: "us-buf.prod.surfshark.com", MultiHop: false},
{Region: "The Americas", Country: "United States", City: "Charlotte", RetroLoc: "US Charlotte", Hostname: "us-clt.prod.surfshark.com", MultiHop: false},
{Region: "The Americas", Country: "United States", City: "Chicago", RetroLoc: "US Chicago", Hostname: "us-chi.prod.surfshark.com", MultiHop: false},
{Region: "The Americas", Country: "United States", City: "Dallas", RetroLoc: "US Dallas", Hostname: "us-dal.prod.surfshark.com", MultiHop: false},
{Region: "The Americas", Country: "United States", City: "Denver", RetroLoc: "US Denver", Hostname: "us-den.prod.surfshark.com", MultiHop: false},
{Region: "The Americas", Country: "United States", City: "Detroit", RetroLoc: "US Gahanna", Hostname: "us-dtw.prod.surfshark.com", MultiHop: false},
{Region: "The Americas", Country: "United States", City: "Houston", RetroLoc: "US Houston", Hostname: "us-hou.prod.surfshark.com", MultiHop: false},
{Region: "The Americas", Country: "United States", City: "Kansas City", RetroLoc: "US Kansas City", Hostname: "us-kan.prod.surfshark.com", MultiHop: false},
{Region: "The Americas", Country: "United States", City: "Las Vegas", RetroLoc: "US Las Vegas", Hostname: "us-las.prod.surfshark.com", MultiHop: false},
{Region: "The Americas", Country: "United States", City: "Latham", RetroLoc: "US Latham", Hostname: "us-ltm.prod.surfshark.com", MultiHop: false},
{Region: "The Americas", Country: "United States", City: "Los Angeles", RetroLoc: "US Los Angeles", Hostname: "us-lax.prod.surfshark.com", MultiHop: false},
{Region: "The Americas", Country: "United States", City: "Miami", RetroLoc: "US Miami", Hostname: "us-mia.prod.surfshark.com", MultiHop: false},
{Region: "The Americas", Country: "United States", City: "New York", RetroLoc: "US New York City mp001", Hostname: "us-nyc-mp001.prod.surfshark.com", MultiHop: false},
{Region: "The Americas", Country: "United States", City: "New York", RetroLoc: "US New York City st001", Hostname: "us-nyc-st001.prod.surfshark.com", MultiHop: false},
{Region: "The Americas", Country: "United States", City: "New York", RetroLoc: "US New York City st002", Hostname: "us-nyc-st002.prod.surfshark.com", MultiHop: false},
{Region: "The Americas", Country: "United States", City: "New York", RetroLoc: "US New York City st003", Hostname: "us-nyc-st003.prod.surfshark.com", MultiHop: false},
{Region: "The Americas", Country: "United States", City: "New York", RetroLoc: "US New York City st004", Hostname: "us-nyc-st004.prod.surfshark.com", MultiHop: false},
{Region: "The Americas", Country: "United States", City: "New York", RetroLoc: "US New York City st005", Hostname: "us-nyc-st005.prod.surfshark.com", MultiHop: false},
{Region: "The Americas", Country: "United States", City: "New York", RetroLoc: "US New York City", Hostname: "us-nyc.prod.surfshark.com", MultiHop: false},
{Region: "The Americas", Country: "United States", City: "Phoenix", RetroLoc: "US Phoenix", Hostname: "us-phx.prod.surfshark.com", MultiHop: false},
{Region: "The Americas", Country: "United States", City: "Salt Lake City", RetroLoc: "US Salt Lake City", Hostname: "us-slc.prod.surfshark.com", MultiHop: false},
{Region: "The Americas", Country: "United States", City: "San Francisco", RetroLoc: "US San Francisco mp001", Hostname: "us-sfo-mp001.prod.surfshark.com", MultiHop: false},
{Region: "The Americas", Country: "United States", City: "San Francisco", RetroLoc: "US San Francisco", Hostname: "us-sfo.prod.surfshark.com", MultiHop: false},
{Region: "The Americas", Country: "United States", City: "Seattle", RetroLoc: "US Seatle", Hostname: "us-sea.prod.surfshark.com", MultiHop: false},
}
}
@@ -0,0 +1,33 @@
package surfshark
import (
"net/http"
"github.com/qdm12/gluetun-servers/pkg/updaters/common"
)
type Updater struct {
client *http.Client
unzipper common.Unzipper
parallelResolver common.ParallelResolver
warner common.Warner
}
func New(client *http.Client, unzipper common.Unzipper,
warner common.Warner, parallelResolver common.ParallelResolver,
) *Updater {
return &Updater{
client: client,
unzipper: unzipper,
parallelResolver: parallelResolver,
warner: warner,
}
}
func (u *Updater) Version() uint16 {
return 4 //nolint:mnd
}
func (u *Updater) Name() string {
return "surfshark"
}
@@ -0,0 +1,75 @@
package surfshark
import (
"context"
"strings"
"github.com/qdm12/gluetun-servers/pkg/updaters/common"
"github.com/qdm12/gluetun-servers/pkg/updaters/openvpn"
"github.com/qdm12/gluetun-servers/pkg/updaters/providers/surfshark/servers"
)
func addOpenVPNServersFromZip(ctx context.Context,
unzipper common.Unzipper, hts hostToServers) (
warnings []string, err error,
) {
const url = "https://my.surfshark.com/vpn/api/v1/server/configurations"
contents, err := unzipper.FetchAndExtract(ctx, url)
if err != nil {
return nil, err
}
hostnamesDone := hts.toHostsSlice()
hostnamesDoneSet := make(map[string]struct{}, len(hostnamesDone))
for _, hostname := range hostnamesDone {
hostnamesDoneSet[hostname] = struct{}{}
}
locationData := servers.LocationData()
hostToLocation := hostToLocation(locationData)
for fileName, content := range contents {
if !strings.HasSuffix(fileName, ".ovpn") {
continue // not an OpenVPN file
}
host, warning, err := openvpn.ExtractHost(content)
if warning != "" {
warnings = append(warnings, warning)
}
if err != nil {
// treat error as warning and go to next file
warning := err.Error() + " in " + fileName
// TODO gather location data for IP address Openvpn files
// and process those when this error triggers.
warnings = append(warnings, warning)
continue
}
_, ok := hostnamesDoneSet[host]
if ok {
continue // already done in API
}
tcp, udp, err := openvpn.ExtractProto(content)
if err != nil {
// treat error as warning and go to next file
warning := err.Error() + " in " + fileName
warnings = append(warnings, warning)
continue
}
data, err := getHostInformation(host, hostToLocation)
if err != nil {
// treat error as warning and go to next file
warning := err.Error()
warnings = append(warnings, warning)
continue
}
hts.addOpenVPN(host, data.Region, data.Country, data.City,
data.RetroLoc, tcp, udp)
}
return warnings, nil
}