expressvpn wip

This commit is contained in:
Quentin McGaw
2026-08-06 01:04:03 +00:00
parent 7ca92b5099
commit c5b9ee89c6
11 changed files with 13564 additions and 197 deletions
@@ -2,6 +2,7 @@ package expressvpn
import ( import (
"errors" "errors"
"net/http"
"net/netip" "net/netip"
"testing" "testing"
@@ -83,7 +84,7 @@ func Test_Provider_GetConnection(t *testing.T) {
unzipper := (common.Unzipper)(nil) unzipper := (common.Unzipper)(nil)
warner := (common.Warner)(nil) warner := (common.Warner)(nil)
parallelResolver := (common.ParallelResolver)(nil) parallelResolver := (common.ParallelResolver)(nil)
provider := New(storage, unzipper, warner, parallelResolver) provider := New(storage, &http.Client{}, unzipper, warner, parallelResolver)
if testCase.panicMessage != "" { if testCase.panicMessage != "" {
assert.PanicsWithValue(t, testCase.panicMessage, func() { assert.PanicsWithValue(t, testCase.panicMessage, func() {
+4 -2
View File
@@ -1,6 +1,8 @@
package expressvpn package expressvpn
import ( import (
"net/http"
"github.com/qdm12/gluetun/internal/constants/providers" "github.com/qdm12/gluetun/internal/constants/providers"
"github.com/qdm12/gluetun/internal/provider/common" "github.com/qdm12/gluetun/internal/provider/common"
"github.com/qdm12/gluetun/internal/provider/expressvpn/updater" "github.com/qdm12/gluetun/internal/provider/expressvpn/updater"
@@ -13,13 +15,13 @@ type Provider struct {
common.Fetcher common.Fetcher
} }
func New(storage common.Storage, unzipper common.Unzipper, updaterWarner common.Warner, func New(storage common.Storage, client *http.Client, unzipper common.Unzipper, updaterWarner common.Warner,
parallelResolver common.ParallelResolver, parallelResolver common.ParallelResolver,
) *Provider { ) *Provider {
return &Provider{ return &Provider{
storage: storage, storage: storage,
connPicker: utils.NewConnectionPicker(), connPicker: utils.NewConnectionPicker(),
Fetcher: updater.New(unzipper, updaterWarner, parallelResolver), Fetcher: updater.New(client, unzipper, updaterWarner, parallelResolver),
} }
} }
@@ -0,0 +1,146 @@
package updater
import (
"fmt"
"strings"
"github.com/qdm12/gluetun/internal/models"
)
// generateCandidateHostnames generates multiple hostname candidates for a given location
// by trying different slug variations, number suffixes, and country aliases.
func generateCandidateHostnames(server models.Server) (hostnames []string, err error) {
country := strings.ToLower(server.Country)
city := strings.ToLower(server.City)
if strings.Contains(country, " (via ") {
destinationEndIndex := strings.Index(country, " (via ")
destination := strings.TrimSpace(country[:destinationEndIndex])
destination = strings.ReplaceAll(destination, " ", "")
sourceEndIndex := strings.Index(country[destinationEndIndex:], ")")
if sourceEndIndex == -1 {
return nil, fmt.Errorf("invalid location format, missing closing parenthesis for source: %q", country)
}
source := strings.TrimSpace(country[destinationEndIndex+len(" (via ") : destinationEndIndex+sourceEndIndex])
source = strings.ReplaceAll(source, " ", "")
sourceAliases := countryNameToAliases(source)
sources := make([]string, 0, 1+len(sourceAliases))
sources = append(sources, source)
sources = append(sources, sourceAliases...)
destinationAliases := countryNameToAliases(destination)
destinations := make([]string, 0, 1+len(destinationAliases))
destinations = append(destinations, destination)
destinations = append(destinations, destinationAliases...)
slugs := make([]string, 0, (1+len(sourceAliases))*(1+len(destinationAliases)))
for _, destination := range destinations {
for _, source := range sources {
slugs = append(slugs, destination+"-"+source)
}
}
if city != "" {
return nil, fmt.Errorf("city %q should be empty for multi-hop country: %s", city, country)
}
return makeNumberedCandidates(slugs, server.Number), nil
}
countrySlug := strings.ReplaceAll(strings.TrimSpace(country), " ", "")
countrySlugAliases := countryNameToAliases(countrySlug)
countrySlugs := make([]string, 0, 1+len(countrySlugAliases))
countrySlugs = append(countrySlugs, countrySlug)
countrySlugs = append(countrySlugs, countrySlugAliases...)
var citySlugs []string
if city != "" {
if strings.Contains(city, " ") {
citySlugs = append(citySlugs, strings.ReplaceAll(city, " ", ""))
citySlugs = append(citySlugs, strings.ReplaceAll(city, " ", "-"))
} else {
citySlugs = append(citySlugs, city)
}
}
var slugs []string
if len(citySlugs) == 0 {
slugs = countrySlugs
} else {
slugs = make([]string, 0, len(countrySlugs)*len(citySlugs))
for _, countrySlug := range countrySlugs {
for _, citySlug := range citySlugs {
slugs = append(slugs, countrySlug+"-"+citySlug)
}
}
}
return makeNumberedCandidates(slugs, server.Number), nil
}
func makeNumberedCandidates(baseSlugs []string, serverNumber uint16) (candidates []string) {
numbersToTry := []uint16{1, 2, 3, 4, 5}
if serverNumber > 0 {
numbersToTry = []uint16{serverNumber}
}
const candidateVariationsPerNumber = 2 // slugN and slug-N
candidateVariationsPerBase := 1 + len(numbersToTry)*candidateVariationsPerNumber // base + (slugN and slug-N for each number)
candidates = make([]string, 0, len(baseSlugs)*candidateVariationsPerBase)
for _, base := range baseSlugs {
candidates = append(candidates, base+hostnameSuffix)
for _, number := range numbersToTry {
candidates = append(candidates,
base+fmt.Sprintf("%d-ca-version-2.expressnetw.com", number),
base+fmt.Sprintf("-%d-ca-version-2.expressnetw.com", number))
}
}
return candidates
}
// hostnameSuffix is appended to all ExpressVPN hostnames.
const hostnameSuffix = "-ca-version-2.expressnetw.com"
// countryNameToAliases maps common country names to alternative names used in hostnames.
func countryNameToAliases(country string) (aliases []string) {
switch country {
case "usa":
return []string{"us"}
case "uk":
return []string{"gb"}
case "netherlands":
return []string{"nl"}
case "switzerland":
return []string{"ch"}
case "japan":
return []string{"jp"}
case "sweden":
return []string{"se"}
case "norway":
return []string{"no"}
case "denmark":
return []string{"dk"}
case "finland":
return []string{"fi"}
case "portugal":
return []string{"pt"}
case "spain":
return []string{"es"}
case "germany":
return []string{"de"}
case "italy":
return []string{"it"}
case "australia":
return []string{"au"}
case "singapore":
return []string{"sg"}
case "taiwan":
return []string{"tw"}
case "southkorea":
return []string{"kr"}
case "northmacedonia":
return []string{"macedonia"}
default:
return nil
}
}
@@ -1,170 +0,0 @@
package updater
import (
"github.com/qdm12/gluetun/internal/models"
)
func hardcodedServers() (servers []models.Server) {
return []models.Server{
{Country: "Albania", Hostname: "albania-ca-version-2.expressnetw.com"},
{Country: "Algeria", Hostname: "algeria-ca-version-2.expressnetw.com"},
{Country: "Andorra", Hostname: "andorra-ca-version-2.expressnetw.com"},
{Country: "Argentina", Hostname: "argentina-ca-version-2.expressnetw.com"},
{Country: "Australia", City: "Adelaide", Hostname: "australia-adelaide--ca-version-2.expressnetw.com"},
{Country: "Australia", City: "Brisbane", Hostname: "australia-brisbane-ca-version-2.expressnetw.com"},
{Country: "Australia", City: "Melbourne", Hostname: "australia-melbourne-ca-version-2.expressnetw.com"},
{Country: "Australia", City: "Perth", Hostname: "australia-perth-ca-version-2.expressnetw.com"},
{Country: "Australia", City: "Sydney", Hostname: "australia-sydney-2-ca-version-2.expressnetw.com"},
{Country: "Australia", City: "Sydney", Hostname: "australia-sydney-ca-version-2.expressnetw.com"},
{Country: "Australia", City: "Woolloomooloo", Hostname: "australia-woolloomooloo-2-ca-version-2.expressnetw.com"},
{Country: "Austria", Hostname: "austria-ca-version-2.expressnetw.com"},
{Country: "Azerbaijan", Hostname: "azerbaijan-ca-version-2.expressnetw.com"},
{Country: "Bahamas", Hostname: "bahamas-ca-version-2.expressnetw.com"},
{Country: "Bangladesh", Hostname: "bangladesh-ca-version-2.expressnetw.com"},
{Country: "Belarus", Hostname: "belarus-ca-version-2.expressnetw.com"},
{Country: "Belgium", Hostname: "belgium-ca-version-2.expressnetw.com"},
{Country: "Bermuda", Hostname: "bermuda-ca-version-2.expressnetw.com"},
{Country: "Bhutan", Hostname: "bhutan-ca-version-2.expressnetw.com"},
{Country: "Bolivia", Hostname: "bolivia-ca-version-2.expressnetw.com"},
{Country: "Brazil", Hostname: "brazil-2-ca-version-2.expressnetw.com"},
{Country: "Brazil", Hostname: "brazil-ca-version-2.expressnetw.com"},
{Country: "Brunei", Hostname: "brunei-ca-version-2.expressnetw.com"},
{Country: "Bulgaria", Hostname: "bulgaria-ca-version-2.expressnetw.com"},
{Country: "Cambodia", Hostname: "cambodia-ca-version-2.expressnetw.com"},
{Country: "Canada", City: "Montreal", Hostname: "canada-montreal-ca-version-2.expressnetw.com"},
{Country: "Canada", City: "Toronto", Hostname: "canada-toronto-2-ca-version-2.expressnetw.com"},
{Country: "Canada", City: "Toronto", Hostname: "canada-toronto-ca-version-2.expressnetw.com"},
{Country: "Cayman Islands", Hostname: "caymanislands-ca-version-2.expressnetw.com"},
{Country: "Chile", Hostname: "chile-ca-version-2.expressnetw.com"},
{Country: "Colombia", Hostname: "colombia-ca-version-2.expressnetw.com"},
{Country: "Costa Rica", Hostname: "costarica-ca-version-2.expressnetw.com"},
{Country: "Croatia", Hostname: "croatia-ca-version-2.expressnetw.com"},
{Country: "Cuba", Hostname: "cuba-ca-version-2.expressnetw.com"},
{Country: "Cyprus", Hostname: "cyprus-ca-version-2.expressnetw.com"},
{Country: "Czech Republic", Hostname: "czechrepublic-ca-version-2.expressnetw.com"},
{Country: "Denmark", Hostname: "denmark-ca-version-2.expressnetw.com"},
{Country: "Dominican Republic", Hostname: "dominicanrepublic-ca-version-2.expressnetw.com"},
{Country: "Ecuador", Hostname: "ecuador-ca-version-2.expressnetw.com"},
{Country: "Egypt", Hostname: "egypt-ca-version-2.expressnetw.com"},
{Country: "Estonia", Hostname: "estonia-ca-version-2.expressnetw.com"},
{Country: "Finland", Hostname: "finland-ca-version-2.expressnetw.com"},
{Country: "France", City: "Alsace", Hostname: "france-alsace-ca-version-2.expressnetw.com"},
{Country: "France", City: "Marseille", Hostname: "france-marseille-ca-version-2.expressnetw.com"},
{Country: "France", City: "Paris", Hostname: "france-paris-1-ca-version-2.expressnetw.com"},
{Country: "France", City: "Paris", Hostname: "france-paris-2-ca-version-2.expressnetw.com"},
{Country: "France", City: "Strasbourg", Hostname: "france-strasbourg-ca-version-2.expressnetw.com"},
{Country: "Georgia", Hostname: "georgia-ca-version-2.expressnetw.com"},
{Country: "Germany", City: "Frankfurt", Hostname: "germany-darmstadt-ca-version-2.expressnetw.com"},
{Country: "Germany", City: "Frankfurt", Hostname: "germany-frankfurt-1-ca-version-2.expressnetw.com"},
{Country: "Germany", City: "Nuremberg", Hostname: "germany-nuremberg-ca-version-2.expressnetw.com"},
{Country: "Ghana", Hostname: "ghana-ca-version-2.expressnetw.com"},
{Country: "Greece", Hostname: "greece-ca-version-2.expressnetw.com"},
{Country: "Guam", Hostname: "guam-ca-version-2.expressnetw.com"},
{Country: "Guatemala", Hostname: "guatemala-ca-version-2.expressnetw.com"},
{Country: "Honduras", Hostname: "honduras-ca-version-2.expressnetw.com"},
{Country: "Hong Kong", Hostname: "hongkong-1-ca-version-2.expressnetw.com"},
{Country: "Hong Kong", Hostname: "hongkong-2-ca-version-2.expressnetw.com"},
{Country: "Hungary", Hostname: "hungary-ca-version-2.expressnetw.com"},
{Country: "Iceland", Hostname: "iceland-ca-version-2.expressnetw.com"},
{Country: "India (via Singapore)", Hostname: "india-sg-ca-version-2.expressnetw.com"},
{Country: "India (via UK)", Hostname: "india-uk-ca-version-2.expressnetw.com"},
{Country: "Indonesia", Hostname: "indonesia-ca-version-2.expressnetw.com"},
{Country: "Ireland", Hostname: "ireland-ca-version-2.expressnetw.com"},
{Country: "Israel", Hostname: "israel-ca-version-2.expressnetw.com"},
{Country: "Italy", City: "Cosenza", Hostname: "italy-cosenza-ca-version-2.expressnetw.com"},
{Country: "Italy", City: "Milan", Hostname: "italy-milan-ca-version-2.expressnetw.com"},
{Country: "Italy", City: "Naples", Hostname: "italy-naples-ca-version-2.expressnetw.com"},
{Country: "Jamaica", Hostname: "jamaica-ca-version-2.expressnetw.com"},
{Country: "Japan", City: "Osaka", Hostname: "japan-osaka-ca-version-2.expressnetw.com"},
{Country: "Japan", City: "Shibuya", Hostname: "japan-shibuya-ca-version-2.expressnetw.com"},
{Country: "Japan", City: "Tokyo", Hostname: "japan-tokyo-ca-version-2.expressnetw.com"},
{Country: "Japan", City: "Yokohama", Hostname: "japan-yokohama-ca-version-2.expressnetw.com"},
{Country: "Kazakhstan", Hostname: "kazakhstan-ca-version-2.expressnetw.com"},
{Country: "Laos", Hostname: "laos-ca-version-2.expressnetw.com"},
{Country: "Latvia", Hostname: "latvia-ca-version-2.expressnetw.com"},
{Country: "Liechtenstein", Hostname: "liechtenstein-ca-version-2.expressnetw.com"},
{Country: "Lithuania", Hostname: "lithuania-ca-version-2.expressnetw.com"},
{Country: "Luxembourg", Hostname: "luxembourg-ca-version-2.expressnetw.com"},
{Country: "Macau", Hostname: "macau-ca-version-2.expressnetw.com"},
{Country: "Malaysia", Hostname: "malaysia-ca-version-2.expressnetw.com"},
{Country: "Mexico", Hostname: "mexico-ca-version-2.expressnetw.com"},
{Country: "Moldova", Hostname: "moldova-ca-version-2.expressnetw.com"},
{Country: "Mongolia", Hostname: "mongolia-ca-version-2.expressnetw.com"},
{Country: "Morocco", Hostname: "morocco-ca-version-2.expressnetw.com"},
{Country: "Myanmar", Hostname: "myanmar-ca-version-2.expressnetw.com"},
{Country: "Nepal", Hostname: "nepal-ca-version-2.expressnetw.com"},
{Country: "Netherlands", City: "Amsterdam", Hostname: "netherlands-amsterdam-ca-version-2.expressnetw.com"},
{Country: "Netherlands", City: "Rotterdam", Hostname: "netherlands-rotterdam-ca-version-2.expressnetw.com"},
{Country: "Netherlands", City: "The Hague", Hostname: "netherlands-thehague-ca-version-2.expressnetw.com"},
{Country: "New Zealand", Hostname: "newzealand-ca-version-2.expressnetw.com"},
{Country: "North Macedonia", Hostname: "macedonia-ca-version-2.expressnetw.com"},
{Country: "Norway", Hostname: "norway-ca-version-2.expressnetw.com"},
{Country: "Panama", Hostname: "panama-ca-version-2.expressnetw.com"},
{Country: "Peru", Hostname: "peru-ca-version-2.expressnetw.com"},
{Country: "Philippines (via Singapore)", Hostname: "ph-via-sing-ca-version-2.expressnetw.com"},
{Country: "Poland", Hostname: "poland-ca-version-2.expressnetw.com"},
{Country: "Portugal", Hostname: "portugal-ca-version-2.expressnetw.com"},
{Country: "Puerto Rico", Hostname: "puertorico-ca-version-2.expressnetw.com"},
{Country: "Romania", Hostname: "romania-ca-version-2.expressnetw.com"},
{Country: "Serbia", Hostname: "serbia-ca-version-2.expressnetw.com"},
{Country: "Singapore", City: "CBD", Hostname: "singapore-cbd-ca-version-2.expressnetw.com"},
{Country: "Singapore", City: "Jurong", Hostname: "singapore-jurong-ca-version-2.expressnetw.com"},
{Country: "Singapore", City: "Marina Bay", Hostname: "singapore-marinabay-ca-version-2.expressnetw.com"},
{Country: "Slovakia", Hostname: "slovakia-ca-version-2.expressnetw.com"},
{Country: "Slovenia", Hostname: "slovenia-ca-version-2.expressnetw.com"},
{Country: "South Africa", Hostname: "southafrica-ca-version-2.expressnetw.com"},
{Country: "South Korea", Hostname: "southkorea2-ca-version-2.expressnetw.com"},
{Country: "Spain", City: "Barcelona", Hostname: "spain-barcelona-ca-version-2.expressnetw.com"},
{Country: "Spain", City: "Barcelona", Hostname: "spain-barcelona2-ca-version-2.expressnetw.com"},
{Country: "Spain", City: "Madrid", Hostname: "spain-ca-version-2.expressnetw.com"},
{Country: "Sri Lanka", Hostname: "srilanka-ca-version-2.expressnetw.com"},
{Country: "Sweden", Hostname: "sweden-ca-version-2.expressnetw.com"},
{Country: "Sweden", Hostname: "sweden2-ca-version-2.expressnetw.com"},
{Country: "Switzerland", Hostname: "switzerland-2-ca-version-2.expressnetw.com"},
{Country: "Switzerland", Hostname: "switzerland-ca-version-2.expressnetw.com"},
{Country: "Taiwan", Hostname: "taiwan-3-ca-version-2.expressnetw.com"},
{Country: "Thailand", Hostname: "thailand-ca-version-2.expressnetw.com"},
{Country: "Trinidad and Tobago", Hostname: "trinidadandtobago-ca-version-2.expressnetw.com"},
{Country: "Turkey", Hostname: "turkey-ca-version-2.expressnetw.com"},
{Country: "UK", City: "Docklands", Hostname: "uk-1-docklands-ca-version-2.expressnetw.com"},
{Country: "UK", City: "East London", Hostname: "uk-east-london-ca-version-2.expressnetw.com"},
{Country: "UK", City: "London", Hostname: "uk-london-ca-version-2.expressnetw.com"},
{Country: "UK", City: "Midlands", Hostname: "uk-midlands-ca-version-2.expressnetw.com"},
{Country: "UK", City: "Tottenham", Hostname: "uk-tottenham-ca-version-2.expressnetw.com"},
{Country: "UK", City: "Wembley", Hostname: "uk-wembley-ca-version-2.expressnetw.com"},
{Country: "Ukraine", Hostname: "ukraine-ca-version-2.expressnetw.com"},
{Country: "Uruguay", Hostname: "uruguay-ca-version-2.expressnetw.com"},
{Country: "USA", City: "Albuquerque", Hostname: "usa-albuquerque-ca-version-2.expressnetw.com"},
{Country: "USA", City: "Atlanta", Hostname: "usa-atlanta-ca-version-2.expressnetw.com"},
{Country: "USA", City: "Boston", Hostname: "us-boston-ca-version-2.expressnetw.com"},
{Country: "USA", City: "Chicago", Hostname: "usa-chicago-ca-version-2.expressnetw.com"},
{Country: "USA", City: "Dallas", Hostname: "usa-dallas-ca-version-2.expressnetw.com"},
{Country: "USA", City: "Denver", Hostname: "usa-denver-ca-version-2.expressnetw.com"},
{Country: "USA", City: "Houston", Hostname: "usa-houston-ca-version-2.expressnetw.com"},
{Country: "USA", City: "Jackson", Hostname: "us-jackson-ca-version-2.expressnetw.com"},
{Country: "USA", City: "Lincoln Park", Hostname: "usa-lincolnpark-ca-version-2.expressnetw.com"},
{Country: "USA", City: "Little Rock", Hostname: "us-littlerock-ca-version-2.expressnetw.com"},
{Country: "USA", City: "Los Angeles", Hostname: "usa-losangeles-2-ca-version-2.expressnetw.com"},
{Country: "USA", City: "Los Angeles", Hostname: "usa-losangeles-3-ca-version-2.expressnetw.com"},
{Country: "USA", City: "Los Angeles", Hostname: "usa-losangeles-ca-version-2.expressnetw.com"},
{Country: "USA", City: "Los Angeles", Hostname: "usa-losangeles5-ca-version-2.expressnetw.com"},
{Country: "USA", City: "Miami", Hostname: "usa-miami-2-ca-version-2.expressnetw.com"},
{Country: "USA", City: "Miami", Hostname: "usa-miami-ca-version-2.expressnetw.com"},
{Country: "USA", City: "New Jersey", Hostname: "usa-newjersey-1-ca-version-2.expressnetw.com"},
{Country: "USA", City: "New Jersey", Hostname: "usa-newjersey-3-ca-version-2.expressnetw.com"},
{Country: "USA", City: "New Jersey", Hostname: "usa-newjersey2-ca-version-2.expressnetw.com"},
{Country: "USA", City: "New Orleans", Hostname: "us-neworleans-ca-version-2.expressnetw.com"},
{Country: "USA", City: "New York", Hostname: "usa-newyork-ca-version-2.expressnetw.com"},
{Country: "USA", City: "Oklahoma City", Hostname: "us-oklahoma-ca-version-2.expressnetw.com"},
{Country: "USA", City: "Phoenix", Hostname: "usa-phoenix-ca-version-2.expressnetw.com"},
{Country: "USA", City: "Salt Lake City", Hostname: "usa-saltlakecity-ca-version-2.expressnetw.com"},
{Country: "USA", City: "San Francisco", Hostname: "usa-sanfrancisco-ca-version-2.expressnetw.com"},
{Country: "USA", City: "Santa Monica", Hostname: "usa-santa-monica-ca-version-2.expressnetw.com"},
{Country: "USA", City: "Seattle", Hostname: "usa-seattle-ca-version-2.expressnetw.com"},
{Country: "USA", City: "Tampa", Hostname: "usa-tampa-1-ca-version-2.expressnetw.com"},
{Country: "USA", City: "Washington DC", Hostname: "usa-washingtondc-ca-version-2.expressnetw.com"},
{Country: "USA", City: "Wichita", Hostname: "us-wichita-ca-version-2.expressnetw.com"},
{Country: "Venezuela", Hostname: "venezuela-ca-version-2.expressnetw.com"},
{Country: "Vietnam", Hostname: "vietnam-ca-version-2.expressnetw.com"},
}
}
@@ -16,7 +16,7 @@ func parallelResolverSettings(hosts []string) (settings resolver.ParallelSetting
Hosts: hosts, Hosts: hosts,
MaxFailRatio: maxFailRatio, MaxFailRatio: maxFailRatio,
Repeat: resolver.RepeatSettings{ Repeat: resolver.RepeatSettings{
MaxDuration: time.Second, MaxDuration: 5 * time.Second,
MaxNoNew: maxNoNew, MaxNoNew: maxNoNew,
MaxFails: maxFails, MaxFails: maxFails,
SortIPs: true, SortIPs: true,
+50 -20
View File
@@ -5,7 +5,6 @@ import (
"fmt" "fmt"
"sort" "sort"
"github.com/qdm12/gluetun/internal/constants/vpn"
"github.com/qdm12/gluetun/internal/models" "github.com/qdm12/gluetun/internal/models"
"github.com/qdm12/gluetun/internal/provider/common" "github.com/qdm12/gluetun/internal/provider/common"
) )
@@ -13,35 +12,66 @@ import (
func (u *Updater) FetchServers(ctx context.Context, minServers int) ( func (u *Updater) FetchServers(ctx context.Context, minServers int) (
servers []models.Server, err error, servers []models.Server, err error,
) { ) {
servers = hardcodedServers() servers, warnings, err := fetchServersFromWebsite(ctx, u.httpClient)
hosts := make([]string, len(servers))
for i := range servers {
hosts[i] = servers[i].Hostname
}
resolveSettings := parallelResolverSettings(hosts)
hostToIPs, warnings, err := u.parallelResolver.Resolve(ctx, resolveSettings)
for _, warning := range warnings { for _, warning := range warnings {
u.warner.Warn(warning) u.warner.Warn(warning)
} }
if err != nil { if err != nil {
return nil, err return nil, fmt.Errorf("fetching servers: %w", err)
} }
i := 0 // Generate candidate hostnames for each location
serverToCandidateHostnames := make(map[*models.Server][]string, len(servers))
var allCandidateHostnames []string
for _, server := range servers { for _, server := range servers {
hostname := server.Hostname candidateHostnames, err := generateCandidateHostnames(server)
server.IPs = hostToIPs[hostname] if err != nil {
if len(server.IPs) == 0 { u.warner.Warn(fmt.Sprintf("generating candidate hostnames for %s - %s: %s",
server.Country, server.City, err))
continue continue
} }
server.VPN = vpn.OpenVPN serverToCandidateHostnames[&server] = candidateHostnames
server.UDP = true // no TCP support allCandidateHostnames = append(allCandidateHostnames, candidateHostnames...)
servers[i] = server
i++
} }
servers = servers[:i]
// Resolve all candidate hostnames in parallel
resolveSettings := parallelResolverSettings(allCandidateHostnames)
hostToIPs, _, err := u.parallelResolver.Resolve(ctx, resolveSettings)
// Ignore resolution warnings since we are mostly bruteforcing DNS records
// so most candidate hostnames will fail resolving.
if err != nil {
return nil, fmt.Errorf("resolving hostnames: %w", err)
}
foundServers := make([]models.Server, 0, len(servers))
for server, candidateHostnames := range serverToCandidateHostnames {
success := false
for _, candidate := range candidateHostnames {
ips := hostToIPs[candidate]
if len(ips) > 0 {
success = true
workingServer := *server
workingServer.Hostname = candidate
workingServer.IPs = ips
foundServers = append(foundServers, workingServer)
}
}
if success {
continue
}
// Log a warning on the bad server location that did not resolve any IPs
serverName := server.Country
if server.City != "" {
serverName += " - " + server.City
}
if server.Number > 0 {
serverName += fmt.Sprintf(" (%d)", server.Number)
}
u.warner.Warn(fmt.Sprintf("no IPs resolved for %s (tried %v)", serverName, candidateHostnames))
}
servers = foundServers
if len(servers) < minServers { if len(servers) < minServers {
return nil, fmt.Errorf("%w: %d and expected at least %d", return nil, fmt.Errorf("%w: %d and expected at least %d",
File diff suppressed because one or more lines are too long
@@ -1,19 +1,23 @@
package updater package updater
import ( import (
"net/http"
"github.com/qdm12/gluetun/internal/provider/common" "github.com/qdm12/gluetun/internal/provider/common"
) )
type Updater struct { type Updater struct {
httpClient *http.Client
unzipper common.Unzipper unzipper common.Unzipper
parallelResolver common.ParallelResolver parallelResolver common.ParallelResolver
warner common.Warner warner common.Warner
} }
func New(unzipper common.Unzipper, warner common.Warner, func New(httpClient *http.Client, unzipper common.Unzipper,
parallelResolver common.ParallelResolver, warner common.Warner, parallelResolver common.ParallelResolver,
) *Updater { ) *Updater {
return &Updater{ return &Updater{
httpClient: httpClient,
unzipper: unzipper, unzipper: unzipper,
parallelResolver: parallelResolver, parallelResolver: parallelResolver,
warner: warner, warner: warner,
@@ -0,0 +1,111 @@
package updater
import (
"context"
"fmt"
"net/http"
"strings"
"github.com/qdm12/gluetun/internal/constants/vpn"
"github.com/qdm12/gluetun/internal/models"
htmlutils "github.com/qdm12/gluetun/internal/updater/html"
"golang.org/x/net/html"
)
func fetchServersFromWebsite(ctx context.Context, client *http.Client) (
servers []models.Server, warnings []string, err error,
) {
const url = "https://www.expressvpn.com/vpn-server"
rootNode, err := htmlutils.Fetch(ctx, client, url)
if err != nil {
return nil, nil, fmt.Errorf("fetching HTML code: %w", err)
}
servers, warnings = parseServerListTable(rootNode)
return servers, warnings, nil
}
func parseServerListTable(rootNode *html.Node) (
servers []models.Server, warnings []string,
) {
// Collect all tr nodes from all tbody elements using recursive walk
var allRows []*html.Node
var walk func(n *html.Node)
walk = func(n *html.Node) {
if n.Data == "tr" {
allRows = append(allRows, n)
return
}
for child := n.FirstChild; child != nil; child = child.NextSibling {
walk(child)
}
}
walk(rootNode)
seenLocations := make(map[string]struct{})
for _, trNode := range allRows {
serverLocation := strings.TrimSpace(htmlutils.Attribute(trNode, "data-server-location"))
if serverLocation == "" {
continue
}
country, city, number, err := parseLocation(serverLocation)
if err != nil {
warnings = append(warnings, fmt.Sprintf("parsing location: %s", err))
continue
}
server := models.Server{
VPN: vpn.OpenVPN,
TCP: true,
UDP: true,
Country: country,
City: city,
Number: number,
}
locationKey := fmt.Sprintf("%s-%s-%d", country, city, number)
if _, ok := seenLocations[locationKey]; ok {
continue
}
seenLocations[locationKey] = struct{}{}
servers = append(servers, server)
}
return servers, warnings
}
func parseLocation(location string) (country, city string, number uint16, err error) {
// Location formats:
// - "Argentina" -> country="Argentina", city="", number=0
// - "Australia - Adelaide" -> country="Australia", city="Adelaide", number=0
// - "Canada - Toronto" -> country="Canada", city="Toronto", number=0
parts := strings.Split(location, " - ")
country = strings.TrimSpace(parts[0])
switch len(parts) {
case 1: // country only
if strings.Contains(country, " [") {
// This can be "country [city]"
parts := strings.SplitN(country, " [", 2)
country = strings.TrimSpace(parts[0])
city = strings.TrimSuffix(strings.TrimSpace(parts[1]), "]")
}
case 2: // country and city
city = strings.TrimSpace(parts[1])
case 3: // country, city, and number
city = strings.TrimSpace(parts[1])
fmt.Sscanf(strings.TrimSpace(parts[2]), "%d", &number)
default:
return "", "", 0, fmt.Errorf("invalid location format: %q", location)
}
// Retro-compatibility transforms
switch strings.ToLower(country) {
case "united states":
country = "USA"
case "united kingdom":
country = "UK"
}
return country, city, number, nil
}
@@ -0,0 +1,206 @@
package updater
import (
"os"
"strings"
"testing"
"github.com/qdm12/gluetun/internal/models"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/net/html"
)
func Test_parseServerListTable(t *testing.T) {
t.Parallel()
htmlContent, err := os.ReadFile("testdata/index.html")
require.NoError(t, err)
rootNode, err := html.Parse(strings.NewReader(string(htmlContent)))
require.NoError(t, err)
servers, warnings := parseServerListTable(rootNode)
require.NoError(t, err)
assert.Empty(t, warnings)
// The ExpressVPN server list should have hundreds of servers
assert.Greater(t, len(servers), 200,
"expected at least 200 servers from the ExpressVPN server list, got %d", len(servers))
// Check that all servers have required fields
for _, server := range servers {
assert.NotEmpty(t, server.Country, "server country should not be empty")
assert.True(t, server.TCP, "TCP should be enabled")
assert.True(t, server.UDP, "UDP should be enabled")
}
// Check some specific servers are found
foundServers := make(map[string]models.Server)
for _, server := range servers {
key := server.Country
if server.City != "" {
key += " - " + server.City
}
foundServers[key] = server
}
// Verify countries without cities
assert.NotEmpty(t, foundServers["Albania"])
assert.NotEmpty(t, foundServers["Singapore"])
}
func Test_parseLocation(t *testing.T) {
t.Parallel()
testCases := map[string]struct {
input string
country string
city string
number uint16
}{
"country only": {
input: "Argentina",
country: "Argentina",
},
"country with city": {
input: "Australia - Adelaide",
country: "Australia",
city: "Adelaide",
},
"country with numbered location": {
input: "Canada - Toronto - 2",
country: "Canada",
city: "Toronto",
number: 2,
},
"virtual location": {
input: "India (via Singapore)",
country: "India (via Singapore)",
},
"country with multiple dashes in city": {
input: "USA - Los Angeles - 1",
country: "USA",
city: "Los Angeles",
number: 1,
},
}
for name, testCase := range testCases {
t.Run(name, func(t *testing.T) {
t.Parallel()
country, city, number, err := parseLocation(testCase.input)
assert.Equal(t, testCase.country, country)
assert.Equal(t, testCase.city, city)
assert.Equal(t, testCase.number, number)
assert.NoError(t, err)
})
}
}
func Test_generateCandidateHostnames(t *testing.T) {
t.Parallel()
testCases := map[string]struct {
input models.Server
mustContain []string
mustNotContain []string
expectedCount int // exact count expected (0 if flexible)
}{
"simple country": {
input: models.Server{Country: "Argentina"},
mustContain: []string{
"argentina-ca-version-2.expressnetw.com",
"argentina1-ca-version-2.expressnetw.com",
"argentina-1-ca-version-2.expressnetw.com",
},
},
"country with city": {
input: models.Server{Country: "USA", City: "New York"},
mustContain: []string{
"usa-newyork-ca-version-2.expressnetw.com",
"us-newyork-ca-version-2.expressnetw.com", // alias
"usa-newyork-1-ca-version-2.expressnetw.com",
"usa-newyork1-ca-version-2.expressnetw.com",
},
},
"North Macedonia abbreviation": {
input: models.Server{Country: "North Macedonia"},
mustContain: []string{
"macedonia-ca-version-2.expressnetw.com",
},
expectedCount: 1, // single hardcoded hostname
},
"India via Singapore": {
input: models.Server{Country: "India (via Singapore)"},
mustContain: []string{
"india-sg-ca-version-2.expressnetw.com",
},
expectedCount: 1, // single hardcoded hostname
},
"India via UK": {
input: models.Server{Country: "India (via UK)"},
mustContain: []string{
"india-uk-ca-version-2.expressnetw.com",
},
expectedCount: 1, // single hardcoded hostname
},
"France Paris": {
input: models.Server{Country: "France", City: "Paris"},
mustContain: []string{
"france-paris-ca-version-2.expressnetw.com",
"france-paris-1-ca-version-2.expressnetw.com",
"france-paris-2-ca-version-2.expressnetw.com",
"france-paris1-ca-version-2.expressnetw.com",
"france-paris2-ca-version-2.expressnetw.com",
},
},
"UK with alias": {
input: models.Server{Country: "UK", City: "London"},
mustContain: []string{
"uk-london-ca-version-2.expressnetw.com",
"gb-london-ca-version-2.expressnetw.com", // alias
},
},
"Germany": {
input: models.Server{Country: "Germany"},
mustContain: []string{
"germany-ca-version-2.expressnetw.com",
"de-ca-version-2.expressnetw.com", // alias
},
},
}
for name, testCase := range testCases {
t.Run(name, func(t *testing.T) {
t.Parallel()
candidates, err := generateCandidateHostnames(testCase.input)
assert.NoError(t, err)
// Check exact count if specified
if testCase.expectedCount > 0 {
assert.Len(t, candidates, testCase.expectedCount,
"expected exactly %d candidates", testCase.expectedCount)
}
// Check required candidates exist
for _, expected := range testCase.mustContain {
assert.Contains(t, candidates, expected,
"expected candidate %q in candidates", expected)
}
// Check excluded candidates don't exist
for _, excluded := range testCase.mustNotContain {
assert.NotContains(t, candidates, excluded,
"did not expect candidate %q in candidates", excluded)
}
// All candidates should end with the expected suffix
for _, candidate := range candidates {
assert.True(t, strings.HasSuffix(candidate, hostnameSuffix),
"candidate %q should end with %q", candidate, hostnameSuffix)
}
})
}
}
+1 -1
View File
@@ -58,7 +58,7 @@ func NewProviders(storage Storage, timeNow func() time.Time,
providers.Airvpn: airvpn.New(storage, client), providers.Airvpn: airvpn.New(storage, client),
providers.Custom: custom.New(extractor), providers.Custom: custom.New(extractor),
providers.Cyberghost: cyberghost.New(storage, updaterWarner, parallelResolver), providers.Cyberghost: cyberghost.New(storage, updaterWarner, parallelResolver),
providers.Expressvpn: expressvpn.New(storage, unzipper, updaterWarner, parallelResolver), providers.Expressvpn: expressvpn.New(storage, client, unzipper, updaterWarner, parallelResolver),
providers.Fastestvpn: fastestvpn.New(storage, client, updaterWarner, parallelResolver), providers.Fastestvpn: fastestvpn.New(storage, client, updaterWarner, parallelResolver),
providers.Giganews: giganews.New(storage, unzipper, updaterWarner, parallelResolver), providers.Giganews: giganews.New(storage, unzipper, updaterWarner, parallelResolver),
providers.HideMyAss: hidemyass.New(storage, client, updaterWarner, parallelResolver), providers.HideMyAss: hidemyass.New(storage, client, updaterWarner, parallelResolver),