chore(updater): move updater packages to pkg/updaters/<name>

This commit is contained in:
Quentin McGaw
2026-04-23 03:47:57 +00:00
parent 5b01324d5f
commit 13503b0ae0
268 changed files with 1602 additions and 1026 deletions
-12
View File
@@ -1,12 +0,0 @@
package html
import "golang.org/x/net/html"
func Attribute(node *html.Node, key string) (value string) {
for _, attribute := range node.Attr {
if attribute.Key == key {
return attribute.Val
}
}
return ""
}
-43
View File
@@ -1,43 +0,0 @@
package html
import (
"container/list"
"fmt"
"golang.org/x/net/html"
)
// BFS returns the node matching the match function and nil
// if no node is found.
func BFS(rootNode *html.Node, match MatchFunc) (node *html.Node) {
visited := make(map[*html.Node]struct{})
queue := list.New()
_ = queue.PushBack(rootNode)
for queue.Len() > 0 {
listElement := queue.Front()
node, ok := queue.Remove(listElement).(*html.Node)
if !ok {
panic(fmt.Sprintf("linked list has bad type %T", listElement.Value))
}
if node == nil {
continue
}
if _, ok := visited[node]; ok {
continue
}
visited[node] = struct{}{}
if match(node) {
return node
}
for child := node.FirstChild; child != nil; child = child.NextSibling {
_ = queue.PushBack(child)
}
}
return nil
}
-22
View File
@@ -1,22 +0,0 @@
package html
import (
"strings"
"golang.org/x/net/html"
)
func HasClassStrings(node *html.Node, classStrings ...string) (match bool) {
targetClasses := make(map[string]struct{}, len(classStrings))
for _, classString := range classStrings {
targetClasses[classString] = struct{}{}
}
classAttribute := Attribute(node, "class")
classes := strings.Fields(classAttribute)
for _, class := range classes {
delete(targetClasses, class)
}
return len(targetClasses) == 0
}
-27
View File
@@ -1,27 +0,0 @@
package html
import (
"bytes"
"fmt"
"golang.org/x/net/html"
)
func WrapError(sentinelError error, node *html.Node) error {
return fmt.Errorf("%w: in HTML code: %s",
sentinelError, mustRenderHTML(node))
}
func WrapWarning(warning string, node *html.Node) string {
return fmt.Sprintf("%s: in HTML code: %s",
warning, mustRenderHTML(node))
}
func mustRenderHTML(node *html.Node) (rendered string) {
stringBuffer := bytes.NewBufferString("")
err := html.Render(stringBuffer, node)
if err != nil {
panic(err)
}
return stringBuffer.String()
}
-41
View File
@@ -1,41 +0,0 @@
package html
import (
"context"
"fmt"
"net/http"
"golang.org/x/net/html"
)
func Fetch(ctx context.Context, client *http.Client, url string) (
rootNode *html.Node, err error,
) {
request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, fmt.Errorf("creating HTTP request: %w", err)
}
response, err := client.Do(request)
if err != nil {
return nil, err
}
if response.StatusCode != http.StatusOK {
return nil, fmt.Errorf("HTTP status code not OK: %d %s",
response.StatusCode, response.Status)
}
rootNode, err = html.Parse(response.Body)
if err != nil {
_ = response.Body.Close()
return nil, fmt.Errorf("parsing HTML code: %w", err)
}
err = response.Body.Close()
if err != nil {
return nil, fmt.Errorf("closing response body: %w", err)
}
return rootNode, nil
}
-94
View File
@@ -1,94 +0,0 @@
package html
import (
"context"
"io"
"net/http"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/net/html"
)
func parseTestHTML(t *testing.T, htmlString string) *html.Node {
t.Helper()
rootNode, err := html.Parse(strings.NewReader(htmlString))
require.NoError(t, err)
return rootNode
}
type roundTripFunc func(r *http.Request) (*http.Response, error)
func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) {
return f(r)
}
func Test_Fetch(t *testing.T) {
t.Parallel()
canceledCtx, cancel := context.WithCancel(context.Background())
cancel()
testCases := map[string]struct {
ctx context.Context
url string
responseStatus int
responseBody io.ReadCloser
rootNode *html.Node
errMessage string
}{
"context canceled": {
ctx: canceledCtx,
url: "https://example.com/path",
errMessage: `Get "https://example.com/path": context canceled`,
},
"response status not ok": {
ctx: context.Background(),
url: "https://example.com/path",
responseStatus: http.StatusNotFound,
errMessage: `HTTP status code not OK: 404 Not Found`,
},
"success": {
ctx: context.Background(),
url: "https://example.com/path",
responseStatus: http.StatusOK,
rootNode: parseTestHTML(t, "some body"),
responseBody: io.NopCloser(strings.NewReader("some body")),
},
}
for name, testCase := range testCases {
t.Run(name, func(t *testing.T) {
t.Parallel()
client := &http.Client{
Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) {
assert.Equal(t, http.MethodGet, r.Method)
assert.Equal(t, r.URL.String(), testCase.url)
ctxErr := r.Context().Err()
if ctxErr != nil {
return nil, ctxErr
}
return &http.Response{
StatusCode: testCase.responseStatus,
Status: http.StatusText(testCase.responseStatus),
Body: testCase.responseBody,
}, nil
}),
}
rootNode, err := Fetch(testCase.ctx, client, testCase.url)
if testCase.errMessage != "" {
assert.EqualError(t, err, testCase.errMessage)
} else {
assert.NoError(t, err)
}
assert.Equal(t, testCase.rootNode, rootNode)
})
}
}
-45
View File
@@ -1,45 +0,0 @@
package html
import (
"golang.org/x/net/html"
)
type MatchFunc func(node *html.Node) (match bool)
func MatchID(id string) MatchFunc {
return func(node *html.Node) (match bool) {
if node == nil {
return false
}
return Attribute(node, "id") == id
}
}
func MatchData(data string) MatchFunc {
return func(node *html.Node) (match bool) {
return node != nil && node.Type == html.ElementNode && node.Data == data
}
}
func DirectChild(parent *html.Node,
matchFunc MatchFunc,
) (child *html.Node) {
for child := parent.FirstChild; child != nil; child = child.NextSibling {
if matchFunc(child) {
return child
}
}
return nil
}
func DirectChildren(parent *html.Node,
matchFunc MatchFunc,
) (children []*html.Node) {
for child := parent.FirstChild; child != nil; child = child.NextSibling {
if matchFunc(child) {
children = append(children, child)
}
}
return children
}
+1 -1
View File
@@ -4,8 +4,8 @@ import (
"context"
"github.com/qdm12/gluetun/internal/configuration/settings"
"github.com/qdm12/gluetun/internal/models"
"github.com/qdm12/gluetun/internal/provider"
"github.com/qdm12/gluetun/pkg/updaters/models"
)
type Providers interface {
-93
View File
@@ -1,93 +0,0 @@
package openvpn
import (
"errors"
"fmt"
"net/netip"
"sort"
"strings"
)
func ExtractProto(b []byte) (tcp, udp bool, err error) {
lines := strings.Split(string(b), "\n")
const protoPrefix = "proto "
for _, line := range lines {
if !strings.HasPrefix(line, protoPrefix) {
continue
}
s := strings.TrimPrefix(line, protoPrefix)
s = strings.TrimSpace(s)
s = strings.ToLower(s)
switch s {
case "tcp", "tcp4", "tcp6", "tcp-client":
return true, false, nil
case "udp", "udp4", "udp6":
return false, true, nil
default:
return false, false, fmt.Errorf("unknown protocol: %s", s)
}
}
// default is UDP if unspecified in openvpn configuration
return false, true, nil
}
func ExtractHost(b []byte) (host, warning string, err error) {
const (
rejectIP = true
rejectDomain = false
)
hosts := extractRemoteHosts(b, rejectIP, rejectDomain)
if len(hosts) == 0 {
return "", "", errors.New("remote host not found")
} else if len(hosts) > 1 {
warning = fmt.Sprintf(
"only using the first host %q and discarding %d other hosts",
hosts[0], len(hosts)-1)
}
return hosts[0], warning, nil
}
func ExtractIPs(b []byte) (ips []netip.Addr, err error) {
const rejectIP, rejectDomain = false, true
ipStrings := extractRemoteHosts(b, rejectIP, rejectDomain)
if len(ipStrings) == 0 {
return nil, errors.New("remote IP not found")
}
sort.Slice(ipStrings, func(i, j int) bool {
return ipStrings[i] < ipStrings[j]
})
ips = make([]netip.Addr, len(ipStrings))
for i := range ipStrings {
ips[i], err = netip.ParseAddr(ipStrings[i])
if err != nil {
return nil, fmt.Errorf("parsing IP address: %w", err)
}
}
return ips, nil
}
func extractRemoteHosts(content []byte, rejectIP, rejectDomain bool) (hosts []string) {
lines := strings.Split(string(content), "\n")
for _, line := range lines {
line = strings.TrimSpace(line)
if !strings.HasPrefix(line, "remote ") {
continue
}
fields := strings.Fields(line)
if len(fields) == 1 || fields[1] == "" {
continue
}
host := fields[1]
_, err := netip.ParseAddr(host)
if (rejectIP && err == nil) ||
(rejectDomain && err != nil) {
continue
}
hosts = append(hosts, host)
}
return hosts
}
-43
View File
@@ -1,43 +0,0 @@
package openvpn
import (
"context"
"fmt"
"io"
"net/http"
)
func FetchFile(ctx context.Context, client *http.Client, url string) (
host string, err error,
) {
b, err := fetchData(ctx, client, url)
if err != nil {
return "", err
}
const rejectIP = true
const rejectDomain = false
hosts := extractRemoteHosts(b, rejectIP, rejectDomain)
if len(hosts) == 0 {
return "", fmt.Errorf("remote host not found for url %s", url)
}
return hosts[0], nil
}
func fetchData(ctx context.Context, client *http.Client, url string) (
b []byte, err error,
) {
request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, err
}
response, err := client.Do(request)
if err != nil {
return nil, err
}
defer response.Body.Close()
return io.ReadAll(response.Body)
}
-67
View File
@@ -1,67 +0,0 @@
package openvpn
import (
"context"
"net/http"
)
// FetchMultiFiles fetches multiple Openvpn files in parallel and
// parses them to extract each of their host. A mapping from host to
// URL is returned.
func FetchMultiFiles(ctx context.Context, client *http.Client, urls []string,
failEarly bool,
) (hostToURL map[string]string, errors []error) {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
hostToURL = make(map[string]string, len(urls))
type Result struct {
url string
host string
}
results := make(chan Result)
defer close(results)
errorsCh := make(chan error)
defer close(errorsCh)
for _, url := range urls {
go func(url string) {
host, err := FetchFile(ctx, client, url)
if err != nil {
errorsCh <- err
return
}
results <- Result{
url: url,
host: host,
}
}(url)
}
for range urls {
select {
case result := <-results:
hostToURL[result.host] = result.url
case err := <-errorsCh:
if !failEarly {
errors = append(errors, err)
break
}
if len(errors) == 0 {
errors = []error{err} // keep only the first error
// stop other operations, this will trigger other errors we ignore
cancel()
}
}
}
if len(errors) > 0 && failEarly {
// we don't care about the result found
return nil, errors
}
return hostToURL, errors
}
+2 -2
View File
@@ -6,8 +6,8 @@ import (
"errors"
"fmt"
"github.com/qdm12/gluetun/internal/models"
"github.com/qdm12/gluetun/internal/provider/common"
"github.com/qdm12/gluetun/pkg/updaters/common"
"github.com/qdm12/gluetun/pkg/updaters/models"
)
type Provider interface {
-10
View File
@@ -1,10 +0,0 @@
package resolver
import (
"context"
"net"
)
type Dialer interface {
Dial(ctx context.Context, network, address string) (net.Conn, error)
}
-20
View File
@@ -1,20 +0,0 @@
package resolver
import (
"net/netip"
)
func uniqueIPsToSlice(uniqueIPs map[string]struct{}) (ips []netip.Addr) {
ips = make([]netip.Addr, 0, len(uniqueIPs))
for key := range uniqueIPs {
ip, err := netip.ParseAddr(key)
if err != nil {
panic(err)
}
if ip.Is4In6() {
ip = netip.AddrFrom4(ip.As4())
}
ips = append(ips, ip)
}
return ips
}
-40
View File
@@ -1,40 +0,0 @@
package resolver
import (
"net/netip"
"testing"
"github.com/stretchr/testify/assert"
)
func Test_uniqueIPsToSlice(t *testing.T) {
t.Parallel()
testCases := map[string]struct {
inputIPs map[string]struct{}
outputIPs []netip.Addr
}{
"nil": {
inputIPs: nil,
outputIPs: []netip.Addr{},
},
"empty": {
inputIPs: map[string]struct{}{},
outputIPs: []netip.Addr{},
},
"single IPv4": {
inputIPs: map[string]struct{}{"1.1.1.1": {}},
outputIPs: []netip.Addr{netip.AddrFrom4([4]byte{1, 1, 1, 1})},
},
"two IPv4s": {
inputIPs: map[string]struct{}{"1.1.1.1": {}, "1.1.2.1": {}},
outputIPs: []netip.Addr{netip.AddrFrom4([4]byte{1, 1, 1, 1}), netip.AddrFrom4([4]byte{1, 1, 2, 1})},
},
}
for name, testCase := range testCases {
t.Run(name, func(t *testing.T) {
t.Parallel()
outputIPs := uniqueIPsToSlice(testCase.inputIPs)
assert.ElementsMatch(t, testCase.outputIPs, outputIPs)
})
}
}
-12
View File
@@ -1,12 +0,0 @@
package resolver
import (
"net"
)
func newResolver(d Dialer) *net.Resolver {
return &net.Resolver{
PreferGo: true,
Dial: d.Dial,
}
}
-105
View File
@@ -1,105 +0,0 @@
package resolver
import (
"context"
"fmt"
"net/netip"
)
type Parallel struct {
repeatResolver *Repeat
}
func NewParallelResolver(dialer Dialer) *Parallel {
return &Parallel{
repeatResolver: NewRepeat(dialer),
}
}
type ParallelSettings struct {
// Hosts to resolve in parallel.
Hosts []string
Repeat RepeatSettings
FailEarly bool
// Maximum ratio of the hosts failing DNS resolution
// divided by the total number of hosts requested.
// This value is between 0 and 1. Note this is only
// applicable if FailEarly is not set to true.
MaxFailRatio float64
}
type parallelResult struct {
host string
IPs []netip.Addr
}
func (pr *Parallel) Resolve(ctx context.Context, settings ParallelSettings) (
hostToIPs map[string][]netip.Addr, warnings []string, err error,
) {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
results := make(chan parallelResult)
defer close(results)
errors := make(chan error)
defer close(errors)
for _, host := range settings.Hosts {
go pr.resolveAsync(ctx, host, settings.Repeat, results, errors)
}
hostToIPs = make(map[string][]netip.Addr, len(settings.Hosts))
maxFails := int(settings.MaxFailRatio * float64(len(settings.Hosts)))
for range settings.Hosts {
select {
case newErr := <-errors:
if settings.FailEarly {
if err == nil {
// only set the error to the first error encountered
// and not the context canceled errors coming after.
err = newErr
cancel()
}
break
}
// do not add warnings coming from the call to cancel()
if len(warnings) < maxFails {
warnings = append(warnings, newErr.Error())
}
if len(warnings) == maxFails {
cancel() // cancel only once when we reach maxFails
}
case result := <-results:
hostToIPs[result.host] = result.IPs
}
}
if err != nil { // fail early
return nil, warnings, err
}
failureRatio := float64(len(warnings)) / float64(len(settings.Hosts))
if failureRatio > settings.MaxFailRatio {
return hostToIPs, warnings,
fmt.Errorf("maximum failure ratio reached: %.2f failure ratio reached", failureRatio)
}
return hostToIPs, warnings, nil
}
func (pr *Parallel) resolveAsync(ctx context.Context, host string,
settings RepeatSettings, results chan<- parallelResult, errors chan<- error,
) {
IPs, err := pr.repeatResolver.Resolve(ctx, host, settings)
if err != nil {
errors <- err
return
}
results <- parallelResult{
host: host,
IPs: IPs,
}
}
-135
View File
@@ -1,135 +0,0 @@
package resolver
import (
"context"
"fmt"
"net"
"net/netip"
"sort"
"time"
)
type Repeat struct {
resolver *net.Resolver
}
func NewRepeat(dialer Dialer) *Repeat {
return &Repeat{
resolver: newResolver(dialer),
}
}
type RepeatSettings struct {
Address string
MaxDuration time.Duration
BetweenDuration time.Duration
MaxNoNew int
// Maximum consecutive DNS resolution failures
MaxFails int
SortIPs bool
}
func (r *Repeat) Resolve(ctx context.Context, host string, settings RepeatSettings) (
ips []netip.Addr, err error,
) {
timedCtx, cancel := context.WithTimeout(ctx, settings.MaxDuration)
defer cancel()
noNewCounter := 0
failCounter := 0
uniqueIPs := make(map[string]struct{})
for err == nil {
// TODO
// - one resolving every 100ms for round robin DNS responses
// - one every second for time based DNS cycling responses
noNewCounter, failCounter, err = r.resolveOnce(ctx, timedCtx, host, settings, uniqueIPs, noNewCounter, failCounter)
}
if len(uniqueIPs) == 0 {
return nil, err
}
ips = uniqueIPsToSlice(uniqueIPs)
if settings.SortIPs {
sort.Slice(ips, func(i, j int) bool {
return ips[i].Compare(ips[j]) < 1
})
}
return ips, nil
}
func (r *Repeat) resolveOnce(ctx, timedCtx context.Context, host string,
settings RepeatSettings, uniqueIPs map[string]struct{}, noNewCounter, failCounter int) (
newNoNewCounter, newFailCounter int, err error,
) {
IPs, err := r.lookupIPs(timedCtx, host)
if err != nil {
failCounter++
if settings.MaxFails > 0 && failCounter == settings.MaxFails {
return noNewCounter, failCounter, fmt.Errorf("reached the maximum number of consecutive failures: "+
"%d failed attempts resolving %s: %s", settings.MaxFails, host, err)
}
// it's fine to fail some of the resolutions
return noNewCounter, failCounter, nil
}
failCounter = 0 // reset the counter if we had no error
anyNew := false
for _, IP := range IPs {
key := IP.String()
if _, ok := uniqueIPs[key]; !ok {
anyNew = true
uniqueIPs[key] = struct{}{}
}
}
if !anyNew {
noNewCounter++
}
if settings.MaxNoNew > 0 && noNewCounter == settings.MaxNoNew {
// we reached the maximum number of resolutions without
// finding any new IP address to our unique IP addresses set.
return noNewCounter, failCounter,
fmt.Errorf("reached the maximum number of no new update: "+
"%d times no updated for %d IP addresses found",
noNewCounter, len(uniqueIPs))
}
timer := time.NewTimer(settings.BetweenDuration)
select {
case <-timer.C:
return noNewCounter, failCounter, nil
case <-ctx.Done():
if !timer.Stop() {
<-timer.C
}
return noNewCounter, failCounter, ctx.Err()
case <-timedCtx.Done():
if err := ctx.Err(); err != nil {
// timedCtx was canceled from its parent context
return noNewCounter, failCounter, err
}
return noNewCounter, failCounter,
fmt.Errorf("reached the timeout: %w", timedCtx.Err())
}
}
func (r *Repeat) lookupIPs(ctx context.Context, host string) (ips []netip.Addr, err error) {
addresses, err := r.resolver.LookupIPAddr(ctx, host)
if err != nil {
return nil, err
}
ips = make([]netip.Addr, 0, len(addresses))
for i := range addresses {
ip, ok := netip.AddrFromSlice(addresses[i].IP)
if !ok {
continue
}
ips = append(ips, ip)
}
return ips, nil
}
-37
View File
@@ -1,37 +0,0 @@
package unzip
import (
"archive/zip"
"bytes"
"io"
"path/filepath"
"strings"
)
func zipExtractAll(zipBytes []byte) (contents map[string][]byte, err error) {
r, err := zip.NewReader(bytes.NewReader(zipBytes), int64(len(zipBytes)))
if err != nil {
return nil, err
}
contents = map[string][]byte{}
for _, zf := range r.File {
fileName := filepath.Base(zf.Name)
if !strings.HasSuffix(fileName, ".ovpn") &&
!strings.HasSuffix(fileName, ".conf") {
continue
}
f, err := zf.Open()
if err != nil {
return nil, err
}
defer f.Close()
contents[fileName], err = io.ReadAll(f)
if err != nil {
return nil, err
}
if err := f.Close(); err != nil {
return nil, err
}
}
return contents, nil
}
-40
View File
@@ -1,40 +0,0 @@
package unzip
import (
"context"
"fmt"
"io"
"net/http"
)
func (u *Unzipper) FetchAndExtract(ctx context.Context, url string) (
contents map[string][]byte, err error,
) {
request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, err
}
request.Header.Set("User-Agent", "gluetun")
response, err := u.client.Do(request)
if err != nil {
return nil, err
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
return nil, fmt.Errorf("HTTP status code not OK: %s: %d %s",
url, response.StatusCode, response.Status)
}
b, err := io.ReadAll(response.Body)
if err != nil {
return nil, err
}
if err := response.Body.Close(); err != nil {
return nil, err
}
return zipExtractAll(b)
}
-15
View File
@@ -1,15 +0,0 @@
package unzip
import (
"net/http"
)
type Unzipper struct {
client *http.Client
}
func New(client *http.Client) *Unzipper {
return &Unzipper{
client: client,
}
}
+2 -2
View File
@@ -6,8 +6,8 @@ import (
"net/http"
"time"
"github.com/qdm12/gluetun/internal/provider/common"
"github.com/qdm12/gluetun/internal/updater/unzip"
"github.com/qdm12/gluetun/pkg/updaters/common"
"github.com/qdm12/gluetun/pkg/updaters/unzip"
"golang.org/x/text/cases"
"golang.org/x/text/language"
)