mirror of
https://github.com/qdm12/gluetun.git
synced 2026-07-24 11:26:25 +02:00
refactor(storage): new storage file structure
- new directory structure containing manifest.json and one json file per provider, by default. - the manifest.json file can specify a filepath for each vpn provider - each vpn provider json data file can contain the `"preferred": true` field to enforce it is used even if outdated, unless there is a version mismatch - `STORAGE_SERVERS_DIRECTORY_PATH` replaces `STORAGE_FILEPATH` (which is now a migration source only). It sets the directory where server manifest and per-provider JSON files are stored (default: `/gluetun/servers/`). - First-run migration: On startup, gluetun checks for the old /gluetun/servers.json file; if found and no new manifest exists, it automatically migrates all data to /gluetun/servers/ directory structure - Silent fallback: If legacy file isn't found, uses the new directory path normally - Legacy cleanup: After successful migration, attempts to remove the old fat JSON file (logs warning only if removal fails, e.g., read-only bind mounts)
This commit is contained in:
Quentin McGaw
@@ -0,0 +1,12 @@
|
||||
package html
|
||||
|
||||
import "golang.org/x/net/html"
|
||||
|
||||
func Attribute(node *html.Node, key string) (value string) {
|
||||
for _, attribute := range node.Attr {
|
||||
if attribute.Key == key {
|
||||
return attribute.Val
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
package html
|
||||
|
||||
import (
|
||||
"container/list"
|
||||
"fmt"
|
||||
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
// BFS returns the node matching the match function and nil
|
||||
// if no node is found.
|
||||
func BFS(rootNode *html.Node, match MatchFunc) (node *html.Node) {
|
||||
visited := make(map[*html.Node]struct{})
|
||||
queue := list.New()
|
||||
_ = queue.PushBack(rootNode)
|
||||
|
||||
for queue.Len() > 0 {
|
||||
listElement := queue.Front()
|
||||
node, ok := queue.Remove(listElement).(*html.Node)
|
||||
if !ok {
|
||||
panic(fmt.Sprintf("linked list has bad type %T", listElement.Value))
|
||||
}
|
||||
|
||||
if node == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
if _, ok := visited[node]; ok {
|
||||
continue
|
||||
}
|
||||
visited[node] = struct{}{}
|
||||
|
||||
if match(node) {
|
||||
return node
|
||||
}
|
||||
|
||||
for child := node.FirstChild; child != nil; child = child.NextSibling {
|
||||
_ = queue.PushBack(child)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
package html
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
func HasClassStrings(node *html.Node, classStrings ...string) (match bool) {
|
||||
targetClasses := make(map[string]struct{}, len(classStrings))
|
||||
for _, classString := range classStrings {
|
||||
targetClasses[classString] = struct{}{}
|
||||
}
|
||||
|
||||
classAttribute := Attribute(node, "class")
|
||||
classes := strings.Fields(classAttribute)
|
||||
for _, class := range classes {
|
||||
delete(targetClasses, class)
|
||||
}
|
||||
|
||||
return len(targetClasses) == 0
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
package html
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
func WrapError(sentinelError error, node *html.Node) error {
|
||||
return fmt.Errorf("%w: in HTML code: %s",
|
||||
sentinelError, mustRenderHTML(node))
|
||||
}
|
||||
|
||||
func WrapWarning(warning string, node *html.Node) string {
|
||||
return fmt.Sprintf("%s: in HTML code: %s",
|
||||
warning, mustRenderHTML(node))
|
||||
}
|
||||
|
||||
func mustRenderHTML(node *html.Node) (rendered string) {
|
||||
stringBuffer := bytes.NewBufferString("")
|
||||
err := html.Render(stringBuffer, node)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return stringBuffer.String()
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
package html
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
func Fetch(ctx context.Context, client *http.Client, url string) (
|
||||
rootNode *html.Node, err error,
|
||||
) {
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("creating HTTP request: %w", err)
|
||||
}
|
||||
|
||||
response, err := client.Do(request)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if response.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("HTTP status code not OK: %d %s",
|
||||
response.StatusCode, response.Status)
|
||||
}
|
||||
|
||||
rootNode, err = html.Parse(response.Body)
|
||||
if err != nil {
|
||||
_ = response.Body.Close()
|
||||
return nil, fmt.Errorf("parsing HTML code: %w", err)
|
||||
}
|
||||
|
||||
err = response.Body.Close()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("closing response body: %w", err)
|
||||
}
|
||||
|
||||
return rootNode, nil
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
package html
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
func parseTestHTML(t *testing.T, htmlString string) *html.Node {
|
||||
t.Helper()
|
||||
rootNode, err := html.Parse(strings.NewReader(htmlString))
|
||||
require.NoError(t, err)
|
||||
return rootNode
|
||||
}
|
||||
|
||||
type roundTripFunc func(r *http.Request) (*http.Response, error)
|
||||
|
||||
func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) {
|
||||
return f(r)
|
||||
}
|
||||
|
||||
func Test_Fetch(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
canceledCtx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
testCases := map[string]struct {
|
||||
ctx context.Context
|
||||
url string
|
||||
responseStatus int
|
||||
responseBody io.ReadCloser
|
||||
rootNode *html.Node
|
||||
errMessage string
|
||||
}{
|
||||
"context canceled": {
|
||||
ctx: canceledCtx,
|
||||
url: "https://example.com/path",
|
||||
errMessage: `Get "https://example.com/path": context canceled`,
|
||||
},
|
||||
"response status not ok": {
|
||||
ctx: context.Background(),
|
||||
url: "https://example.com/path",
|
||||
responseStatus: http.StatusNotFound,
|
||||
errMessage: `HTTP status code not OK: 404 Not Found`,
|
||||
},
|
||||
"success": {
|
||||
ctx: context.Background(),
|
||||
url: "https://example.com/path",
|
||||
responseStatus: http.StatusOK,
|
||||
rootNode: parseTestHTML(t, "some body"),
|
||||
responseBody: io.NopCloser(strings.NewReader("some body")),
|
||||
},
|
||||
}
|
||||
|
||||
for name, testCase := range testCases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
client := &http.Client{
|
||||
Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) {
|
||||
assert.Equal(t, http.MethodGet, r.Method)
|
||||
assert.Equal(t, r.URL.String(), testCase.url)
|
||||
|
||||
ctxErr := r.Context().Err()
|
||||
if ctxErr != nil {
|
||||
return nil, ctxErr
|
||||
}
|
||||
|
||||
return &http.Response{
|
||||
StatusCode: testCase.responseStatus,
|
||||
Status: http.StatusText(testCase.responseStatus),
|
||||
Body: testCase.responseBody,
|
||||
}, nil
|
||||
}),
|
||||
}
|
||||
|
||||
rootNode, err := Fetch(testCase.ctx, client, testCase.url)
|
||||
|
||||
if testCase.errMessage != "" {
|
||||
assert.EqualError(t, err, testCase.errMessage)
|
||||
} else {
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
assert.Equal(t, testCase.rootNode, rootNode)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package html
|
||||
|
||||
import (
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
type MatchFunc func(node *html.Node) (match bool)
|
||||
|
||||
func MatchID(id string) MatchFunc {
|
||||
return func(node *html.Node) (match bool) {
|
||||
if node == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
return Attribute(node, "id") == id
|
||||
}
|
||||
}
|
||||
|
||||
func MatchData(data string) MatchFunc {
|
||||
return func(node *html.Node) (match bool) {
|
||||
return node != nil && node.Type == html.ElementNode && node.Data == data
|
||||
}
|
||||
}
|
||||
|
||||
func DirectChild(parent *html.Node,
|
||||
matchFunc MatchFunc,
|
||||
) (child *html.Node) {
|
||||
for child := parent.FirstChild; child != nil; child = child.NextSibling {
|
||||
if matchFunc(child) {
|
||||
return child
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func DirectChildren(parent *html.Node,
|
||||
matchFunc MatchFunc,
|
||||
) (children []*html.Node) {
|
||||
for child := parent.FirstChild; child != nil; child = child.NextSibling {
|
||||
if matchFunc(child) {
|
||||
children = append(children, child)
|
||||
}
|
||||
}
|
||||
return children
|
||||
}
|
||||
Reference in New Issue
Block a user