refactor(storage): new storage file structure

- new directory structure containing manifest.json and one json file per provider, by default.
- the manifest.json file can specify a filepath for each vpn provider
- each vpn provider json data file can contain the `"preferred": true` field to enforce it is used even if outdated, unless there is a version mismatch
- `STORAGE_SERVERS_DIRECTORY_PATH` replaces `STORAGE_FILEPATH` (which is now a migration source only). It sets the directory where server manifest and per-provider JSON files are stored (default: `/gluetun/servers/`).
- First-run migration: On startup, gluetun checks for the old /gluetun/servers.json file; if found and no new manifest exists, it automatically migrates all data to /gluetun/servers/ directory structure
- Silent fallback: If legacy file isn't found, uses the new directory path normally
- Legacy cleanup: After successful migration, attempts to remove the old fat JSON file (logs warning only if removal fails, e.g., read-only bind mounts)

gluetun-servers/pkg/updaters/common/errors.go

@@ -0,0 +1,8 @@
+package common
+
+import "errors"
+
+var (
+	ErrNotEnoughServers   = errors.New("not enough servers found")
+	ErrCredentialsMissing = errors.New("credentials are missing")
+)

gluetun-servers/pkg/updaters/common/interfaces.go

@@ -0,0 +1,36 @@
+package common
+
+import (
+	"context"
+	"net/netip"
+
+	"github.com/qdm12/gluetun-servers/pkg/models"
+	pubipmodels "github.com/qdm12/gluetun-servers/pkg/publicip/models"
+	"github.com/qdm12/gluetun-servers/pkg/updaters/resolver"
+)
+
+type Fetcher interface {
+	FetchServers(ctx context.Context, minServers int) (servers []models.Server, err error)
+	Version() uint16
+	Name() string
+}
+
+type ParallelResolver interface {
+	Resolve(ctx context.Context, settings resolver.ParallelSettings) (
+		hostToIPs map[string][]netip.Addr, warnings []string, err error)
+}
+
+type Unzipper interface {
+	FetchAndExtract(ctx context.Context, url string) (
+		contents map[string][]byte, err error)
+}
+
+type Warner interface {
+	Warn(s string)
+}
+
+type IPFetcher interface {
+	String() string
+	CanFetchAnyIP() bool
+	FetchInfo(ctx context.Context, ip netip.Addr) (result pubipmodels.PublicIP, err error)
+}

gluetun-servers/pkg/updaters/common/mocks.go

@@ -0,0 +1,126 @@
+// Code generated by MockGen. DO NOT EDIT.
+// Source: github.com/qdm12/gluetun-servers/pkg/updaters/common (interfaces: ParallelResolver,Unzipper,Warner)
+
+// Package common is a generated GoMock package.
+package common
+
+import (
+	context "context"
+	netip "net/netip"
+	reflect "reflect"
+
+	gomock "github.com/golang/mock/gomock"
+	resolver "github.com/qdm12/gluetun-servers/pkg/updaters/resolver"
+)
+
+// MockParallelResolver is a mock of ParallelResolver interface.
+type MockParallelResolver struct {
+	ctrl     *gomock.Controller
+	recorder *MockParallelResolverMockRecorder
+}
+
+// MockParallelResolverMockRecorder is the mock recorder for MockParallelResolver.
+type MockParallelResolverMockRecorder struct {
+	mock *MockParallelResolver
+}
+
+// NewMockParallelResolver creates a new mock instance.
+func NewMockParallelResolver(ctrl *gomock.Controller) *MockParallelResolver {
+	mock := &MockParallelResolver{ctrl: ctrl}
+	mock.recorder = &MockParallelResolverMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockParallelResolver) EXPECT() *MockParallelResolverMockRecorder {
+	return m.recorder
+}
+
+// Resolve mocks base method.
+func (m *MockParallelResolver) Resolve(arg0 context.Context, arg1 resolver.ParallelSettings) (map[string][]netip.Addr, []string, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Resolve", arg0, arg1)
+	ret0, _ := ret[0].(map[string][]netip.Addr)
+	ret1, _ := ret[1].([]string)
+	ret2, _ := ret[2].(error)
+	return ret0, ret1, ret2
+}
+
+// Resolve indicates an expected call of Resolve.
+func (mr *MockParallelResolverMockRecorder) Resolve(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Resolve", reflect.TypeOf((*MockParallelResolver)(nil).Resolve), arg0, arg1)
+}
+
+// MockUnzipper is a mock of Unzipper interface.
+type MockUnzipper struct {
+	ctrl     *gomock.Controller
+	recorder *MockUnzipperMockRecorder
+}
+
+// MockUnzipperMockRecorder is the mock recorder for MockUnzipper.
+type MockUnzipperMockRecorder struct {
+	mock *MockUnzipper
+}
+
+// NewMockUnzipper creates a new mock instance.
+func NewMockUnzipper(ctrl *gomock.Controller) *MockUnzipper {
+	mock := &MockUnzipper{ctrl: ctrl}
+	mock.recorder = &MockUnzipperMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockUnzipper) EXPECT() *MockUnzipperMockRecorder {
+	return m.recorder
+}
+
+// FetchAndExtract mocks base method.
+func (m *MockUnzipper) FetchAndExtract(arg0 context.Context, arg1 string) (map[string][]byte, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "FetchAndExtract", arg0, arg1)
+	ret0, _ := ret[0].(map[string][]byte)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// FetchAndExtract indicates an expected call of FetchAndExtract.
+func (mr *MockUnzipperMockRecorder) FetchAndExtract(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FetchAndExtract", reflect.TypeOf((*MockUnzipper)(nil).FetchAndExtract), arg0, arg1)
+}
+
+// MockWarner is a mock of Warner interface.
+type MockWarner struct {
+	ctrl     *gomock.Controller
+	recorder *MockWarnerMockRecorder
+}
+
+// MockWarnerMockRecorder is the mock recorder for MockWarner.
+type MockWarnerMockRecorder struct {
+	mock *MockWarner
+}
+
+// NewMockWarner creates a new mock instance.
+func NewMockWarner(ctrl *gomock.Controller) *MockWarner {
+	mock := &MockWarner{ctrl: ctrl}
+	mock.recorder = &MockWarnerMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockWarner) EXPECT() *MockWarnerMockRecorder {
+	return m.recorder
+}
+
+// Warn mocks base method.
+func (m *MockWarner) Warn(arg0 string) {
+	m.ctrl.T.Helper()
+	m.ctrl.Call(m, "Warn", arg0)
+}
+
+// Warn indicates an expected call of Warn.
+func (mr *MockWarnerMockRecorder) Warn(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Warn", reflect.TypeOf((*MockWarner)(nil).Warn), arg0)
+}

gluetun-servers/pkg/updaters/common/mocks_generate_test.go

@@ -0,0 +1,6 @@
+package common
+
+// Exceptionally, these mocks are exported since they are used by all
+// provider subpackages tests, and it reduces test code duplication a lot.
+// Note mocks.go might need to be removed before re-generating it.
+//go:generate mockgen -destination=mocks.go -package $GOPACKAGE . ParallelResolver,Unzipper,Warner