feat(storage): storage file structure changes (#3301)

- migrate persisted server data storage from `/gluetun/servers.json` to `/gluetun/servers/`
- add `STORAGE_SERVERS_ENABLED=on` to enable or disable on-disk server data storage
- add `STORAGE_SERVERS_DIRECTORY_PATH=/gluetun/servers` to configure where per-provider server files are stored
- keep backward compatibility with legacy `STORAGE_FILEPATH=/gluetun/servers.json`
- automatically read and migrate legacy `/gluetun/servers.json` into the new `/gluetun/servers/` layout when needed
- try to remove the legacy servers file after a successful migration to the new storage directory
- switch persisted server data from one large JSON file to a manifest plus per-provider JSON files
- add `UPDATER_PREFER_DIRECT_DOWNLOAD` to allow preferring direct download of provider server data
- keep deprecated updater flags `-enduser` and `-maintainer` as no-op warnings for backward compatibility
- preserve compatibility checks so persisted server data is discarded when its schema version no longer matches the built-in data
- allow preferred persisted provider data to override built-in data when versions match
- servers data now lives at https://github.com/qdm12/gluetun-servers/tree/main/pkg/servers

internal/storage/servers.go

@@ -2,6 +2,8 @@ package storage
 
 import (
 	"fmt"
+	"os"
+	"path/filepath"
 	"time"
 
 	"github.com/qdm12/gluetun/internal/constants/providers"
@@ -10,12 +12,12 @@ import (
 
 // SetServers sets the given servers for the given provider
 // in the storage in-memory map and saves all the servers
-// to file.
+// to files.
 // Note the servers given are not copied so the caller must
 // NOT MUTATE them after calling this method.
 func (s *Storage) SetServers(provider string, servers []models.Server) (err error) {
 	if provider == providers.Custom {
-		return
+		return nil
 	}
 
 	s.mergedMutex.Lock()
@@ -26,10 +28,24 @@ func (s *Storage) SetServers(provider string, servers []models.Server) (err erro
 	serversObject.Servers = servers
 	s.mergedServers.ProviderToServers[provider] = serversObject
 
-	err = s.flushToFile(s.filepath)
+	if !s.disk {
+		return nil
+	}
+
+	manifestPath := filepath.Join(s.directoryPath, manifestFilename)
+	err = s.flushToFile(manifestPath)
 	if err != nil {
 		return fmt.Errorf("saving servers to file: %w", err)
 	}
+
+	if !s.hasLegacy() {
+		return nil
+	}
+	s.logger.Infof("removing legacy %s which is now migrated to %s", s.legacyFilepath, s.directoryPath)
+	err = os.Remove(s.legacyFilepath)
+	if err != nil {
+		s.logger.Warn("failed removing legacy servers file " + s.legacyFilepath + ": " + err.Error())
+	}
 	return nil
 }