From 02ef79dcb29ca7a4a962bf7a626f8a41789017f6 Mon Sep 17 00:00:00 2001 From: jeffvli Date: Wed, 12 Oct 2022 13:39:59 -0700 Subject: [PATCH] Optimize jellyfin scanner - Include changes / unfinished subsonic scanner --- src/server/queue/index.ts | 7 +- src/server/queue/jellyfin/index.ts | 7 + src/server/queue/jellyfin/jellyfin-tasks.ts | 631 ------------------ .../{jellyfin-api.ts => jellyfin.api.ts} | 37 +- src/server/queue/jellyfin/jellyfin.scanner.ts | 453 +++++++++++++ .../{jellyfin-types.ts => jellyfin.types.ts} | 161 ++++- src/server/queue/jellyfin/jellyfin.utils.ts | 278 ++++++++ src/server/queue/queues/index.ts | 5 + src/server/queue/queues/scanner.queue.ts | 57 ++ src/server/queue/scanner-queue.ts | 74 -- src/server/queue/subsonic/index.ts | 7 + .../{subsonic-api.ts => subsonic.api.ts} | 32 +- ...{subsonic-tasks.ts => subsonic.scanner.ts} | 364 ++++------ .../{subsonic-types.ts => subsonic.types.ts} | 2 +- 14 files changed, 1161 insertions(+), 954 deletions(-) create mode 100644 src/server/queue/jellyfin/index.ts delete mode 100644 src/server/queue/jellyfin/jellyfin-tasks.ts rename src/server/queue/jellyfin/{jellyfin-api.ts => jellyfin.api.ts} (69%) create mode 100644 src/server/queue/jellyfin/jellyfin.scanner.ts rename src/server/queue/jellyfin/{jellyfin-types.ts => jellyfin.types.ts} (58%) create mode 100644 src/server/queue/jellyfin/jellyfin.utils.ts create mode 100644 src/server/queue/queues/index.ts create mode 100644 src/server/queue/queues/scanner.queue.ts delete mode 100644 src/server/queue/scanner-queue.ts create mode 100644 src/server/queue/subsonic/index.ts rename src/server/queue/subsonic/{subsonic-api.ts => subsonic.api.ts} (81%) rename src/server/queue/subsonic/{subsonic-tasks.ts => subsonic.scanner.ts} (50%) rename src/server/queue/subsonic/{subsonic-types.ts => subsonic.types.ts} (98%) diff --git a/src/server/queue/index.ts b/src/server/queue/index.ts index 0e16dd6fa..d9083af73 100644 --- a/src/server/queue/index.ts +++ b/src/server/queue/index.ts @@ -1,5 +1,2 @@ -export * from './subsonic/subsonic-api'; -export * from './subsonic/subsonic-tasks'; -export * from './jellyfin/jellyfin-api'; -export * from './jellyfin/jellyfin-tasks'; -export * from './scanner-queue'; +export * from './subsonic'; +export * from './jellyfin'; diff --git a/src/server/queue/jellyfin/index.ts b/src/server/queue/jellyfin/index.ts new file mode 100644 index 000000000..11d5d8330 --- /dev/null +++ b/src/server/queue/jellyfin/index.ts @@ -0,0 +1,7 @@ +import { jellyfinApi } from './jellyfin.api'; +import { jellyfinScanner } from './jellyfin.scanner'; + +export const jellyfin = { + api: jellyfinApi, + scanner: jellyfinScanner, +}; diff --git a/src/server/queue/jellyfin/jellyfin-tasks.ts b/src/server/queue/jellyfin/jellyfin-tasks.ts deleted file mode 100644 index d9381ff3f..000000000 --- a/src/server/queue/jellyfin/jellyfin-tasks.ts +++ /dev/null @@ -1,631 +0,0 @@ -import uniqBy from 'lodash/uniqBy'; -import { prisma } from '../../lib'; -import { Server, ServerFolder, Task } from '../../types/types'; -import { groupByProperty, uniqueArray } from '../../utils'; -import { completeTask, q } from '../scanner-queue'; -import { jellyfinApi } from './jellyfin-api'; -import { JFSong } from './jellyfin-types'; - -const scanGenres = async ( - server: Server, - serverFolder: ServerFolder, - task: Task -) => { - const taskId = `[${server.name} (${serverFolder.name})] genres`; - - q.push({ - fn: async () => { - await prisma.task.update({ - data: { message: 'Scanning genres' }, - where: { id: task.id }, - }); - - const genres = await jellyfinApi.getGenres(server, { - parentId: serverFolder.remoteId, - }); - - const genresCreate = genres.Items.map((genre) => { - return { name: genre.Name }; - }); - - await prisma.genre.createMany({ - data: genresCreate, - skipDuplicates: true, - }); - - return { task }; - }, - id: taskId, - }); -}; - -const scanAlbumArtists = async ( - server: Server, - serverFolder: ServerFolder, - task: Task -) => { - const taskId = `[${server.name} (${serverFolder.name})] album artists`; - - q.push({ - fn: async () => { - await prisma.task.update({ - data: { message: 'Scanning album artists' }, - where: { id: task.id }, - }); - const albumArtists = await jellyfinApi.getAlbumArtists(server, { - fields: 'Genres,DateCreated,ExternalUrls,Overview', - parentId: serverFolder.remoteId, - }); - - for (const albumArtist of albumArtists.Items) { - const genresConnectOrCreate = albumArtist.Genres.map((genre) => { - return { create: { name: genre }, where: { name: genre } }; - }); - - const imagesConnectOrCreate = []; - for (const [key, value] of Object.entries(albumArtist.ImageTags)) { - imagesConnectOrCreate.push({ - create: { name: key, url: value }, - where: { uniqueImageId: { name: key, url: value } }, - }); - } - - const externalsConnectOrCreate = albumArtist.ExternalUrls.map( - (external) => { - return { - create: { name: external.Name, url: external.Url }, - where: { - uniqueExternalId: { name: external.Name, url: external.Url }, - }, - }; - } - ); - - await prisma.albumArtist.upsert({ - create: { - biography: albumArtist.Overview, - externals: { connectOrCreate: externalsConnectOrCreate }, - genres: { connectOrCreate: genresConnectOrCreate }, - images: { connectOrCreate: imagesConnectOrCreate }, - name: albumArtist.Name, - remoteCreatedAt: albumArtist.DateCreated, - remoteId: albumArtist.Id, - serverFolders: { connect: { id: serverFolder.id } }, - serverId: server.id, - }, - update: { - biography: albumArtist.Overview, - deleted: false, - externals: { connectOrCreate: externalsConnectOrCreate }, - genres: { connectOrCreate: genresConnectOrCreate }, - images: { connectOrCreate: imagesConnectOrCreate }, - name: albumArtist.Name, - remoteCreatedAt: albumArtist.DateCreated, - remoteId: albumArtist.Id, - serverFolders: { connect: { id: serverFolder.id } }, - serverId: server.id, - }, - where: { - uniqueAlbumArtistId: { - remoteId: albumArtist.Id, - serverId: server.id, - }, - }, - }); - } - - return { task }; - }, - id: taskId, - }); -}; - -const scanAlbums = async ( - server: Server, - serverFolder: ServerFolder, - task: Task -) => { - const check = await jellyfinApi.getAlbums(server, { - enableUserData: false, - includeItemTypes: 'MusicAlbum', - limit: 1, - parentId: serverFolder.remoteId, - recursive: true, - }); - - const albumCount = check.TotalRecordCount; - const chunkSize = 5000; - const albumChunkCount = Math.ceil(albumCount / chunkSize); - - const taskId = `(${task.id}) [${server.name} (${serverFolder.name})] albums`; - - q.push({ - fn: async () => { - await prisma.task.update({ - data: { message: 'Scanning albums' }, - where: { id: task.id }, - }); - - for (let i = 0; i < albumChunkCount; i += 1) { - const albums = await jellyfinApi.getAlbums(server, { - enableImageTypes: 'Primary,Logo,Backdrop', - enableUserData: false, - fields: 'Genres,DateCreated,ExternalUrls,Overview', - imageTypeLimit: 1, - limit: chunkSize, - parentId: serverFolder.remoteId, - recursive: true, - startIndex: i * chunkSize, - }); - - for (const album of albums.Items) { - const genresConnectOrCreate = album.Genres.map((genre) => { - return { create: { name: genre }, where: { name: genre } }; - }); - - const imagesConnectOrCreate = []; - for (const [key, value] of Object.entries(album.ImageTags)) { - imagesConnectOrCreate.push({ - create: { name: key, url: value }, - where: { uniqueImageId: { name: key, url: value } }, - }); - } - - const externalsConnectOrCreate = album.ExternalUrls.map( - (external) => { - return { - create: { name: external.Name, url: external.Url }, - where: { - uniqueExternalId: { name: external.Name, url: external.Url }, - }, - }; - } - ); - - const albumArtist = - album.AlbumArtists.length > 0 - ? await prisma.albumArtist.findUnique({ - where: { - uniqueAlbumArtistId: { - remoteId: album.AlbumArtists && album.AlbumArtists[0].Id, - serverId: server.id, - }, - }, - }) - : undefined; - - await prisma.album.upsert({ - create: { - albumArtistId: albumArtist?.id, - date: album.PremiereDate, - externals: { connectOrCreate: externalsConnectOrCreate }, - genres: { connectOrCreate: genresConnectOrCreate }, - images: { connectOrCreate: imagesConnectOrCreate }, - name: album.Name, - remoteCreatedAt: album.DateCreated, - remoteId: album.Id, - serverFolders: { connect: { id: serverFolder.id } }, - serverId: server.id, - year: album.ProductionYear, - }, - update: { - albumArtistId: albumArtist?.id, - date: album.PremiereDate, - deleted: false, - externals: { connectOrCreate: externalsConnectOrCreate }, - genres: { connectOrCreate: genresConnectOrCreate }, - images: { connectOrCreate: imagesConnectOrCreate }, - name: album.Name, - remoteCreatedAt: album.DateCreated, - remoteId: album.Id, - serverFolders: { connect: { id: serverFolder.id } }, - serverId: server.id, - year: album.ProductionYear, - }, - where: { - uniqueAlbumId: { - remoteId: album.Id, - serverId: server.id, - }, - }, - }); - } - - const currentTask = await prisma.task.findUnique({ - where: { id: task.id }, - }); - - const newCount = - Number(currentTask?.progress || 0) + Number(albums.Items.length); - - await prisma.task.update({ - data: { progress: String(newCount) }, - where: { id: task.id }, - }); - } - - return { task }; - }, - id: taskId, - }); -}; - -const insertSongGroup = async ( - server: Server, - serverFolder: ServerFolder, - songs: JFSong[], - remoteAlbumId: string -) => { - const songsUpsert = songs.map((song) => { - const artistsConnectOrCreate = song.ArtistItems.map((artist) => { - return { - create: { - name: artist.Name, - remoteId: artist.Id, - serverFolders: { connect: { id: serverFolder.id } }, - serverId: server.id, - }, - where: { - uniqueArtistId: { - remoteId: artist.Id, - serverId: server.id, - }, - }, - }; - }); - - const genresConnectOrCreate = song.Genres.map((genre) => { - return { create: { name: genre }, where: { name: genre } }; - }); - - const imagesConnectOrCreate = []; - for (const [key, value] of Object.entries(song.ImageTags)) { - imagesConnectOrCreate.push({ - create: { name: key, url: value }, - where: { uniqueImageId: { name: key, url: value } }, - }); - } - - const externalsConnectOrCreate = song.ExternalUrls.map((external) => { - return { - create: { name: external.Name, url: external.Url }, - where: { - uniqueExternalId: { name: external.Name, url: external.Url }, - }, - }; - }); - - const pathSplit = song.MediaSources[0].Path.split('/'); - const parentPath = pathSplit.slice(0, pathSplit.length - 1).join('/'); - - return { - create: { - artists: { connectOrCreate: artistsConnectOrCreate }, - bitRate: Math.floor(song.MediaSources[0].Bitrate / 1000), - container: song.MediaSources[0].Container, - date: song.PremiereDate, - disc: song.ParentIndexNumber, - duration: Math.floor(song.MediaSources[0].RunTimeTicks / 1e7), - externals: { connectOrCreate: externalsConnectOrCreate }, - folders: { - connect: { - uniqueFolderId: { path: parentPath, serverId: server.id }, - }, - }, - genres: { connectOrCreate: genresConnectOrCreate }, - images: { connectOrCreate: imagesConnectOrCreate }, - name: song.Name, - remoteCreatedAt: song.DateCreated, - remoteId: song.Id, - serverFolders: { connect: { id: serverFolder.id } }, - serverId: server.id, - track: song.IndexNumber, - year: song.ProductionYear, - }, - update: { - artists: { connectOrCreate: artistsConnectOrCreate }, - bitRate: Math.floor(song.MediaSources[0].Bitrate / 1000), - container: song.MediaSources[0].Container, - date: song.PremiereDate, - disc: song.ParentIndexNumber, - duration: Math.floor(song.MediaSources[0].RunTimeTicks / 1e7), - externals: { connectOrCreate: externalsConnectOrCreate }, - folders: { - connect: { - uniqueFolderId: { path: parentPath, serverId: server.id }, - }, - }, - genres: { connectOrCreate: genresConnectOrCreate }, - images: { connectOrCreate: imagesConnectOrCreate }, - name: song.Name, - remoteCreatedAt: song.DateCreated, - remoteId: song.Id, - serverFolders: { connect: { id: serverFolder.id } }, - serverId: server.id, - track: song.IndexNumber, - year: song.ProductionYear, - }, - where: { - uniqueSongId: { - remoteId: song.Id, - serverId: server.id, - }, - }, - }; - }); - - const artists = uniqBy( - songs.flatMap((song) => { - return song.ArtistItems.map((artist) => ({ - name: artist.Name, - remoteId: artist.Id, - serverFolders: { connect: { id: serverFolder.id } }, - serverId: server.id, - })); - }), - 'remoteId' - ); - - const uniqueArtistIds = songs - .flatMap((song) => { - return song.ArtistItems.flatMap((artist) => artist.Id); - }) - .filter(uniqueArray); - - const artistsConnect = uniqueArtistIds.map((artistId) => { - return { - uniqueArtistId: { - remoteId: artistId!, - serverId: server.id, - }, - }; - }); - - artists.forEach(async (artist) => { - await prisma.artist.upsert({ - create: artist, - update: artist, - where: { - uniqueArtistId: { - remoteId: artist.remoteId, - serverId: server.id, - }, - }, - }); - }); - - await prisma.$transaction([ - prisma.artist.updateMany({ - data: { deleted: false }, - where: { - remoteId: { in: uniqueArtistIds }, - serverId: server.id, - }, - }), - prisma.album.update({ - data: { - artists: { connect: artistsConnect }, - deleted: false, - songs: { upsert: songsUpsert }, - }, - where: { - uniqueAlbumId: { - remoteId: remoteAlbumId, - serverId: server.id, - }, - }, - }), - ]); -}; - -const scanSongs = async ( - server: Server, - serverFolder: ServerFolder, - task: Task -) => { - const check = await jellyfinApi.getSongs(server, { - enableUserData: false, - limit: 0, - parentId: serverFolder.remoteId, - recursive: true, - }); - - // Fetch in chunks - const songCount = check.TotalRecordCount; - const chunkSize = 10000; - const songChunkCount = Math.ceil(songCount / chunkSize); - const taskId = `[${server.name} (${serverFolder.name})] songs`; - - q.push({ - fn: async () => { - await prisma.task.update({ - data: { message: 'Scanning songs' }, - where: { id: task.id }, - }); - - for (let i = 0; i < songChunkCount; i += 1) { - const songs = await jellyfinApi.getSongs(server, { - enableImageTypes: 'Primary,Logo,Backdrop', - enableUserData: false, - fields: 'Genres,DateCreated,ExternalUrls,MediaSources', - imageTypeLimit: 1, - limit: chunkSize, - parentId: serverFolder.remoteId, - recursive: true, - sortBy: 'DateCreated,Album', - sortOrder: 'Descending', - startIndex: i * chunkSize, - }); - - const folderGroups = songs.Items.map((song) => { - const songPaths = song.MediaSources[0].Path.split('/'); - const paths = []; - for (let b = 0; b < songPaths.length - 1; b += 1) { - paths.push({ - name: songPaths[b], - path: songPaths.slice(0, b + 1).join('/'), - }); - } - - return paths; - }); - - const uniqueFolders = uniqBy( - folderGroups.flatMap((folder) => folder).filter((f) => f.path !== ''), - 'path' - ); - - const folderSave: any[] = []; - - for (let f = 0; f < uniqueFolders.length; f += 1) { - const t = await prisma.folder.upsert({ - create: { - name: uniqueFolders[f].name, - path: uniqueFolders[f].path, - serverFolders: { - connect: { - uniqueServerFolderId: { - remoteId: serverFolder.remoteId, - serverId: server.id, - }, - }, - }, - serverId: server.id, - }, - update: { - name: uniqueFolders[f].name, - path: uniqueFolders[f].path, - serverFolders: { - connect: { - uniqueServerFolderId: { - remoteId: serverFolder.remoteId, - serverId: server.id, - }, - }, - }, - }, - where: { - uniqueFolderId: { - path: uniqueFolders[f].path, - serverId: server.id, - }, - }, - }); - - folderSave.push(t); - } - - folderSave.forEach(async (f) => { - if (f.parentId) return; - - const pathSplit = f.path.split('/'); - const parentPath = pathSplit.slice(0, pathSplit.length - 1).join('/'); - - const parentPathData = folderSave.find( - (save) => save.path === parentPath - ); - - if (parentPathData) { - await prisma.folder.update({ - data: { - parentId: parentPathData.id, - }, - where: { id: f.id }, - }); - } - }); - - const albumSongGroups = groupByProperty(songs.Items, 'AlbumId'); - - const keys = Object.keys(albumSongGroups); - - for (let b = 0; b < keys.length; b += 1) { - const songGroup = albumSongGroups[keys[b]]; - await insertSongGroup(server, serverFolder, songGroup, keys[b]); - - const currentTask = await prisma.task.findUnique({ - where: { id: task.id }, - }); - - const newCount = - Number(currentTask?.progress || 0) + Number(songGroup.length); - - await prisma.task.update({ - data: { progress: String(newCount) }, - where: { id: task.id }, - }); - } - } - - return { completed: true, task }; - }, - id: taskId, - }); -}; - -const checkDeleted = async ( - server: Server, - serverFolder: ServerFolder, - task: Task -) => { - q.push({ - fn: async () => { - await prisma.$transaction([ - prisma.albumArtist.updateMany({ - data: { deleted: true }, - where: { - serverFolders: { some: { id: serverFolder.id } }, - serverId: server.id, - updatedAt: { lte: task.createdAt }, - }, - }), - prisma.artist.updateMany({ - data: { deleted: true }, - where: { - serverFolders: { some: { id: serverFolder.id } }, - serverId: server.id, - updatedAt: { lte: task.createdAt }, - }, - }), - prisma.album.updateMany({ - data: { deleted: true }, - where: { - serverFolders: { some: { id: serverFolder.id } }, - serverId: server.id, - updatedAt: { lte: task.createdAt }, - }, - }), - prisma.song.updateMany({ - data: { deleted: true }, - where: { - serverFolders: { some: { id: serverFolder.id } }, - serverId: server.id, - updatedAt: { lte: task.createdAt }, - }, - }), - ]); - }, - id: `${task.id}-difference`, - }); -}; - -const scanAll = async ( - server: Server, - serverFolder: ServerFolder, - task: Task -) => { - await scanGenres(server, serverFolder, task); - await scanAlbumArtists(server, serverFolder, task); - await scanAlbums(server, serverFolder, task); - await scanSongs(server, serverFolder, task); - await checkDeleted(server, serverFolder, task); - await completeTask(task); -}; - -export const jellyfinTasks = { - scanAlbumArtists, - scanAlbums, - scanAll, - scanGenres, - scanSongs, -}; diff --git a/src/server/queue/jellyfin/jellyfin-api.ts b/src/server/queue/jellyfin/jellyfin.api.ts similarity index 69% rename from src/server/queue/jellyfin/jellyfin-api.ts rename to src/server/queue/jellyfin/jellyfin.api.ts index 268dc7e5c..fb92d50dc 100644 --- a/src/server/queue/jellyfin/jellyfin-api.ts +++ b/src/server/queue/jellyfin/jellyfin.api.ts @@ -1,17 +1,43 @@ +import { Server } from '@prisma/client'; import axios from 'axios'; -import { Server } from '../../types/types'; import { JFAlbumArtistsResponse, JFAlbumsResponse, JFArtistsResponse, + JFAuthenticate, + JFCollectionType, JFGenreResponse, + JFItemType, JFMusicFoldersResponse, JFRequestParams, JFSongsResponse, -} from './jellyfin-types'; +} from './jellyfin.types'; export const api = axios.create({}); +export const authenticate = async (options: { + password: string; + url: string; + username: string; +}) => { + const { password, url, username } = options; + const cleanServerUrl = url.replace(/\/$/, ''); + + console.log('cleanServerUrl', cleanServerUrl); + + const { data } = await api.post( + `${cleanServerUrl}/users/authenticatebyname`, + { pw: password, username }, + { + headers: { + 'X-Emby-Authorization': `MediaBrowser Client="Sonixd", Device="PC", DeviceId="Sonixd", Version="1.0.0-alpha1"`, + }, + } + ); + + return data; +}; + export const getMusicFolders = async (server: Partial) => { const { data } = await api.get( `${server.url}/users/${server.remoteUserId}/items`, @@ -19,7 +45,7 @@ export const getMusicFolders = async (server: Partial) => { ); const musicFolders = data.Items.filter( - (folder) => folder.CollectionType === 'music' + (folder) => folder.CollectionType === JFCollectionType.MUSIC ); return musicFolders; @@ -63,7 +89,7 @@ export const getAlbums = async (server: Server, params: JFRequestParams) => { `${server.url}/users/${server.remoteUserId}/items`, { headers: { 'X-MediaBrowser-Token': server.token }, - params: { includeItemTypes: 'MusicAlbum', ...params }, + params: { includeItemTypes: JFItemType.MUSICALBUM, ...params }, } ); @@ -75,7 +101,7 @@ export const getSongs = async (server: Server, params: JFRequestParams) => { `${server.url}/users/${server.remoteUserId}/items`, { headers: { 'X-MediaBrowser-Token': server.token }, - params: { includeItemTypes: 'Audio', ...params }, + params: { includeItemTypes: JFItemType.AUDIO, ...params }, } ); @@ -83,6 +109,7 @@ export const getSongs = async (server: Server, params: JFRequestParams) => { }; export const jellyfinApi = { + authenticate, getAlbumArtists, getAlbums, getArtists, diff --git a/src/server/queue/jellyfin/jellyfin.scanner.ts b/src/server/queue/jellyfin/jellyfin.scanner.ts new file mode 100644 index 000000000..fca8ca695 --- /dev/null +++ b/src/server/queue/jellyfin/jellyfin.scanner.ts @@ -0,0 +1,453 @@ +import { + ExternalSource, + Folder, + ImageType, + Server, + ServerFolder, + Task, +} from '@prisma/client'; +import uniqBy from 'lodash/uniqBy'; +import { prisma } from '../../lib'; +import { groupByProperty } from '../../utils'; +import { queue } from '../queues'; +import { jellyfinApi } from './jellyfin.api'; +import { JFExternalType, JFImageType, JFItemType } from './jellyfin.types'; +import { jellyfinUtils } from './jellyfin.utils'; + +const scanGenres = async (options: { + server: Server; + serverFolder: ServerFolder; + task: Task; +}) => { + await prisma.task.update({ + data: { message: 'Scanning genres' }, + where: { id: options.task.id }, + }); + + const genres = await jellyfinApi.getGenres(options.server, { + parentId: options.serverFolder.remoteId, + }); + + const genresCreate = genres.Items.map((genre) => { + return { name: genre.Name }; + }); + + await prisma.genre.createMany({ + data: genresCreate, + skipDuplicates: true, + }); +}; + +const scanAlbumArtists = async ( + server: Server, + serverFolder: ServerFolder, + task: Task +) => { + await prisma.task.update({ + data: { message: 'Scanning album artists' }, + where: { id: task.id }, + }); + + const albumArtists = await jellyfinApi.getAlbumArtists(server, { + fields: 'Genres,DateCreated,ExternalUrls,Overview', + parentId: serverFolder.remoteId, + }); + + await jellyfinUtils.insertGenres(albumArtists.Items); + await jellyfinUtils.insertImages(albumArtists.Items); + await jellyfinUtils.insertExternals(albumArtists.Items); + + for (const albumArtist of albumArtists.Items) { + const genresConnect = albumArtist.Genres.map((genre) => ({ name: genre })); + + const imagesConnectOrCreate = []; + for (const backdrop of albumArtist.BackdropImageTags) { + imagesConnectOrCreate.push({ + create: { remoteUrl: backdrop, type: ImageType.BACKDROP }, + where: { + uniqueImageId: { remoteUrl: backdrop, type: ImageType.BACKDROP }, + }, + }); + } + + const imagesConnect = []; + for (const [key, value] of Object.entries(albumArtist.ImageTags)) { + if (key === JFImageType.PRIMARY) { + imagesConnect.push({ + uniqueImageId: { remoteUrl: value, type: ImageType.PRIMARY }, + }); + } + if (key === JFImageType.LOGO) { + imagesConnect.push({ + uniqueImageId: { remoteUrl: value, type: ImageType.LOGO }, + }); + } + } + + const externalsConnect = albumArtist.ExternalUrls.map((external) => ({ + uniqueExternalId: { + source: + external.Name === JFExternalType.MUSICBRAINZ + ? ExternalSource.MUSICBRAINZ + : ExternalSource.THEAUDIODB, + value: external.Url.split('/').pop() || '', + }, + })); + + await prisma.albumArtist.upsert({ + create: { + biography: albumArtist.Overview, + externals: { connect: externalsConnect }, + genres: { connect: genresConnect }, + images: { + connect: imagesConnect, + connectOrCreate: imagesConnectOrCreate, + }, + name: albumArtist.Name, + remoteCreatedAt: albumArtist.DateCreated, + remoteId: albumArtist.Id, + serverFolders: { connect: { id: serverFolder.id } }, + serverId: server.id, + sortName: albumArtist.Name, + }, + update: { + biography: albumArtist.Overview, + deleted: false, + externals: { connect: externalsConnect }, + genres: { connect: genresConnect }, + images: { connectOrCreate: imagesConnectOrCreate }, + name: albumArtist.Name, + remoteCreatedAt: albumArtist.DateCreated, + remoteId: albumArtist.Id, + serverFolders: { connect: { id: serverFolder.id } }, + serverId: server.id, + sortName: albumArtist.Name, + }, + where: { + uniqueAlbumArtistId: { + remoteId: albumArtist.Id, + serverId: server.id, + }, + }, + }); + } +}; + +const scanAlbums = async ( + server: Server, + serverFolder: ServerFolder, + task: Task +) => { + const check = await jellyfinApi.getAlbums(server, { + enableUserData: false, + includeItemTypes: JFItemType.MUSICALBUM, + limit: 1, + parentId: serverFolder.remoteId, + recursive: true, + }); + + const albumCount = check.TotalRecordCount; + const chunkSize = 5000; + const albumChunkCount = Math.ceil(albumCount / chunkSize); + + await prisma.task.update({ + data: { message: 'Scanning albums' }, + where: { id: task.id }, + }); + + for (let i = 0; i < albumChunkCount; i += 1) { + const albums = await jellyfinApi.getAlbums(server, { + enableImageTypes: 'Primary,Logo,Backdrop', + enableUserData: false, + fields: 'Genres,DateCreated,ExternalUrls,Overview', + imageTypeLimit: 1, + limit: chunkSize, + parentId: serverFolder.remoteId, + recursive: true, + startIndex: i * chunkSize, + }); + + await jellyfinUtils.insertGenres(albums.Items); + await jellyfinUtils.insertImages(albums.Items); + await jellyfinUtils.insertExternals(albums.Items); + + for (const album of albums.Items) { + const genresConnect = album.Genres.map((genre) => ({ name: genre })); + + const imagesConnect = []; + for (const [key, value] of Object.entries(album.ImageTags)) { + if (key === JFImageType.PRIMARY) { + imagesConnect.push({ + uniqueImageId: { remoteUrl: value, type: ImageType.PRIMARY }, + }); + } + if (key === JFImageType.LOGO) { + imagesConnect.push({ + uniqueImageId: { remoteUrl: value, type: ImageType.LOGO }, + }); + } + } + + const externalsConnect = album.ExternalUrls.map((external) => ({ + uniqueExternalId: { + source: + external.Name === JFExternalType.MUSICBRAINZ + ? ExternalSource.MUSICBRAINZ + : ExternalSource.THEAUDIODB, + value: external.Url.split('/').pop() || '', + }, + })); + + const albumArtist = + album.AlbumArtists.length > 0 + ? await prisma.albumArtist.findUnique({ + where: { + uniqueAlbumArtistId: { + remoteId: album.AlbumArtists && album.AlbumArtists[0].Id, + serverId: server.id, + }, + }, + }) + : undefined; + + await prisma.album.upsert({ + create: { + albumArtistId: albumArtist?.id, + externals: { connect: externalsConnect }, + genres: { connect: genresConnect }, + images: { connect: imagesConnect }, + name: album.Name, + releaseDate: album.PremiereDate, + releaseYear: album.ProductionYear, + remoteCreatedAt: album.DateCreated, + remoteId: album.Id, + serverFolders: { connect: { id: serverFolder.id } }, + serverId: server.id, + sortName: album.Name, + }, + update: { + albumArtistId: albumArtist?.id, + deleted: false, + externals: { connect: externalsConnect }, + genres: { connect: genresConnect }, + images: { connect: imagesConnect }, + name: album.Name, + releaseDate: album.PremiereDate, + releaseYear: album.ProductionYear, + remoteCreatedAt: album.DateCreated, + remoteId: album.Id, + serverFolders: { connect: { id: serverFolder.id } }, + serverId: server.id, + sortName: album.Name, + }, + where: { + uniqueAlbumId: { + remoteId: album.Id, + serverId: server.id, + }, + }, + }); + } + } +}; + +const scanSongs = async ( + server: Server, + serverFolder: ServerFolder, + task: Task +) => { + const check = await jellyfinApi.getSongs(server, { + enableUserData: false, + limit: 0, + parentId: serverFolder.remoteId, + recursive: true, + }); + + const songCount = check.TotalRecordCount; + const chunkSize = 5000; + const songChunkCount = Math.ceil(songCount / chunkSize); + + await prisma.task.update({ + data: { message: 'Scanning songs' }, + where: { id: task.id }, + }); + + for (let i = 0; i < songChunkCount; i += 1) { + const songs = await jellyfinApi.getSongs(server, { + enableImageTypes: 'Primary,Logo,Backdrop', + enableUserData: false, + fields: 'Genres,DateCreated,ExternalUrls,MediaSources,SortName', + imageTypeLimit: 1, + limit: chunkSize, + parentId: serverFolder.remoteId, + recursive: true, + sortBy: 'DateCreated,Album', + sortOrder: 'Descending', + startIndex: i * chunkSize, + }); + + const folderGroups = songs.Items.map((song) => { + const songPaths = song.MediaSources[0].Path.split('/'); + const paths = []; + for (let b = 0; b < songPaths.length - 1; b += 1) { + paths.push({ + name: songPaths[b], + path: songPaths.slice(0, b + 1).join('/'), + }); + } + + return paths; + }); + + const uniqueFolders = uniqBy( + folderGroups.flatMap((folder) => folder).filter((f) => f.path !== ''), + 'path' + ); + + const createdFolders: Folder[] = []; + for (const folder of uniqueFolders) { + const createdFolder = await prisma.folder.upsert({ + create: { + name: folder.name, + path: folder.path, + serverFolders: { + connect: { + uniqueServerFolderId: { + remoteId: serverFolder.remoteId, + serverId: server.id, + }, + }, + }, + serverId: server.id, + }, + update: { + name: folder.name, + path: folder.path, + serverFolders: { + connect: { + uniqueServerFolderId: { + remoteId: serverFolder.remoteId, + serverId: server.id, + }, + }, + }, + }, + where: { + uniqueFolderId: { + path: folder.path, + serverId: server.id, + }, + }, + }); + + createdFolders.push(createdFolder); + } + + for (const folder of createdFolders) { + if (folder.parentId) break; + + const pathSplit = folder.path.split('/'); + const parentPath = pathSplit.slice(0, pathSplit.length - 1).join('/'); + + const parentPathData = createdFolders.find( + (save) => save.path === parentPath + ); + + if (parentPathData) { + await prisma.folder.update({ + data: { + parentId: parentPathData.id, + }, + where: { id: folder.id }, + }); + } + } + + await jellyfinUtils.insertArtists(server, serverFolder, songs.Items); + await jellyfinUtils.insertImages(songs.Items); + await jellyfinUtils.insertExternals(songs.Items); + + const albumSongGroups = groupByProperty(songs.Items, 'AlbumId'); + const keys = Object.keys(albumSongGroups); + + for (const key of keys) { + const songGroup = albumSongGroups[key]; + await jellyfinUtils.insertSongGroup(server, serverFolder, songGroup, key); + } + } +}; + +const checkDeleted = async ( + server: Server, + serverFolder: ServerFolder, + task: Task +) => { + await prisma.$transaction([ + prisma.albumArtist.updateMany({ + data: { deleted: true }, + where: { + serverFolders: { some: { id: serverFolder.id } }, + serverId: server.id, + updatedAt: { lte: task.createdAt }, + }, + }), + prisma.artist.updateMany({ + data: { deleted: true }, + where: { + serverFolders: { some: { id: serverFolder.id } }, + serverId: server.id, + updatedAt: { lte: task.createdAt }, + }, + }), + prisma.album.updateMany({ + data: { deleted: true }, + where: { + serverFolders: { some: { id: serverFolder.id } }, + serverId: server.id, + updatedAt: { lte: task.createdAt }, + }, + }), + prisma.song.updateMany({ + data: { deleted: true }, + where: { + serverFolders: { some: { id: serverFolder.id } }, + serverId: server.id, + updatedAt: { lte: task.createdAt }, + }, + }), + ]); +}; + +const scanAll = async ( + server: Server, + serverFolders: ServerFolder[], + task: Task +) => { + queue.scanner.push({ + fn: async () => { + await prisma.task.update({ + data: { message: 'Beginning scan...' }, + where: { id: task.id }, + }); + + for (const serverFolder of serverFolders) { + await scanGenres({ server, serverFolder, task }); + await scanAlbumArtists(server, serverFolder, task); + await scanAlbums(server, serverFolder, task); + await scanSongs(server, serverFolder, task); + await checkDeleted(server, serverFolder, task); + } + + return { task }; + }, + id: task.id, + }); +}; + +export const jellyfinScanner = { + scanAlbumArtists, + scanAlbums, + scanAll, + scanGenres, + scanSongs, +}; diff --git a/src/server/queue/jellyfin/jellyfin-types.ts b/src/server/queue/jellyfin/jellyfin.types.ts similarity index 58% rename from src/server/queue/jellyfin/jellyfin-types.ts rename to src/server/queue/jellyfin/jellyfin.types.ts index 7976ecb3d..2054bd136 100644 --- a/src/server/queue/jellyfin/jellyfin-types.ts +++ b/src/server/queue/jellyfin/jellyfin.types.ts @@ -49,7 +49,7 @@ export interface JFRequestParams { } export interface JFMusicFolder { - BackdropImageTags: any[]; + BackdropImageTags: string[]; ChannelId: null; CollectionType: string; Id: string; @@ -68,7 +68,7 @@ export interface JFGenre { ChannelId: null; Id: string; ImageBlurHashes: any; - ImageTags: any; + ImageTags: ImageTags; LocationType: string; Name: string; ServerId: string; @@ -84,7 +84,7 @@ export interface JFAlbumArtist { Genres: string[]; Id: string; ImageBlurHashes: any; - ImageTags: string[]; + ImageTags: ImageTags; LocationType: string; Name: string; Overview?: string; @@ -113,13 +113,13 @@ export interface JFArtist { export interface JFAlbum { AlbumArtist: string; - AlbumArtists: GenericItem[]; - ArtistItems: GenericItem[]; + AlbumArtists: JFGenericItem[]; + ArtistItems: JFGenericItem[]; Artists: string[]; ChannelId: null; DateCreated: string; ExternalUrls: ExternalURL[]; - GenreItems: GenericItem[]; + GenreItems: JFGenericItem[]; Genres: string[]; Id: string; ImageBlurHashes: ImageBlurHashes; @@ -139,16 +139,16 @@ export interface JFAlbum { export interface JFSong { Album: string; AlbumArtist: string; - AlbumArtists: GenericItem[]; + AlbumArtists: JFGenericItem[]; AlbumId: string; AlbumPrimaryImageTag: string; - ArtistItems: GenericItem[]; + ArtistItems: JFGenericItem[]; Artists: string[]; BackdropImageTags: string[]; ChannelId: null; DateCreated: string; ExternalUrls: ExternalURL[]; - GenreItems: GenericItem[]; + GenreItems: JFGenericItem[]; Genres: string[]; Id: string; ImageBlurHashes: ImageBlurHashes; @@ -164,6 +164,7 @@ export interface JFSong { ProductionYear: number; RunTimeTicks: number; ServerId: string; + SortName: string; Type: string; } @@ -196,7 +197,7 @@ interface GenreItem { Name: string; } -interface GenericItem { +export interface JFGenericItem { Id: string; Name: string; } @@ -261,3 +262,143 @@ interface MediaStream { Type: string; Width?: number; } + +export enum JFExternalType { + MUSICBRAINZ = 'MusicBrainz', + THEAUDIODB = 'TheAudioDb', +} + +export enum JFImageType { + LOGO = 'Logo', + PRIMARY = 'Primary', +} + +export enum JFItemType { + AUDIO = 'Audio', + MUSICALBUM = 'MusicAlbum', +} + +export enum JFCollectionType { + MUSIC = 'music', + PLAYLISTS = 'playlists', +} + +export interface JFAuthenticate { + AccessToken: string; + ServerId: string; + SessionInfo: SessionInfo; + User: User; +} + +interface SessionInfo { + AdditionalUsers: any[]; + ApplicationVersion: string; + Capabilities: Capabilities; + Client: string; + DeviceId: string; + DeviceName: string; + HasCustomDeviceName: boolean; + Id: string; + IsActive: boolean; + LastActivityDate: string; + LastPlaybackCheckIn: string; + NowPlayingQueue: any[]; + NowPlayingQueueFullItems: any[]; + PlayState: PlayState; + PlayableMediaTypes: any[]; + RemoteEndPoint: string; + ServerId: string; + SupportedCommands: any[]; + SupportsMediaControl: boolean; + SupportsRemoteControl: boolean; + UserId: string; + UserName: string; +} + +interface Capabilities { + PlayableMediaTypes: any[]; + SupportedCommands: any[]; + SupportsContentUploading: boolean; + SupportsMediaControl: boolean; + SupportsPersistentIdentifier: boolean; + SupportsSync: boolean; +} + +interface PlayState { + CanSeek: boolean; + IsMuted: boolean; + IsPaused: boolean; + RepeatMode: string; +} + +interface User { + Configuration: Configuration; + EnableAutoLogin: boolean; + HasConfiguredEasyPassword: boolean; + HasConfiguredPassword: boolean; + HasPassword: boolean; + Id: string; + LastActivityDate: string; + LastLoginDate: string; + Name: string; + Policy: Policy; + ServerId: string; +} + +interface Configuration { + DisplayCollectionsView: boolean; + DisplayMissingEpisodes: boolean; + EnableLocalPassword: boolean; + EnableNextEpisodeAutoPlay: boolean; + GroupedFolders: any[]; + HidePlayedInLatest: boolean; + LatestItemsExcludes: any[]; + MyMediaExcludes: any[]; + OrderedViews: any[]; + PlayDefaultAudioTrack: boolean; + RememberAudioSelections: boolean; + RememberSubtitleSelections: boolean; + SubtitleLanguagePreference: string; + SubtitleMode: string; +} + +interface Policy { + AccessSchedules: any[]; + AuthenticationProviderId: string; + BlockUnratedItems: any[]; + BlockedChannels: any[]; + BlockedMediaFolders: any[]; + BlockedTags: any[]; + EnableAllChannels: boolean; + EnableAllDevices: boolean; + EnableAllFolders: boolean; + EnableAudioPlaybackTranscoding: boolean; + EnableContentDeletion: boolean; + EnableContentDeletionFromFolders: any[]; + EnableContentDownloading: boolean; + EnableLiveTvAccess: boolean; + EnableLiveTvManagement: boolean; + EnableMediaConversion: boolean; + EnableMediaPlayback: boolean; + EnablePlaybackRemuxing: boolean; + EnablePublicSharing: boolean; + EnableRemoteAccess: boolean; + EnableRemoteControlOfOtherUsers: boolean; + EnableSharedDeviceControl: boolean; + EnableSyncTranscoding: boolean; + EnableUserPreferenceAccess: boolean; + EnableVideoPlaybackTranscoding: boolean; + EnabledChannels: any[]; + EnabledDevices: any[]; + EnabledFolders: any[]; + ForceRemoteSourceTranscoding: boolean; + InvalidLoginAttemptCount: number; + IsAdministrator: boolean; + IsDisabled: boolean; + IsHidden: boolean; + LoginAttemptsBeforeLockout: number; + MaxActiveSessions: number; + PasswordResetProviderId: string; + RemoteClientBitrateLimit: number; + SyncPlayAccess: string; +} diff --git a/src/server/queue/jellyfin/jellyfin.utils.ts b/src/server/queue/jellyfin/jellyfin.utils.ts new file mode 100644 index 000000000..1f13985d0 --- /dev/null +++ b/src/server/queue/jellyfin/jellyfin.utils.ts @@ -0,0 +1,278 @@ +import { + ExternalSource, + ExternalType, + ImageType, + Prisma, + Server, + ServerFolder, +} from '@prisma/client'; +import { prisma } from '../../lib'; +import { uniqueArray } from '../../utils'; +import { + JFAlbum, + JFAlbumArtist, + JFExternalType, + JFImageType, + JFSong, +} from './jellyfin.types'; + +const insertGenres = async (items: JFSong[] | JFAlbum[] | JFAlbumArtist[]) => { + const genresCreateMany = items + .flatMap((item) => item.GenreItems) + .map((genre) => ({ name: genre.Name })); + + await prisma.genre.createMany({ + data: genresCreateMany, + skipDuplicates: true, + }); +}; + +const insertArtists = async ( + server: Server, + serverFolder: ServerFolder, + items: JFSong[] | JFAlbum[] +) => { + const artistItems = items.flatMap((item) => item.ArtistItems); + + const createMany = artistItems.map((artist) => ({ + name: artist.Name, + remoteId: artist.Id, + serverId: server.id, + sortName: '', + })); + + await prisma.artist.createMany({ + data: createMany, + skipDuplicates: true, + }); + + for (const artist of artistItems) { + await prisma.artist.update({ + data: { serverFolders: { connect: { id: serverFolder.id } } }, + where: { + uniqueArtistId: { + remoteId: artist.Id, + serverId: server.id, + }, + }, + }); + } +}; + +const insertImages = async (items: JFSong[] | JFAlbum[] | JFAlbumArtist[]) => { + const imageItems = items.flatMap((item) => item.ImageTags); + + const createMany: Prisma.ImageCreateManyInput[] = []; + + for (const image of imageItems) { + if (image.Logo) { + createMany.push({ + remoteUrl: image.Logo, + type: ImageType.LOGO, + }); + } + if (image.Primary) { + createMany.push({ + remoteUrl: image.Primary, + type: ImageType.PRIMARY, + }); + } + } + + await prisma.image.createMany({ + data: createMany, + skipDuplicates: true, + }); +}; + +const insertExternals = async ( + items: JFSong[] | JFAlbum[] | JFAlbumArtist[] +) => { + const externalItems = items.flatMap((item) => item.ExternalUrls); + const createMany: Prisma.ExternalCreateManyInput[] = []; + + for (const external of externalItems) { + if ( + external.Name === JFExternalType.MUSICBRAINZ || + external.Name === JFExternalType.THEAUDIODB + ) { + const source = + external.Name === JFExternalType.MUSICBRAINZ + ? ExternalSource.MUSICBRAINZ + : ExternalSource.THEAUDIODB; + + const value = external.Url.split('/').pop() || ''; + + createMany.push({ source, type: ExternalType.ID, value }); + } + } + + await prisma.external.createMany({ + data: createMany, + skipDuplicates: true, + }); +}; + +const insertSongGroup = async ( + server: Server, + serverFolder: ServerFolder, + songs: JFSong[], + remoteAlbumId: string +) => { + const songsUpsert: Prisma.SongUpsertWithWhereUniqueWithoutAlbumInput[] = + songs.map((song) => { + const genresConnect = song.Genres.map((genre) => ({ name: genre })); + + const artistsConnect = song.ArtistItems.map((artist) => ({ + uniqueArtistId: { + remoteId: artist.Id, + serverId: server.id, + }, + })); + + const externalsConnect = song.ExternalUrls.map((external) => ({ + uniqueExternalId: { + source: + external.Name === JFExternalType.MUSICBRAINZ + ? ExternalSource.MUSICBRAINZ + : ExternalSource.THEAUDIODB, + value: external.Url.split('/').pop() || '', + }, + })); + + const imagesConnect = []; + for (const [key, value] of Object.entries(song.ImageTags)) { + if (key === JFImageType.PRIMARY) { + imagesConnect.push({ + uniqueImageId: { remoteUrl: value, type: ImageType.PRIMARY }, + }); + } + if (key === JFImageType.LOGO) { + imagesConnect.push({ + uniqueImageId: { remoteUrl: value, type: ImageType.LOGO }, + }); + } + } + + const pathSplit = song.MediaSources[0].Path.split('/'); + const parentPath = pathSplit.slice(0, pathSplit.length - 1).join('/'); + + return { + create: { + artists: { connect: artistsConnect }, + bitRate: Math.floor(song.MediaSources[0].Bitrate / 1e3), + container: song.MediaSources[0].Container, + discNumber: song.ParentIndexNumber, + duration: Math.floor(song.MediaSources[0].RunTimeTicks / 1e7), + externals: { connect: externalsConnect }, + folders: { + connect: { + uniqueFolderId: { path: parentPath, serverId: server.id }, + }, + }, + genres: { connect: genresConnect }, + images: { connect: imagesConnect }, + name: song.Name, + releaseDate: song.PremiereDate, + releaseYear: song.ProductionYear, + remoteCreatedAt: song.DateCreated, + remoteId: song.Id, + serverFolders: { connect: { id: serverFolder.id } }, + serverId: server.id, + size: String(song.MediaSources[0].Size), + sortName: song.Name, + trackNumber: song.IndexNumber, + }, + update: { + artists: { connect: artistsConnect }, + bitRate: Math.floor(song.MediaSources[0].Bitrate / 1e3), + container: song.MediaSources[0].Container, + discNumber: song.ParentIndexNumber, + duration: Math.floor(song.MediaSources[0].RunTimeTicks / 1e7), + externals: { connect: externalsConnect }, + folders: { + connect: { + uniqueFolderId: { path: parentPath, serverId: server.id }, + }, + }, + genres: { connect: genresConnect }, + images: { connect: imagesConnect }, + name: song.Name, + releaseDate: song.PremiereDate, + releaseYear: song.ProductionYear, + remoteCreatedAt: song.DateCreated, + remoteId: song.Id, + serverFolders: { connect: { id: serverFolder.id } }, + serverId: server.id, + sortName: song.Name, + trackNumber: song.IndexNumber, + }, + where: { + uniqueSongId: { + remoteId: song.Id, + serverId: server.id, + }, + }, + }; + }); + + // const artists = uniqBy( + // songs.flatMap((song) => { + // return song.ArtistItems.map((artist) => ({ + // deleted: false, + // name: artist.Name, + // remoteId: artist.Id, + // serverFolders: { connect: { id: serverFolder.id } }, + // serverId: server.id, + // sortName: '', + // })); + // }), + // 'remoteId' + // ); + + // for (const artist of artists) { + // await prisma.artist.upsert({ + // create: artist, + // update: artist, + // where: { + // uniqueArtistId: { + // remoteId: artist.remoteId, + // serverId: server.id, + // }, + // }, + // }); + // } + + const uniqueArtistIds = songs + .flatMap((song) => song.ArtistItems.flatMap((artist) => artist.Id)) + .filter(uniqueArray); + + const artistsConnect = uniqueArtistIds.map((artistId) => ({ + uniqueArtistId: { + remoteId: artistId, + serverId: server.id, + }, + })); + + await prisma.album.update({ + data: { + artists: { connect: artistsConnect }, + deleted: false, + songs: { upsert: songsUpsert }, + }, + where: { + uniqueAlbumId: { + remoteId: remoteAlbumId, + serverId: server.id, + }, + }, + }); +}; + +export const jellyfinUtils = { + insertArtists, + insertExternals, + insertGenres, + insertImages, + insertSongGroup, +}; diff --git a/src/server/queue/queues/index.ts b/src/server/queue/queues/index.ts new file mode 100644 index 000000000..fe1e80b4b --- /dev/null +++ b/src/server/queue/queues/index.ts @@ -0,0 +1,5 @@ +import { scannerQueue } from './scanner.queue'; + +export const queue = { + scanner: scannerQueue, +}; diff --git a/src/server/queue/queues/scanner.queue.ts b/src/server/queue/queues/scanner.queue.ts new file mode 100644 index 000000000..a0462581c --- /dev/null +++ b/src/server/queue/queues/scanner.queue.ts @@ -0,0 +1,57 @@ +import { Task } from '@prisma/client'; +import Queue from 'better-queue'; +import { prisma } from '../../lib'; + +interface QueueTask { + fn: any; + id: string; + task: Task; +} + +export const scannerQueue: Queue = new Queue( + async (task: QueueTask, cb: any) => { + const result = await task.fn(); + return cb(null, result); + }, + { + afterProcessDelay: 1000, + cancelIfRunning: true, + concurrent: 1, + filo: false, + maxRetries: 5, + maxTimeout: 600000, + retryDelay: 2000, + } +); + +scannerQueue.on('task_finish', async (taskId) => { + await prisma.task.update({ + data: { + completed: true, + isError: false, + progress: null, + }, + where: { id: taskId }, + }); +}); + +scannerQueue.on('task_failed', async (taskId, errorMessage) => { + const dbTaskId = taskId.split('(')[1].split(')')[0]; + + console.log('errorMessage', errorMessage); + await prisma.task.update({ + data: { + completed: true, + isError: true, + message: errorMessage, + }, + where: { id: dbTaskId }, + }); +}); + +scannerQueue.on('drain', async () => { + await prisma.task.updateMany({ + data: { completed: true, progress: null }, + where: { completed: false }, + }); +}); diff --git a/src/server/queue/scanner-queue.ts b/src/server/queue/scanner-queue.ts deleted file mode 100644 index 1ccbfa058..000000000 --- a/src/server/queue/scanner-queue.ts +++ /dev/null @@ -1,74 +0,0 @@ -import Queue from 'better-queue'; -import { prisma } from '../lib'; -import { Task } from '../types/types'; - -interface QueueTask { - fn: any; - id: string; - task: Task; -} - -// interface QueueResult { -// completed?: boolean; -// message?: string; -// task: Task; -// } - -export const q: Queue = new Queue( - async (task: QueueTask, cb: any) => { - const result = await task.fn(); - return cb(null, result); - }, - { - afterProcessDelay: 1000, - cancelIfRunning: true, - concurrent: 1, - filo: false, - maxRetries: 5, - maxTimeout: 600000, - retryDelay: 2000, - } -); - -// q.on('task_finish', async (_taskId, result: QueueResult) => {}); - -q.on('task_failed', async (taskId, errorMessage) => { - const dbTaskId = taskId.split('(')[1].split(')')[0]; - await prisma.task.update({ - data: { - completed: true, - inProgress: false, - isError: true, - message: errorMessage, - }, - where: { id: dbTaskId }, - }); -}); - -q.on('drain', async () => { - await prisma.task.updateMany({ - data: { completed: true, inProgress: false }, - where: { - OR: [{ inProgress: true }, { completed: false }], - }, - }); -}); - -export const completeTask = async (task: Task) => { - q.push({ - fn: async () => { - await prisma.task.update({ - data: { - completed: true, - inProgress: false, - message: 'Completed', - progress: '', - }, - where: { id: task.id }, - }); - - return { task }; - }, - id: `${task.id}-complete`, - }); -}; diff --git a/src/server/queue/subsonic/index.ts b/src/server/queue/subsonic/index.ts new file mode 100644 index 000000000..07a5b7f87 --- /dev/null +++ b/src/server/queue/subsonic/index.ts @@ -0,0 +1,7 @@ +import { subsonicApi } from './subsonic.api'; +import { subsonicScanner } from './subsonic.scanner'; + +export const subsonic = { + api: subsonicApi, + scanner: subsonicScanner, +}; diff --git a/src/server/queue/subsonic/subsonic-api.ts b/src/server/queue/subsonic/subsonic.api.ts similarity index 81% rename from src/server/queue/subsonic/subsonic-api.ts rename to src/server/queue/subsonic/subsonic.api.ts index f6bc3f57d..6cb2c3ee0 100644 --- a/src/server/queue/subsonic/subsonic-api.ts +++ b/src/server/queue/subsonic/subsonic.api.ts @@ -1,5 +1,7 @@ +import { Server } from '@prisma/client'; import axios from 'axios'; -import { Server } from '../../types/types'; +import md5 from 'md5'; +import { randomString } from '../../utils'; import { SSAlbumListEntry, SSAlbumListResponse, @@ -10,7 +12,7 @@ import { SSArtistsResponse, SSGenresResponse, SSMusicFoldersResponse, -} from './subsonic-types'; +} from './subsonic.types'; const api = axios.create({ validateStatus: (status) => status >= 200, @@ -26,6 +28,31 @@ api.interceptors.response.use( } ); +const authenticate = async (options: { + legacy?: boolean; + password: string; + url: string; + username: string; +}) => { + let token; + + const cleanServerUrl = options.url.replace(/\/$/, ''); + + if (options.legacy) { + token = `u=${options.username}&p=${options.password}`; + } else { + const salt = randomString(12); + const hash = md5(options.password + salt); + token = `u=${options.username}&s=${salt}&t=${hash}`; + } + + const { data } = await api.get( + `${cleanServerUrl}/rest/ping.view?v=1.13.0&c=sonixd&f=json&${token}` + ); + + return { token, ...data }; +}; + const getMusicFolders = async (server: Partial) => { const { data } = await api.get( `${server.url}/rest/getMusicFolders.view?v=1.13.0&c=sonixd&f=json&${server.token}` @@ -120,6 +147,7 @@ const getArtistInfo = async (server: Server, id: string) => { }; export const subsonicApi = { + authenticate, getAlbum, getAlbums, getArtistInfo, diff --git a/src/server/queue/subsonic/subsonic-tasks.ts b/src/server/queue/subsonic/subsonic.scanner.ts similarity index 50% rename from src/server/queue/subsonic/subsonic-tasks.ts rename to src/server/queue/subsonic/subsonic.scanner.ts index 027801600..91038406f 100644 --- a/src/server/queue/subsonic/subsonic-tasks.ts +++ b/src/server/queue/subsonic/subsonic.scanner.ts @@ -1,207 +1,152 @@ /* eslint-disable no-await-in-loop */ +import { Server, ServerFolder, Task } from '@prisma/client'; import { prisma, throttle } from '../../lib'; -import { Server, ServerFolder, Task } from '../../types/types'; import { groupByProperty, uniqueArray } from '../../utils'; -import { completeTask, q } from '../scanner-queue'; -import { subsonicApi } from './subsonic-api'; -import { SSAlbumListEntry } from './subsonic-types'; +import { subsonicApi } from './subsonic.api'; +import { SSAlbumListEntry } from './subsonic.types'; export const scanGenres = async ( server: Server, serverFolder: ServerFolder ) => { - const taskId = `[${server.name} (${serverFolder.name})] genres`; + const res = await subsonicApi.getGenres(server); - q.push({ - fn: async () => { - const task = await prisma.task.create({ - data: { - inProgress: true, - name: taskId, - serverFolderId: serverFolder.id, - }, - }); - - const res = await subsonicApi.getGenres(server); - - const genres = res.genres.genre.map((genre) => { - return { name: genre.value }; - }); - - const createdGenres = await prisma.genre.createMany({ - data: genres, - skipDuplicates: true, - }); - - const message = `Imported ${createdGenres.count} new genres.`; - - return { message, task }; - }, - id: taskId, + const genres = res.genres.genre.map((genre) => { + return { name: genre.value }; }); + + const createdGenres = await prisma.genre.createMany({ + data: genres, + skipDuplicates: true, + }); + + const message = `Imported ${createdGenres.count} new genres.`; }; export const scanAlbumArtists = async ( server: Server, serverFolder: ServerFolder ) => { - const taskId = `[${server.name} (${serverFolder.name})] album artists`; + const artists = await subsonicApi.getArtists(server, serverFolder.remoteId); - q.push({ - fn: async () => { - const task = await prisma.task.create({ - data: { - inProgress: true, - name: taskId, - serverFolderId: serverFolder.id, + for (const artist of artists) { + await prisma.albumArtist.upsert({ + create: { + name: artist.name, + remoteId: artist.id, + serverId: server.id, + }, + update: { + name: artist.name, + remoteId: artist.id, + serverId: server.id, + }, + where: { + uniqueAlbumArtistId: { + remoteId: artist.id, + serverId: server.id, }, - }); + }, + }); - const artists = await subsonicApi.getArtists( - server, - serverFolder.remoteId - ); + await prisma.artist.upsert({ + create: { + name: artist.name, + remoteId: artist.id, + serverId: server.id, + }, + update: { + name: artist.name, + remoteId: artist.id, + serverId: server.id, + }, + where: { + uniqueArtistId: { + remoteId: artist.id, + serverId: server.id, + }, + }, + }); + } - for (const artist of artists) { - await prisma.albumArtist.upsert({ - create: { - name: artist.name, - remoteId: artist.id, - serverId: server.id, - }, - update: { - name: artist.name, - remoteId: artist.id, - serverId: server.id, - }, - where: { - uniqueAlbumArtistId: { - remoteId: artist.id, - serverId: server.id, - }, - }, - }); - - await prisma.artist.upsert({ - create: { - name: artist.name, - remoteId: artist.id, - serverId: server.id, - }, - update: { - name: artist.name, - remoteId: artist.id, - serverId: server.id, - }, - where: { - uniqueArtistId: { - remoteId: artist.id, - serverId: server.id, - }, - }, - }); - } - - const message = `Scanned ${artists.length} album artists.`; - - return { message, task }; - }, - id: taskId, - }); + const message = `Scanned ${artists.length} album artists.`; }; export const scanAlbums = async ( server: Server, serverFolder: ServerFolder ) => { - const taskId = `[${server.name} (${serverFolder.name})] albums`; + const promises: any[] = []; + const albums = await subsonicApi.getAlbums(server, { + musicFolderId: serverFolder.id, + offset: 0, + size: 500, + type: 'newest', + }); - q.push({ - fn: async () => { - const task = await prisma.task.create({ - data: { - inProgress: true, - name: taskId, - serverFolderId: serverFolder.id, + const albumArtistGroups = groupByProperty(albums, 'artistId'); + + const addAlbums = async ( + a: SSAlbumListEntry[], + albumArtistRemoteId: string + ) => { + const albumArtist = await prisma.albumArtist.findUnique({ + where: { + uniqueAlbumArtistId: { + remoteId: albumArtistRemoteId, + serverId: server.id, }, - }); + }, + }); - const promises: any[] = []; - const albums = await subsonicApi.getAlbums(server, { - musicFolderId: serverFolder.id, - offset: 0, - size: 500, - type: 'newest', - }); + if (albumArtist) { + a.forEach(async (album) => { + const imagesConnectOrCreate = album.coverArt + ? { + create: { name: 'Primary', url: album.coverArt }, + where: { + uniqueImageId: { name: 'Primary', url: album.coverArt }, + }, + } + : []; - const albumArtistGroups = groupByProperty(albums, 'artistId'); - - const addAlbums = async ( - a: SSAlbumListEntry[], - albumArtistRemoteId: string - ) => { - const albumArtist = await prisma.albumArtist.findUnique({ + await prisma.album.upsert({ + create: { + albumArtistId: albumArtist.id, + images: { connectOrCreate: imagesConnectOrCreate }, + name: album.title, + remoteCreatedAt: album.created, + remoteId: album.id, + serverId: server.id, + year: album.year, + }, + update: { + albumArtistId: albumArtist.id, + images: { connectOrCreate: imagesConnectOrCreate }, + name: album.title, + remoteCreatedAt: album.created, + remoteId: album.id, + serverId: server.id, + year: album.year, + }, where: { - uniqueAlbumArtistId: { - remoteId: albumArtistRemoteId, + uniqueAlbumId: { + remoteId: album.id, serverId: server.id, }, }, }); - - if (albumArtist) { - a.forEach(async (album) => { - const imagesConnectOrCreate = album.coverArt - ? { - create: { name: 'Primary', url: album.coverArt }, - where: { - uniqueImageId: { name: 'Primary', url: album.coverArt }, - }, - } - : []; - - await prisma.album.upsert({ - create: { - albumArtistId: albumArtist.id, - images: { connectOrCreate: imagesConnectOrCreate }, - name: album.title, - remoteCreatedAt: album.created, - remoteId: album.id, - serverId: server.id, - year: album.year, - }, - update: { - albumArtistId: albumArtist.id, - images: { connectOrCreate: imagesConnectOrCreate }, - name: album.title, - remoteCreatedAt: album.created, - remoteId: album.id, - serverId: server.id, - year: album.year, - }, - where: { - uniqueAlbumId: { - remoteId: album.id, - serverId: server.id, - }, - }, - }); - }); - } - }; - - Object.keys(albumArtistGroups).forEach((key) => { - promises.push(addAlbums(albumArtistGroups[key], key)); }); + } + }; - await Promise.all(promises); - - const message = `Scanned ${albums.length} albums.`; - - return { message, task }; - }, - id: taskId, + Object.keys(albumArtistGroups).forEach((key) => { + promises.push(addAlbums(albumArtistGroups[key], key)); }); + + await Promise.all(promises); + + const message = `Scanned ${albums.length} albums.`; }; const throttledAlbumFetch = throttle( @@ -315,36 +260,19 @@ export const scanAlbumDetail = async ( ) => { const taskId = `[${server.name} (${serverFolder.name})] albums detail`; - q.push({ - fn: async () => { - const task = await prisma.task.create({ - data: { - inProgress: true, - name: taskId, - serverFolderId: serverFolder.id, - }, - }); - - const promises = []; - const dbAlbums = await prisma.album.findMany({ - where: { - serverId: server.id, - }, - }); - - for (let i = 0; i < dbAlbums.length; i += 1) { - promises.push( - throttledAlbumFetch(server, serverFolder, dbAlbums[i], i) - ); - } - - await Promise.all(promises); - const message = `Scanned ${dbAlbums.length} albums.`; - - return { message, task }; + const promises = []; + const dbAlbums = await prisma.album.findMany({ + where: { + serverId: server.id, }, - id: taskId, }); + + for (let i = 0; i < dbAlbums.length; i += 1) { + promises.push(throttledAlbumFetch(server, serverFolder, dbAlbums[i], i)); + } + + await Promise.all(promises); + const message = `Scanned ${dbAlbums.length} albums.`; }; const throttledArtistDetailFetch = throttle( @@ -409,38 +337,23 @@ export const scanAlbumArtistDetail = async ( ) => { const taskId = `[${server.name} (${serverFolder.name})] artists detail`; - q.push({ - fn: async () => { - const task = await prisma.task.create({ - data: { - inProgress: true, - name: taskId, - serverFolderId: serverFolder.id, - }, - }); - - const promises = []; - const dbArtists = await prisma.albumArtist.findMany({ - where: { - serverId: server.id, - }, - }); - - for (let i = 0; i < dbArtists.length; i += 1) { - promises.push( - throttledArtistDetailFetch( - server, - dbArtists[i].id, - dbArtists[i].remoteId, - i - ) - ); - } - - return { task }; + const promises = []; + const dbArtists = await prisma.albumArtist.findMany({ + where: { + serverId: server.id, }, - id: taskId, }); + + for (let i = 0; i < dbArtists.length; i += 1) { + promises.push( + throttledArtistDetailFetch( + server, + dbArtists[i].id, + dbArtists[i].remoteId, + i + ) + ); + } }; const scanAll = async ( @@ -453,10 +366,9 @@ const scanAll = async ( await scanAlbumArtistDetail(server, serverFolder); await scanAlbums(server, serverFolder); await scanAlbumDetail(server, serverFolder); - await completeTask(task); }; -export const subsonicTasks = { +export const subsonicScanner = { scanAll, scanGenres, }; diff --git a/src/server/queue/subsonic/subsonic-types.ts b/src/server/queue/subsonic/subsonic.types.ts similarity index 98% rename from src/server/queue/subsonic/subsonic-types.ts rename to src/server/queue/subsonic/subsonic.types.ts index 85c7c0c3f..3dfc0b892 100644 --- a/src/server/queue/subsonic/subsonic-types.ts +++ b/src/server/queue/subsonic/subsonic.types.ts @@ -127,7 +127,7 @@ export interface SSSong { export interface SSAlbumsParams { fromYear?: number; genre?: string; - musicFolderId?: number; + musicFolderId?: string; offset?: number; size?: number; toYear?: number;