From e2808e0bd4af91496fa62ab623a44eaf51718016 Mon Sep 17 00:00:00 2001 From: jeffvli Date: Fri, 14 Oct 2022 20:26:53 -0700 Subject: [PATCH] Update scanners --- src/server/queue/jellyfin/jellyfin.api.ts | 2 - src/server/queue/jellyfin/jellyfin.scanner.ts | 94 +++-- src/server/queue/jellyfin/jellyfin.utils.ts | 72 +++- src/server/queue/navidrome/index.ts | 7 + src/server/queue/navidrome/navidrome.api.ts | 83 ++++ .../queue/navidrome/navidrome.normalize.ts | 59 +++ .../queue/navidrome/navidrome.scanner.ts | 376 ++++++++++++++++++ src/server/queue/navidrome/navidrome.types.ts | 169 ++++++++ src/server/queue/navidrome/navidrome.utils.ts | 125 ++++++ src/server/queue/subsonic/subsonic.scanner.ts | 127 +----- 10 files changed, 962 insertions(+), 152 deletions(-) create mode 100644 src/server/queue/navidrome/index.ts create mode 100644 src/server/queue/navidrome/navidrome.api.ts create mode 100644 src/server/queue/navidrome/navidrome.normalize.ts create mode 100644 src/server/queue/navidrome/navidrome.scanner.ts create mode 100644 src/server/queue/navidrome/navidrome.types.ts create mode 100644 src/server/queue/navidrome/navidrome.utils.ts diff --git a/src/server/queue/jellyfin/jellyfin.api.ts b/src/server/queue/jellyfin/jellyfin.api.ts index fb92d50dc..0c28255b3 100644 --- a/src/server/queue/jellyfin/jellyfin.api.ts +++ b/src/server/queue/jellyfin/jellyfin.api.ts @@ -23,8 +23,6 @@ export const authenticate = async (options: { const { password, url, username } = options; const cleanServerUrl = url.replace(/\/$/, ''); - console.log('cleanServerUrl', cleanServerUrl); - const { data } = await api.post( `${cleanServerUrl}/users/authenticatebyname`, { pw: password, username }, diff --git a/src/server/queue/jellyfin/jellyfin.scanner.ts b/src/server/queue/jellyfin/jellyfin.scanner.ts index fca8ca695..4e9456b6e 100644 --- a/src/server/queue/jellyfin/jellyfin.scanner.ts +++ b/src/server/queue/jellyfin/jellyfin.scanner.ts @@ -48,6 +48,7 @@ const scanAlbumArtists = async ( where: { id: task.id }, }); + // TODO: Possibly need to scan without the parentId to get all artists, since Jellyfin may link an album to an artist of a different folder const albumArtists = await jellyfinApi.getAlbumArtists(server, { fields: 'Genres,DateCreated,ExternalUrls,Overview', parentId: serverFolder.remoteId, @@ -70,16 +71,21 @@ const scanAlbumArtists = async ( }); } - const imagesConnect = []; for (const [key, value] of Object.entries(albumArtist.ImageTags)) { if (key === JFImageType.PRIMARY) { - imagesConnect.push({ - uniqueImageId: { remoteUrl: value, type: ImageType.PRIMARY }, + imagesConnectOrCreate.push({ + create: { remoteUrl: value, type: ImageType.PRIMARY }, + where: { + uniqueImageId: { remoteUrl: value, type: ImageType.PRIMARY }, + }, }); } if (key === JFImageType.LOGO) { - imagesConnect.push({ - uniqueImageId: { remoteUrl: value, type: ImageType.LOGO }, + imagesConnectOrCreate.push({ + create: { remoteUrl: value, type: ImageType.LOGO }, + where: { + uniqueImageId: { remoteUrl: value, type: ImageType.LOGO }, + }, }); } } @@ -100,7 +106,6 @@ const scanAlbumArtists = async ( externals: { connect: externalsConnect }, genres: { connect: genresConnect }, images: { - connect: imagesConnect, connectOrCreate: imagesConnectOrCreate, }, name: albumArtist.Name, @@ -115,7 +120,9 @@ const scanAlbumArtists = async ( deleted: false, externals: { connect: externalsConnect }, genres: { connect: genresConnect }, - images: { connectOrCreate: imagesConnectOrCreate }, + images: { + connectOrCreate: imagesConnectOrCreate, + }, name: albumArtist.Name, remoteCreatedAt: albumArtist.DateCreated, remoteId: albumArtist.Id, @@ -174,16 +181,22 @@ const scanAlbums = async ( for (const album of albums.Items) { const genresConnect = album.Genres.map((genre) => ({ name: genre })); - const imagesConnect = []; + const imagesConnectOrCreate = []; for (const [key, value] of Object.entries(album.ImageTags)) { if (key === JFImageType.PRIMARY) { - imagesConnect.push({ - uniqueImageId: { remoteUrl: value, type: ImageType.PRIMARY }, + imagesConnectOrCreate.push({ + create: { remoteUrl: value, type: ImageType.PRIMARY }, + where: { + uniqueImageId: { remoteUrl: value, type: ImageType.PRIMARY }, + }, }); } if (key === JFImageType.LOGO) { - imagesConnect.push({ - uniqueImageId: { remoteUrl: value, type: ImageType.LOGO }, + imagesConnectOrCreate.push({ + create: { remoteUrl: value, type: ImageType.LOGO }, + where: { + uniqueImageId: { remoteUrl: value, type: ImageType.LOGO }, + }, }); } } @@ -198,24 +211,53 @@ const scanAlbums = async ( }, })); - const albumArtist = - album.AlbumArtists.length > 0 - ? await prisma.albumArtist.findUnique({ - where: { - uniqueAlbumArtistId: { - remoteId: album.AlbumArtists && album.AlbumArtists[0].Id, - serverId: server.id, - }, + const remoteAlbumArtists = album.AlbumArtists; + + const albumArtists = await prisma.albumArtist.findMany({ + where: { + remoteId: { in: remoteAlbumArtists.map((artist) => artist.Id) }, + }, + }); + + const albumArtistsConnect = []; + for (const albumArtist of remoteAlbumArtists) { + const invalid = !albumArtists.find( + (artist) => artist.remoteId === albumArtist.Id + ); + + if (invalid) { + // If Jellyfin returns an invalid album artist, we'll just use the first matching one + const foundAlternate = await prisma.albumArtist.findFirst({ + where: { + name: albumArtist.Name, + serverId: server.id, + }, + }); + + if (foundAlternate) { + albumArtistsConnect.push({ + uniqueAlbumArtistId: { + remoteId: foundAlternate.remoteId, + serverId: server.id, }, - }) - : undefined; + }); + } + } else { + albumArtistsConnect.push({ + uniqueAlbumArtistId: { + remoteId: albumArtist.Id, + serverId: server.id, + }, + }); + } + } await prisma.album.upsert({ create: { - albumArtistId: albumArtist?.id, + albumArtists: { connect: albumArtistsConnect }, externals: { connect: externalsConnect }, genres: { connect: genresConnect }, - images: { connect: imagesConnect }, + images: { connectOrCreate: imagesConnectOrCreate }, name: album.Name, releaseDate: album.PremiereDate, releaseYear: album.ProductionYear, @@ -226,11 +268,11 @@ const scanAlbums = async ( sortName: album.Name, }, update: { - albumArtistId: albumArtist?.id, + albumArtists: { connect: albumArtistsConnect }, deleted: false, externals: { connect: externalsConnect }, genres: { connect: genresConnect }, - images: { connect: imagesConnect }, + images: { connectOrCreate: imagesConnectOrCreate }, name: album.Name, releaseDate: album.PremiereDate, releaseYear: album.ProductionYear, diff --git a/src/server/queue/jellyfin/jellyfin.utils.ts b/src/server/queue/jellyfin/jellyfin.utils.ts index 4bd937406..d4b14f437 100644 --- a/src/server/queue/jellyfin/jellyfin.utils.ts +++ b/src/server/queue/jellyfin/jellyfin.utils.ts @@ -6,6 +6,7 @@ import { Server, ServerFolder, } from '@prisma/client'; +import uniqBy from 'lodash/uniqBy'; import { prisma } from '../../lib'; import { uniqueArray } from '../../utils'; import { @@ -32,13 +33,16 @@ const insertArtists = async ( serverFolder: ServerFolder, items: JFSong[] | JFAlbum[] ) => { - const artistItems = items.flatMap((item) => item.ArtistItems); + const artistItems = uniqBy( + items.flatMap((item) => item.ArtistItems), + 'Id' + ); const createMany = artistItems.map((artist) => ({ name: artist.Name, remoteId: artist.Id, serverId: server.id, - sortName: '', + sortName: artist.Name, })); await prisma.artist.createMany({ @@ -60,7 +64,10 @@ const insertArtists = async ( }; const insertImages = async (items: JFSong[] | JFAlbum[] | JFAlbumArtist[]) => { - const imageItems = items.flatMap((item) => item.ImageTags); + const imageItems = uniqBy( + items.flatMap((item) => item.ImageTags), + 'Id' + ); const createMany: Prisma.ImageCreateManyInput[] = []; @@ -88,7 +95,10 @@ const insertImages = async (items: JFSong[] | JFAlbum[] | JFAlbumArtist[]) => { const insertExternals = async ( items: JFSong[] | JFAlbum[] | JFAlbumArtist[] ) => { - const externalItems = items.flatMap((item) => item.ExternalUrls); + const externalItems = uniqBy( + items.flatMap((item) => item.ExternalUrls), + 'Url' + ); const createMany: Prisma.ExternalCreateManyInput[] = []; for (const external of externalItems) { @@ -119,6 +129,32 @@ const insertSongGroup = async ( songs: JFSong[], remoteAlbumId: string ) => { + const remoteAlbumArtist = + songs[0].AlbumArtists.length > 0 ? songs[0].AlbumArtists[0] : undefined; + + let albumArtist = remoteAlbumArtist?.Id + ? await prisma.albumArtist.findUnique({ + where: { + uniqueAlbumArtistId: { + remoteId: remoteAlbumArtist.Id, + serverId: server.id, + }, + }, + }) + : undefined; + + // If Jellyfin returns an invalid album artist, we'll just use the first matching one + if (remoteAlbumArtist && !albumArtist) { + albumArtist = await prisma.albumArtist.findFirst({ + where: { + name: remoteAlbumArtist?.Name, + serverId: server.id, + }, + }); + } + + const albumArtistId = albumArtist ? albumArtist.id : undefined; + const songsUpsert: Prisma.SongUpsertWithWhereUniqueWithoutAlbumInput[] = songs.map((song) => { const genresConnect = song.Genres.map((genre) => ({ name: genre })); @@ -140,16 +176,28 @@ const insertSongGroup = async ( }, })); - const imagesConnect = []; + const imagesConnectOrCreate = []; for (const [key, value] of Object.entries(song.ImageTags)) { if (key === JFImageType.PRIMARY) { - imagesConnect.push({ - uniqueImageId: { remoteUrl: value, type: ImageType.PRIMARY }, + imagesConnectOrCreate.push({ + create: { + remoteUrl: value, + type: ImageType.PRIMARY, + }, + where: { + uniqueImageId: { remoteUrl: value, type: ImageType.PRIMARY }, + }, }); } if (key === JFImageType.LOGO) { - imagesConnect.push({ - uniqueImageId: { remoteUrl: value, type: ImageType.LOGO }, + imagesConnectOrCreate.push({ + create: { + remoteUrl: value, + type: ImageType.LOGO, + }, + where: { + uniqueImageId: { remoteUrl: value, type: ImageType.LOGO }, + }, }); } } @@ -159,6 +207,7 @@ const insertSongGroup = async ( return { create: { + albumArtistId, artists: { connect: artistsConnect }, bitRate: Math.floor(song.MediaSources[0].Bitrate / 1e3), container: song.MediaSources[0].Container, @@ -172,7 +221,7 @@ const insertSongGroup = async ( }, }, genres: { connect: genresConnect }, - images: { connect: imagesConnect }, + images: { connectOrCreate: imagesConnectOrCreate }, name: song.Name, releaseDate: song.PremiereDate, releaseYear: song.ProductionYear, @@ -185,6 +234,7 @@ const insertSongGroup = async ( trackNumber: song.IndexNumber, }, update: { + albumArtistId, artists: { connect: artistsConnect }, bitRate: Math.floor(song.MediaSources[0].Bitrate / 1e3), container: song.MediaSources[0].Container, @@ -198,7 +248,7 @@ const insertSongGroup = async ( }, }, genres: { connect: genresConnect }, - images: { connect: imagesConnect }, + images: { connectOrCreate: imagesConnectOrCreate }, name: song.Name, releaseDate: song.PremiereDate, releaseYear: song.ProductionYear, diff --git a/src/server/queue/navidrome/index.ts b/src/server/queue/navidrome/index.ts new file mode 100644 index 000000000..e21add18c --- /dev/null +++ b/src/server/queue/navidrome/index.ts @@ -0,0 +1,7 @@ +import { navidromeApi } from './navidrome.api'; +import { navidromeScanner } from './navidrome.scanner'; + +export const navidrome = { + api: navidromeApi, + scanner: navidromeScanner, +}; diff --git a/src/server/queue/navidrome/navidrome.api.ts b/src/server/queue/navidrome/navidrome.api.ts new file mode 100644 index 000000000..3993aff08 --- /dev/null +++ b/src/server/queue/navidrome/navidrome.api.ts @@ -0,0 +1,83 @@ +import { Server } from '@prisma/client'; +import axios from 'axios'; +import { + NDAlbumListResponse, + NDGenreListResponse, + NDAlbumListParams, + NDGenreListParams, + NDSongListParams, + NDSongListResponse, + NDArtistListResponse, + NDAuthenticate, +} from './navidrome.types'; + +const api = axios.create(); + +const authenticate = async (options: { + password: string; + url: string; + username: string; +}) => { + const { password, url, username } = options; + const cleanServerUrl = url.replace(/\/$/, ''); + + const { data } = await api.post( + `${cleanServerUrl}/auth/login`, + { password, username } + ); + + return data; +}; + +const getGenres = async (server: Server, params?: NDGenreListParams) => { + const { data } = await api.get( + `${server.url}/api/genre`, + { + headers: { 'x-nd-authorization': `Bearer ${server.token}` }, + params, + } + ); + + return data; +}; + +const getArtists = async (server: Server, params?: NDGenreListParams) => { + const { data } = await api.get( + `${server.url}/api/artist`, + { + headers: { 'x-nd-authorization': `Bearer ${server.token}` }, + params, + } + ); + + return data; +}; + +const getAlbums = async (server: Server, params?: NDAlbumListParams) => { + const { data } = await api.get( + `${server.url}/api/album`, + { + headers: { 'x-nd-authorization': `Bearer ${server.token}` }, + params, + } + ); + + return data; +}; + +const getSongs = async (server: Server, params?: NDSongListParams) => { + const { data } = await api.get(`${server.url}/api/song`, { + headers: { 'x-nd-authorization': `Bearer ${server.token}` }, + params, + }); + + return data; +}; + +export const navidromeApi = { + authenticate, + getAlbums, + getArtists, + getGenres, + getSongs, +}; diff --git a/src/server/queue/navidrome/navidrome.normalize.ts b/src/server/queue/navidrome/navidrome.normalize.ts new file mode 100644 index 000000000..2546e9ef2 --- /dev/null +++ b/src/server/queue/navidrome/navidrome.normalize.ts @@ -0,0 +1,59 @@ +import { + NormalizedAlbum, + NormalizedArtist, + NormalizedGenre, + NormalizedSong, +} from '../api/types'; +import { NDAlbum, NDArtist, NDGenre, NDSong } from './navidrome.types'; + +const genre = (genre: NDGenre): NormalizedGenre => { + return { + id: genre.id, + name: genre.name, + }; +}; + +const artist = (artist: NDArtist): NormalizedArtist => { + return { + biography: artist.biography, + genres: artist.genres.map(genre), + id: artist.id, + name: artist.name, + }; +}; + +const album = (album: NDAlbum): NormalizedAlbum => { + return { + albumArtistId: album.albumArtistId, + createdAt: album.createdAt, + genres: album.genres.map(genre), + id: album.id, + name: album.name, + year: album.minYear, + }; +}; + +const song = (song: NDSong): NormalizedSong => { + return { + albumId: song.albumId, + artists: [{ id: song.artistId, name: song.artist }], + bitRate: song.bitRate, + container: song.suffix, + createdAt: song.createdAt, + disc: song.discNumber, + duration: song.duration, + genres: song.genres.map(genre), + id: song.id, + name: song.title, + path: song.path, + track: song.trackNumber, + year: song.year, + }; +}; + +export const navidromeNormalize = { + album, + artist, + genre, + song, +}; diff --git a/src/server/queue/navidrome/navidrome.scanner.ts b/src/server/queue/navidrome/navidrome.scanner.ts new file mode 100644 index 000000000..6c0ff3ade --- /dev/null +++ b/src/server/queue/navidrome/navidrome.scanner.ts @@ -0,0 +1,376 @@ +/* eslint-disable no-await-in-loop */ +import { + ExternalSource, + ExternalType, + Folder, + ImageType, + Server, + ServerFolder, + Task, +} from '@prisma/client'; +import uniqBy from 'lodash/uniqBy'; +import { prisma } from '../../lib'; +import { groupByProperty } from '../../utils'; +import { queue } from '../queues'; +import { navidromeApi } from './navidrome.api'; +import { navidromeUtils } from './navidrome.utils'; + +const CHUNK_SIZE = 5000; + +export const scanGenres = async (server: Server, task: Task) => { + await prisma.task.update({ + data: { message: 'Scanning genres' }, + where: { id: task.id }, + }); + + const res = await navidromeApi.getGenres(server); + + const genres = res.map((genre) => { + return { name: genre.name }; + }); + + await prisma.genre.createMany({ + data: genres, + skipDuplicates: true, + }); +}; + +export const scanAlbumArtists = async ( + server: Server, + serverFolder: ServerFolder +) => { + const artists = await navidromeApi.getArtists(server); + + const externalsCreateMany = artists + .filter((artist) => artist.mbzArtistId) + .map((artist) => ({ + source: ExternalSource.MUSICBRAINZ, + type: ExternalType.ID, + value: artist.mbzArtistId, + })); + + await prisma.external.createMany({ + data: externalsCreateMany, + skipDuplicates: true, + }); + + for (const artist of artists) { + const genresConnect = artist.genres + ? artist.genres.map((genre) => ({ name: genre.name })) + : undefined; + + const externalsConnect = artist.mbzArtistId + ? { + uniqueExternalId: { + source: ExternalSource.MUSICBRAINZ, + value: artist.mbzArtistId, + }, + } + : undefined; + + await prisma.albumArtist.upsert({ + create: { + deleted: false, + externals: { connect: externalsConnect }, + genres: { connect: genresConnect }, + name: artist.name, + remoteId: artist.id, + serverFolders: { connect: { id: serverFolder.id } }, + serverId: server.id, + sortName: artist.name, + }, + update: { + deleted: false, + externals: { connect: externalsConnect }, + genres: { connect: genresConnect }, + name: artist.name, + remoteId: artist.id, + serverFolders: { connect: { id: serverFolder.id } }, + serverId: server.id, + sortName: artist.name, + }, + where: { + uniqueAlbumArtistId: { + remoteId: artist.id, + serverId: server.id, + }, + }, + }); + } +}; + +export const scanAlbums = async ( + server: Server, + serverFolder: ServerFolder +) => { + let start = 0; + let count = 5000; + do { + const albums = await navidromeApi.getAlbums(server, { + _end: start + CHUNK_SIZE, + _start: start, + }); + + const imagesCreateMany = albums + .filter((album) => album.coverArtId) + .map((album) => ({ + remoteUrl: album.coverArtId, + type: ImageType.PRIMARY, + })); + + await prisma.image.createMany({ + data: imagesCreateMany, + skipDuplicates: true, + }); + + const artistIds = ( + await prisma.artist.findMany({ + select: { remoteId: true }, + where: { serverId: server.id }, + }) + ).map((artist) => artist.remoteId); + + for (const album of albums) { + const imagesConnect = album.coverArtId + ? { + uniqueImageId: { + remoteUrl: album.coverArtId, + type: ImageType.PRIMARY, + }, + } + : undefined; + + const genresConnect = album.genres + ? album.genres.map((genre) => ({ name: genre.name })) + : undefined; + + const validArtistIds = []; + const ndArtistIds = album.allArtistIds.split(' '); + + for (const artistId of ndArtistIds) { + if (artistIds.includes(artistId)) { + validArtistIds.push(artistId); + } + } + + const artistsConnect = validArtistIds.map((id) => ({ + uniqueArtistId: { + remoteId: id, + serverId: server.id, + }, + })); + + const albumArtistConnect = album.artistId + ? { + uniqueAlbumArtistId: { + remoteId: album.artistId, + serverId: server.id, + }, + } + : undefined; + + await prisma.album.upsert({ + create: { + albumArtists: { connect: albumArtistConnect }, + artists: { connect: artistsConnect }, + deleted: false, + genres: { connect: genresConnect }, + images: { connect: imagesConnect }, + name: album.name, + releaseDate: album?.minYear + ? new Date(album.minYear, 0).toISOString() + : undefined, + releaseYear: album.minYear, + remoteCreatedAt: album.createdAt, + remoteId: album.id, + serverFolders: { connect: { id: serverFolder.id } }, + serverId: server.id, + sortName: album.name, + }, + update: { + albumArtists: { connect: albumArtistConnect }, + artists: { connect: artistsConnect }, + deleted: false, + genres: { connect: genresConnect }, + images: { connect: imagesConnect }, + name: album.name, + releaseDate: album?.minYear + ? new Date(album.minYear, 0).toISOString() + : undefined, + releaseYear: album.minYear, + remoteCreatedAt: album.createdAt, + remoteId: album.id, + serverFolders: { connect: { id: serverFolder.id } }, + serverId: server.id, + sortName: album.name, + }, + where: { + uniqueAlbumId: { + remoteId: album.id, + serverId: server.id, + }, + }, + }); + } + + start += CHUNK_SIZE; + count = albums.length; + } while (count === CHUNK_SIZE); +}; + +const scanSongs = async (server: Server, serverFolder: ServerFolder) => { + let start = 0; + let count = 5000; + do { + const songs = await navidromeApi.getSongs(server, { + _end: start + CHUNK_SIZE, + _start: start, + }); + + const externalsCreateMany = []; + const genresCreateMany = []; + for (const song of songs) { + if (song.mbzTrackId) { + externalsCreateMany.push({ + source: ExternalSource.MUSICBRAINZ, + type: ExternalType.ID, + value: song.mbzTrackId, + }); + } + + if (song.genres?.length > 0) { + genresCreateMany.push( + ...song.genres.map((genre) => ({ name: genre.name })) + ); + } + } + + await prisma.external.createMany({ + data: externalsCreateMany, + skipDuplicates: true, + }); + + await prisma.genre.createMany({ + data: genresCreateMany, + skipDuplicates: true, + }); + + const folderGroups = songs.map((song) => { + const songPaths = song.path.split('/'); + const paths = []; + for (let b = 0; b < songPaths.length - 1; b += 1) { + paths.push({ + name: songPaths[b], + path: songPaths.slice(0, b + 1).join('/'), + }); + } + + return paths; + }); + + const uniqueFolders = uniqBy( + folderGroups.flatMap((folder) => folder).filter((f) => f.path !== ''), + 'path' + ); + + const createdFolders: Folder[] = []; + for (const folder of uniqueFolders) { + const createdFolder = await prisma.folder.upsert({ + create: { + name: folder.name, + path: folder.path, + serverFolders: { + connect: { + uniqueServerFolderId: { + remoteId: serverFolder.remoteId, + serverId: server.id, + }, + }, + }, + serverId: server.id, + }, + update: { + name: folder.name, + path: folder.path, + serverFolders: { + connect: { + uniqueServerFolderId: { + remoteId: serverFolder.remoteId, + serverId: server.id, + }, + }, + }, + }, + where: { + uniqueFolderId: { + path: folder.path, + serverId: server.id, + }, + }, + }); + + createdFolders.push(createdFolder); + } + + for (const folder of createdFolders) { + if (folder.parentId) break; + + const pathSplit = folder.path.split('/'); + const parentPath = pathSplit.slice(0, pathSplit.length - 1).join('/'); + + const parentPathData = createdFolders.find( + (save) => save.path === parentPath + ); + + if (parentPathData) { + await prisma.folder.update({ + data: { + parentId: parentPathData.id, + }, + where: { id: folder.id }, + }); + } + } + + const albumSongGroups = groupByProperty(songs, 'albumId'); + const albumIds = Object.keys(albumSongGroups); + + for (const id of albumIds) { + const songGroup = albumSongGroups[id]; + await navidromeUtils.insertSongGroup(server, serverFolder, songGroup, id); + } + + start += CHUNK_SIZE; + count = songs.length; + } while (count === CHUNK_SIZE); +}; + +const scanAll = async ( + server: Server, + serverFolders: ServerFolder[], + task: Task +) => { + queue.scanner.push({ + fn: async () => { + await prisma.task.update({ + data: { message: 'Beginning scan...' }, + where: { id: task.id }, + }); + + for (const serverFolder of serverFolders) { + await scanGenres(server, task); + await scanAlbumArtists(server, serverFolder); + await scanAlbums(server, serverFolder); + await scanSongs(server, serverFolder); + } + + return { task }; + }, + id: task.id, + }); +}; + +export const navidromeScanner = { + scanAll, + scanGenres, +}; diff --git a/src/server/queue/navidrome/navidrome.types.ts b/src/server/queue/navidrome/navidrome.types.ts new file mode 100644 index 000000000..7d8d529e3 --- /dev/null +++ b/src/server/queue/navidrome/navidrome.types.ts @@ -0,0 +1,169 @@ +export type NDAuthenticate = { + id: string; + isAdmin: boolean; + name: string; + subsonicSalt: string; + subsonicToken: string; + token: string; + username: string; +}; + +export type NDGenre = { + id: string; + name: string; +}; + +export type NDAlbum = { + albumArtist: string; + albumArtistId: string; + allArtistIds: string; + artist: string; + artistId: string; + compilation: boolean; + coverArtId: string; + coverArtPath: string; + createdAt: string; + duration: number; + fullText: string; + genre: string; + genres: NDGenre[]; + id: string; + maxYear: number; + mbzAlbumArtistId: string; + mbzAlbumId: string; + minYear: number; + name: string; + orderAlbumArtistName: string; + orderAlbumName: string; + playCount: number; + playDate: string; + rating: number; + size: number; + songCount: number; + sortAlbumArtistName: string; + sortArtistName: string; + starred: boolean; + starredAt: string; + updatedAt: string; +}; + +export type NDSong = { + album: string; + albumArtist: string; + albumArtistId: string; + albumId: string; + artist: string; + artistId: string; + bitRate: number; + bookmarkPosition: number; + channels: number; + compilation: boolean; + createdAt: string; + discNumber: number; + duration: number; + fullText: string; + genre: string; + genres: NDGenre[]; + hasCoverArt: boolean; + id: string; + mbzAlbumArtistId: string; + mbzAlbumId: string; + mbzArtistId: string; + mbzTrackId: string; + orderAlbumArtistName: string; + orderAlbumName: string; + orderArtistName: string; + orderTitle: string; + path: string; + playCount: number; + playDate: string; + rating: number; + size: number; + sortAlbumArtistName: string; + sortArtistName: string; + starred: boolean; + starredAt: string; + suffix: string; + title: string; + trackNumber: number; + updatedAt: string; + year: number; +}; + +export type NDArtist = { + albumCount: number; + biography: string; + externalInfoUpdatedAt: string; + externalUrl: string; + fullText: string; + genres: NDGenre[]; + id: string; + largeImageUrl: string; + mbzArtistId: string; + mediumImageUrl: string; + name: string; + orderArtistName: string; + playCount: number; + playDate: string; + rating: number; + size: number; + smallImageUrl: string; + songCount: number; + starred: boolean; + starredAt: string; +}; + +export type NDGenreListResponse = NDGenre[]; + +export type NDAlbumListResponse = NDAlbum[]; + +export type NDSongListResponse = NDSong[]; + +export type NDArtistListResponse = NDArtist[]; + +export type NDPagination = { + _end?: number; + _start?: number; +}; + +export type NDOrder = { + _order?: 'ASC' | 'DESC'; +}; + +export enum NDGenreSort { + NAME = 'name', +} + +export type NDGenreListParams = { + _sort?: NDGenreSort; + id?: string; +} & NDPagination & + NDOrder; + +export enum NDAlbumSort { + ARTIST = 'artist', + MAX_YEAR = 'max_year', + NAME = 'name', + RANDOM = 'random', + RECENTLY_ADDED = 'recently_added', +} + +export type NDAlbumListParams = { + _sort?: NDAlbumSort; + artist_id?: string; + compilation?: boolean; + genre_id?: string; + has_rating?: boolean; + id?: string; + name?: string; + recently_played?: boolean; + starred?: boolean; + year?: number; +} & NDPagination & + NDOrder; + +export type NDSongListParams = { + genre_id?: string; + starred?: boolean; +} & NDPagination & + NDOrder; diff --git a/src/server/queue/navidrome/navidrome.utils.ts b/src/server/queue/navidrome/navidrome.utils.ts new file mode 100644 index 000000000..6aad9627e --- /dev/null +++ b/src/server/queue/navidrome/navidrome.utils.ts @@ -0,0 +1,125 @@ +import { ExternalSource, Server, ServerFolder } from '@prisma/client'; +import { prisma } from '../../lib'; +import { NDSong } from './navidrome.types'; + +const insertSongGroup = async ( + server: Server, + serverFolder: ServerFolder, + songs: NDSong[], + remoteAlbumId: string +) => { + const songsWithArtistIds = songs.filter((song) => song.artistId); + const artistId = + songsWithArtistIds.length > 0 ? songsWithArtistIds[0].artistId : undefined; + + const albumArtist = artistId + ? await prisma.albumArtist.findUnique({ + where: { + uniqueAlbumArtistId: { + remoteId: artistId, + serverId: server.id, + }, + }, + }) + : undefined; + + const songsUpsert = songs.map((song) => { + const genresConnect = song.genres + ? song.genres.map((genre) => ({ name: genre.name })) + : undefined; + + const externalsConnect = song.mbzTrackId + ? { + uniqueExternalId: { + source: ExternalSource.MUSICBRAINZ, + value: song.mbzTrackId, + }, + } + : undefined; + + const pathSplit = song.path.split('/'); + const parentPath = pathSplit.slice(0, pathSplit.length - 1).join('/'); + + return { + create: { + albumArtistId: albumArtist?.id, + artistName: !song.artistId ? song.artist : undefined, + bitRate: song.bitRate, + container: song.suffix, + deleted: false, + discNumber: song.discNumber, + duration: song.duration, + externals: { connect: externalsConnect }, + folders: { + connect: { + uniqueFolderId: { path: parentPath, serverId: server.id }, + }, + }, + genres: { connect: genresConnect }, + name: song.title, + releaseDate: song?.year + ? new Date(song.year, 0).toISOString() + : undefined, + releaseYear: song?.year, + remoteCreatedAt: song.createdAt, + remoteId: song.id, + serverFolders: { connect: { id: serverFolder.id } }, + serverId: server.id, + size: song.size, + sortName: song.title, + trackNumber: song.trackNumber, + }, + update: { + albumArtistId: albumArtist?.id, + artistName: !song.artistId ? song.artist : undefined, + bitRate: song.bitRate, + container: song.suffix, + deleted: false, + discNumber: song.discNumber, + duration: song.duration, + externals: { connect: externalsConnect }, + folders: { + connect: { + uniqueFolderId: { path: parentPath, serverId: server.id }, + }, + }, + genres: { connect: genresConnect }, + name: song.title, + releaseDate: song?.year + ? new Date(song.year, 0).toISOString() + : undefined, + releaseYear: song?.year, + remoteCreatedAt: song.createdAt, + remoteId: song.id, + serverFolders: { connect: { id: serverFolder.id } }, + serverId: server.id, + size: song.size, + sortName: song.title, + trackNumber: song.trackNumber, + }, + where: { + uniqueSongId: { + remoteId: song.id, + serverId: server.id, + }, + }, + }; + }); + + await prisma.album.update({ + data: { + deleted: false, + songs: { upsert: songsUpsert }, + }, + where: { + uniqueAlbumId: { + remoteId: remoteAlbumId, + serverId: server.id, + }, + }, + }); +}; + +export const navidromeUtils = { + insertSongGroup, +}; diff --git a/src/server/queue/subsonic/subsonic.scanner.ts b/src/server/queue/subsonic/subsonic.scanner.ts index ec08dc1f9..c21799bbc 100644 --- a/src/server/queue/subsonic/subsonic.scanner.ts +++ b/src/server/queue/subsonic/subsonic.scanner.ts @@ -35,12 +35,14 @@ export const scanAlbumArtists = async ( create: { name: artist.name, remoteId: artist.id, + serverFolders: { connect: { id: serverFolder.id } }, serverId: server.id, sortName: artist.name, }, update: { name: artist.name, remoteId: artist.id, + serverFolders: { connect: { id: serverFolder.id } }, serverId: server.id, sortName: artist.name, }, @@ -51,27 +53,6 @@ export const scanAlbumArtists = async ( }, }, }); - - await prisma.artist.upsert({ - create: { - name: artist.name, - remoteId: artist.id, - serverId: server.id, - sortName: artist.name, - }, - update: { - name: artist.name, - remoteId: artist.id, - serverId: server.id, - sortName: artist.name, - }, - where: { - uniqueArtistId: { - remoteId: artist.id, - serverId: server.id, - }, - }, - }); } }; @@ -98,20 +79,18 @@ export const scanAlbums = async ( } : undefined; - const albumArtist = album.artistId - ? await prisma.albumArtist.findUnique({ - where: { - uniqueAlbumArtistId: { - remoteId: album.artistId, - serverId: server.id, - }, + const albumArtistConnect = album.artistId + ? { + uniqueAlbumArtistId: { + remoteId: album.artistId, + serverId: server.id, }, - }) + } : undefined; await prisma.album.upsert({ create: { - albumArtistId: albumArtist?.id, + albumArtists: { connect: albumArtistConnect }, genres: { connect: album.genre ? { name: album.genre } : undefined }, images: { connect: imagesConnect }, name: album.title, @@ -126,7 +105,7 @@ export const scanAlbums = async ( sortName: album.title, }, update: { - albumArtistId: albumArtist?.id, + albumArtists: { connect: albumArtistConnect }, genres: { connect: album.genre ? { name: album.genre } : undefined }, images: { connect: imagesConnect }, name: album.title, @@ -168,9 +147,9 @@ const throttledAlbumFetch = throttle( } : undefined; - const artistsConnect = song.artistId + const albumArtistsConnect = song.artistId ? { - uniqueArtistId: { + uniqueAlbumArtistId: { remoteId: song.artistId, serverId: server.id, }, @@ -179,8 +158,8 @@ const throttledAlbumFetch = throttle( return { create: { + albumArtists: { connect: albumArtistsConnect }, artistName: !song.artistId ? song.artist : undefined, - artists: { connect: artistsConnect }, bitRate: song.bitRate, container: song.suffix, discNumber: song.discNumber, @@ -201,8 +180,8 @@ const throttledAlbumFetch = throttle( trackNumber: song.track, }, update: { + albumArtists: { connect: albumArtistsConnect }, artistName: !song.artistId ? song.artist : undefined, - artists: { connect: artistsConnect }, bitRate: song.bitRate, container: song.suffix, discNumber: song.discNumber, @@ -278,83 +257,6 @@ export const scanAlbumDetail = async ( await Promise.all(promises); }; -// const throttledArtistDetailFetch = throttle( -// async ( -// server: Server, -// artistId: string, -// artistRemoteId: string, -// i: number -// ) => { -// console.log('artisdetail', i); - -// const artistInfo = await subsonicApi.getArtistInfo(server, artistRemoteId); - -// const externalsConnectOrCreate = []; -// if (artistInfo.artistInfo2.lastFmUrl) { -// externalsConnectOrCreate.push({ -// create: { -// name: 'Last.fm', -// url: artistInfo.artistInfo2.lastFmUrl, -// }, -// where: { -// uniqueExternalId: { -// name: 'Last.fm', -// url: artistInfo.artistInfo2.lastFmUrl, -// }, -// }, -// }); -// } - -// if (artistInfo.artistInfo2.musicBrainzId) { -// externalsConnectOrCreate.push({ -// create: { -// name: 'MusicBrainz', -// url: `https://musicbrainz.org/artist/${artistInfo.artistInfo2.musicBrainzId}`, -// }, -// where: { -// uniqueExternalId: { -// name: 'MusicBrainz', -// url: `https://musicbrainz.org/artist/${artistInfo.artistInfo2.musicBrainzId}`, -// }, -// }, -// }); -// } - -// try { -// await prisma.albumArtist.update({ -// data: { -// biography: artistInfo.artistInfo2.biography, -// // externals: { connectOrCreate: externalsConnectOrCreate }, -// }, -// where: { id: artistId }, -// }); -// } catch (err) { -// console.log(err); -// } -// } -// ); - -// export const scanAlbumArtistDetail = async ( -// server: Server, -// serverFolder: ServerFolder -// ) => { -// const promises = []; -// const dbArtists = await prisma.albumArtist.findMany({ -// where: { serverId: server.id }, -// }); - -// for (let i = 0; i < dbArtists.length; i += 1) { -// promises.push( -// throttledArtistDetailFetch( -// server, -// dbArtists[i].id, -// dbArtists[i].remoteId, -// i -// ) -// ); -// } -// }; - const scanAll = async ( server: Server, serverFolders: ServerFolder[], @@ -370,7 +272,6 @@ const scanAll = async ( for (const serverFolder of serverFolders) { await scanGenres(server, task); await scanAlbumArtists(server, serverFolder); - // await scanAlbumArtistDetail(server, serverFolder); await scanAlbums(server, serverFolder); await scanAlbumDetail(server, serverFolder); }