Update scanners

src/server/queue/jellyfin/jellyfin.api.ts

@@ -23,8 +23,6 @@ export const authenticate = async (options: {
   const { password, url, username } = options;
   const cleanServerUrl = url.replace(/\/$/, '');
 
-  console.log('cleanServerUrl', cleanServerUrl);
-
   const { data } = await api.post<JFAuthenticate>(
     `${cleanServerUrl}/users/authenticatebyname`,
     { pw: password, username },

src/server/queue/jellyfin/jellyfin.scanner.ts

@@ -48,6 +48,7 @@ const scanAlbumArtists = async (
     where: { id: task.id },
   });
 
+  // TODO: Possibly need to scan without the parentId to get all artists, since Jellyfin may link an album to an artist of a different folder
   const albumArtists = await jellyfinApi.getAlbumArtists(server, {
     fields: 'Genres,DateCreated,ExternalUrls,Overview',
     parentId: serverFolder.remoteId,
@@ -70,16 +71,21 @@ const scanAlbumArtists = async (
       });
     }
 
-    const imagesConnect = [];
     for (const [key, value] of Object.entries(albumArtist.ImageTags)) {
       if (key === JFImageType.PRIMARY) {
-        imagesConnect.push({
+        imagesConnectOrCreate.push({
-          uniqueImageId: { remoteUrl: value, type: ImageType.PRIMARY },
+          create: { remoteUrl: value, type: ImageType.PRIMARY },
+          where: {
+            uniqueImageId: { remoteUrl: value, type: ImageType.PRIMARY },
+          },
         });
       }
       if (key === JFImageType.LOGO) {
-        imagesConnect.push({
+        imagesConnectOrCreate.push({
-          uniqueImageId: { remoteUrl: value, type: ImageType.LOGO },
+          create: { remoteUrl: value, type: ImageType.LOGO },
+          where: {
+            uniqueImageId: { remoteUrl: value, type: ImageType.LOGO },
+          },
         });
       }
     }
@@ -100,7 +106,6 @@ const scanAlbumArtists = async (
         externals: { connect: externalsConnect },
         genres: { connect: genresConnect },
         images: {
-          connect: imagesConnect,
           connectOrCreate: imagesConnectOrCreate,
         },
         name: albumArtist.Name,
@@ -115,7 +120,9 @@ const scanAlbumArtists = async (
         deleted: false,
         externals: { connect: externalsConnect },
         genres: { connect: genresConnect },
-        images: { connectOrCreate: imagesConnectOrCreate },
+        images: {
+          connectOrCreate: imagesConnectOrCreate,
+        },
         name: albumArtist.Name,
         remoteCreatedAt: albumArtist.DateCreated,
         remoteId: albumArtist.Id,
@@ -174,16 +181,22 @@ const scanAlbums = async (
     for (const album of albums.Items) {
       const genresConnect = album.Genres.map((genre) => ({ name: genre }));
 
-      const imagesConnect = [];
+      const imagesConnectOrCreate = [];
       for (const [key, value] of Object.entries(album.ImageTags)) {
         if (key === JFImageType.PRIMARY) {
-          imagesConnect.push({
+          imagesConnectOrCreate.push({
-            uniqueImageId: { remoteUrl: value, type: ImageType.PRIMARY },
+            create: { remoteUrl: value, type: ImageType.PRIMARY },
+            where: {
+              uniqueImageId: { remoteUrl: value, type: ImageType.PRIMARY },
+            },
           });
         }
         if (key === JFImageType.LOGO) {
-          imagesConnect.push({
+          imagesConnectOrCreate.push({
-            uniqueImageId: { remoteUrl: value, type: ImageType.LOGO },
+            create: { remoteUrl: value, type: ImageType.LOGO },
+            where: {
+              uniqueImageId: { remoteUrl: value, type: ImageType.LOGO },
+            },
           });
         }
       }
@@ -198,24 +211,53 @@ const scanAlbums = async (
         },
       }));
 
-      const albumArtist =
+      const remoteAlbumArtists = album.AlbumArtists;
-        album.AlbumArtists.length > 0
+
-          ? await prisma.albumArtist.findUnique({
+      const albumArtists = await prisma.albumArtist.findMany({
-              where: {
+        where: {
-                uniqueAlbumArtistId: {
+          remoteId: { in: remoteAlbumArtists.map((artist) => artist.Id) },
-                  remoteId: album.AlbumArtists && album.AlbumArtists[0].Id,
+        },
-                  serverId: server.id,
+      });
-                },
+
+      const albumArtistsConnect = [];
+      for (const albumArtist of remoteAlbumArtists) {
+        const invalid = !albumArtists.find(
+          (artist) => artist.remoteId === albumArtist.Id
+        );
+
+        if (invalid) {
+          // If Jellyfin returns an invalid album artist, we'll just use the first matching one
+          const foundAlternate = await prisma.albumArtist.findFirst({
+            where: {
+              name: albumArtist.Name,
+              serverId: server.id,
+            },
+          });
+
+          if (foundAlternate) {
+            albumArtistsConnect.push({
+              uniqueAlbumArtistId: {
+                remoteId: foundAlternate.remoteId,
+                serverId: server.id,
               },
-            })
+            });
-          : undefined;
+          }
+        } else {
+          albumArtistsConnect.push({
+            uniqueAlbumArtistId: {
+              remoteId: albumArtist.Id,
+              serverId: server.id,
+            },
+          });
+        }
+      }
 
       await prisma.album.upsert({
         create: {
-          albumArtistId: albumArtist?.id,
+          albumArtists: { connect: albumArtistsConnect },
           externals: { connect: externalsConnect },
           genres: { connect: genresConnect },
-          images: { connect: imagesConnect },
+          images: { connectOrCreate: imagesConnectOrCreate },
           name: album.Name,
           releaseDate: album.PremiereDate,
           releaseYear: album.ProductionYear,
@@ -226,11 +268,11 @@ const scanAlbums = async (
           sortName: album.Name,
         },
         update: {
-          albumArtistId: albumArtist?.id,
+          albumArtists: { connect: albumArtistsConnect },
           deleted: false,
           externals: { connect: externalsConnect },
           genres: { connect: genresConnect },
-          images: { connect: imagesConnect },
+          images: { connectOrCreate: imagesConnectOrCreate },
           name: album.Name,
           releaseDate: album.PremiereDate,
           releaseYear: album.ProductionYear,

src/server/queue/jellyfin/jellyfin.utils.ts

@@ -6,6 +6,7 @@ import {
   Server,
   ServerFolder,
 } from '@prisma/client';
+import uniqBy from 'lodash/uniqBy';
 import { prisma } from '../../lib';
 import { uniqueArray } from '../../utils';
 import {
@@ -32,13 +33,16 @@ const insertArtists = async (
   serverFolder: ServerFolder,
   items: JFSong[] | JFAlbum[]
 ) => {
-  const artistItems = items.flatMap((item) => item.ArtistItems);
+  const artistItems = uniqBy(
+    items.flatMap((item) => item.ArtistItems),
+    'Id'
+  );
 
   const createMany = artistItems.map((artist) => ({
     name: artist.Name,
     remoteId: artist.Id,
     serverId: server.id,
-    sortName: '',
+    sortName: artist.Name,
   }));
 
   await prisma.artist.createMany({
@@ -60,7 +64,10 @@ const insertArtists = async (
 };
 
 const insertImages = async (items: JFSong[] | JFAlbum[] | JFAlbumArtist[]) => {
-  const imageItems = items.flatMap((item) => item.ImageTags);
+  const imageItems = uniqBy(
+    items.flatMap((item) => item.ImageTags),
+    'Id'
+  );
 
   const createMany: Prisma.ImageCreateManyInput[] = [];
 
@@ -88,7 +95,10 @@ const insertImages = async (items: JFSong[] | JFAlbum[] | JFAlbumArtist[]) => {
 const insertExternals = async (
   items: JFSong[] | JFAlbum[] | JFAlbumArtist[]
 ) => {
-  const externalItems = items.flatMap((item) => item.ExternalUrls);
+  const externalItems = uniqBy(
+    items.flatMap((item) => item.ExternalUrls),
+    'Url'
+  );
   const createMany: Prisma.ExternalCreateManyInput[] = [];
 
   for (const external of externalItems) {
@@ -119,6 +129,32 @@ const insertSongGroup = async (
   songs: JFSong[],
   remoteAlbumId: string
 ) => {
+  const remoteAlbumArtist =
+    songs[0].AlbumArtists.length > 0 ? songs[0].AlbumArtists[0] : undefined;
+
+  let albumArtist = remoteAlbumArtist?.Id
+    ? await prisma.albumArtist.findUnique({
+        where: {
+          uniqueAlbumArtistId: {
+            remoteId: remoteAlbumArtist.Id,
+            serverId: server.id,
+          },
+        },
+      })
+    : undefined;
+
+  // If Jellyfin returns an invalid album artist, we'll just use the first matching one
+  if (remoteAlbumArtist && !albumArtist) {
+    albumArtist = await prisma.albumArtist.findFirst({
+      where: {
+        name: remoteAlbumArtist?.Name,
+        serverId: server.id,
+      },
+    });
+  }
+
+  const albumArtistId = albumArtist ? albumArtist.id : undefined;
+
   const songsUpsert: Prisma.SongUpsertWithWhereUniqueWithoutAlbumInput[] =
     songs.map((song) => {
       const genresConnect = song.Genres.map((genre) => ({ name: genre }));
@@ -140,16 +176,28 @@ const insertSongGroup = async (
         },
       }));
 
-      const imagesConnect = [];
+      const imagesConnectOrCreate = [];
       for (const [key, value] of Object.entries(song.ImageTags)) {
         if (key === JFImageType.PRIMARY) {
-          imagesConnect.push({
+          imagesConnectOrCreate.push({
-            uniqueImageId: { remoteUrl: value, type: ImageType.PRIMARY },
+            create: {
+              remoteUrl: value,
+              type: ImageType.PRIMARY,
+            },
+            where: {
+              uniqueImageId: { remoteUrl: value, type: ImageType.PRIMARY },
+            },
           });
         }
         if (key === JFImageType.LOGO) {
-          imagesConnect.push({
+          imagesConnectOrCreate.push({
-            uniqueImageId: { remoteUrl: value, type: ImageType.LOGO },
+            create: {
+              remoteUrl: value,
+              type: ImageType.LOGO,
+            },
+            where: {
+              uniqueImageId: { remoteUrl: value, type: ImageType.LOGO },
+            },
           });
         }
       }
@@ -159,6 +207,7 @@ const insertSongGroup = async (
 
       return {
         create: {
+          albumArtistId,
           artists: { connect: artistsConnect },
           bitRate: Math.floor(song.MediaSources[0].Bitrate / 1e3),
           container: song.MediaSources[0].Container,
@@ -172,7 +221,7 @@ const insertSongGroup = async (
             },
           },
           genres: { connect: genresConnect },
-          images: { connect: imagesConnect },
+          images: { connectOrCreate: imagesConnectOrCreate },
           name: song.Name,
           releaseDate: song.PremiereDate,
           releaseYear: song.ProductionYear,
@@ -185,6 +234,7 @@ const insertSongGroup = async (
           trackNumber: song.IndexNumber,
         },
         update: {
+          albumArtistId,
           artists: { connect: artistsConnect },
           bitRate: Math.floor(song.MediaSources[0].Bitrate / 1e3),
           container: song.MediaSources[0].Container,
@@ -198,7 +248,7 @@ const insertSongGroup = async (
             },
           },
           genres: { connect: genresConnect },
-          images: { connect: imagesConnect },
+          images: { connectOrCreate: imagesConnectOrCreate },
           name: song.Name,
           releaseDate: song.PremiereDate,
           releaseYear: song.ProductionYear,

src/server/queue/navidrome/index.ts

@@ -0,0 +1,7 @@
+import { navidromeApi } from './navidrome.api';
+import { navidromeScanner } from './navidrome.scanner';
+
+export const navidrome = {
+  api: navidromeApi,
+  scanner: navidromeScanner,
+};

src/server/queue/navidrome/navidrome.api.ts

@@ -0,0 +1,83 @@
+import { Server } from '@prisma/client';
+import axios from 'axios';
+import {
+  NDAlbumListResponse,
+  NDGenreListResponse,
+  NDAlbumListParams,
+  NDGenreListParams,
+  NDSongListParams,
+  NDSongListResponse,
+  NDArtistListResponse,
+  NDAuthenticate,
+} from './navidrome.types';
+
+const api = axios.create();
+
+const authenticate = async (options: {
+  password: string;
+  url: string;
+  username: string;
+}) => {
+  const { password, url, username } = options;
+  const cleanServerUrl = url.replace(/\/$/, '');
+
+  const { data } = await api.post<NDAuthenticate>(
+    `${cleanServerUrl}/auth/login`,
+    { password, username }
+  );
+
+  return data;
+};
+
+const getGenres = async (server: Server, params?: NDGenreListParams) => {
+  const { data } = await api.get<NDGenreListResponse>(
+    `${server.url}/api/genre`,
+    {
+      headers: { 'x-nd-authorization': `Bearer ${server.token}` },
+      params,
+    }
+  );
+
+  return data;
+};
+
+const getArtists = async (server: Server, params?: NDGenreListParams) => {
+  const { data } = await api.get<NDArtistListResponse>(
+    `${server.url}/api/artist`,
+    {
+      headers: { 'x-nd-authorization': `Bearer ${server.token}` },
+      params,
+    }
+  );
+
+  return data;
+};
+
+const getAlbums = async (server: Server, params?: NDAlbumListParams) => {
+  const { data } = await api.get<NDAlbumListResponse>(
+    `${server.url}/api/album`,
+    {
+      headers: { 'x-nd-authorization': `Bearer ${server.token}` },
+      params,
+    }
+  );
+
+  return data;
+};
+
+const getSongs = async (server: Server, params?: NDSongListParams) => {
+  const { data } = await api.get<NDSongListResponse>(`${server.url}/api/song`, {
+    headers: { 'x-nd-authorization': `Bearer ${server.token}` },
+    params,
+  });
+
+  return data;
+};
+
+export const navidromeApi = {
+  authenticate,
+  getAlbums,
+  getArtists,
+  getGenres,
+  getSongs,
+};

src/server/queue/navidrome/navidrome.normalize.ts

@@ -0,0 +1,59 @@
+import {
+  NormalizedAlbum,
+  NormalizedArtist,
+  NormalizedGenre,
+  NormalizedSong,
+} from '../api/types';
+import { NDAlbum, NDArtist, NDGenre, NDSong } from './navidrome.types';
+
+const genre = (genre: NDGenre): NormalizedGenre => {
+  return {
+    id: genre.id,
+    name: genre.name,
+  };
+};
+
+const artist = (artist: NDArtist): NormalizedArtist => {
+  return {
+    biography: artist.biography,
+    genres: artist.genres.map(genre),
+    id: artist.id,
+    name: artist.name,
+  };
+};
+
+const album = (album: NDAlbum): NormalizedAlbum => {
+  return {
+    albumArtistId: album.albumArtistId,
+    createdAt: album.createdAt,
+    genres: album.genres.map(genre),
+    id: album.id,
+    name: album.name,
+    year: album.minYear,
+  };
+};
+
+const song = (song: NDSong): NormalizedSong => {
+  return {
+    albumId: song.albumId,
+    artists: [{ id: song.artistId, name: song.artist }],
+    bitRate: song.bitRate,
+    container: song.suffix,
+    createdAt: song.createdAt,
+    disc: song.discNumber,
+    duration: song.duration,
+    genres: song.genres.map(genre),
+    id: song.id,
+    name: song.title,
+    path: song.path,
+    track: song.trackNumber,
+    year: song.year,
+  };
+};
+
+export const navidromeNormalize = {
+  album,
+  artist,
+  genre,
+  song,
+};

src/server/queue/navidrome/navidrome.scanner.ts

@@ -0,0 +1,376 @@
+/* eslint-disable no-await-in-loop */
+import {
+  ExternalSource,
+  ExternalType,
+  Folder,
+  ImageType,
+  Server,
+  ServerFolder,
+  Task,
+} from '@prisma/client';
+import uniqBy from 'lodash/uniqBy';
+import { prisma } from '../../lib';
+import { groupByProperty } from '../../utils';
+import { queue } from '../queues';
+import { navidromeApi } from './navidrome.api';
+import { navidromeUtils } from './navidrome.utils';
+
+const CHUNK_SIZE = 5000;
+
+export const scanGenres = async (server: Server, task: Task) => {
+  await prisma.task.update({
+    data: { message: 'Scanning genres' },
+    where: { id: task.id },
+  });
+
+  const res = await navidromeApi.getGenres(server);
+
+  const genres = res.map((genre) => {
+    return { name: genre.name };
+  });
+
+  await prisma.genre.createMany({
+    data: genres,
+    skipDuplicates: true,
+  });
+};
+
+export const scanAlbumArtists = async (
+  server: Server,
+  serverFolder: ServerFolder
+) => {
+  const artists = await navidromeApi.getArtists(server);
+
+  const externalsCreateMany = artists
+    .filter((artist) => artist.mbzArtistId)
+    .map((artist) => ({
+      source: ExternalSource.MUSICBRAINZ,
+      type: ExternalType.ID,
+      value: artist.mbzArtistId,
+    }));
+
+  await prisma.external.createMany({
+    data: externalsCreateMany,
+    skipDuplicates: true,
+  });
+
+  for (const artist of artists) {
+    const genresConnect = artist.genres
+      ? artist.genres.map((genre) => ({ name: genre.name }))
+      : undefined;
+
+    const externalsConnect = artist.mbzArtistId
+      ? {
+          uniqueExternalId: {
+            source: ExternalSource.MUSICBRAINZ,
+            value: artist.mbzArtistId,
+          },
+        }
+      : undefined;
+
+    await prisma.albumArtist.upsert({
+      create: {
+        deleted: false,
+        externals: { connect: externalsConnect },
+        genres: { connect: genresConnect },
+        name: artist.name,
+        remoteId: artist.id,
+        serverFolders: { connect: { id: serverFolder.id } },
+        serverId: server.id,
+        sortName: artist.name,
+      },
+      update: {
+        deleted: false,
+        externals: { connect: externalsConnect },
+        genres: { connect: genresConnect },
+        name: artist.name,
+        remoteId: artist.id,
+        serverFolders: { connect: { id: serverFolder.id } },
+        serverId: server.id,
+        sortName: artist.name,
+      },
+      where: {
+        uniqueAlbumArtistId: {
+          remoteId: artist.id,
+          serverId: server.id,
+        },
+      },
+    });
+  }
+};
+
+export const scanAlbums = async (
+  server: Server,
+  serverFolder: ServerFolder
+) => {
+  let start = 0;
+  let count = 5000;
+  do {
+    const albums = await navidromeApi.getAlbums(server, {
+      _end: start + CHUNK_SIZE,
+      _start: start,
+    });
+
+    const imagesCreateMany = albums
+      .filter((album) => album.coverArtId)
+      .map((album) => ({
+        remoteUrl: album.coverArtId,
+        type: ImageType.PRIMARY,
+      }));
+
+    await prisma.image.createMany({
+      data: imagesCreateMany,
+      skipDuplicates: true,
+    });
+
+    const artistIds = (
+      await prisma.artist.findMany({
+        select: { remoteId: true },
+        where: { serverId: server.id },
+      })
+    ).map((artist) => artist.remoteId);
+
+    for (const album of albums) {
+      const imagesConnect = album.coverArtId
+        ? {
+            uniqueImageId: {
+              remoteUrl: album.coverArtId,
+              type: ImageType.PRIMARY,
+            },
+          }
+        : undefined;
+
+      const genresConnect = album.genres
+        ? album.genres.map((genre) => ({ name: genre.name }))
+        : undefined;
+
+      const validArtistIds = [];
+      const ndArtistIds = album.allArtistIds.split(' ');
+
+      for (const artistId of ndArtistIds) {
+        if (artistIds.includes(artistId)) {
+          validArtistIds.push(artistId);
+        }
+      }
+
+      const artistsConnect = validArtistIds.map((id) => ({
+        uniqueArtistId: {
+          remoteId: id,
+          serverId: server.id,
+        },
+      }));
+
+      const albumArtistConnect = album.artistId
+        ? {
+            uniqueAlbumArtistId: {
+              remoteId: album.artistId,
+              serverId: server.id,
+            },
+          }
+        : undefined;
+
+      await prisma.album.upsert({
+        create: {
+          albumArtists: { connect: albumArtistConnect },
+          artists: { connect: artistsConnect },
+          deleted: false,
+          genres: { connect: genresConnect },
+          images: { connect: imagesConnect },
+          name: album.name,
+          releaseDate: album?.minYear
+            ? new Date(album.minYear, 0).toISOString()
+            : undefined,
+          releaseYear: album.minYear,
+          remoteCreatedAt: album.createdAt,
+          remoteId: album.id,
+          serverFolders: { connect: { id: serverFolder.id } },
+          serverId: server.id,
+          sortName: album.name,
+        },
+        update: {
+          albumArtists: { connect: albumArtistConnect },
+          artists: { connect: artistsConnect },
+          deleted: false,
+          genres: { connect: genresConnect },
+          images: { connect: imagesConnect },
+          name: album.name,
+          releaseDate: album?.minYear
+            ? new Date(album.minYear, 0).toISOString()
+            : undefined,
+          releaseYear: album.minYear,
+          remoteCreatedAt: album.createdAt,
+          remoteId: album.id,
+          serverFolders: { connect: { id: serverFolder.id } },
+          serverId: server.id,
+          sortName: album.name,
+        },
+        where: {
+          uniqueAlbumId: {
+            remoteId: album.id,
+            serverId: server.id,
+          },
+        },
+      });
+    }
+
+    start += CHUNK_SIZE;
+    count = albums.length;
+  } while (count === CHUNK_SIZE);
+};
+
+const scanSongs = async (server: Server, serverFolder: ServerFolder) => {
+  let start = 0;
+  let count = 5000;
+  do {
+    const songs = await navidromeApi.getSongs(server, {
+      _end: start + CHUNK_SIZE,
+      _start: start,
+    });
+
+    const externalsCreateMany = [];
+    const genresCreateMany = [];
+    for (const song of songs) {
+      if (song.mbzTrackId) {
+        externalsCreateMany.push({
+          source: ExternalSource.MUSICBRAINZ,
+          type: ExternalType.ID,
+          value: song.mbzTrackId,
+        });
+      }
+
+      if (song.genres?.length > 0) {
+        genresCreateMany.push(
+          ...song.genres.map((genre) => ({ name: genre.name }))
+        );
+      }
+    }
+
+    await prisma.external.createMany({
+      data: externalsCreateMany,
+      skipDuplicates: true,
+    });
+
+    await prisma.genre.createMany({
+      data: genresCreateMany,
+      skipDuplicates: true,
+    });
+
+    const folderGroups = songs.map((song) => {
+      const songPaths = song.path.split('/');
+      const paths = [];
+      for (let b = 0; b < songPaths.length - 1; b += 1) {
+        paths.push({
+          name: songPaths[b],
+          path: songPaths.slice(0, b + 1).join('/'),
+        });
+      }
+
+      return paths;
+    });
+
+    const uniqueFolders = uniqBy(
+      folderGroups.flatMap((folder) => folder).filter((f) => f.path !== ''),
+      'path'
+    );
+
+    const createdFolders: Folder[] = [];
+    for (const folder of uniqueFolders) {
+      const createdFolder = await prisma.folder.upsert({
+        create: {
+          name: folder.name,
+          path: folder.path,
+          serverFolders: {
+            connect: {
+              uniqueServerFolderId: {
+                remoteId: serverFolder.remoteId,
+                serverId: server.id,
+              },
+            },
+          },
+          serverId: server.id,
+        },
+        update: {
+          name: folder.name,
+          path: folder.path,
+          serverFolders: {
+            connect: {
+              uniqueServerFolderId: {
+                remoteId: serverFolder.remoteId,
+                serverId: server.id,
+              },
+            },
+          },
+        },
+        where: {
+          uniqueFolderId: {
+            path: folder.path,
+            serverId: server.id,
+          },
+        },
+      });
+
+      createdFolders.push(createdFolder);
+    }
+
+    for (const folder of createdFolders) {
+      if (folder.parentId) break;
+
+      const pathSplit = folder.path.split('/');
+      const parentPath = pathSplit.slice(0, pathSplit.length - 1).join('/');
+
+      const parentPathData = createdFolders.find(
+        (save) => save.path === parentPath
+      );
+
+      if (parentPathData) {
+        await prisma.folder.update({
+          data: {
+            parentId: parentPathData.id,
+          },
+          where: { id: folder.id },
+        });
+      }
+    }
+
+    const albumSongGroups = groupByProperty(songs, 'albumId');
+    const albumIds = Object.keys(albumSongGroups);
+
+    for (const id of albumIds) {
+      const songGroup = albumSongGroups[id];
+      await navidromeUtils.insertSongGroup(server, serverFolder, songGroup, id);
+    }
+
+    start += CHUNK_SIZE;
+    count = songs.length;
+  } while (count === CHUNK_SIZE);
+};
+
+const scanAll = async (
+  server: Server,
+  serverFolders: ServerFolder[],
+  task: Task
+) => {
+  queue.scanner.push({
+    fn: async () => {
+      await prisma.task.update({
+        data: { message: 'Beginning scan...' },
+        where: { id: task.id },
+      });
+
+      for (const serverFolder of serverFolders) {
+        await scanGenres(server, task);
+        await scanAlbumArtists(server, serverFolder);
+        await scanAlbums(server, serverFolder);
+        await scanSongs(server, serverFolder);
+      }
+
+      return { task };
+    },
+    id: task.id,
+  });
+};
+
+export const navidromeScanner = {
+  scanAll,
+  scanGenres,
+};

src/server/queue/navidrome/navidrome.types.ts

@@ -0,0 +1,169 @@
+export type NDAuthenticate = {
+  id: string;
+  isAdmin: boolean;
+  name: string;
+  subsonicSalt: string;
+  subsonicToken: string;
+  token: string;
+  username: string;
+};
+
+export type NDGenre = {
+  id: string;
+  name: string;
+};
+
+export type NDAlbum = {
+  albumArtist: string;
+  albumArtistId: string;
+  allArtistIds: string;
+  artist: string;
+  artistId: string;
+  compilation: boolean;
+  coverArtId: string;
+  coverArtPath: string;
+  createdAt: string;
+  duration: number;
+  fullText: string;
+  genre: string;
+  genres: NDGenre[];
+  id: string;
+  maxYear: number;
+  mbzAlbumArtistId: string;
+  mbzAlbumId: string;
+  minYear: number;
+  name: string;
+  orderAlbumArtistName: string;
+  orderAlbumName: string;
+  playCount: number;
+  playDate: string;
+  rating: number;
+  size: number;
+  songCount: number;
+  sortAlbumArtistName: string;
+  sortArtistName: string;
+  starred: boolean;
+  starredAt: string;
+  updatedAt: string;
+};
+
+export type NDSong = {
+  album: string;
+  albumArtist: string;
+  albumArtistId: string;
+  albumId: string;
+  artist: string;
+  artistId: string;
+  bitRate: number;
+  bookmarkPosition: number;
+  channels: number;
+  compilation: boolean;
+  createdAt: string;
+  discNumber: number;
+  duration: number;
+  fullText: string;
+  genre: string;
+  genres: NDGenre[];
+  hasCoverArt: boolean;
+  id: string;
+  mbzAlbumArtistId: string;
+  mbzAlbumId: string;
+  mbzArtistId: string;
+  mbzTrackId: string;
+  orderAlbumArtistName: string;
+  orderAlbumName: string;
+  orderArtistName: string;
+  orderTitle: string;
+  path: string;
+  playCount: number;
+  playDate: string;
+  rating: number;
+  size: number;
+  sortAlbumArtistName: string;
+  sortArtistName: string;
+  starred: boolean;
+  starredAt: string;
+  suffix: string;
+  title: string;
+  trackNumber: number;
+  updatedAt: string;
+  year: number;
+};
+
+export type NDArtist = {
+  albumCount: number;
+  biography: string;
+  externalInfoUpdatedAt: string;
+  externalUrl: string;
+  fullText: string;
+  genres: NDGenre[];
+  id: string;
+  largeImageUrl: string;
+  mbzArtistId: string;
+  mediumImageUrl: string;
+  name: string;
+  orderArtistName: string;
+  playCount: number;
+  playDate: string;
+  rating: number;
+  size: number;
+  smallImageUrl: string;
+  songCount: number;
+  starred: boolean;
+  starredAt: string;
+};
+
+export type NDGenreListResponse = NDGenre[];
+
+export type NDAlbumListResponse = NDAlbum[];
+
+export type NDSongListResponse = NDSong[];
+
+export type NDArtistListResponse = NDArtist[];
+
+export type NDPagination = {
+  _end?: number;
+  _start?: number;
+};
+
+export type NDOrder = {
+  _order?: 'ASC' | 'DESC';
+};
+
+export enum NDGenreSort {
+  NAME = 'name',
+}
+
+export type NDGenreListParams = {
+  _sort?: NDGenreSort;
+  id?: string;
+} & NDPagination &
+  NDOrder;
+
+export enum NDAlbumSort {
+  ARTIST = 'artist',
+  MAX_YEAR = 'max_year',
+  NAME = 'name',
+  RANDOM = 'random',
+  RECENTLY_ADDED = 'recently_added',
+}
+
+export type NDAlbumListParams = {
+  _sort?: NDAlbumSort;
+  artist_id?: string;
+  compilation?: boolean;
+  genre_id?: string;
+  has_rating?: boolean;
+  id?: string;
+  name?: string;
+  recently_played?: boolean;
+  starred?: boolean;
+  year?: number;
+} & NDPagination &
+  NDOrder;
+
+export type NDSongListParams = {
+  genre_id?: string;
+  starred?: boolean;
+} & NDPagination &
+  NDOrder;

src/server/queue/navidrome/navidrome.utils.ts

@@ -0,0 +1,125 @@
+import { ExternalSource, Server, ServerFolder } from '@prisma/client';
+import { prisma } from '../../lib';
+import { NDSong } from './navidrome.types';
+
+const insertSongGroup = async (
+  server: Server,
+  serverFolder: ServerFolder,
+  songs: NDSong[],
+  remoteAlbumId: string
+) => {
+  const songsWithArtistIds = songs.filter((song) => song.artistId);
+  const artistId =
+    songsWithArtistIds.length > 0 ? songsWithArtistIds[0].artistId : undefined;
+
+  const albumArtist = artistId
+    ? await prisma.albumArtist.findUnique({
+        where: {
+          uniqueAlbumArtistId: {
+            remoteId: artistId,
+            serverId: server.id,
+          },
+        },
+      })
+    : undefined;
+
+  const songsUpsert = songs.map((song) => {
+    const genresConnect = song.genres
+      ? song.genres.map((genre) => ({ name: genre.name }))
+      : undefined;
+
+    const externalsConnect = song.mbzTrackId
+      ? {
+          uniqueExternalId: {
+            source: ExternalSource.MUSICBRAINZ,
+            value: song.mbzTrackId,
+          },
+        }
+      : undefined;
+
+    const pathSplit = song.path.split('/');
+    const parentPath = pathSplit.slice(0, pathSplit.length - 1).join('/');
+
+    return {
+      create: {
+        albumArtistId: albumArtist?.id,
+        artistName: !song.artistId ? song.artist : undefined,
+        bitRate: song.bitRate,
+        container: song.suffix,
+        deleted: false,
+        discNumber: song.discNumber,
+        duration: song.duration,
+        externals: { connect: externalsConnect },
+        folders: {
+          connect: {
+            uniqueFolderId: { path: parentPath, serverId: server.id },
+          },
+        },
+        genres: { connect: genresConnect },
+        name: song.title,
+        releaseDate: song?.year
+          ? new Date(song.year, 0).toISOString()
+          : undefined,
+        releaseYear: song?.year,
+        remoteCreatedAt: song.createdAt,
+        remoteId: song.id,
+        serverFolders: { connect: { id: serverFolder.id } },
+        serverId: server.id,
+        size: song.size,
+        sortName: song.title,
+        trackNumber: song.trackNumber,
+      },
+      update: {
+        albumArtistId: albumArtist?.id,
+        artistName: !song.artistId ? song.artist : undefined,
+        bitRate: song.bitRate,
+        container: song.suffix,
+        deleted: false,
+        discNumber: song.discNumber,
+        duration: song.duration,
+        externals: { connect: externalsConnect },
+        folders: {
+          connect: {
+            uniqueFolderId: { path: parentPath, serverId: server.id },
+          },
+        },
+        genres: { connect: genresConnect },
+        name: song.title,
+        releaseDate: song?.year
+          ? new Date(song.year, 0).toISOString()
+          : undefined,
+        releaseYear: song?.year,
+        remoteCreatedAt: song.createdAt,
+        remoteId: song.id,
+        serverFolders: { connect: { id: serverFolder.id } },
+        serverId: server.id,
+        size: song.size,
+        sortName: song.title,
+        trackNumber: song.trackNumber,
+      },
+      where: {
+        uniqueSongId: {
+          remoteId: song.id,
+          serverId: server.id,
+        },
+      },
+    };
+  });
+
+  await prisma.album.update({
+    data: {
+      deleted: false,
+      songs: { upsert: songsUpsert },
+    },
+    where: {
+      uniqueAlbumId: {
+        remoteId: remoteAlbumId,
+        serverId: server.id,
+      },
+    },
+  });
+};
+
+export const navidromeUtils = {
+  insertSongGroup,
+};

src/server/queue/subsonic/subsonic.scanner.ts

@@ -35,12 +35,14 @@ export const scanAlbumArtists = async (
       create: {
         name: artist.name,
         remoteId: artist.id,
+        serverFolders: { connect: { id: serverFolder.id } },
         serverId: server.id,
         sortName: artist.name,
       },
       update: {
         name: artist.name,
         remoteId: artist.id,
+        serverFolders: { connect: { id: serverFolder.id } },
         serverId: server.id,
         sortName: artist.name,
       },
@@ -51,27 +53,6 @@ export const scanAlbumArtists = async (
         },
       },
     });
-
-    await prisma.artist.upsert({
-      create: {
-        name: artist.name,
-        remoteId: artist.id,
-        serverId: server.id,
-        sortName: artist.name,
-      },
-      update: {
-        name: artist.name,
-        remoteId: artist.id,
-        serverId: server.id,
-        sortName: artist.name,
-      },
-      where: {
-        uniqueArtistId: {
-          remoteId: artist.id,
-          serverId: server.id,
-        },
-      },
-    });
   }
 };
 
@@ -98,20 +79,18 @@ export const scanAlbums = async (
         }
       : undefined;
 
-    const albumArtist = album.artistId
+    const albumArtistConnect = album.artistId
-      ? await prisma.albumArtist.findUnique({
+      ? {
-          where: {
+          uniqueAlbumArtistId: {
-            uniqueAlbumArtistId: {
+            remoteId: album.artistId,
-              remoteId: album.artistId,
+            serverId: server.id,
-              serverId: server.id,
-            },
           },
-        })
+        }
       : undefined;
 
     await prisma.album.upsert({
       create: {
-        albumArtistId: albumArtist?.id,
+        albumArtists: { connect: albumArtistConnect },
         genres: { connect: album.genre ? { name: album.genre } : undefined },
         images: { connect: imagesConnect },
         name: album.title,
@@ -126,7 +105,7 @@ export const scanAlbums = async (
         sortName: album.title,
       },
       update: {
-        albumArtistId: albumArtist?.id,
+        albumArtists: { connect: albumArtistConnect },
         genres: { connect: album.genre ? { name: album.genre } : undefined },
         images: { connect: imagesConnect },
         name: album.title,
@@ -168,9 +147,9 @@ const throttledAlbumFetch = throttle(
             }
           : undefined;
 
-        const artistsConnect = song.artistId
+        const albumArtistsConnect = song.artistId
           ? {
-              uniqueArtistId: {
+              uniqueAlbumArtistId: {
                 remoteId: song.artistId,
                 serverId: server.id,
               },
@@ -179,8 +158,8 @@ const throttledAlbumFetch = throttle(
 
         return {
           create: {
+            albumArtists: { connect: albumArtistsConnect },
             artistName: !song.artistId ? song.artist : undefined,
-            artists: { connect: artistsConnect },
             bitRate: song.bitRate,
             container: song.suffix,
             discNumber: song.discNumber,
@@ -201,8 +180,8 @@ const throttledAlbumFetch = throttle(
             trackNumber: song.track,
           },
           update: {
+            albumArtists: { connect: albumArtistsConnect },
             artistName: !song.artistId ? song.artist : undefined,
-            artists: { connect: artistsConnect },
             bitRate: song.bitRate,
             container: song.suffix,
             discNumber: song.discNumber,
@@ -278,83 +257,6 @@ export const scanAlbumDetail = async (
   await Promise.all(promises);
 };
 
-// const throttledArtistDetailFetch = throttle(
-//   async (
-//     server: Server,
-//     artistId: string,
-//     artistRemoteId: string,
-//     i: number
-//   ) => {
-//     console.log('artisdetail', i);
-
-//     const artistInfo = await subsonicApi.getArtistInfo(server, artistRemoteId);
-
-//     const externalsConnectOrCreate = [];
-//     if (artistInfo.artistInfo2.lastFmUrl) {
-//       externalsConnectOrCreate.push({
-//         create: {
-//           name: 'Last.fm',
-//           url: artistInfo.artistInfo2.lastFmUrl,
-//         },
-//         where: {
-//           uniqueExternalId: {
-//             name: 'Last.fm',
-//             url: artistInfo.artistInfo2.lastFmUrl,
-//           },
-//         },
-//       });
-//     }
-
-//     if (artistInfo.artistInfo2.musicBrainzId) {
-//       externalsConnectOrCreate.push({
-//         create: {
-//           name: 'MusicBrainz',
-//           url: `https://musicbrainz.org/artist/${artistInfo.artistInfo2.musicBrainzId}`,
-//         },
-//         where: {
-//           uniqueExternalId: {
-//             name: 'MusicBrainz',
-//             url: `https://musicbrainz.org/artist/${artistInfo.artistInfo2.musicBrainzId}`,
-//           },
-//         },
-//       });
-//     }
-
-//     try {
-//       await prisma.albumArtist.update({
-//         data: {
-//           biography: artistInfo.artistInfo2.biography,
-//           // externals: { connectOrCreate: externalsConnectOrCreate },
-//         },
-//         where: { id: artistId },
-//       });
-//     } catch (err) {
-//       console.log(err);
-//     }
-//   }
-// );
-
-// export const scanAlbumArtistDetail = async (
-//   server: Server,
-//   serverFolder: ServerFolder
-// ) => {
-//   const promises = [];
-//   const dbArtists = await prisma.albumArtist.findMany({
-//     where: { serverId: server.id },
-//   });
-
-//   for (let i = 0; i < dbArtists.length; i += 1) {
-//     promises.push(
-//       throttledArtistDetailFetch(
-//         server,
-//         dbArtists[i].id,
-//         dbArtists[i].remoteId,
-//         i
-//       )
-//     );
-//   }
-// };
-
 const scanAll = async (
   server: Server,
   serverFolders: ServerFolder[],
@@ -370,7 +272,6 @@ const scanAll = async (
       for (const serverFolder of serverFolders) {
         await scanGenres(server, task);
         await scanAlbumArtists(server, serverFolder);
-        // await scanAlbumArtistDetail(server, serverFolder);
         await scanAlbums(server, serverFolder);
         await scanAlbumDetail(server, serverFolder);
       }