vanja/feishin
1
0
Fork 0
mirror of https://github.com/jeffvli/feishin.git synced 2026-05-09 20:29:36 +02:00
Code Issues Packages Projects Releases Wiki Activity

Update subsonic scanner for new schema

Browse Source
This commit is contained in:
jeffvli
2022-10-12 20:47:23 -07:00
parent 0772566637
commit 968e80a6d8
3 changed files with 244 additions and 195 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/server/queue/subsonic/subsonic.scanner.ts
+206 -193
View File
@@ -1,26 +1,27 @@
/* eslint-disable no-await-in-loop */ /* eslint-disable no-await-in-loop */
import { Server, ServerFolder, Task } from '@prisma/client'; import { ImageType, Server, ServerFolder, Task } from '@prisma/client';
import { prisma, throttle } from '../../lib'; import { prisma, throttle } from '../../lib';
import { groupByProperty, uniqueArray } from '../../utils'; import { uniqueArray } from '../../utils';
import { queue } from '../queues';
import { subsonicApi } from './subsonic.api'; import { subsonicApi } from './subsonic.api';
import { SSAlbumListEntry } from './subsonic.types'; import { subsonicUtils } from './subsonic.utils';
export const scanGenres = async (server: Server, task: Task) => {
await prisma.task.update({
data: { message: 'Scanning genres' },
where: { id: task.id },
});
export const scanGenres = async (
server: Server,
serverFolder: ServerFolder
) => {
const res = await subsonicApi.getGenres(server); const res = await subsonicApi.getGenres(server);
const genres = res.genres.genre.map((genre) => { const genres = res.genres.genre.map((genre) => {
return { name: genre.value }; return { name: genre.value };
}); });
const createdGenres = await prisma.genre.createMany({ await prisma.genre.createMany({
data: genres, data: genres,
skipDuplicates: true, skipDuplicates: true,
}); });
const message = `Imported ${createdGenres.count} new genres.`;
}; };
export const scanAlbumArtists = async ( export const scanAlbumArtists = async (
@@ -35,11 +36,13 @@ export const scanAlbumArtists = async (
name: artist.name, name: artist.name,
remoteId: artist.id, remoteId: artist.id,
serverId: server.id, serverId: server.id,
sortName: artist.name,
}, },
update: { update: {
name: artist.name, name: artist.name,
remoteId: artist.id, remoteId: artist.id,
serverId: server.id, serverId: server.id,
sortName: artist.name,
}, },
where: { where: {
uniqueAlbumArtistId: { uniqueAlbumArtistId: {
@@ -54,11 +57,13 @@ export const scanAlbumArtists = async (
name: artist.name, name: artist.name,
remoteId: artist.id, remoteId: artist.id,
serverId: server.id, serverId: server.id,
sortName: artist.name,
}, },
update: { update: {
name: artist.name, name: artist.name,
remoteId: artist.id, remoteId: artist.id,
serverId: server.id, serverId: server.id,
sortName: artist.name,
}, },
where: { where: {
uniqueArtistId: { uniqueArtistId: {
@@ -68,15 +73,12 @@ export const scanAlbumArtists = async (
}, },
}); });
} }
const message = `Scanned ${artists.length} album artists.`;
}; };
export const scanAlbums = async ( export const scanAlbums = async (
server: Server, server: Server,
serverFolder: ServerFolder serverFolder: ServerFolder
) => { ) => {
const promises: any[] = [];
const albums = await subsonicApi.getAlbums(server, { const albums = await subsonicApi.getAlbums(server, {
musicFolderId: serverFolder.id, musicFolderId: serverFolder.id,
offset: 0, offset: 0,
@@ -84,92 +86,87 @@ export const scanAlbums = async (
type: 'newest', type: 'newest',
}); });
const albumArtistGroups = groupByProperty(albums, 'artistId'); await subsonicUtils.insertImages(albums);
const addAlbums = async ( for (const album of albums) {
a: SSAlbumListEntry[], const imagesConnect = album.coverArt
albumArtistRemoteId: string ? {
) => { uniqueImageId: {
const albumArtist = await prisma.albumArtist.findUnique({ remoteUrl: album.coverArt,
type: ImageType.PRIMARY,
},
}
: undefined;
const albumArtist = album.artistId
? await prisma.albumArtist.findUnique({
where: {
uniqueAlbumArtistId: {
remoteId: album.artistId,
serverId: server.id,
},
},
})
: undefined;
await prisma.album.upsert({
create: {
albumArtistId: albumArtist?.id,
genres: { connect: album.genre ? { name: album.genre } : undefined },
images: { connect: imagesConnect },
name: album.title,
releaseDate: album?.year
? new Date(album.year, 0).toISOString()
: undefined,
releaseYear: album.year,
remoteCreatedAt: album.created,
remoteId: album.id,
serverFolders: { connect: { id: serverFolder.id } },
serverId: server.id,
sortName: album.title,
},
update: {
albumArtistId: albumArtist?.id,
genres: { connect: album.genre ? { name: album.genre } : undefined },
images: { connect: imagesConnect },
name: album.title,
releaseDate: album?.year
? new Date(album.year, 0).toISOString()
: undefined,
releaseYear: album.year,
remoteCreatedAt: album.created,
remoteId: album.id,
serverFolders: { connect: { id: serverFolder.id } },
serverId: server.id,
sortName: album.title,
},
where: { where: {
uniqueAlbumArtistId: { uniqueAlbumId: {
remoteId: albumArtistRemoteId, remoteId: album.id,
serverId: server.id, serverId: server.id,
}, },
}, },
}); });
}
if (albumArtist) {
a.forEach(async (album) => {
const imagesConnectOrCreate = album.coverArt
? {
create: { name: 'Primary', url: album.coverArt },
where: {
uniqueImageId: { name: 'Primary', url: album.coverArt },
},
}
: [];
await prisma.album.upsert({
create: {
albumArtistId: albumArtist.id,
images: { connectOrCreate: imagesConnectOrCreate },
name: album.title,
remoteCreatedAt: album.created,
remoteId: album.id,
serverId: server.id,
year: album.year,
},
update: {
albumArtistId: albumArtist.id,
images: { connectOrCreate: imagesConnectOrCreate },
name: album.title,
remoteCreatedAt: album.created,
remoteId: album.id,
serverId: server.id,
year: album.year,
},
where: {
uniqueAlbumId: {
remoteId: album.id,
serverId: server.id,
},
},
});
});
}
};
Object.keys(albumArtistGroups).forEach((key) => {
promises.push(addAlbums(albumArtistGroups[key], key));
});
await Promise.all(promises);
const message = `Scanned ${albums.length} albums.`;
}; };
const throttledAlbumFetch = throttle( const throttledAlbumFetch = throttle(
async (server: Server, serverFolder: ServerFolder, album: any, i: number) => { async (server: Server, serverFolder: ServerFolder, album: any) => {
const albumRes = await subsonicApi.getAlbum(server, album.remoteId); const albumRes = await subsonicApi.getAlbum(server, album.remoteId);
console.log('fetch', i);
if (albumRes) { if (albumRes) {
await subsonicUtils.insertSongImages(albumRes);
const songsUpsert = albumRes.album.song.map((song) => { const songsUpsert = albumRes.album.song.map((song) => {
const genresConnectOrCreate = song.genre const genresConnect = song.genre ? { name: song.genre } : undefined;
? {
create: { name: song.genre },
where: { name: song.genre },
}
: [];
const imagesConnectOrCreate = song.coverArt const imagesConnect = song.coverArt
? { ? {
create: { name: 'Primary', url: song.coverArt }, uniqueImageId: {
where: { uniqueImageId: { name: 'Primary', url: song.coverArt } }, remoteUrl: song.coverArt,
type: ImageType.PRIMARY,
},
} }
: []; : undefined;
const artistsConnect = song.artistId const artistsConnect = song.artistId
? { ? {
@@ -178,7 +175,7 @@ const throttledAlbumFetch = throttle(
serverId: server.id, serverId: server.id,
}, },
} }
: []; : undefined;
return { return {
create: { create: {
@@ -186,31 +183,44 @@ const throttledAlbumFetch = throttle(
artists: { connect: artistsConnect }, artists: { connect: artistsConnect },
bitRate: song.bitRate, bitRate: song.bitRate,
container: song.suffix, container: song.suffix,
disc: song.discNumber, discNumber: song.discNumber,
duration: song.duration, duration: song.duration,
genres: { connectOrCreate: genresConnectOrCreate }, genres: { connect: genresConnect },
images: { connectOrCreate: imagesConnectOrCreate }, images: { connect: imagesConnect },
name: song.title, name: song.title,
releaseDate: song?.year
? new Date(song.year, 0).toISOString()
: undefined,
releaseYear: song.year,
remoteCreatedAt: song.created, remoteCreatedAt: song.created,
remoteId: song.id, remoteId: song.id,
serverFolders: { connect: { id: serverFolder.id } },
serverId: server.id, serverId: server.id,
track: song.track, size: song.size,
year: song.year, sortName: song.title,
trackNumber: song.track,
}, },
update: { update: {
artistName: !song.artistId ? song.artist : undefined, artistName: !song.artistId ? song.artist : undefined,
artists: { connect: artistsConnect }, artists: { connect: artistsConnect },
bitRate: song.bitRate, bitRate: song.bitRate,
container: song.suffix, container: song.suffix,
disc: song.discNumber, discNumber: song.discNumber,
duration: song.duration, duration: song.duration,
genres: { connectOrCreate: genresConnectOrCreate }, genres: { connect: genresConnect },
images: { connectOrCreate: imagesConnectOrCreate }, images: { connect: imagesConnect },
name: song.title, name: song.title,
releaseDate: song?.year
? new Date(song.year, 0).toISOString()
: undefined,
releaseYear: song.year,
remoteCreatedAt: song.created, remoteCreatedAt: song.created,
remoteId: song.id, remoteId: song.id,
track: song.track, serverFolders: { connect: { id: serverFolder.id } },
year: song.year, serverId: server.id,
size: song.size,
sortName: song.title,
trackNumber: song.track,
}, },
where: { where: {
uniqueSongId: { uniqueSongId: {
@@ -234,22 +244,18 @@ const throttledAlbumFetch = throttle(
}; };
}); });
try { await prisma.album.update({
await prisma.album.update({ data: {
data: { artists: { connect: artistsConnect },
artists: { connect: artistsConnect }, songs: { upsert: songsUpsert },
songs: { upsert: songsUpsert }, },
where: {
uniqueAlbumId: {
remoteId: albumRes.album.id,
serverId: server.id,
}, },
where: { },
uniqueAlbumId: { });
remoteId: albumRes.album.id,
serverId: server.id,
},
},
});
} catch (err) {
console.log(err);
}
} }
} }
); );
@@ -258,8 +264,6 @@ export const scanAlbumDetail = async (
server: Server, server: Server,
serverFolder: ServerFolder serverFolder: ServerFolder
) => { ) => {
const taskId = `[${server.name} (${serverFolder.name})] albums detail`;
const promises = []; const promises = [];
const dbAlbums = await prisma.album.findMany({ const dbAlbums = await prisma.album.findMany({
where: { where: {
@@ -268,104 +272,113 @@ export const scanAlbumDetail = async (
}); });
for (let i = 0; i < dbAlbums.length; i += 1) { for (let i = 0; i < dbAlbums.length; i += 1) {
promises.push(throttledAlbumFetch(server, serverFolder, dbAlbums[i], i)); promises.push(throttledAlbumFetch(server, serverFolder, dbAlbums[i]));
} }
await Promise.all(promises); await Promise.all(promises);
const message = `Scanned ${dbAlbums.length} albums.`;
}; };
const throttledArtistDetailFetch = throttle( // const throttledArtistDetailFetch = throttle(
async ( // async (
server: Server, // server: Server,
artistId: number, // artistId: string,
artistRemoteId: string, // artistRemoteId: string,
i: number // i: number
) => { // ) => {
console.log('artisdetail', i); // console.log('artisdetail', i);
const artistInfo = await subsonicApi.getArtistInfo(server, artistRemoteId); // const artistInfo = await subsonicApi.getArtistInfo(server, artistRemoteId);
const externalsConnectOrCreate = []; // const externalsConnectOrCreate = [];
if (artistInfo.artistInfo2.lastFmUrl) { // if (artistInfo.artistInfo2.lastFmUrl) {
externalsConnectOrCreate.push({ // externalsConnectOrCreate.push({
create: { // create: {
name: 'Last.fm', // name: 'Last.fm',
url: artistInfo.artistInfo2.lastFmUrl, // url: artistInfo.artistInfo2.lastFmUrl,
}, // },
where: { // where: {
uniqueExternalId: { // uniqueExternalId: {
name: 'Last.fm', // name: 'Last.fm',
url: artistInfo.artistInfo2.lastFmUrl, // url: artistInfo.artistInfo2.lastFmUrl,
}, // },
}, // },
}); // });
} // }
if (artistInfo.artistInfo2.musicBrainzId) { // if (artistInfo.artistInfo2.musicBrainzId) {
externalsConnectOrCreate.push({ // externalsConnectOrCreate.push({
create: { // create: {
name: 'MusicBrainz', // name: 'MusicBrainz',
url: `https://musicbrainz.org/artist/${artistInfo.artistInfo2.musicBrainzId}`, // url: `https://musicbrainz.org/artist/${artistInfo.artistInfo2.musicBrainzId}`,
}, // },
where: { // where: {
uniqueExternalId: { // uniqueExternalId: {
name: 'MusicBrainz', // name: 'MusicBrainz',
url: `https://musicbrainz.org/artist/${artistInfo.artistInfo2.musicBrainzId}`, // url: `https://musicbrainz.org/artist/${artistInfo.artistInfo2.musicBrainzId}`,
}, // },
}, // },
}); // });
} // }
try { // try {
await prisma.albumArtist.update({ // await prisma.albumArtist.update({
data: { // data: {
biography: artistInfo.artistInfo2.biography, // biography: artistInfo.artistInfo2.biography,
externals: { connectOrCreate: externalsConnectOrCreate }, // // externals: { connectOrCreate: externalsConnectOrCreate },
}, // },
where: { id: artistId }, // where: { id: artistId },
}); // });
} catch (err) { // } catch (err) {
console.log(err); // console.log(err);
} // }
} // }
); // );
export const scanAlbumArtistDetail = async ( // export const scanAlbumArtistDetail = async (
server: Server, // server: Server,
serverFolder: ServerFolder // serverFolder: ServerFolder
) => { // ) => {
const taskId = `[${server.name} (${serverFolder.name})] artists detail`; // const promises = [];
// const dbArtists = await prisma.albumArtist.findMany({
// where: { serverId: server.id },
// });
const promises = []; // for (let i = 0; i < dbArtists.length; i += 1) {
const dbArtists = await prisma.albumArtist.findMany({ // promises.push(
where: { // throttledArtistDetailFetch(
serverId: server.id, // server,
}, // dbArtists[i].id,
}); // dbArtists[i].remoteId,
// i
for (let i = 0; i < dbArtists.length; i += 1) { // )
promises.push( // );
throttledArtistDetailFetch( // }
server, // };
dbArtists[i].id,
dbArtists[i].remoteId,
i
)
);
}
};
const scanAll = async ( const scanAll = async (
server: Server, server: Server,
serverFolder: ServerFolder, serverFolders: ServerFolder[],
task: Task task: Task
) => { ) => {
await scanGenres(server, serverFolder); queue.scanner.push({
await scanAlbumArtists(server, serverFolder); fn: async () => {
await scanAlbumArtistDetail(server, serverFolder); await prisma.task.update({
await scanAlbums(server, serverFolder); data: { message: 'Beginning scan...' },
await scanAlbumDetail(server, serverFolder); where: { id: task.id },
});
for (const serverFolder of serverFolders) {
await scanGenres(server, task);
await scanAlbumArtists(server, serverFolder);
// await scanAlbumArtistDetail(server, serverFolder);
await scanAlbums(server, serverFolder);
await scanAlbumDetail(server, serverFolder);
}
return { task };
},
id: task.id,
});
}; };
export const subsonicScanner = { export const subsonicScanner = {
src/server/queue/subsonic/subsonic.types.ts
+2 -2
View File
@@ -54,8 +54,8 @@ export interface SSMusicFolder {
} }
export interface SSGenre { export interface SSGenre {
albumCount: number; albumCount?: number;
songCount: number; songCount?: number;
value: string; value: string;
} }
src/server/queue/subsonic/subsonic.utils.ts
+36
View File
@@ -0,0 +1,36 @@
import { ImageType } from '@prisma/client';
import { prisma } from '../../lib';
import { SSAlbumListEntry, SSAlbumResponse } from './subsonic.types';
const insertImages = async (items: SSAlbumListEntry[]) => {
const createMany = items
.filter((item) => item.coverArt)
.map((item) => ({
remoteUrl: item.coverArt,
type: ImageType.PRIMARY,
}));
await prisma.image.createMany({
data: createMany,
skipDuplicates: true,
});
};
const insertSongImages = async (item: SSAlbumResponse) => {
const createMany = item.album.song
.filter((song) => song.coverArt)
.map((song) => ({
remoteUrl: song.coverArt,
type: ImageType.PRIMARY,
}));
await prisma.image.createMany({
data: createMany,
skipDuplicates: true,
});
};
export const subsonicUtils = {
insertImages,
insertSongImages,
};