mirror of
https://github.com/jeffvli/feishin.git
synced 2026-06-10 22:32:17 +02:00
Optimize jellyfin scanner
- Include changes / unfinished subsonic scanner
This commit is contained in:
jeffvli
@@ -0,0 +1,453 @@
|
||||
import {
|
||||
ExternalSource,
|
||||
Folder,
|
||||
ImageType,
|
||||
Server,
|
||||
ServerFolder,
|
||||
Task,
|
||||
} from '@prisma/client';
|
||||
import uniqBy from 'lodash/uniqBy';
|
||||
import { prisma } from '../../lib';
|
||||
import { groupByProperty } from '../../utils';
|
||||
import { queue } from '../queues';
|
||||
import { jellyfinApi } from './jellyfin.api';
|
||||
import { JFExternalType, JFImageType, JFItemType } from './jellyfin.types';
|
||||
import { jellyfinUtils } from './jellyfin.utils';
|
||||
|
||||
const scanGenres = async (options: {
|
||||
server: Server;
|
||||
serverFolder: ServerFolder;
|
||||
task: Task;
|
||||
}) => {
|
||||
await prisma.task.update({
|
||||
data: { message: 'Scanning genres' },
|
||||
where: { id: options.task.id },
|
||||
});
|
||||
|
||||
const genres = await jellyfinApi.getGenres(options.server, {
|
||||
parentId: options.serverFolder.remoteId,
|
||||
});
|
||||
|
||||
const genresCreate = genres.Items.map((genre) => {
|
||||
return { name: genre.Name };
|
||||
});
|
||||
|
||||
await prisma.genre.createMany({
|
||||
data: genresCreate,
|
||||
skipDuplicates: true,
|
||||
});
|
||||
};
|
||||
|
||||
const scanAlbumArtists = async (
|
||||
server: Server,
|
||||
serverFolder: ServerFolder,
|
||||
task: Task
|
||||
) => {
|
||||
await prisma.task.update({
|
||||
data: { message: 'Scanning album artists' },
|
||||
where: { id: task.id },
|
||||
});
|
||||
|
||||
const albumArtists = await jellyfinApi.getAlbumArtists(server, {
|
||||
fields: 'Genres,DateCreated,ExternalUrls,Overview',
|
||||
parentId: serverFolder.remoteId,
|
||||
});
|
||||
|
||||
await jellyfinUtils.insertGenres(albumArtists.Items);
|
||||
await jellyfinUtils.insertImages(albumArtists.Items);
|
||||
await jellyfinUtils.insertExternals(albumArtists.Items);
|
||||
|
||||
for (const albumArtist of albumArtists.Items) {
|
||||
const genresConnect = albumArtist.Genres.map((genre) => ({ name: genre }));
|
||||
|
||||
const imagesConnectOrCreate = [];
|
||||
for (const backdrop of albumArtist.BackdropImageTags) {
|
||||
imagesConnectOrCreate.push({
|
||||
create: { remoteUrl: backdrop, type: ImageType.BACKDROP },
|
||||
where: {
|
||||
uniqueImageId: { remoteUrl: backdrop, type: ImageType.BACKDROP },
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
const imagesConnect = [];
|
||||
for (const [key, value] of Object.entries(albumArtist.ImageTags)) {
|
||||
if (key === JFImageType.PRIMARY) {
|
||||
imagesConnect.push({
|
||||
uniqueImageId: { remoteUrl: value, type: ImageType.PRIMARY },
|
||||
});
|
||||
}
|
||||
if (key === JFImageType.LOGO) {
|
||||
imagesConnect.push({
|
||||
uniqueImageId: { remoteUrl: value, type: ImageType.LOGO },
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const externalsConnect = albumArtist.ExternalUrls.map((external) => ({
|
||||
uniqueExternalId: {
|
||||
source:
|
||||
external.Name === JFExternalType.MUSICBRAINZ
|
||||
? ExternalSource.MUSICBRAINZ
|
||||
: ExternalSource.THEAUDIODB,
|
||||
value: external.Url.split('/').pop() || '',
|
||||
},
|
||||
}));
|
||||
|
||||
await prisma.albumArtist.upsert({
|
||||
create: {
|
||||
biography: albumArtist.Overview,
|
||||
externals: { connect: externalsConnect },
|
||||
genres: { connect: genresConnect },
|
||||
images: {
|
||||
connect: imagesConnect,
|
||||
connectOrCreate: imagesConnectOrCreate,
|
||||
},
|
||||
name: albumArtist.Name,
|
||||
remoteCreatedAt: albumArtist.DateCreated,
|
||||
remoteId: albumArtist.Id,
|
||||
serverFolders: { connect: { id: serverFolder.id } },
|
||||
serverId: server.id,
|
||||
sortName: albumArtist.Name,
|
||||
},
|
||||
update: {
|
||||
biography: albumArtist.Overview,
|
||||
deleted: false,
|
||||
externals: { connect: externalsConnect },
|
||||
genres: { connect: genresConnect },
|
||||
images: { connectOrCreate: imagesConnectOrCreate },
|
||||
name: albumArtist.Name,
|
||||
remoteCreatedAt: albumArtist.DateCreated,
|
||||
remoteId: albumArtist.Id,
|
||||
serverFolders: { connect: { id: serverFolder.id } },
|
||||
serverId: server.id,
|
||||
sortName: albumArtist.Name,
|
||||
},
|
||||
where: {
|
||||
uniqueAlbumArtistId: {
|
||||
remoteId: albumArtist.Id,
|
||||
serverId: server.id,
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
const scanAlbums = async (
|
||||
server: Server,
|
||||
serverFolder: ServerFolder,
|
||||
task: Task
|
||||
) => {
|
||||
const check = await jellyfinApi.getAlbums(server, {
|
||||
enableUserData: false,
|
||||
includeItemTypes: JFItemType.MUSICALBUM,
|
||||
limit: 1,
|
||||
parentId: serverFolder.remoteId,
|
||||
recursive: true,
|
||||
});
|
||||
|
||||
const albumCount = check.TotalRecordCount;
|
||||
const chunkSize = 5000;
|
||||
const albumChunkCount = Math.ceil(albumCount / chunkSize);
|
||||
|
||||
await prisma.task.update({
|
||||
data: { message: 'Scanning albums' },
|
||||
where: { id: task.id },
|
||||
});
|
||||
|
||||
for (let i = 0; i < albumChunkCount; i += 1) {
|
||||
const albums = await jellyfinApi.getAlbums(server, {
|
||||
enableImageTypes: 'Primary,Logo,Backdrop',
|
||||
enableUserData: false,
|
||||
fields: 'Genres,DateCreated,ExternalUrls,Overview',
|
||||
imageTypeLimit: 1,
|
||||
limit: chunkSize,
|
||||
parentId: serverFolder.remoteId,
|
||||
recursive: true,
|
||||
startIndex: i * chunkSize,
|
||||
});
|
||||
|
||||
await jellyfinUtils.insertGenres(albums.Items);
|
||||
await jellyfinUtils.insertImages(albums.Items);
|
||||
await jellyfinUtils.insertExternals(albums.Items);
|
||||
|
||||
for (const album of albums.Items) {
|
||||
const genresConnect = album.Genres.map((genre) => ({ name: genre }));
|
||||
|
||||
const imagesConnect = [];
|
||||
for (const [key, value] of Object.entries(album.ImageTags)) {
|
||||
if (key === JFImageType.PRIMARY) {
|
||||
imagesConnect.push({
|
||||
uniqueImageId: { remoteUrl: value, type: ImageType.PRIMARY },
|
||||
});
|
||||
}
|
||||
if (key === JFImageType.LOGO) {
|
||||
imagesConnect.push({
|
||||
uniqueImageId: { remoteUrl: value, type: ImageType.LOGO },
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const externalsConnect = album.ExternalUrls.map((external) => ({
|
||||
uniqueExternalId: {
|
||||
source:
|
||||
external.Name === JFExternalType.MUSICBRAINZ
|
||||
? ExternalSource.MUSICBRAINZ
|
||||
: ExternalSource.THEAUDIODB,
|
||||
value: external.Url.split('/').pop() || '',
|
||||
},
|
||||
}));
|
||||
|
||||
const albumArtist =
|
||||
album.AlbumArtists.length > 0
|
||||
? await prisma.albumArtist.findUnique({
|
||||
where: {
|
||||
uniqueAlbumArtistId: {
|
||||
remoteId: album.AlbumArtists && album.AlbumArtists[0].Id,
|
||||
serverId: server.id,
|
||||
},
|
||||
},
|
||||
})
|
||||
: undefined;
|
||||
|
||||
await prisma.album.upsert({
|
||||
create: {
|
||||
albumArtistId: albumArtist?.id,
|
||||
externals: { connect: externalsConnect },
|
||||
genres: { connect: genresConnect },
|
||||
images: { connect: imagesConnect },
|
||||
name: album.Name,
|
||||
releaseDate: album.PremiereDate,
|
||||
releaseYear: album.ProductionYear,
|
||||
remoteCreatedAt: album.DateCreated,
|
||||
remoteId: album.Id,
|
||||
serverFolders: { connect: { id: serverFolder.id } },
|
||||
serverId: server.id,
|
||||
sortName: album.Name,
|
||||
},
|
||||
update: {
|
||||
albumArtistId: albumArtist?.id,
|
||||
deleted: false,
|
||||
externals: { connect: externalsConnect },
|
||||
genres: { connect: genresConnect },
|
||||
images: { connect: imagesConnect },
|
||||
name: album.Name,
|
||||
releaseDate: album.PremiereDate,
|
||||
releaseYear: album.ProductionYear,
|
||||
remoteCreatedAt: album.DateCreated,
|
||||
remoteId: album.Id,
|
||||
serverFolders: { connect: { id: serverFolder.id } },
|
||||
serverId: server.id,
|
||||
sortName: album.Name,
|
||||
},
|
||||
where: {
|
||||
uniqueAlbumId: {
|
||||
remoteId: album.Id,
|
||||
serverId: server.id,
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const scanSongs = async (
|
||||
server: Server,
|
||||
serverFolder: ServerFolder,
|
||||
task: Task
|
||||
) => {
|
||||
const check = await jellyfinApi.getSongs(server, {
|
||||
enableUserData: false,
|
||||
limit: 0,
|
||||
parentId: serverFolder.remoteId,
|
||||
recursive: true,
|
||||
});
|
||||
|
||||
const songCount = check.TotalRecordCount;
|
||||
const chunkSize = 5000;
|
||||
const songChunkCount = Math.ceil(songCount / chunkSize);
|
||||
|
||||
await prisma.task.update({
|
||||
data: { message: 'Scanning songs' },
|
||||
where: { id: task.id },
|
||||
});
|
||||
|
||||
for (let i = 0; i < songChunkCount; i += 1) {
|
||||
const songs = await jellyfinApi.getSongs(server, {
|
||||
enableImageTypes: 'Primary,Logo,Backdrop',
|
||||
enableUserData: false,
|
||||
fields: 'Genres,DateCreated,ExternalUrls,MediaSources,SortName',
|
||||
imageTypeLimit: 1,
|
||||
limit: chunkSize,
|
||||
parentId: serverFolder.remoteId,
|
||||
recursive: true,
|
||||
sortBy: 'DateCreated,Album',
|
||||
sortOrder: 'Descending',
|
||||
startIndex: i * chunkSize,
|
||||
});
|
||||
|
||||
const folderGroups = songs.Items.map((song) => {
|
||||
const songPaths = song.MediaSources[0].Path.split('/');
|
||||
const paths = [];
|
||||
for (let b = 0; b < songPaths.length - 1; b += 1) {
|
||||
paths.push({
|
||||
name: songPaths[b],
|
||||
path: songPaths.slice(0, b + 1).join('/'),
|
||||
});
|
||||
}
|
||||
|
||||
return paths;
|
||||
});
|
||||
|
||||
const uniqueFolders = uniqBy(
|
||||
folderGroups.flatMap((folder) => folder).filter((f) => f.path !== ''),
|
||||
'path'
|
||||
);
|
||||
|
||||
const createdFolders: Folder[] = [];
|
||||
for (const folder of uniqueFolders) {
|
||||
const createdFolder = await prisma.folder.upsert({
|
||||
create: {
|
||||
name: folder.name,
|
||||
path: folder.path,
|
||||
serverFolders: {
|
||||
connect: {
|
||||
uniqueServerFolderId: {
|
||||
remoteId: serverFolder.remoteId,
|
||||
serverId: server.id,
|
||||
},
|
||||
},
|
||||
},
|
||||
serverId: server.id,
|
||||
},
|
||||
update: {
|
||||
name: folder.name,
|
||||
path: folder.path,
|
||||
serverFolders: {
|
||||
connect: {
|
||||
uniqueServerFolderId: {
|
||||
remoteId: serverFolder.remoteId,
|
||||
serverId: server.id,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
where: {
|
||||
uniqueFolderId: {
|
||||
path: folder.path,
|
||||
serverId: server.id,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
createdFolders.push(createdFolder);
|
||||
}
|
||||
|
||||
for (const folder of createdFolders) {
|
||||
if (folder.parentId) break;
|
||||
|
||||
const pathSplit = folder.path.split('/');
|
||||
const parentPath = pathSplit.slice(0, pathSplit.length - 1).join('/');
|
||||
|
||||
const parentPathData = createdFolders.find(
|
||||
(save) => save.path === parentPath
|
||||
);
|
||||
|
||||
if (parentPathData) {
|
||||
await prisma.folder.update({
|
||||
data: {
|
||||
parentId: parentPathData.id,
|
||||
},
|
||||
where: { id: folder.id },
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
await jellyfinUtils.insertArtists(server, serverFolder, songs.Items);
|
||||
await jellyfinUtils.insertImages(songs.Items);
|
||||
await jellyfinUtils.insertExternals(songs.Items);
|
||||
|
||||
const albumSongGroups = groupByProperty(songs.Items, 'AlbumId');
|
||||
const keys = Object.keys(albumSongGroups);
|
||||
|
||||
for (const key of keys) {
|
||||
const songGroup = albumSongGroups[key];
|
||||
await jellyfinUtils.insertSongGroup(server, serverFolder, songGroup, key);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const checkDeleted = async (
|
||||
server: Server,
|
||||
serverFolder: ServerFolder,
|
||||
task: Task
|
||||
) => {
|
||||
await prisma.$transaction([
|
||||
prisma.albumArtist.updateMany({
|
||||
data: { deleted: true },
|
||||
where: {
|
||||
serverFolders: { some: { id: serverFolder.id } },
|
||||
serverId: server.id,
|
||||
updatedAt: { lte: task.createdAt },
|
||||
},
|
||||
}),
|
||||
prisma.artist.updateMany({
|
||||
data: { deleted: true },
|
||||
where: {
|
||||
serverFolders: { some: { id: serverFolder.id } },
|
||||
serverId: server.id,
|
||||
updatedAt: { lte: task.createdAt },
|
||||
},
|
||||
}),
|
||||
prisma.album.updateMany({
|
||||
data: { deleted: true },
|
||||
where: {
|
||||
serverFolders: { some: { id: serverFolder.id } },
|
||||
serverId: server.id,
|
||||
updatedAt: { lte: task.createdAt },
|
||||
},
|
||||
}),
|
||||
prisma.song.updateMany({
|
||||
data: { deleted: true },
|
||||
where: {
|
||||
serverFolders: { some: { id: serverFolder.id } },
|
||||
serverId: server.id,
|
||||
updatedAt: { lte: task.createdAt },
|
||||
},
|
||||
}),
|
||||
]);
|
||||
};
|
||||
|
||||
const scanAll = async (
|
||||
server: Server,
|
||||
serverFolders: ServerFolder[],
|
||||
task: Task
|
||||
) => {
|
||||
queue.scanner.push({
|
||||
fn: async () => {
|
||||
await prisma.task.update({
|
||||
data: { message: 'Beginning scan...' },
|
||||
where: { id: task.id },
|
||||
});
|
||||
|
||||
for (const serverFolder of serverFolders) {
|
||||
await scanGenres({ server, serverFolder, task });
|
||||
await scanAlbumArtists(server, serverFolder, task);
|
||||
await scanAlbums(server, serverFolder, task);
|
||||
await scanSongs(server, serverFolder, task);
|
||||
await checkDeleted(server, serverFolder, task);
|
||||
}
|
||||
|
||||
return { task };
|
||||
},
|
||||
id: task.id,
|
||||
});
|
||||
};
|
||||
|
||||
export const jellyfinScanner = {
|
||||
scanAlbumArtists,
|
||||
scanAlbums,
|
||||
scanAll,
|
||||
scanGenres,
|
||||
scanSongs,
|
||||
};
|
||||
Reference in New Issue
Block a user