vanja/feishin
1
0
Fork 0
mirror of https://github.com/jeffvli/feishin.git synced 2026-05-10 04:30:25 +02:00
Code Issues Packages Projects Releases Wiki Activity

Optimize jellyfin scanner

Browse Source 
- Include changes / unfinished subsonic scanner
This commit is contained in:
jeffvli
2022-10-12 13:39:59 -07:00
parent 8aedd94033
commit 02ef79dcb2
14 changed files with 1161 additions and 954 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/server/queue/index.ts
+2 -5
View File
@@ -1,5 +1,2 @@
export * from './subsonic/subsonic-api'; export * from './subsonic';
export * from './subsonic/subsonic-tasks'; export * from './jellyfin';
export * from './jellyfin/jellyfin-api';
export * from './jellyfin/jellyfin-tasks';
export * from './scanner-queue';
src/server/queue/jellyfin/index.ts
+7
View File
@@ -0,0 +1,7 @@
import { jellyfinApi } from './jellyfin.api';
import { jellyfinScanner } from './jellyfin.scanner';
export const jellyfin = {
api: jellyfinApi,
scanner: jellyfinScanner,
};
src/server/queue/jellyfin/jellyfin-tasks.ts
-631
View File
@@ -1,631 +0,0 @@
import uniqBy from 'lodash/uniqBy';
import { prisma } from '../../lib';
import { Server, ServerFolder, Task } from '../../types/types';
import { groupByProperty, uniqueArray } from '../../utils';
import { completeTask, q } from '../scanner-queue';
import { jellyfinApi } from './jellyfin-api';
import { JFSong } from './jellyfin-types';
const scanGenres = async (
server: Server,
serverFolder: ServerFolder,
task: Task
) => {
const taskId = `[${server.name} (${serverFolder.name})] genres`;
q.push({
fn: async () => {
await prisma.task.update({
data: { message: 'Scanning genres' },
where: { id: task.id },
});
const genres = await jellyfinApi.getGenres(server, {
parentId: serverFolder.remoteId,
});
const genresCreate = genres.Items.map((genre) => {
return { name: genre.Name };
});
await prisma.genre.createMany({
data: genresCreate,
skipDuplicates: true,
});
return { task };
},
id: taskId,
});
};
const scanAlbumArtists = async (
server: Server,
serverFolder: ServerFolder,
task: Task
) => {
const taskId = `[${server.name} (${serverFolder.name})] album artists`;
q.push({
fn: async () => {
await prisma.task.update({
data: { message: 'Scanning album artists' },
where: { id: task.id },
});
const albumArtists = await jellyfinApi.getAlbumArtists(server, {
fields: 'Genres,DateCreated,ExternalUrls,Overview',
parentId: serverFolder.remoteId,
});
for (const albumArtist of albumArtists.Items) {
const genresConnectOrCreate = albumArtist.Genres.map((genre) => {
return { create: { name: genre }, where: { name: genre } };
});
const imagesConnectOrCreate = [];
for (const [key, value] of Object.entries(albumArtist.ImageTags)) {
imagesConnectOrCreate.push({
create: { name: key, url: value },
where: { uniqueImageId: { name: key, url: value } },
});
}
const externalsConnectOrCreate = albumArtist.ExternalUrls.map(
(external) => {
return {
create: { name: external.Name, url: external.Url },
where: {
uniqueExternalId: { name: external.Name, url: external.Url },
},
};
}
);
await prisma.albumArtist.upsert({
create: {
biography: albumArtist.Overview,
externals: { connectOrCreate: externalsConnectOrCreate },
genres: { connectOrCreate: genresConnectOrCreate },
images: { connectOrCreate: imagesConnectOrCreate },
name: albumArtist.Name,
remoteCreatedAt: albumArtist.DateCreated,
remoteId: albumArtist.Id,
serverFolders: { connect: { id: serverFolder.id } },
serverId: server.id,
},
update: {
biography: albumArtist.Overview,
deleted: false,
externals: { connectOrCreate: externalsConnectOrCreate },
genres: { connectOrCreate: genresConnectOrCreate },
images: { connectOrCreate: imagesConnectOrCreate },
name: albumArtist.Name,
remoteCreatedAt: albumArtist.DateCreated,
remoteId: albumArtist.Id,
serverFolders: { connect: { id: serverFolder.id } },
serverId: server.id,
},
where: {
uniqueAlbumArtistId: {
remoteId: albumArtist.Id,
serverId: server.id,
},
},
});
}
return { task };
},
id: taskId,
});
};
const scanAlbums = async (
server: Server,
serverFolder: ServerFolder,
task: Task
) => {
const check = await jellyfinApi.getAlbums(server, {
enableUserData: false,
includeItemTypes: 'MusicAlbum',
limit: 1,
parentId: serverFolder.remoteId,
recursive: true,
});
const albumCount = check.TotalRecordCount;
const chunkSize = 5000;
const albumChunkCount = Math.ceil(albumCount / chunkSize);
const taskId = `(${task.id}) [${server.name} (${serverFolder.name})] albums`;
q.push({
fn: async () => {
await prisma.task.update({
data: { message: 'Scanning albums' },
where: { id: task.id },
});
for (let i = 0; i < albumChunkCount; i += 1) {
const albums = await jellyfinApi.getAlbums(server, {
enableImageTypes: 'Primary,Logo,Backdrop',
enableUserData: false,
fields: 'Genres,DateCreated,ExternalUrls,Overview',
imageTypeLimit: 1,
limit: chunkSize,
parentId: serverFolder.remoteId,
recursive: true,
startIndex: i * chunkSize,
});
for (const album of albums.Items) {
const genresConnectOrCreate = album.Genres.map((genre) => {
return { create: { name: genre }, where: { name: genre } };
});
const imagesConnectOrCreate = [];
for (const [key, value] of Object.entries(album.ImageTags)) {
imagesConnectOrCreate.push({
create: { name: key, url: value },
where: { uniqueImageId: { name: key, url: value } },
});
}
const externalsConnectOrCreate = album.ExternalUrls.map(
(external) => {
return {
create: { name: external.Name, url: external.Url },
where: {
uniqueExternalId: { name: external.Name, url: external.Url },
},
};
}
);
const albumArtist =
album.AlbumArtists.length > 0
? await prisma.albumArtist.findUnique({
where: {
uniqueAlbumArtistId: {
remoteId: album.AlbumArtists && album.AlbumArtists[0].Id,
serverId: server.id,
},
},
})
: undefined;
await prisma.album.upsert({
create: {
albumArtistId: albumArtist?.id,
date: album.PremiereDate,
externals: { connectOrCreate: externalsConnectOrCreate },
genres: { connectOrCreate: genresConnectOrCreate },
images: { connectOrCreate: imagesConnectOrCreate },
name: album.Name,
remoteCreatedAt: album.DateCreated,
remoteId: album.Id,
serverFolders: { connect: { id: serverFolder.id } },
serverId: server.id,
year: album.ProductionYear,
},
update: {
albumArtistId: albumArtist?.id,
date: album.PremiereDate,
deleted: false,
externals: { connectOrCreate: externalsConnectOrCreate },
genres: { connectOrCreate: genresConnectOrCreate },
images: { connectOrCreate: imagesConnectOrCreate },
name: album.Name,
remoteCreatedAt: album.DateCreated,
remoteId: album.Id,
serverFolders: { connect: { id: serverFolder.id } },
serverId: server.id,
year: album.ProductionYear,
},
where: {
uniqueAlbumId: {
remoteId: album.Id,
serverId: server.id,
},
},
});
}
const currentTask = await prisma.task.findUnique({
where: { id: task.id },
});
const newCount =
Number(currentTask?.progress || 0) + Number(albums.Items.length);
await prisma.task.update({
data: { progress: String(newCount) },
where: { id: task.id },
});
}
return { task };
},
id: taskId,
});
};
const insertSongGroup = async (
server: Server,
serverFolder: ServerFolder,
songs: JFSong[],
remoteAlbumId: string
) => {
const songsUpsert = songs.map((song) => {
const artistsConnectOrCreate = song.ArtistItems.map((artist) => {
return {
create: {
name: artist.Name,
remoteId: artist.Id,
serverFolders: { connect: { id: serverFolder.id } },
serverId: server.id,
},
where: {
uniqueArtistId: {
remoteId: artist.Id,
serverId: server.id,
},
},
};
});
const genresConnectOrCreate = song.Genres.map((genre) => {
return { create: { name: genre }, where: { name: genre } };
});
const imagesConnectOrCreate = [];
for (const [key, value] of Object.entries(song.ImageTags)) {
imagesConnectOrCreate.push({
create: { name: key, url: value },
where: { uniqueImageId: { name: key, url: value } },
});
}
const externalsConnectOrCreate = song.ExternalUrls.map((external) => {
return {
create: { name: external.Name, url: external.Url },
where: {
uniqueExternalId: { name: external.Name, url: external.Url },
},
};
});
const pathSplit = song.MediaSources[0].Path.split('/');
const parentPath = pathSplit.slice(0, pathSplit.length - 1).join('/');
return {
create: {
artists: { connectOrCreate: artistsConnectOrCreate },
bitRate: Math.floor(song.MediaSources[0].Bitrate / 1000),
container: song.MediaSources[0].Container,
date: song.PremiereDate,
disc: song.ParentIndexNumber,
duration: Math.floor(song.MediaSources[0].RunTimeTicks / 1e7),
externals: { connectOrCreate: externalsConnectOrCreate },
folders: {
connect: {
uniqueFolderId: { path: parentPath, serverId: server.id },
},
},
genres: { connectOrCreate: genresConnectOrCreate },
images: { connectOrCreate: imagesConnectOrCreate },
name: song.Name,
remoteCreatedAt: song.DateCreated,
remoteId: song.Id,
serverFolders: { connect: { id: serverFolder.id } },
serverId: server.id,
track: song.IndexNumber,
year: song.ProductionYear,
},
update: {
artists: { connectOrCreate: artistsConnectOrCreate },
bitRate: Math.floor(song.MediaSources[0].Bitrate / 1000),
container: song.MediaSources[0].Container,
date: song.PremiereDate,
disc: song.ParentIndexNumber,
duration: Math.floor(song.MediaSources[0].RunTimeTicks / 1e7),
externals: { connectOrCreate: externalsConnectOrCreate },
folders: {
connect: {
uniqueFolderId: { path: parentPath, serverId: server.id },
},
},
genres: { connectOrCreate: genresConnectOrCreate },
images: { connectOrCreate: imagesConnectOrCreate },
name: song.Name,
remoteCreatedAt: song.DateCreated,
remoteId: song.Id,
serverFolders: { connect: { id: serverFolder.id } },
serverId: server.id,
track: song.IndexNumber,
year: song.ProductionYear,
},
where: {
uniqueSongId: {
remoteId: song.Id,
serverId: server.id,
},
},
};
});
const artists = uniqBy(
songs.flatMap((song) => {
return song.ArtistItems.map((artist) => ({
name: artist.Name,
remoteId: artist.Id,
serverFolders: { connect: { id: serverFolder.id } },
serverId: server.id,
}));
}),
'remoteId'
);
const uniqueArtistIds = songs
.flatMap((song) => {
return song.ArtistItems.flatMap((artist) => artist.Id);
})
.filter(uniqueArray);
const artistsConnect = uniqueArtistIds.map((artistId) => {
return {
uniqueArtistId: {
remoteId: artistId!,
serverId: server.id,
},
};
});
artists.forEach(async (artist) => {
await prisma.artist.upsert({
create: artist,
update: artist,
where: {
uniqueArtistId: {
remoteId: artist.remoteId,
serverId: server.id,
},
},
});
});
await prisma.$transaction([
prisma.artist.updateMany({
data: { deleted: false },
where: {
remoteId: { in: uniqueArtistIds },
serverId: server.id,
},
}),
prisma.album.update({
data: {
artists: { connect: artistsConnect },
deleted: false,
songs: { upsert: songsUpsert },
},
where: {
uniqueAlbumId: {
remoteId: remoteAlbumId,
serverId: server.id,
},
},
}),
]);
};
const scanSongs = async (
server: Server,
serverFolder: ServerFolder,
task: Task
) => {
const check = await jellyfinApi.getSongs(server, {
enableUserData: false,
limit: 0,
parentId: serverFolder.remoteId,
recursive: true,
});
// Fetch in chunks
const songCount = check.TotalRecordCount;
const chunkSize = 10000;
const songChunkCount = Math.ceil(songCount / chunkSize);
const taskId = `[${server.name} (${serverFolder.name})] songs`;
q.push({
fn: async () => {
await prisma.task.update({
data: { message: 'Scanning songs' },
where: { id: task.id },
});
for (let i = 0; i < songChunkCount; i += 1) {
const songs = await jellyfinApi.getSongs(server, {
enableImageTypes: 'Primary,Logo,Backdrop',
enableUserData: false,
fields: 'Genres,DateCreated,ExternalUrls,MediaSources',
imageTypeLimit: 1,
limit: chunkSize,
parentId: serverFolder.remoteId,
recursive: true,
sortBy: 'DateCreated,Album',
sortOrder: 'Descending',
startIndex: i * chunkSize,
});
const folderGroups = songs.Items.map((song) => {
const songPaths = song.MediaSources[0].Path.split('/');
const paths = [];
for (let b = 0; b < songPaths.length - 1; b += 1) {
paths.push({
name: songPaths[b],
path: songPaths.slice(0, b + 1).join('/'),
});
}
return paths;
});
const uniqueFolders = uniqBy(
folderGroups.flatMap((folder) => folder).filter((f) => f.path !== ''),
'path'
);
const folderSave: any[] = [];
for (let f = 0; f < uniqueFolders.length; f += 1) {
const t = await prisma.folder.upsert({
create: {
name: uniqueFolders[f].name,
path: uniqueFolders[f].path,
serverFolders: {
connect: {
uniqueServerFolderId: {
remoteId: serverFolder.remoteId,
serverId: server.id,
},
},
},
serverId: server.id,
},
update: {
name: uniqueFolders[f].name,
path: uniqueFolders[f].path,
serverFolders: {
connect: {
uniqueServerFolderId: {
remoteId: serverFolder.remoteId,
serverId: server.id,
},
},
},
},
where: {
uniqueFolderId: {
path: uniqueFolders[f].path,
serverId: server.id,
},
},
});
folderSave.push(t);
}
folderSave.forEach(async (f) => {
if (f.parentId) return;
const pathSplit = f.path.split('/');
const parentPath = pathSplit.slice(0, pathSplit.length - 1).join('/');
const parentPathData = folderSave.find(
(save) => save.path === parentPath
);
if (parentPathData) {
await prisma.folder.update({
data: {
parentId: parentPathData.id,
},
where: { id: f.id },
});
}
});
const albumSongGroups = groupByProperty(songs.Items, 'AlbumId');
const keys = Object.keys(albumSongGroups);
for (let b = 0; b < keys.length; b += 1) {
const songGroup = albumSongGroups[keys[b]];
await insertSongGroup(server, serverFolder, songGroup, keys[b]);
const currentTask = await prisma.task.findUnique({
where: { id: task.id },
});
const newCount =
Number(currentTask?.progress || 0) + Number(songGroup.length);
await prisma.task.update({
data: { progress: String(newCount) },
where: { id: task.id },
});
}
}
return { completed: true, task };
},
id: taskId,
});
};
const checkDeleted = async (
server: Server,
serverFolder: ServerFolder,
task: Task
) => {
q.push({
fn: async () => {
await prisma.$transaction([
prisma.albumArtist.updateMany({
data: { deleted: true },
where: {
serverFolders: { some: { id: serverFolder.id } },
serverId: server.id,
updatedAt: { lte: task.createdAt },
},
}),
prisma.artist.updateMany({
data: { deleted: true },
where: {
serverFolders: { some: { id: serverFolder.id } },
serverId: server.id,
updatedAt: { lte: task.createdAt },
},
}),
prisma.album.updateMany({
data: { deleted: true },
where: {
serverFolders: { some: { id: serverFolder.id } },
serverId: server.id,
updatedAt: { lte: task.createdAt },
},
}),
prisma.song.updateMany({
data: { deleted: true },
where: {
serverFolders: { some: { id: serverFolder.id } },
serverId: server.id,
updatedAt: { lte: task.createdAt },
},
}),
]);
},
id: `${task.id}-difference`,
});
};
const scanAll = async (
server: Server,
serverFolder: ServerFolder,
task: Task
) => {
await scanGenres(server, serverFolder, task);
await scanAlbumArtists(server, serverFolder, task);
await scanAlbums(server, serverFolder, task);
await scanSongs(server, serverFolder, task);
await checkDeleted(server, serverFolder, task);
await completeTask(task);
};
export const jellyfinTasks = {
scanAlbumArtists,
scanAlbums,
scanAll,
scanGenres,
scanSongs,
};
src/server/queue/jellyfin/jellyfin-api.ts → src/server/queue/jellyfin/jellyfin.api.ts
+32 -5
View File
@@ -1,17 +1,43 @@
import { Server } from '@prisma/client';
import axios from 'axios'; import axios from 'axios';
import { Server } from '../../types/types';
import { import {
JFAlbumArtistsResponse, JFAlbumArtistsResponse,
JFAlbumsResponse, JFAlbumsResponse,
JFArtistsResponse, JFArtistsResponse,
JFAuthenticate,
JFCollectionType,
JFGenreResponse, JFGenreResponse,
JFItemType,
JFMusicFoldersResponse, JFMusicFoldersResponse,
JFRequestParams, JFRequestParams,
JFSongsResponse, JFSongsResponse,
} from './jellyfin-types'; } from './jellyfin.types';
export const api = axios.create({}); export const api = axios.create({});
export const authenticate = async (options: {
password: string;
url: string;
username: string;
}) => {
const { password, url, username } = options;
const cleanServerUrl = url.replace(/\/$/, '');
console.log('cleanServerUrl', cleanServerUrl);
const { data } = await api.post<JFAuthenticate>(
`${cleanServerUrl}/users/authenticatebyname`,
{ pw: password, username },
{
headers: {
'X-Emby-Authorization': `MediaBrowser Client="Sonixd", Device="PC", DeviceId="Sonixd", Version="1.0.0-alpha1"`,
},
}
);
return data;
};
export const getMusicFolders = async (server: Partial<Server>) => { export const getMusicFolders = async (server: Partial<Server>) => {
const { data } = await api.get<JFMusicFoldersResponse>( const { data } = await api.get<JFMusicFoldersResponse>(
`${server.url}/users/${server.remoteUserId}/items`, `${server.url}/users/${server.remoteUserId}/items`,
@@ -19,7 +45,7 @@ export const getMusicFolders = async (server: Partial<Server>) => {
); );
const musicFolders = data.Items.filter( const musicFolders = data.Items.filter(
(folder) => folder.CollectionType === 'music' (folder) => folder.CollectionType === JFCollectionType.MUSIC
); );
return musicFolders; return musicFolders;
@@ -63,7 +89,7 @@ export const getAlbums = async (server: Server, params: JFRequestParams) => {
`${server.url}/users/${server.remoteUserId}/items`, `${server.url}/users/${server.remoteUserId}/items`,
{ {
headers: { 'X-MediaBrowser-Token': server.token }, headers: { 'X-MediaBrowser-Token': server.token },
params: { includeItemTypes: 'MusicAlbum', ...params }, params: { includeItemTypes: JFItemType.MUSICALBUM, ...params },
} }
); );
@@ -75,7 +101,7 @@ export const getSongs = async (server: Server, params: JFRequestParams) => {
`${server.url}/users/${server.remoteUserId}/items`, `${server.url}/users/${server.remoteUserId}/items`,
{ {
headers: { 'X-MediaBrowser-Token': server.token }, headers: { 'X-MediaBrowser-Token': server.token },
params: { includeItemTypes: 'Audio', ...params }, params: { includeItemTypes: JFItemType.AUDIO, ...params },
} }
); );
@@ -83,6 +109,7 @@ export const getSongs = async (server: Server, params: JFRequestParams) => {
}; };
export const jellyfinApi = { export const jellyfinApi = {
authenticate,
getAlbumArtists, getAlbumArtists,
getAlbums, getAlbums,
getArtists, getArtists,
src/server/queue/jellyfin/jellyfin.scanner.ts
+453
View File
@@ -0,0 +1,453 @@
import {
ExternalSource,
Folder,
ImageType,
Server,
ServerFolder,
Task,
} from '@prisma/client';
import uniqBy from 'lodash/uniqBy';
import { prisma } from '../../lib';
import { groupByProperty } from '../../utils';
import { queue } from '../queues';
import { jellyfinApi } from './jellyfin.api';
import { JFExternalType, JFImageType, JFItemType } from './jellyfin.types';
import { jellyfinUtils } from './jellyfin.utils';
const scanGenres = async (options: {
server: Server;
serverFolder: ServerFolder;
task: Task;
}) => {
await prisma.task.update({
data: { message: 'Scanning genres' },
where: { id: options.task.id },
});
const genres = await jellyfinApi.getGenres(options.server, {
parentId: options.serverFolder.remoteId,
});
const genresCreate = genres.Items.map((genre) => {
return { name: genre.Name };
});
await prisma.genre.createMany({
data: genresCreate,
skipDuplicates: true,
});
};
const scanAlbumArtists = async (
server: Server,
serverFolder: ServerFolder,
task: Task
) => {
await prisma.task.update({
data: { message: 'Scanning album artists' },
where: { id: task.id },
});
const albumArtists = await jellyfinApi.getAlbumArtists(server, {
fields: 'Genres,DateCreated,ExternalUrls,Overview',
parentId: serverFolder.remoteId,
});
await jellyfinUtils.insertGenres(albumArtists.Items);
await jellyfinUtils.insertImages(albumArtists.Items);
await jellyfinUtils.insertExternals(albumArtists.Items);
for (const albumArtist of albumArtists.Items) {
const genresConnect = albumArtist.Genres.map((genre) => ({ name: genre }));
const imagesConnectOrCreate = [];
for (const backdrop of albumArtist.BackdropImageTags) {
imagesConnectOrCreate.push({
create: { remoteUrl: backdrop, type: ImageType.BACKDROP },
where: {
uniqueImageId: { remoteUrl: backdrop, type: ImageType.BACKDROP },
},
});
}
const imagesConnect = [];
for (const [key, value] of Object.entries(albumArtist.ImageTags)) {
if (key === JFImageType.PRIMARY) {
imagesConnect.push({
uniqueImageId: { remoteUrl: value, type: ImageType.PRIMARY },
});
}
if (key === JFImageType.LOGO) {
imagesConnect.push({
uniqueImageId: { remoteUrl: value, type: ImageType.LOGO },
});
}
}
const externalsConnect = albumArtist.ExternalUrls.map((external) => ({
uniqueExternalId: {
source:
external.Name === JFExternalType.MUSICBRAINZ
? ExternalSource.MUSICBRAINZ
: ExternalSource.THEAUDIODB,
value: external.Url.split('/').pop() || '',
},
}));
await prisma.albumArtist.upsert({
create: {
biography: albumArtist.Overview,
externals: { connect: externalsConnect },
genres: { connect: genresConnect },
images: {
connect: imagesConnect,
connectOrCreate: imagesConnectOrCreate,
},
name: albumArtist.Name,
remoteCreatedAt: albumArtist.DateCreated,
remoteId: albumArtist.Id,
serverFolders: { connect: { id: serverFolder.id } },
serverId: server.id,
sortName: albumArtist.Name,
},
update: {
biography: albumArtist.Overview,
deleted: false,
externals: { connect: externalsConnect },
genres: { connect: genresConnect },
images: { connectOrCreate: imagesConnectOrCreate },
name: albumArtist.Name,
remoteCreatedAt: albumArtist.DateCreated,
remoteId: albumArtist.Id,
serverFolders: { connect: { id: serverFolder.id } },
serverId: server.id,
sortName: albumArtist.Name,
},
where: {
uniqueAlbumArtistId: {
remoteId: albumArtist.Id,
serverId: server.id,
},
},
});
}
};
const scanAlbums = async (
server: Server,
serverFolder: ServerFolder,
task: Task
) => {
const check = await jellyfinApi.getAlbums(server, {
enableUserData: false,
includeItemTypes: JFItemType.MUSICALBUM,
limit: 1,
parentId: serverFolder.remoteId,
recursive: true,
});
const albumCount = check.TotalRecordCount;
const chunkSize = 5000;
const albumChunkCount = Math.ceil(albumCount / chunkSize);
await prisma.task.update({
data: { message: 'Scanning albums' },
where: { id: task.id },
});
for (let i = 0; i < albumChunkCount; i += 1) {
const albums = await jellyfinApi.getAlbums(server, {
enableImageTypes: 'Primary,Logo,Backdrop',
enableUserData: false,
fields: 'Genres,DateCreated,ExternalUrls,Overview',
imageTypeLimit: 1,
limit: chunkSize,
parentId: serverFolder.remoteId,
recursive: true,
startIndex: i * chunkSize,
});
await jellyfinUtils.insertGenres(albums.Items);
await jellyfinUtils.insertImages(albums.Items);
await jellyfinUtils.insertExternals(albums.Items);
for (const album of albums.Items) {
const genresConnect = album.Genres.map((genre) => ({ name: genre }));
const imagesConnect = [];
for (const [key, value] of Object.entries(album.ImageTags)) {
if (key === JFImageType.PRIMARY) {
imagesConnect.push({
uniqueImageId: { remoteUrl: value, type: ImageType.PRIMARY },
});
}
if (key === JFImageType.LOGO) {
imagesConnect.push({
uniqueImageId: { remoteUrl: value, type: ImageType.LOGO },
});
}
}
const externalsConnect = album.ExternalUrls.map((external) => ({
uniqueExternalId: {
source:
external.Name === JFExternalType.MUSICBRAINZ
? ExternalSource.MUSICBRAINZ
: ExternalSource.THEAUDIODB,
value: external.Url.split('/').pop() || '',
},
}));
const albumArtist =
album.AlbumArtists.length > 0
? await prisma.albumArtist.findUnique({
where: {
uniqueAlbumArtistId: {
remoteId: album.AlbumArtists && album.AlbumArtists[0].Id,
serverId: server.id,
},
},
})
: undefined;
await prisma.album.upsert({
create: {
albumArtistId: albumArtist?.id,
externals: { connect: externalsConnect },
genres: { connect: genresConnect },
images: { connect: imagesConnect },
name: album.Name,
releaseDate: album.PremiereDate,
releaseYear: album.ProductionYear,
remoteCreatedAt: album.DateCreated,
remoteId: album.Id,
serverFolders: { connect: { id: serverFolder.id } },
serverId: server.id,
sortName: album.Name,
},
update: {
albumArtistId: albumArtist?.id,
deleted: false,
externals: { connect: externalsConnect },
genres: { connect: genresConnect },
images: { connect: imagesConnect },
name: album.Name,
releaseDate: album.PremiereDate,
releaseYear: album.ProductionYear,
remoteCreatedAt: album.DateCreated,
remoteId: album.Id,
serverFolders: { connect: { id: serverFolder.id } },
serverId: server.id,
sortName: album.Name,
},
where: {
uniqueAlbumId: {
remoteId: album.Id,
serverId: server.id,
},
},
});
}
}
};
const scanSongs = async (
server: Server,
serverFolder: ServerFolder,
task: Task
) => {
const check = await jellyfinApi.getSongs(server, {
enableUserData: false,
limit: 0,
parentId: serverFolder.remoteId,
recursive: true,
});
const songCount = check.TotalRecordCount;
const chunkSize = 5000;
const songChunkCount = Math.ceil(songCount / chunkSize);
await prisma.task.update({
data: { message: 'Scanning songs' },
where: { id: task.id },
});
for (let i = 0; i < songChunkCount; i += 1) {
const songs = await jellyfinApi.getSongs(server, {
enableImageTypes: 'Primary,Logo,Backdrop',
enableUserData: false,
fields: 'Genres,DateCreated,ExternalUrls,MediaSources,SortName',
imageTypeLimit: 1,
limit: chunkSize,
parentId: serverFolder.remoteId,
recursive: true,
sortBy: 'DateCreated,Album',
sortOrder: 'Descending',
startIndex: i * chunkSize,
});
const folderGroups = songs.Items.map((song) => {
const songPaths = song.MediaSources[0].Path.split('/');
const paths = [];
for (let b = 0; b < songPaths.length - 1; b += 1) {
paths.push({
name: songPaths[b],
path: songPaths.slice(0, b + 1).join('/'),
});
}
return paths;
});
const uniqueFolders = uniqBy(
folderGroups.flatMap((folder) => folder).filter((f) => f.path !== ''),
'path'
);
const createdFolders: Folder[] = [];
for (const folder of uniqueFolders) {
const createdFolder = await prisma.folder.upsert({
create: {
name: folder.name,
path: folder.path,
serverFolders: {
connect: {
uniqueServerFolderId: {
remoteId: serverFolder.remoteId,
serverId: server.id,
},
},
},
serverId: server.id,
},
update: {
name: folder.name,
path: folder.path,
serverFolders: {
connect: {
uniqueServerFolderId: {
remoteId: serverFolder.remoteId,
serverId: server.id,
},
},
},
},
where: {
uniqueFolderId: {
path: folder.path,
serverId: server.id,
},
},
});
createdFolders.push(createdFolder);
}
for (const folder of createdFolders) {
if (folder.parentId) break;
const pathSplit = folder.path.split('/');
const parentPath = pathSplit.slice(0, pathSplit.length - 1).join('/');
const parentPathData = createdFolders.find(
(save) => save.path === parentPath
);
if (parentPathData) {
await prisma.folder.update({
data: {
parentId: parentPathData.id,
},
where: { id: folder.id },
});
}
}
await jellyfinUtils.insertArtists(server, serverFolder, songs.Items);
await jellyfinUtils.insertImages(songs.Items);
await jellyfinUtils.insertExternals(songs.Items);
const albumSongGroups = groupByProperty(songs.Items, 'AlbumId');
const keys = Object.keys(albumSongGroups);
for (const key of keys) {
const songGroup = albumSongGroups[key];
await jellyfinUtils.insertSongGroup(server, serverFolder, songGroup, key);
}
}
};
const checkDeleted = async (
server: Server,
serverFolder: ServerFolder,
task: Task
) => {
await prisma.$transaction([
prisma.albumArtist.updateMany({
data: { deleted: true },
where: {
serverFolders: { some: { id: serverFolder.id } },
serverId: server.id,
updatedAt: { lte: task.createdAt },
},
}),
prisma.artist.updateMany({
data: { deleted: true },
where: {
serverFolders: { some: { id: serverFolder.id } },
serverId: server.id,
updatedAt: { lte: task.createdAt },
},
}),
prisma.album.updateMany({
data: { deleted: true },
where: {
serverFolders: { some: { id: serverFolder.id } },
serverId: server.id,
updatedAt: { lte: task.createdAt },
},
}),
prisma.song.updateMany({
data: { deleted: true },
where: {
serverFolders: { some: { id: serverFolder.id } },
serverId: server.id,
updatedAt: { lte: task.createdAt },
},
}),
]);
};
const scanAll = async (
server: Server,
serverFolders: ServerFolder[],
task: Task
) => {
queue.scanner.push({
fn: async () => {
await prisma.task.update({
data: { message: 'Beginning scan...' },
where: { id: task.id },
});
for (const serverFolder of serverFolders) {
await scanGenres({ server, serverFolder, task });
await scanAlbumArtists(server, serverFolder, task);
await scanAlbums(server, serverFolder, task);
await scanSongs(server, serverFolder, task);
await checkDeleted(server, serverFolder, task);
}
return { task };
},
id: task.id,
});
};
export const jellyfinScanner = {
scanAlbumArtists,
scanAlbums,
scanAll,
scanGenres,
scanSongs,
};
src/server/queue/jellyfin/jellyfin-types.ts → src/server/queue/jellyfin/jellyfin.types.ts
+151 -10
View File
@@ -49,7 +49,7 @@ export interface JFRequestParams {
} }
export interface JFMusicFolder { export interface JFMusicFolder {
BackdropImageTags: any[]; BackdropImageTags: string[];
ChannelId: null; ChannelId: null;
CollectionType: string; CollectionType: string;
Id: string; Id: string;
@@ -68,7 +68,7 @@ export interface JFGenre {
ChannelId: null; ChannelId: null;
Id: string; Id: string;
ImageBlurHashes: any; ImageBlurHashes: any;
ImageTags: any; ImageTags: ImageTags;
LocationType: string; LocationType: string;
Name: string; Name: string;
ServerId: string; ServerId: string;
@@ -84,7 +84,7 @@ export interface JFAlbumArtist {
Genres: string[]; Genres: string[];
Id: string; Id: string;
ImageBlurHashes: any; ImageBlurHashes: any;
ImageTags: string[]; ImageTags: ImageTags;
LocationType: string; LocationType: string;
Name: string; Name: string;
Overview?: string; Overview?: string;
@@ -113,13 +113,13 @@ export interface JFArtist {
export interface JFAlbum { export interface JFAlbum {
AlbumArtist: string; AlbumArtist: string;
AlbumArtists: GenericItem[]; AlbumArtists: JFGenericItem[];
ArtistItems: GenericItem[]; ArtistItems: JFGenericItem[];
Artists: string[]; Artists: string[];
ChannelId: null; ChannelId: null;
DateCreated: string; DateCreated: string;
ExternalUrls: ExternalURL[]; ExternalUrls: ExternalURL[];
GenreItems: GenericItem[]; GenreItems: JFGenericItem[];
Genres: string[]; Genres: string[];
Id: string; Id: string;
ImageBlurHashes: ImageBlurHashes; ImageBlurHashes: ImageBlurHashes;
@@ -139,16 +139,16 @@ export interface JFAlbum {
export interface JFSong { export interface JFSong {
Album: string; Album: string;
AlbumArtist: string; AlbumArtist: string;
AlbumArtists: GenericItem[]; AlbumArtists: JFGenericItem[];
AlbumId: string; AlbumId: string;
AlbumPrimaryImageTag: string; AlbumPrimaryImageTag: string;
ArtistItems: GenericItem[]; ArtistItems: JFGenericItem[];
Artists: string[]; Artists: string[];
BackdropImageTags: string[]; BackdropImageTags: string[];
ChannelId: null; ChannelId: null;
DateCreated: string; DateCreated: string;
ExternalUrls: ExternalURL[]; ExternalUrls: ExternalURL[];
GenreItems: GenericItem[]; GenreItems: JFGenericItem[];
Genres: string[]; Genres: string[];
Id: string; Id: string;
ImageBlurHashes: ImageBlurHashes; ImageBlurHashes: ImageBlurHashes;
@@ -164,6 +164,7 @@ export interface JFSong {
ProductionYear: number; ProductionYear: number;
RunTimeTicks: number; RunTimeTicks: number;
ServerId: string; ServerId: string;
SortName: string;
Type: string; Type: string;
} }
@@ -196,7 +197,7 @@ interface GenreItem {
Name: string; Name: string;
} }
interface GenericItem { export interface JFGenericItem {
Id: string; Id: string;
Name: string; Name: string;
} }
@@ -261,3 +262,143 @@ interface MediaStream {
Type: string; Type: string;
Width?: number; Width?: number;
} }
export enum JFExternalType {
MUSICBRAINZ = 'MusicBrainz',
THEAUDIODB = 'TheAudioDb',
}
export enum JFImageType {
LOGO = 'Logo',
PRIMARY = 'Primary',
}
export enum JFItemType {
AUDIO = 'Audio',
MUSICALBUM = 'MusicAlbum',
}
export enum JFCollectionType {
MUSIC = 'music',
PLAYLISTS = 'playlists',
}
export interface JFAuthenticate {
AccessToken: string;
ServerId: string;
SessionInfo: SessionInfo;
User: User;
}
interface SessionInfo {
AdditionalUsers: any[];
ApplicationVersion: string;
Capabilities: Capabilities;
Client: string;
DeviceId: string;
DeviceName: string;
HasCustomDeviceName: boolean;
Id: string;
IsActive: boolean;
LastActivityDate: string;
LastPlaybackCheckIn: string;
NowPlayingQueue: any[];
NowPlayingQueueFullItems: any[];
PlayState: PlayState;
PlayableMediaTypes: any[];
RemoteEndPoint: string;
ServerId: string;
SupportedCommands: any[];
SupportsMediaControl: boolean;
SupportsRemoteControl: boolean;
UserId: string;
UserName: string;
}
interface Capabilities {
PlayableMediaTypes: any[];
SupportedCommands: any[];
SupportsContentUploading: boolean;
SupportsMediaControl: boolean;
SupportsPersistentIdentifier: boolean;
SupportsSync: boolean;
}
interface PlayState {
CanSeek: boolean;
IsMuted: boolean;
IsPaused: boolean;
RepeatMode: string;
}
interface User {
Configuration: Configuration;
EnableAutoLogin: boolean;
HasConfiguredEasyPassword: boolean;
HasConfiguredPassword: boolean;
HasPassword: boolean;
Id: string;
LastActivityDate: string;
LastLoginDate: string;
Name: string;
Policy: Policy;
ServerId: string;
}
interface Configuration {
DisplayCollectionsView: boolean;
DisplayMissingEpisodes: boolean;
EnableLocalPassword: boolean;
EnableNextEpisodeAutoPlay: boolean;
GroupedFolders: any[];
HidePlayedInLatest: boolean;
LatestItemsExcludes: any[];
MyMediaExcludes: any[];
OrderedViews: any[];
PlayDefaultAudioTrack: boolean;
RememberAudioSelections: boolean;
RememberSubtitleSelections: boolean;
SubtitleLanguagePreference: string;
SubtitleMode: string;
}
interface Policy {
AccessSchedules: any[];
AuthenticationProviderId: string;
BlockUnratedItems: any[];
BlockedChannels: any[];
BlockedMediaFolders: any[];
BlockedTags: any[];
EnableAllChannels: boolean;
EnableAllDevices: boolean;
EnableAllFolders: boolean;
EnableAudioPlaybackTranscoding: boolean;
EnableContentDeletion: boolean;
EnableContentDeletionFromFolders: any[];
EnableContentDownloading: boolean;
EnableLiveTvAccess: boolean;
EnableLiveTvManagement: boolean;
EnableMediaConversion: boolean;
EnableMediaPlayback: boolean;
EnablePlaybackRemuxing: boolean;
EnablePublicSharing: boolean;
EnableRemoteAccess: boolean;
EnableRemoteControlOfOtherUsers: boolean;
EnableSharedDeviceControl: boolean;
EnableSyncTranscoding: boolean;
EnableUserPreferenceAccess: boolean;
EnableVideoPlaybackTranscoding: boolean;
EnabledChannels: any[];
EnabledDevices: any[];
EnabledFolders: any[];
ForceRemoteSourceTranscoding: boolean;
InvalidLoginAttemptCount: number;
IsAdministrator: boolean;
IsDisabled: boolean;
IsHidden: boolean;
LoginAttemptsBeforeLockout: number;
MaxActiveSessions: number;
PasswordResetProviderId: string;
RemoteClientBitrateLimit: number;
SyncPlayAccess: string;
}
src/server/queue/jellyfin/jellyfin.utils.ts
+278
View File
@@ -0,0 +1,278 @@
import {
ExternalSource,
ExternalType,
ImageType,
Prisma,
Server,
ServerFolder,
} from '@prisma/client';
import { prisma } from '../../lib';
import { uniqueArray } from '../../utils';
import {
JFAlbum,
JFAlbumArtist,
JFExternalType,
JFImageType,
JFSong,
} from './jellyfin.types';
const insertGenres = async (items: JFSong[] | JFAlbum[] | JFAlbumArtist[]) => {
const genresCreateMany = items
.flatMap((item) => item.GenreItems)
.map((genre) => ({ name: genre.Name }));
await prisma.genre.createMany({
data: genresCreateMany,
skipDuplicates: true,
});
};
const insertArtists = async (
server: Server,
serverFolder: ServerFolder,
items: JFSong[] | JFAlbum[]
) => {
const artistItems = items.flatMap((item) => item.ArtistItems);
const createMany = artistItems.map((artist) => ({
name: artist.Name,
remoteId: artist.Id,
serverId: server.id,
sortName: '',
}));
await prisma.artist.createMany({
data: createMany,
skipDuplicates: true,
});
for (const artist of artistItems) {
await prisma.artist.update({
data: { serverFolders: { connect: { id: serverFolder.id } } },
where: {
uniqueArtistId: {
remoteId: artist.Id,
serverId: server.id,
},
},
});
}
};
const insertImages = async (items: JFSong[] | JFAlbum[] | JFAlbumArtist[]) => {
const imageItems = items.flatMap((item) => item.ImageTags);
const createMany: Prisma.ImageCreateManyInput[] = [];
for (const image of imageItems) {
if (image.Logo) {
createMany.push({
remoteUrl: image.Logo,
type: ImageType.LOGO,
});
}
if (image.Primary) {
createMany.push({
remoteUrl: image.Primary,
type: ImageType.PRIMARY,
});
}
}
await prisma.image.createMany({
data: createMany,
skipDuplicates: true,
});
};
const insertExternals = async (
items: JFSong[] | JFAlbum[] | JFAlbumArtist[]
) => {
const externalItems = items.flatMap((item) => item.ExternalUrls);
const createMany: Prisma.ExternalCreateManyInput[] = [];
for (const external of externalItems) {
if (
external.Name === JFExternalType.MUSICBRAINZ ||
external.Name === JFExternalType.THEAUDIODB
) {
const source =
external.Name === JFExternalType.MUSICBRAINZ
? ExternalSource.MUSICBRAINZ
: ExternalSource.THEAUDIODB;
const value = external.Url.split('/').pop() || '';
createMany.push({ source, type: ExternalType.ID, value });
}
}
await prisma.external.createMany({
data: createMany,
skipDuplicates: true,
});
};
const insertSongGroup = async (
server: Server,
serverFolder: ServerFolder,
songs: JFSong[],
remoteAlbumId: string
) => {
const songsUpsert: Prisma.SongUpsertWithWhereUniqueWithoutAlbumInput[] =
songs.map((song) => {
const genresConnect = song.Genres.map((genre) => ({ name: genre }));
const artistsConnect = song.ArtistItems.map((artist) => ({
uniqueArtistId: {
remoteId: artist.Id,
serverId: server.id,
},
}));
const externalsConnect = song.ExternalUrls.map((external) => ({
uniqueExternalId: {
source:
external.Name === JFExternalType.MUSICBRAINZ
? ExternalSource.MUSICBRAINZ
: ExternalSource.THEAUDIODB,
value: external.Url.split('/').pop() || '',
},
}));
const imagesConnect = [];
for (const [key, value] of Object.entries(song.ImageTags)) {
if (key === JFImageType.PRIMARY) {
imagesConnect.push({
uniqueImageId: { remoteUrl: value, type: ImageType.PRIMARY },
});
}
if (key === JFImageType.LOGO) {
imagesConnect.push({
uniqueImageId: { remoteUrl: value, type: ImageType.LOGO },
});
}
}
const pathSplit = song.MediaSources[0].Path.split('/');
const parentPath = pathSplit.slice(0, pathSplit.length - 1).join('/');
return {
create: {
artists: { connect: artistsConnect },
bitRate: Math.floor(song.MediaSources[0].Bitrate / 1e3),
container: song.MediaSources[0].Container,
discNumber: song.ParentIndexNumber,
duration: Math.floor(song.MediaSources[0].RunTimeTicks / 1e7),
externals: { connect: externalsConnect },
folders: {
connect: {
uniqueFolderId: { path: parentPath, serverId: server.id },
},
},
genres: { connect: genresConnect },
images: { connect: imagesConnect },
name: song.Name,
releaseDate: song.PremiereDate,
releaseYear: song.ProductionYear,
remoteCreatedAt: song.DateCreated,
remoteId: song.Id,
serverFolders: { connect: { id: serverFolder.id } },
serverId: server.id,
size: String(song.MediaSources[0].Size),
sortName: song.Name,
trackNumber: song.IndexNumber,
},
update: {
artists: { connect: artistsConnect },
bitRate: Math.floor(song.MediaSources[0].Bitrate / 1e3),
container: song.MediaSources[0].Container,
discNumber: song.ParentIndexNumber,
duration: Math.floor(song.MediaSources[0].RunTimeTicks / 1e7),
externals: { connect: externalsConnect },
folders: {
connect: {
uniqueFolderId: { path: parentPath, serverId: server.id },
},
},
genres: { connect: genresConnect },
images: { connect: imagesConnect },
name: song.Name,
releaseDate: song.PremiereDate,
releaseYear: song.ProductionYear,
remoteCreatedAt: song.DateCreated,
remoteId: song.Id,
serverFolders: { connect: { id: serverFolder.id } },
serverId: server.id,
sortName: song.Name,
trackNumber: song.IndexNumber,
},
where: {
uniqueSongId: {
remoteId: song.Id,
serverId: server.id,
},
},
};
});
// const artists = uniqBy(
// songs.flatMap((song) => {
// return song.ArtistItems.map((artist) => ({
// deleted: false,
// name: artist.Name,
// remoteId: artist.Id,
// serverFolders: { connect: { id: serverFolder.id } },
// serverId: server.id,
// sortName: '',
// }));
// }),
// 'remoteId'
// );
// for (const artist of artists) {
// await prisma.artist.upsert({
// create: artist,
// update: artist,
// where: {
// uniqueArtistId: {
// remoteId: artist.remoteId,
// serverId: server.id,
// },
// },
// });
// }
const uniqueArtistIds = songs
.flatMap((song) => song.ArtistItems.flatMap((artist) => artist.Id))
.filter(uniqueArray);
const artistsConnect = uniqueArtistIds.map((artistId) => ({
uniqueArtistId: {
remoteId: artistId,
serverId: server.id,
},
}));
await prisma.album.update({
data: {
artists: { connect: artistsConnect },
deleted: false,
songs: { upsert: songsUpsert },
},
where: {
uniqueAlbumId: {
remoteId: remoteAlbumId,
serverId: server.id,
},
},
});
};
export const jellyfinUtils = {
insertArtists,
insertExternals,
insertGenres,
insertImages,
insertSongGroup,
};
src/server/queue/queues/index.ts
+5
View File
@@ -0,0 +1,5 @@
import { scannerQueue } from './scanner.queue';
export const queue = {
scanner: scannerQueue,
};
src/server/queue/queues/scanner.queue.ts
+57
View File
@@ -0,0 +1,57 @@
import { Task } from '@prisma/client';
import Queue from 'better-queue';
import { prisma } from '../../lib';
interface QueueTask {
fn: any;
id: string;
task: Task;
}
export const scannerQueue: Queue = new Queue(
async (task: QueueTask, cb: any) => {
const result = await task.fn();
return cb(null, result);
},
{
afterProcessDelay: 1000,
cancelIfRunning: true,
concurrent: 1,
filo: false,
maxRetries: 5,
maxTimeout: 600000,
retryDelay: 2000,
}
);
scannerQueue.on('task_finish', async (taskId) => {
await prisma.task.update({
data: {
completed: true,
isError: false,
progress: null,
},
where: { id: taskId },
});
});
scannerQueue.on('task_failed', async (taskId, errorMessage) => {
const dbTaskId = taskId.split('(')[1].split(')')[0];
console.log('errorMessage', errorMessage);
await prisma.task.update({
data: {
completed: true,
isError: true,
message: errorMessage,
},
where: { id: dbTaskId },
});
});
scannerQueue.on('drain', async () => {
await prisma.task.updateMany({
data: { completed: true, progress: null },
where: { completed: false },
});
});
src/server/queue/scanner-queue.ts
-74
View File
@@ -1,74 +0,0 @@
import Queue from 'better-queue';
import { prisma } from '../lib';
import { Task } from '../types/types';
interface QueueTask {
fn: any;
id: string;
task: Task;
}
// interface QueueResult {
// completed?: boolean;
// message?: string;
// task: Task;
// }
export const q: Queue = new Queue(
async (task: QueueTask, cb: any) => {
const result = await task.fn();
return cb(null, result);
},
{
afterProcessDelay: 1000,
cancelIfRunning: true,
concurrent: 1,
filo: false,
maxRetries: 5,
maxTimeout: 600000,
retryDelay: 2000,
}
);
// q.on('task_finish', async (_taskId, result: QueueResult) => {});
q.on('task_failed', async (taskId, errorMessage) => {
const dbTaskId = taskId.split('(')[1].split(')')[0];
await prisma.task.update({
data: {
completed: true,
inProgress: false,
isError: true,
message: errorMessage,
},
where: { id: dbTaskId },
});
});
q.on('drain', async () => {
await prisma.task.updateMany({
data: { completed: true, inProgress: false },
where: {
OR: [{ inProgress: true }, { completed: false }],
},
});
});
export const completeTask = async (task: Task) => {
q.push({
fn: async () => {
await prisma.task.update({
data: {
completed: true,
inProgress: false,
message: 'Completed',
progress: '',
},
where: { id: task.id },
});
return { task };
},
id: `${task.id}-complete`,
});
};
src/server/queue/subsonic/index.ts
+7
View File
@@ -0,0 +1,7 @@
import { subsonicApi } from './subsonic.api';
import { subsonicScanner } from './subsonic.scanner';
export const subsonic = {
api: subsonicApi,
scanner: subsonicScanner,
};
src/server/queue/subsonic/subsonic-api.ts → src/server/queue/subsonic/subsonic.api.ts
+30 -2
View File
@@ -1,5 +1,7 @@
import { Server } from '@prisma/client';
import axios from 'axios'; import axios from 'axios';
import { Server } from '../../types/types'; import md5 from 'md5';
import { randomString } from '../../utils';
import { import {
SSAlbumListEntry, SSAlbumListEntry,
SSAlbumListResponse, SSAlbumListResponse,
@@ -10,7 +12,7 @@ import {
SSArtistsResponse, SSArtistsResponse,
SSGenresResponse, SSGenresResponse,
SSMusicFoldersResponse, SSMusicFoldersResponse,
} from './subsonic-types'; } from './subsonic.types';
const api = axios.create({ const api = axios.create({
validateStatus: (status) => status >= 200, validateStatus: (status) => status >= 200,
@@ -26,6 +28,31 @@ api.interceptors.response.use(
} }
); );
const authenticate = async (options: {
legacy?: boolean;
password: string;
url: string;
username: string;
}) => {
let token;
const cleanServerUrl = options.url.replace(/\/$/, '');
if (options.legacy) {
token = `u=${options.username}&p=${options.password}`;
} else {
const salt = randomString(12);
const hash = md5(options.password + salt);
token = `u=${options.username}&s=${salt}&t=${hash}`;
}
const { data } = await api.get(
`${cleanServerUrl}/rest/ping.view?v=1.13.0&c=sonixd&f=json&${token}`
);
return { token, ...data };
};
const getMusicFolders = async (server: Partial<Server>) => { const getMusicFolders = async (server: Partial<Server>) => {
const { data } = await api.get<SSMusicFoldersResponse>( const { data } = await api.get<SSMusicFoldersResponse>(
`${server.url}/rest/getMusicFolders.view?v=1.13.0&c=sonixd&f=json&${server.token}` `${server.url}/rest/getMusicFolders.view?v=1.13.0&c=sonixd&f=json&${server.token}`
@@ -120,6 +147,7 @@ const getArtistInfo = async (server: Server, id: string) => {
}; };
export const subsonicApi = { export const subsonicApi = {
authenticate,
getAlbum, getAlbum,
getAlbums, getAlbums,
getArtistInfo, getArtistInfo,
src/server/queue/subsonic/subsonic-tasks.ts → src/server/queue/subsonic/subsonic.scanner.ts
+6 -94
View File
@@ -1,27 +1,14 @@
/* eslint-disable no-await-in-loop */ /* eslint-disable no-await-in-loop */
import { Server, ServerFolder, Task } from '@prisma/client';
import { prisma, throttle } from '../../lib'; import { prisma, throttle } from '../../lib';
import { Server, ServerFolder, Task } from '../../types/types';
import { groupByProperty, uniqueArray } from '../../utils'; import { groupByProperty, uniqueArray } from '../../utils';
import { completeTask, q } from '../scanner-queue'; import { subsonicApi } from './subsonic.api';
import { subsonicApi } from './subsonic-api'; import { SSAlbumListEntry } from './subsonic.types';
import { SSAlbumListEntry } from './subsonic-types';
export const scanGenres = async ( export const scanGenres = async (
server: Server, server: Server,
serverFolder: ServerFolder serverFolder: ServerFolder
) => { ) => {
const taskId = `[${server.name} (${serverFolder.name})] genres`;
q.push({
fn: async () => {
const task = await prisma.task.create({
data: {
inProgress: true,
name: taskId,
serverFolderId: serverFolder.id,
},
});
const res = await subsonicApi.getGenres(server); const res = await subsonicApi.getGenres(server);
const genres = res.genres.genre.map((genre) => { const genres = res.genres.genre.map((genre) => {
@@ -34,33 +21,13 @@ export const scanGenres = async (
}); });
const message = `Imported ${createdGenres.count} new genres.`; const message = `Imported ${createdGenres.count} new genres.`;
return { message, task };
},
id: taskId,
});
}; };
export const scanAlbumArtists = async ( export const scanAlbumArtists = async (
server: Server, server: Server,
serverFolder: ServerFolder serverFolder: ServerFolder
) => { ) => {
const taskId = `[${server.name} (${serverFolder.name})] album artists`; const artists = await subsonicApi.getArtists(server, serverFolder.remoteId);
q.push({
fn: async () => {
const task = await prisma.task.create({
data: {
inProgress: true,
name: taskId,
serverFolderId: serverFolder.id,
},
});
const artists = await subsonicApi.getArtists(
server,
serverFolder.remoteId
);
for (const artist of artists) { for (const artist of artists) {
await prisma.albumArtist.upsert({ await prisma.albumArtist.upsert({
@@ -103,29 +70,12 @@ export const scanAlbumArtists = async (
} }
const message = `Scanned ${artists.length} album artists.`; const message = `Scanned ${artists.length} album artists.`;
return { message, task };
},
id: taskId,
});
}; };
export const scanAlbums = async ( export const scanAlbums = async (
server: Server, server: Server,
serverFolder: ServerFolder serverFolder: ServerFolder
) => { ) => {
const taskId = `[${server.name} (${serverFolder.name})] albums`;
q.push({
fn: async () => {
const task = await prisma.task.create({
data: {
inProgress: true,
name: taskId,
serverFolderId: serverFolder.id,
},
});
const promises: any[] = []; const promises: any[] = [];
const albums = await subsonicApi.getAlbums(server, { const albums = await subsonicApi.getAlbums(server, {
musicFolderId: serverFolder.id, musicFolderId: serverFolder.id,
@@ -197,11 +147,6 @@ export const scanAlbums = async (
await Promise.all(promises); await Promise.all(promises);
const message = `Scanned ${albums.length} albums.`; const message = `Scanned ${albums.length} albums.`;
return { message, task };
},
id: taskId,
});
}; };
const throttledAlbumFetch = throttle( const throttledAlbumFetch = throttle(
@@ -315,16 +260,6 @@ export const scanAlbumDetail = async (
) => { ) => {
const taskId = `[${server.name} (${serverFolder.name})] albums detail`; const taskId = `[${server.name} (${serverFolder.name})] albums detail`;
q.push({
fn: async () => {
const task = await prisma.task.create({
data: {
inProgress: true,
name: taskId,
serverFolderId: serverFolder.id,
},
});
const promises = []; const promises = [];
const dbAlbums = await prisma.album.findMany({ const dbAlbums = await prisma.album.findMany({
where: { where: {
@@ -333,18 +268,11 @@ export const scanAlbumDetail = async (
}); });
for (let i = 0; i < dbAlbums.length; i += 1) { for (let i = 0; i < dbAlbums.length; i += 1) {
promises.push( promises.push(throttledAlbumFetch(server, serverFolder, dbAlbums[i], i));
throttledAlbumFetch(server, serverFolder, dbAlbums[i], i)
);
} }
await Promise.all(promises); await Promise.all(promises);
const message = `Scanned ${dbAlbums.length} albums.`; const message = `Scanned ${dbAlbums.length} albums.`;
return { message, task };
},
id: taskId,
});
}; };
const throttledArtistDetailFetch = throttle( const throttledArtistDetailFetch = throttle(
@@ -409,16 +337,6 @@ export const scanAlbumArtistDetail = async (
) => { ) => {
const taskId = `[${server.name} (${serverFolder.name})] artists detail`; const taskId = `[${server.name} (${serverFolder.name})] artists detail`;
q.push({
fn: async () => {
const task = await prisma.task.create({
data: {
inProgress: true,
name: taskId,
serverFolderId: serverFolder.id,
},
});
const promises = []; const promises = [];
const dbArtists = await prisma.albumArtist.findMany({ const dbArtists = await prisma.albumArtist.findMany({
where: { where: {
@@ -436,11 +354,6 @@ export const scanAlbumArtistDetail = async (
) )
); );
} }
return { task };
},
id: taskId,
});
}; };
const scanAll = async ( const scanAll = async (
@@ -453,10 +366,9 @@ const scanAll = async (
await scanAlbumArtistDetail(server, serverFolder); await scanAlbumArtistDetail(server, serverFolder);
await scanAlbums(server, serverFolder); await scanAlbums(server, serverFolder);
await scanAlbumDetail(server, serverFolder); await scanAlbumDetail(server, serverFolder);
await completeTask(task);
}; };
export const subsonicTasks = { export const subsonicScanner = {
scanAll, scanAll,
scanGenres, scanGenres,
}; };
src/server/queue/subsonic/subsonic-types.ts → src/server/queue/subsonic/subsonic.types.ts
+1 -1
View File
@@ -127,7 +127,7 @@ export interface SSSong {
export interface SSAlbumsParams { export interface SSAlbumsParams {
fromYear?: number; fromYear?: number;
genre?: string; genre?: string;
musicFolderId?: number; musicFolderId?: string;
offset?: number; offset?: number;
size?: number; size?: number;
toYear?: number; toYear?: number;