add initial files

This commit is contained in:
jeffvli
2022-07-25 19:40:16 -07:00
commit e8b612c974
283 changed files with 62820 additions and 0 deletions
+92
View File
@@ -0,0 +1,92 @@
import axios from 'axios';
import { Server } from '../../types/types';
import {
JFAlbumArtistsResponse,
JFAlbumsResponse,
JFArtistsResponse,
JFGenreResponse,
JFMusicFoldersResponse,
JFRequestParams,
JFSongsResponse,
} from './jellyfin-types';
export const api = axios.create({});
export const getMusicFolders = async (server: Partial<Server>) => {
const { data } = await api.get<JFMusicFoldersResponse>(
`${server.url}/users/${server.remoteUserId}/items`,
{ headers: { 'X-MediaBrowser-Token': server.token! } }
);
const musicFolders = data.Items.filter(
(folder) => folder.CollectionType === 'music'
);
return musicFolders;
};
export const getGenres = async (server: Server, params: JFRequestParams) => {
const { data } = await api.get<JFGenreResponse>(`${server.url}/genres`, {
headers: { 'X-MediaBrowser-Token': server.token },
params,
});
return data;
};
export const getAlbumArtists = async (
server: Server,
params: JFRequestParams
) => {
const { data } = await api.get<JFAlbumArtistsResponse>(
`${server.url}/artists/albumArtists`,
{
headers: { 'X-MediaBrowser-Token': server.token },
params,
}
);
return data;
};
export const getArtists = async (server: Server, params: JFRequestParams) => {
const { data } = await api.get<JFArtistsResponse>(`${server.url}/artists`, {
headers: { 'X-MediaBrowser-Token': server.token },
params,
});
return data;
};
export const getAlbums = async (server: Server, params: JFRequestParams) => {
const { data } = await api.get<JFAlbumsResponse>(
`${server.url}/users/${server.remoteUserId}/items`,
{
headers: { 'X-MediaBrowser-Token': server.token },
params: { includeItemTypes: 'MusicAlbum', ...params },
}
);
return data;
};
export const getSongs = async (server: Server, params: JFRequestParams) => {
const { data } = await api.get<JFSongsResponse>(
`${server.url}/users/${server.remoteUserId}/items`,
{
headers: { 'X-MediaBrowser-Token': server.token },
params: { includeItemTypes: 'Audio', ...params },
}
);
return data;
};
export const jellyfinApi = {
getAlbumArtists,
getAlbums,
getArtists,
getGenres,
getMusicFolders,
getSongs,
};
+631
View File
@@ -0,0 +1,631 @@
import uniqBy from 'lodash/uniqBy';
import { prisma } from '../../lib';
import { Server, ServerFolder, Task } from '../../types/types';
import { groupByProperty, uniqueArray } from '../../utils';
import { completeTask, q } from '../scanner-queue';
import { jellyfinApi } from './jellyfin-api';
import { JFSong } from './jellyfin-types';
const scanGenres = async (
server: Server,
serverFolder: ServerFolder,
task: Task
) => {
const taskId = `[${server.name} (${serverFolder.name})] genres`;
q.push({
fn: async () => {
await prisma.task.update({
data: { message: 'Scanning genres' },
where: { id: task.id },
});
const genres = await jellyfinApi.getGenres(server, {
parentId: serverFolder.remoteId,
});
const genresCreate = genres.Items.map((genre) => {
return { name: genre.Name };
});
await prisma.genre.createMany({
data: genresCreate,
skipDuplicates: true,
});
return { task };
},
id: taskId,
});
};
const scanAlbumArtists = async (
server: Server,
serverFolder: ServerFolder,
task: Task
) => {
const taskId = `[${server.name} (${serverFolder.name})] album artists`;
q.push({
fn: async () => {
await prisma.task.update({
data: { message: 'Scanning album artists' },
where: { id: task.id },
});
const albumArtists = await jellyfinApi.getAlbumArtists(server, {
fields: 'Genres,DateCreated,ExternalUrls,Overview',
parentId: serverFolder.remoteId,
});
for (const albumArtist of albumArtists.Items) {
const genresConnectOrCreate = albumArtist.Genres.map((genre) => {
return { create: { name: genre }, where: { name: genre } };
});
const imagesConnectOrCreate = [];
for (const [key, value] of Object.entries(albumArtist.ImageTags)) {
imagesConnectOrCreate.push({
create: { name: key, url: value },
where: { uniqueImageId: { name: key, url: value } },
});
}
const externalsConnectOrCreate = albumArtist.ExternalUrls.map(
(external) => {
return {
create: { name: external.Name, url: external.Url },
where: {
uniqueExternalId: { name: external.Name, url: external.Url },
},
};
}
);
await prisma.albumArtist.upsert({
create: {
biography: albumArtist.Overview,
externals: { connectOrCreate: externalsConnectOrCreate },
genres: { connectOrCreate: genresConnectOrCreate },
images: { connectOrCreate: imagesConnectOrCreate },
name: albumArtist.Name,
remoteCreatedAt: albumArtist.DateCreated,
remoteId: albumArtist.Id,
serverFolders: { connect: { id: serverFolder.id } },
serverId: server.id,
},
update: {
biography: albumArtist.Overview,
deleted: false,
externals: { connectOrCreate: externalsConnectOrCreate },
genres: { connectOrCreate: genresConnectOrCreate },
images: { connectOrCreate: imagesConnectOrCreate },
name: albumArtist.Name,
remoteCreatedAt: albumArtist.DateCreated,
remoteId: albumArtist.Id,
serverFolders: { connect: { id: serverFolder.id } },
serverId: server.id,
},
where: {
uniqueAlbumArtistId: {
remoteId: albumArtist.Id,
serverId: server.id,
},
},
});
}
return { task };
},
id: taskId,
});
};
const scanAlbums = async (
server: Server,
serverFolder: ServerFolder,
task: Task
) => {
const check = await jellyfinApi.getAlbums(server, {
enableUserData: false,
includeItemTypes: 'MusicAlbum',
limit: 1,
parentId: serverFolder.remoteId,
recursive: true,
});
const albumCount = check.TotalRecordCount;
const chunkSize = 5000;
const albumChunkCount = Math.ceil(albumCount / chunkSize);
const taskId = `(${task.id}) [${server.name} (${serverFolder.name})] albums`;
q.push({
fn: async () => {
await prisma.task.update({
data: { message: 'Scanning albums' },
where: { id: task.id },
});
for (let i = 0; i < albumChunkCount; i += 1) {
const albums = await jellyfinApi.getAlbums(server, {
enableImageTypes: 'Primary,Logo,Backdrop',
enableUserData: false,
fields: 'Genres,DateCreated,ExternalUrls,Overview',
imageTypeLimit: 1,
limit: chunkSize,
parentId: serverFolder.remoteId,
recursive: true,
startIndex: i * chunkSize,
});
for (const album of albums.Items) {
const genresConnectOrCreate = album.Genres.map((genre) => {
return { create: { name: genre }, where: { name: genre } };
});
const imagesConnectOrCreate = [];
for (const [key, value] of Object.entries(album.ImageTags)) {
imagesConnectOrCreate.push({
create: { name: key, url: value },
where: { uniqueImageId: { name: key, url: value } },
});
}
const externalsConnectOrCreate = album.ExternalUrls.map(
(external) => {
return {
create: { name: external.Name, url: external.Url },
where: {
uniqueExternalId: { name: external.Name, url: external.Url },
},
};
}
);
const albumArtist =
album.AlbumArtists.length > 0
? await prisma.albumArtist.findUnique({
where: {
uniqueAlbumArtistId: {
remoteId: album.AlbumArtists && album.AlbumArtists[0].Id,
serverId: server.id,
},
},
})
: undefined;
await prisma.album.upsert({
create: {
albumArtistId: albumArtist?.id,
date: album.PremiereDate,
externals: { connectOrCreate: externalsConnectOrCreate },
genres: { connectOrCreate: genresConnectOrCreate },
images: { connectOrCreate: imagesConnectOrCreate },
name: album.Name,
remoteCreatedAt: album.DateCreated,
remoteId: album.Id,
serverFolders: { connect: { id: serverFolder.id } },
serverId: server.id,
year: album.ProductionYear,
},
update: {
albumArtistId: albumArtist?.id,
date: album.PremiereDate,
deleted: false,
externals: { connectOrCreate: externalsConnectOrCreate },
genres: { connectOrCreate: genresConnectOrCreate },
images: { connectOrCreate: imagesConnectOrCreate },
name: album.Name,
remoteCreatedAt: album.DateCreated,
remoteId: album.Id,
serverFolders: { connect: { id: serverFolder.id } },
serverId: server.id,
year: album.ProductionYear,
},
where: {
uniqueAlbumId: {
remoteId: album.Id,
serverId: server.id,
},
},
});
}
const currentTask = await prisma.task.findUnique({
where: { id: task.id },
});
const newCount =
Number(currentTask?.progress || 0) + Number(albums.Items.length);
await prisma.task.update({
data: { progress: String(newCount) },
where: { id: task.id },
});
}
return { task };
},
id: taskId,
});
};
const insertSongGroup = async (
server: Server,
serverFolder: ServerFolder,
songs: JFSong[],
remoteAlbumId: string
) => {
const songsUpsert = songs.map((song) => {
const artistsConnectOrCreate = song.ArtistItems.map((artist) => {
return {
create: {
name: artist.Name,
remoteId: artist.Id,
serverFolders: { connect: { id: serverFolder.id } },
serverId: server.id,
},
where: {
uniqueArtistId: {
remoteId: artist.Id,
serverId: server.id,
},
},
};
});
const genresConnectOrCreate = song.Genres.map((genre) => {
return { create: { name: genre }, where: { name: genre } };
});
const imagesConnectOrCreate = [];
for (const [key, value] of Object.entries(song.ImageTags)) {
imagesConnectOrCreate.push({
create: { name: key, url: value },
where: { uniqueImageId: { name: key, url: value } },
});
}
const externalsConnectOrCreate = song.ExternalUrls.map((external) => {
return {
create: { name: external.Name, url: external.Url },
where: {
uniqueExternalId: { name: external.Name, url: external.Url },
},
};
});
const pathSplit = song.MediaSources[0].Path.split('/');
const parentPath = pathSplit.slice(0, pathSplit.length - 1).join('/');
return {
create: {
artists: { connectOrCreate: artistsConnectOrCreate },
bitRate: Math.floor(song.MediaSources[0].Bitrate / 1000),
container: song.MediaSources[0].Container,
date: song.PremiereDate,
disc: song.ParentIndexNumber,
duration: Math.floor(song.MediaSources[0].RunTimeTicks / 1e7),
externals: { connectOrCreate: externalsConnectOrCreate },
folders: {
connect: {
uniqueFolderId: { path: parentPath, serverId: server.id },
},
},
genres: { connectOrCreate: genresConnectOrCreate },
images: { connectOrCreate: imagesConnectOrCreate },
name: song.Name,
remoteCreatedAt: song.DateCreated,
remoteId: song.Id,
serverFolders: { connect: { id: serverFolder.id } },
serverId: server.id,
track: song.IndexNumber,
year: song.ProductionYear,
},
update: {
artists: { connectOrCreate: artistsConnectOrCreate },
bitRate: Math.floor(song.MediaSources[0].Bitrate / 1000),
container: song.MediaSources[0].Container,
date: song.PremiereDate,
disc: song.ParentIndexNumber,
duration: Math.floor(song.MediaSources[0].RunTimeTicks / 1e7),
externals: { connectOrCreate: externalsConnectOrCreate },
folders: {
connect: {
uniqueFolderId: { path: parentPath, serverId: server.id },
},
},
genres: { connectOrCreate: genresConnectOrCreate },
images: { connectOrCreate: imagesConnectOrCreate },
name: song.Name,
remoteCreatedAt: song.DateCreated,
remoteId: song.Id,
serverFolders: { connect: { id: serverFolder.id } },
serverId: server.id,
track: song.IndexNumber,
year: song.ProductionYear,
},
where: {
uniqueSongId: {
remoteId: song.Id,
serverId: server.id,
},
},
};
});
const artists = uniqBy(
songs.flatMap((song) => {
return song.ArtistItems.map((artist) => ({
name: artist.Name,
remoteId: artist.Id,
serverFolders: { connect: { id: serverFolder.id } },
serverId: server.id,
}));
}),
'remoteId'
);
const uniqueArtistIds = songs
.flatMap((song) => {
return song.ArtistItems.flatMap((artist) => artist.Id);
})
.filter(uniqueArray);
const artistsConnect = uniqueArtistIds.map((artistId) => {
return {
uniqueArtistId: {
remoteId: artistId!,
serverId: server.id,
},
};
});
artists.forEach(async (artist) => {
await prisma.artist.upsert({
create: artist,
update: artist,
where: {
uniqueArtistId: {
remoteId: artist.remoteId,
serverId: server.id,
},
},
});
});
await prisma.$transaction([
prisma.artist.updateMany({
data: { deleted: false },
where: {
remoteId: { in: uniqueArtistIds },
serverId: server.id,
},
}),
prisma.album.update({
data: {
artists: { connect: artistsConnect },
deleted: false,
songs: { upsert: songsUpsert },
},
where: {
uniqueAlbumId: {
remoteId: remoteAlbumId,
serverId: server.id,
},
},
}),
]);
};
const scanSongs = async (
server: Server,
serverFolder: ServerFolder,
task: Task
) => {
const check = await jellyfinApi.getSongs(server, {
enableUserData: false,
limit: 0,
parentId: serverFolder.remoteId,
recursive: true,
});
// Fetch in chunks
const songCount = check.TotalRecordCount;
const chunkSize = 10000;
const songChunkCount = Math.ceil(songCount / chunkSize);
const taskId = `[${server.name} (${serverFolder.name})] songs`;
q.push({
fn: async () => {
await prisma.task.update({
data: { message: 'Scanning songs' },
where: { id: task.id },
});
for (let i = 0; i < songChunkCount; i += 1) {
const songs = await jellyfinApi.getSongs(server, {
enableImageTypes: 'Primary,Logo,Backdrop',
enableUserData: false,
fields: 'Genres,DateCreated,ExternalUrls,MediaSources',
imageTypeLimit: 1,
limit: chunkSize,
parentId: serverFolder.remoteId,
recursive: true,
sortBy: 'DateCreated,Album',
sortOrder: 'Descending',
startIndex: i * chunkSize,
});
const folderGroups = songs.Items.map((song) => {
const songPaths = song.MediaSources[0].Path.split('/');
const paths = [];
for (let b = 0; b < songPaths.length - 1; b += 1) {
paths.push({
name: songPaths[b],
path: songPaths.slice(0, b + 1).join('/'),
});
}
return paths;
});
const uniqueFolders = uniqBy(
folderGroups.flatMap((folder) => folder).filter((f) => f.path !== ''),
'path'
);
const folderSave: any[] = [];
for (let f = 0; f < uniqueFolders.length; f += 1) {
const t = await prisma.folder.upsert({
create: {
name: uniqueFolders[f].name,
path: uniqueFolders[f].path,
serverFolders: {
connect: {
uniqueServerFolderId: {
remoteId: serverFolder.remoteId,
serverId: server.id,
},
},
},
serverId: server.id,
},
update: {
name: uniqueFolders[f].name,
path: uniqueFolders[f].path,
serverFolders: {
connect: {
uniqueServerFolderId: {
remoteId: serverFolder.remoteId,
serverId: server.id,
},
},
},
},
where: {
uniqueFolderId: {
path: uniqueFolders[f].path,
serverId: server.id,
},
},
});
folderSave.push(t);
}
folderSave.forEach(async (f) => {
if (f.parentId) return;
const pathSplit = f.path.split('/');
const parentPath = pathSplit.slice(0, pathSplit.length - 1).join('/');
const parentPathData = folderSave.find(
(save) => save.path === parentPath
);
if (parentPathData) {
await prisma.folder.update({
data: {
parentId: parentPathData.id,
},
where: { id: f.id },
});
}
});
const albumSongGroups = groupByProperty(songs.Items, 'AlbumId');
const keys = Object.keys(albumSongGroups);
for (let b = 0; b < keys.length; b += 1) {
const songGroup = albumSongGroups[keys[b]];
await insertSongGroup(server, serverFolder, songGroup, keys[b]);
const currentTask = await prisma.task.findUnique({
where: { id: task.id },
});
const newCount =
Number(currentTask?.progress || 0) + Number(songGroup.length);
await prisma.task.update({
data: { progress: String(newCount) },
where: { id: task.id },
});
}
}
return { completed: true, task };
},
id: taskId,
});
};
const checkDeleted = async (
server: Server,
serverFolder: ServerFolder,
task: Task
) => {
q.push({
fn: async () => {
await prisma.$transaction([
prisma.albumArtist.updateMany({
data: { deleted: true },
where: {
serverFolders: { some: { id: serverFolder.id } },
serverId: server.id,
updatedAt: { lte: task.createdAt },
},
}),
prisma.artist.updateMany({
data: { deleted: true },
where: {
serverFolders: { some: { id: serverFolder.id } },
serverId: server.id,
updatedAt: { lte: task.createdAt },
},
}),
prisma.album.updateMany({
data: { deleted: true },
where: {
serverFolders: { some: { id: serverFolder.id } },
serverId: server.id,
updatedAt: { lte: task.createdAt },
},
}),
prisma.song.updateMany({
data: { deleted: true },
where: {
serverFolders: { some: { id: serverFolder.id } },
serverId: server.id,
updatedAt: { lte: task.createdAt },
},
}),
]);
},
id: `${task.id}-difference`,
});
};
const scanAll = async (
server: Server,
serverFolder: ServerFolder,
task: Task
) => {
await scanGenres(server, serverFolder, task);
await scanAlbumArtists(server, serverFolder, task);
await scanAlbums(server, serverFolder, task);
await scanSongs(server, serverFolder, task);
await checkDeleted(server, serverFolder, task);
await completeTask(task);
};
export const jellyfinTasks = {
scanAlbumArtists,
scanAlbums,
scanAll,
scanGenres,
scanSongs,
};
+263
View File
@@ -0,0 +1,263 @@
export interface JFBaseResponse {
StartIndex: number;
TotalRecordCount: number;
}
export interface JFMusicFoldersResponse extends JFBaseResponse {
Items: JFMusicFolder[];
}
export interface JFGenreResponse extends JFBaseResponse {
Items: JFGenre[];
}
export interface JFAlbumArtistsResponse extends JFBaseResponse {
Items: JFAlbumArtist[];
}
export interface JFArtistsResponse extends JFBaseResponse {
Items: JFAlbumArtist[];
}
export interface JFAlbumsResponse extends JFBaseResponse {
Items: JFAlbum[];
}
export interface JFSongsResponse extends JFBaseResponse {
Items: JFSong[];
}
export interface JFRequestParams {
albumArtistIds?: string;
artistIds?: string;
enableImageTypes?: string;
enableTotalRecordCount?: boolean;
enableUserData?: boolean;
excludeItemTypes?: string;
fields?: string;
imageTypeLimit?: number;
includeItemTypes?: string;
isFavorite?: boolean;
limit?: number;
parentId?: string;
recursive?: boolean;
searchTerm?: string;
sortBy?: string;
sortOrder?: 'Ascending' | 'Descending';
startIndex?: number;
userId?: string;
}
export interface JFMusicFolder {
BackdropImageTags: any[];
ChannelId: null;
CollectionType: string;
Id: string;
ImageBlurHashes: ImageBlurHashes;
ImageTags: ImageTags;
IsFolder: boolean;
LocationType: string;
Name: string;
ServerId: string;
Type: string;
UserData: UserData;
}
export interface JFGenre {
BackdropImageTags: any[];
ChannelId: null;
Id: string;
ImageBlurHashes: any;
ImageTags: any;
LocationType: string;
Name: string;
ServerId: string;
Type: string;
}
export interface JFAlbumArtist {
BackdropImageTags: string[];
ChannelId: null;
DateCreated: string;
ExternalUrls: ExternalURL[];
GenreItems: GenreItem[];
Genres: string[];
Id: string;
ImageBlurHashes: any;
ImageTags: string[];
LocationType: string;
Name: string;
Overview?: string;
RunTimeTicks: number;
ServerId: string;
Type: string;
}
export interface JFArtist {
BackdropImageTags: string[];
ChannelId: null;
DateCreated: string;
ExternalUrls: ExternalURL[];
GenreItems: GenreItem[];
Genres: string[];
Id: string;
ImageBlurHashes: any;
ImageTags: string[];
LocationType: string;
Name: string;
Overview?: string;
RunTimeTicks: number;
ServerId: string;
Type: string;
}
export interface JFAlbum {
AlbumArtist: string;
AlbumArtists: GenericItem[];
ArtistItems: GenericItem[];
Artists: string[];
ChannelId: null;
DateCreated: string;
ExternalUrls: ExternalURL[];
GenreItems: GenericItem[];
Genres: string[];
Id: string;
ImageBlurHashes: ImageBlurHashes;
ImageTags: ImageTags;
IsFolder: boolean;
LocationType: string;
Name: string;
ParentLogoImageTag: string;
ParentLogoItemId: string;
PremiereDate?: string;
ProductionYear: number;
RunTimeTicks: number;
ServerId: string;
Type: string;
}
export interface JFSong {
Album: string;
AlbumArtist: string;
AlbumArtists: GenericItem[];
AlbumId: string;
AlbumPrimaryImageTag: string;
ArtistItems: GenericItem[];
Artists: string[];
BackdropImageTags: string[];
ChannelId: null;
DateCreated: string;
ExternalUrls: ExternalURL[];
GenreItems: GenericItem[];
Genres: string[];
Id: string;
ImageBlurHashes: ImageBlurHashes;
ImageTags: ImageTags;
IndexNumber: number;
IsFolder: boolean;
LocationType: string;
MediaSources: MediaSources[];
MediaType: string;
Name: string;
ParentIndexNumber: number;
PremiereDate?: string;
ProductionYear: number;
RunTimeTicks: number;
ServerId: string;
Type: string;
}
interface ImageBlurHashes {
Backdrop?: any;
Logo?: any;
Primary?: any;
}
interface ImageTags {
Logo?: string;
Primary?: string;
}
interface UserData {
IsFavorite: boolean;
Key: string;
PlayCount: number;
PlaybackPositionTicks: number;
Played: boolean;
}
interface ExternalURL {
Name: string;
Url: string;
}
interface GenreItem {
Id: string;
Name: string;
}
interface GenericItem {
Id: string;
Name: string;
}
interface MediaSources {
Bitrate: number;
Container: string;
DefaultAudioStreamIndex: number;
ETag: string;
Formats: any[];
GenPtsInput: boolean;
Id: string;
IgnoreDts: boolean;
IgnoreIndex: boolean;
IsInfiniteStream: boolean;
IsRemote: boolean;
MediaAttachments: any[];
MediaStreams: MediaStream[];
Name: string;
Path: string;
Protocol: string;
ReadAtNativeFramerate: boolean;
RequiredHttpHeaders: any;
RequiresClosing: boolean;
RequiresLooping: boolean;
RequiresOpening: boolean;
RunTimeTicks: number;
Size: number;
SupportsDirectPlay: boolean;
SupportsDirectStream: boolean;
SupportsProbing: boolean;
SupportsTranscoding: boolean;
Type: string;
}
interface MediaStream {
AspectRatio?: string;
BitDepth?: number;
BitRate?: number;
ChannelLayout?: string;
Channels?: number;
Codec: string;
CodecTimeBase: string;
ColorSpace?: string;
Comment?: string;
DisplayTitle?: string;
Height?: number;
Index: number;
IsDefault: boolean;
IsExternal: boolean;
IsForced: boolean;
IsInterlaced: boolean;
IsTextSubtitleStream: boolean;
Level: number;
PixelFormat?: string;
Profile?: string;
RealFrameRate?: number;
RefFrames?: number;
SampleRate?: number;
SupportsExternalStream: boolean;
TimeBase: string;
Type: string;
Width?: number;
}