Fixed: Improve times for refreshing movies

Co-authored-by: Kevin LAMBERT <kev993@gmail.com>
This commit is contained in:
Bogdan 2025-04-07 14:41:01 +03:00
parent 9fa75f0539
commit 9231a0e526
14 changed files with 174 additions and 104 deletions

View file

@ -32,11 +32,16 @@ namespace NzbDrone.Core.Test.MovieTests.AlternativeTitleServiceTests
.With(m => m.CleanTitle = "myothertitle")
.With(m => m.Id = 1)
.Build();
Mocker.GetMock<IAlternativeTitleRepository>()
.Setup(x => x.FindByCleanTitles(It.IsAny<List<string>>()))
.Returns(new List<AlternativeTitle>());
}
private void GivenExistingTitles(params AlternativeTitle[] titles)
{
Mocker.GetMock<IAlternativeTitleRepository>().Setup(r => r.FindByMovieMetadataId(_movie.Id))
Mocker.GetMock<IAlternativeTitleRepository>()
.Setup(r => r.FindByMovieMetadataId(_movie.Id))
.Returns(titles.ToList());
}
@ -52,7 +57,7 @@ namespace NzbDrone.Core.Test.MovieTests.AlternativeTitleServiceTests
Subject.UpdateTitles(titles, _movie);
Mocker.GetMock<IAlternativeTitleRepository>().Verify(r => r.InsertMany(inserts), Times.Once());
Mocker.GetMock<IAlternativeTitleRepository>().Verify(r => r.UpdateMany(updates), Times.Once());
Mocker.GetMock<IAlternativeTitleRepository>().Verify(r => r.UpdateMany(new List<AlternativeTitle>()), Times.Once());
Mocker.GetMock<IAlternativeTitleRepository>().Verify(r => r.DeleteMany(deletes), Times.Once());
}
@ -96,13 +101,18 @@ namespace NzbDrone.Core.Test.MovieTests.AlternativeTitleServiceTests
public void should_update_with_correct_id()
{
var existingTitle = Builder<AlternativeTitle>.CreateNew().With(t => t.Id = 2).Build();
GivenExistingTitles(existingTitle);
var updateTitle = existingTitle.JsonClone();
updateTitle.Id = 0;
Subject.UpdateTitles(new List<AlternativeTitle> { updateTitle }, _movie);
var result = Subject.UpdateTitles(new List<AlternativeTitle> { updateTitle }, _movie);
Mocker.GetMock<IAlternativeTitleRepository>().Verify(r => r.UpdateMany(It.Is<IList<AlternativeTitle>>(list => list.First().Id == existingTitle.Id)), Times.Once());
result.Should().HaveCount(1);
result.First().Id.Should().Be(existingTitle.Id);
Mocker.GetMock<IAlternativeTitleRepository>().Verify(r => r.UpdateMany(It.Is<IList<AlternativeTitle>>(l => l.Count == 0)), Times.Once());
}
}
}

View file

@ -53,7 +53,7 @@ namespace NzbDrone.Core.Test.MovieTests.AlternativeTitleServiceTests
Subject.UpdateCredits(titles, _movie);
Mocker.GetMock<ICreditRepository>().Verify(r => r.InsertMany(inserts), Times.Once());
Mocker.GetMock<ICreditRepository>().Verify(r => r.UpdateMany(updates), Times.Once());
Mocker.GetMock<ICreditRepository>().Verify(r => r.UpdateMany(new List<Credit>()), Times.Once());
Mocker.GetMock<ICreditRepository>().Verify(r => r.DeleteMany(deletes), Times.Once());
}
@ -91,9 +91,12 @@ namespace NzbDrone.Core.Test.MovieTests.AlternativeTitleServiceTests
var updateCredit = existingCredit.JsonClone();
updateCredit.Id = 0;
Subject.UpdateCredits(new List<Credit> { updateCredit }, _movie);
var result = Subject.UpdateCredits(new List<Credit> { updateCredit }, _movie);
Mocker.GetMock<ICreditRepository>().Verify(r => r.UpdateMany(It.Is<IList<Credit>>(list => list.First().Id == existingCredit.Id)), Times.Once());
result.Should().HaveCount(1);
result.First().Id.Should().Be(existingCredit.Id);
Mocker.GetMock<ICreditRepository>().Verify(r => r.UpdateMany(It.Is<IList<Credit>>(l => l.Count == 0)), Times.Once());
}
}
}

View file

@ -1,4 +1,4 @@
using System.Diagnostics;
using System.Diagnostics;
namespace NzbDrone.Core.Datastore
{

View file

@ -617,7 +617,7 @@ namespace NzbDrone.Core.MetadataSource.SkyHook
var newAlternativeTitle = new AlternativeTitle
{
Title = arg.Title,
SourceType = SourceType.TMDB,
SourceType = SourceType.Tmdb,
CleanTitle = arg.Title.CleanMovieTitle()
};

View file

@ -1,9 +1,8 @@
using NzbDrone.Core.Datastore;
using NzbDrone.Core.Parser;
namespace NzbDrone.Core.Movies.AlternativeTitles
{
public class AlternativeTitle : ModelBase
public class AlternativeTitle : Entity<AlternativeTitle>
{
public SourceType SourceType { get; set; }
public int MovieMetadataId { get; set; }
@ -14,39 +13,22 @@ namespace NzbDrone.Core.Movies.AlternativeTitles
{
}
public AlternativeTitle(string title, SourceType sourceType = SourceType.TMDB, int sourceId = 0)
public AlternativeTitle(string title, SourceType sourceType = SourceType.Tmdb)
{
Title = title;
CleanTitle = title.CleanMovieTitle();
SourceType = sourceType;
}
public override bool Equals(object obj)
{
var item = obj as AlternativeTitle;
if (item == null)
{
return false;
}
return item.CleanTitle == CleanTitle;
}
public override int GetHashCode()
{
return CleanTitle.GetHashCode();
}
public override string ToString()
{
return Title;
return $"{Title} [{CleanTitle}]";
}
}
public enum SourceType
{
TMDB = 0,
Tmdb = 0,
Mappings = 1,
User = 2,
Indexer = 3

View file

@ -7,6 +7,7 @@ namespace NzbDrone.Core.Movies.AlternativeTitles
public interface IAlternativeTitleRepository : IBasicRepository<AlternativeTitle>
{
List<AlternativeTitle> FindByMovieMetadataId(int movieId);
List<AlternativeTitle> FindByCleanTitles(List<string> cleanTitles);
void DeleteForMovies(List<int> movieIds);
}
@ -22,6 +23,11 @@ namespace NzbDrone.Core.Movies.AlternativeTitles
return Query(x => x.MovieMetadataId == movieId);
}
public List<AlternativeTitle> FindByCleanTitles(List<string> cleanTitles)
{
return Query(x => cleanTitles.Contains(x.CleanTitle));
}
public void DeleteForMovies(List<int> movieIds)
{
Delete(x => movieIds.Contains(x.MovieMetadataId));

View file

@ -1,7 +1,6 @@
using System.Collections.Generic;
using System.Linq;
using NLog;
using NzbDrone.Core.Configuration;
using NzbDrone.Core.Messaging.Events;
using NzbDrone.Core.Movies.Events;
@ -20,18 +19,11 @@ namespace NzbDrone.Core.Movies.AlternativeTitles
public class AlternativeTitleService : IAlternativeTitleService, IHandleAsync<MoviesDeletedEvent>
{
private readonly IAlternativeTitleRepository _titleRepo;
private readonly IConfigService _configService;
private readonly IEventAggregator _eventAggregator;
private readonly Logger _logger;
public AlternativeTitleService(IAlternativeTitleRepository titleRepo,
IEventAggregator eventAggregator,
IConfigService configService,
Logger logger)
public AlternativeTitleService(IAlternativeTitleRepository titleRepo, Logger logger)
{
_titleRepo = titleRepo;
_eventAggregator = eventAggregator;
_configService = configService;
_logger = logger;
}
@ -82,18 +74,45 @@ namespace NzbDrone.Core.Movies.AlternativeTitles
titles = titles.DistinctBy(t => t.CleanTitle).ToList();
// Make sure we are not adding titles that exist for other movies (until language PR goes in)
titles = titles.Where(t => !_titleRepo.All().Any(e => e.CleanTitle == t.CleanTitle && e.MovieMetadataId != t.MovieMetadataId)).ToList();
var allTitlesByCleanTitles = _titleRepo.FindByCleanTitles(titles.Select(t => t.CleanTitle).ToList());
titles = titles.Where(t => !allTitlesByCleanTitles.Any(e => e.CleanTitle == t.CleanTitle && e.MovieMetadataId != t.MovieMetadataId)).ToList();
// Now find titles to delete, update and insert.
var existingTitles = _titleRepo.FindByMovieMetadataId(movieMetadataId);
var insert = titles.Where(t => !existingTitles.Contains(t));
var update = existingTitles.Where(t => titles.Contains(t));
var delete = existingTitles.Where(t => !titles.Contains(t));
var updateList = new List<AlternativeTitle>();
var addList = new List<AlternativeTitle>();
var upToDateCount = 0;
_titleRepo.DeleteMany(delete.ToList());
_titleRepo.UpdateMany(update.ToList());
_titleRepo.InsertMany(insert.ToList());
foreach (var title in titles)
{
var existingTitle = existingTitles.FirstOrDefault(x => x.CleanTitle == title.CleanTitle);
if (existingTitle != null)
{
existingTitles.Remove(existingTitle);
title.UseDbFieldsFrom(existingTitle);
if (!title.Equals(existingTitle))
{
updateList.Add(title);
}
else
{
upToDateCount++;
}
}
else
{
addList.Add(title);
}
}
_titleRepo.DeleteMany(existingTitles);
_titleRepo.UpdateMany(updateList);
_titleRepo.InsertMany(addList);
_logger.Debug("[{0}] {1} alternative titles up to date; Updating {2}, Adding {3}, Deleting {4} entries.", movieMetadata.Title, upToDateCount, updateList.Count, addList.Count, existingTitles.Count);
return titles;
}

View file

@ -1,9 +1,8 @@
using System.Collections.Generic;
using NzbDrone.Core.Datastore;
namespace NzbDrone.Core.Movies.Credits
{
public class Credit : ModelBase
public class Credit : Entity<Credit>
{
public Credit()
{

View file

@ -1,5 +1,6 @@
using System.Collections.Generic;
using System.Linq;
using NLog;
using NzbDrone.Core.Messaging.Events;
using NzbDrone.Core.Movies.Events;
@ -18,10 +19,12 @@ namespace NzbDrone.Core.Movies.Credits
public class CreditService : ICreditService, IHandleAsync<MoviesDeletedEvent>
{
private readonly ICreditRepository _creditRepo;
private readonly Logger _logger;
public CreditService(ICreditRepository creditRepo)
public CreditService(ICreditRepository creditRepo, Logger logger)
{
_creditRepo = creditRepo;
_logger = logger;
}
public List<Credit> GetAllCreditsForMovieMetadata(int movieMetadataId)
@ -64,21 +67,45 @@ namespace NzbDrone.Core.Movies.Credits
// First update the movie ids so we can correlate them later.
credits.ForEach(t => t.MovieMetadataId = movieMetadataId);
// Now find credits to delete, update and insert.
var existingCredits = _creditRepo.FindByMovieMetadataId(movieMetadataId);
// Should never have multiple credits with same credit_id, but check to ensure incase TMDB is on fritz
// Should never have multiple credits with same credit_id, but check to ensure in case TMDB is on fritz
var dupeFreeCredits = credits.DistinctBy(m => m.CreditTmdbId).ToList();
dupeFreeCredits.ForEach(c => c.Id = existingCredits.FirstOrDefault(t => t.CreditTmdbId == c.CreditTmdbId)?.Id ?? 0);
var existingCredits = _creditRepo.FindByMovieMetadataId(movieMetadataId);
var insert = dupeFreeCredits.Where(t => t.Id == 0).ToList();
var update = dupeFreeCredits.Where(t => t.Id > 0).ToList();
var delete = existingCredits.Where(t => !dupeFreeCredits.Any(c => c.CreditTmdbId == t.CreditTmdbId)).ToList();
var updateList = new List<Credit>();
var addList = new List<Credit>();
var upToDateCount = 0;
_creditRepo.DeleteMany(delete);
_creditRepo.UpdateMany(update);
_creditRepo.InsertMany(insert);
foreach (var credit in dupeFreeCredits)
{
var existingCredit = existingCredits.FirstOrDefault(x => x.CreditTmdbId == credit.CreditTmdbId);
if (existingCredit != null)
{
existingCredits.Remove(existingCredit);
credit.UseDbFieldsFrom(existingCredit);
if (!credit.Equals(existingCredit))
{
updateList.Add(credit);
}
else
{
upToDateCount++;
}
}
else
{
addList.Add(credit);
}
}
_creditRepo.DeleteMany(existingCredits);
_creditRepo.UpdateMany(updateList);
_creditRepo.InsertMany(addList);
_logger.Debug("[{0}] {1} credits up to date; Updating {2}, Adding {3}, Deleting {4} entries.", movieMetadata.Title, upToDateCount, updateList.Count, addList.Count, existingCredits.Count);
return credits;
}

View file

@ -13,7 +13,7 @@ namespace NzbDrone.Core.Movies
List<MovieMetadata> FindById(List<int> tmdbIds);
List<MovieMetadata> GetMoviesWithCollections();
List<MovieMetadata> GetMoviesByCollectionTmdbId(int collectionId);
bool UpsertMany(List<MovieMetadata> data);
bool UpsertMany(List<MovieMetadata> metadatas);
}
public class MovieMetadataRepository : BasicRepository<MovieMetadata>, IMovieMetadataRepository
@ -51,40 +51,43 @@ namespace NzbDrone.Core.Movies
return Query(x => x.CollectionTmdbId == collectionId);
}
public bool UpsertMany(List<MovieMetadata> data)
public bool UpsertMany(List<MovieMetadata> metadatas)
{
var existingMetadata = FindById(data.Select(x => x.TmdbId).ToList());
var updateMetadataList = new List<MovieMetadata>();
var addMetadataList = new List<MovieMetadata>();
var upToDateMetadataCount = 0;
var updateList = new List<MovieMetadata>();
var addList = new List<MovieMetadata>();
var upToDateCount = 0;
foreach (var meta in data)
var existingMetadatas = FindById(metadatas.Select(x => x.TmdbId).ToList());
foreach (var metadata in metadatas)
{
var existing = existingMetadata.SingleOrDefault(x => x.TmdbId == meta.TmdbId);
if (existing != null)
var existingMetadata = existingMetadatas.SingleOrDefault(x => x.TmdbId == metadata.TmdbId);
if (existingMetadata != null)
{
meta.UseDbFieldsFrom(existing);
if (!meta.Equals(existing))
metadata.UseDbFieldsFrom(existingMetadata);
if (!metadata.Equals(existingMetadata))
{
updateMetadataList.Add(meta);
updateList.Add(metadata);
}
else
{
upToDateMetadataCount++;
upToDateCount++;
}
}
else
{
addMetadataList.Add(meta);
addList.Add(metadata);
}
}
UpdateMany(updateMetadataList);
InsertMany(addMetadataList);
UpdateMany(updateList);
InsertMany(addList);
_logger.Debug($"{upToDateMetadataCount} movie metadata up to date; Updating {updateMetadataList.Count}, Adding {addMetadataList.Count} movie metadata entries.");
_logger.Debug("{0} movie metadata up to date; Updating {1}, Adding {2} entries.", upToDateCount, updateList.Count, addList.Count);
return updateMetadataList.Count > 0 || addMetadataList.Count > 0;
return updateList.Count > 0 || addList.Count > 0;
}
}
}

View file

@ -12,7 +12,7 @@ namespace NzbDrone.Core.Movies
public RatingChild Trakt { get; set; }
}
public class RatingChild
public class RatingChild : MemberwiseEquatable<RatingChild>
{
public int Votes { get; set; }
public decimal Value { get; set; }

View file

@ -4,7 +4,6 @@ using System.Linq;
using NLog;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Instrumentation.Extensions;
using NzbDrone.Core.AutoTagging;
using NzbDrone.Core.Configuration;
using NzbDrone.Core.Exceptions;
using NzbDrone.Core.MediaFiles;
@ -30,13 +29,12 @@ namespace NzbDrone.Core.Movies
private readonly IMovieMetadataService _movieMetadataService;
private readonly IRootFolderService _folderService;
private readonly IMovieTranslationService _movieTranslationService;
private readonly IAlternativeTitleService _titleService;
private readonly IAlternativeTitleService _alternativeTitleService;
private readonly ICreditService _creditService;
private readonly IEventAggregator _eventAggregator;
private readonly IDiskScanService _diskScanService;
private readonly ICheckIfMovieShouldBeRefreshed _checkIfMovieShouldBeRefreshed;
private readonly IConfigService _configService;
private readonly IAutoTaggingService _autoTaggingService;
private readonly Logger _logger;
public RefreshMovieService(IProvideMovieInfo movieInfo,
@ -45,13 +43,12 @@ namespace NzbDrone.Core.Movies
IMovieMetadataService movieMetadataService,
IRootFolderService folderService,
IMovieTranslationService movieTranslationService,
IAlternativeTitleService titleService,
IAlternativeTitleService alternativeTitleService,
ICreditService creditService,
IEventAggregator eventAggregator,
IDiskScanService diskScanService,
ICheckIfMovieShouldBeRefreshed checkIfMovieShouldBeRefreshed,
IConfigService configService,
IAutoTaggingService autoTaggingService,
Logger logger)
{
_movieInfo = movieInfo;
@ -60,13 +57,12 @@ namespace NzbDrone.Core.Movies
_movieMetadataService = movieMetadataService;
_folderService = folderService;
_movieTranslationService = movieTranslationService;
_titleService = titleService;
_alternativeTitleService = alternativeTitleService;
_creditService = creditService;
_eventAggregator = eventAggregator;
_diskScanService = diskScanService;
_checkIfMovieShouldBeRefreshed = checkIfMovieShouldBeRefreshed;
_configService = configService;
_autoTaggingService = autoTaggingService;
_logger = logger;
}
@ -161,11 +157,12 @@ namespace NzbDrone.Core.Movies
movieMetadata.CollectionTitle = null;
}
movieMetadata.AlternativeTitles = _titleService.UpdateTitles(movieInfo.AlternativeTitles, movieMetadata);
movieMetadata.AlternativeTitles = _alternativeTitleService.UpdateTitles(movieInfo.AlternativeTitles, movieMetadata);
_movieTranslationService.UpdateTranslations(movieInfo.Translations, movieMetadata);
_creditService.UpdateCredits(credits, movieMetadata);
_movieMetadataService.Upsert(movieMetadata);
_creditService.UpdateCredits(credits, movieMetadata);
movie.MovieMetadata = movieMetadata;
@ -252,19 +249,19 @@ namespace NzbDrone.Core.Movies
else
{
// TODO refresh all moviemetadata here, even if not used by a Movie
var allMovie = _movieService.GetAllMovies().OrderBy(c => c.MovieMetadata.Value.SortTitle).ToList();
var allMovies = _movieService.GetAllMovies();
var updatedTMDBMovies = new HashSet<int>();
var updatedTmdbMovies = new HashSet<int>();
if (message.LastStartTime.HasValue && message.LastStartTime.Value.AddDays(14) > DateTime.UtcNow)
{
updatedTMDBMovies = _movieInfo.GetChangedMovies(message.LastStartTime.Value);
updatedTmdbMovies = _movieInfo.GetChangedMovies(message.LastStartTime.Value);
}
foreach (var movie in allMovie)
foreach (var movie in allMovies)
{
var movieLocal = movie;
if ((updatedTMDBMovies.Count == 0 && _checkIfMovieShouldBeRefreshed.ShouldRefresh(movie.MovieMetadata)) || updatedTMDBMovies.Contains(movie.TmdbId) || message.Trigger == CommandTrigger.Manual)
if ((updatedTmdbMovies.Count == 0 && _checkIfMovieShouldBeRefreshed.ShouldRefresh(movie.MovieMetadata)) || updatedTmdbMovies.Contains(movie.TmdbId) || message.Trigger == CommandTrigger.Manual)
{
try
{

View file

@ -1,9 +1,8 @@
using NzbDrone.Core.Datastore;
using NzbDrone.Core.Languages;
namespace NzbDrone.Core.Movies.Translations
{
public class MovieTranslation : ModelBase
public class MovieTranslation : Entity<MovieTranslation>
{
public int MovieMetadataId { get; set; }
public string Title { get; set; }

View file

@ -57,15 +57,40 @@ namespace NzbDrone.Core.Movies.Translations
// Now find translations to delete, update and insert
var existingTranslations = _translationRepo.FindByMovieMetadataId(movieMetadataId);
translations.ForEach(c => c.Id = existingTranslations.FirstOrDefault(t => t.Language == c.Language)?.Id ?? 0);
var updateList = new List<MovieTranslation>();
var addList = new List<MovieTranslation>();
var upToDateCount = 0;
var insert = translations.Where(t => t.Id == 0).ToList();
var update = translations.Where(t => t.Id > 0).ToList();
var delete = existingTranslations.Where(t => !translations.Any(c => c.Language == t.Language)).ToList();
foreach (var translation in translations)
{
var existingTranslation = existingTranslations.FirstOrDefault(x => x.Language == translation.Language);
_translationRepo.DeleteMany(delete.ToList());
_translationRepo.UpdateMany(update.ToList());
_translationRepo.InsertMany(insert.ToList());
if (existingTranslation != null)
{
existingTranslations.Remove(existingTranslation);
translation.UseDbFieldsFrom(existingTranslation);
if (!translation.Equals(existingTranslation))
{
updateList.Add(translation);
}
else
{
upToDateCount++;
}
}
else
{
addList.Add(translation);
}
}
_translationRepo.DeleteMany(existingTranslations);
_translationRepo.UpdateMany(updateList);
_translationRepo.InsertMany(addList);
_logger.Debug("[{0}] {1} translations up to date; Updating {2}, Adding {3}, Deleting {4} entries.", movieMetadata.Title, upToDateCount, updateList.Count, addList.Count, existingTranslations.Count);
return translations;
}