Fixed: (AnimeTorrents) Add DownloadableOnly/FreeleechOnly settings

src/NzbDrone.Core/Datastore/Migration/030_animetorrents_use_custom_config_contract.cs

@@ -0,0 +1,14 @@
+using FluentMigrator;
+using NzbDrone.Core.Datastore.Migration.Framework;
+
+namespace NzbDrone.Core.Datastore.Migration
+{
+    [Migration(030)]
+    public class animetorrents_use_custom_config_contract : NzbDroneMigrationBase
+    {
+        protected override void MainDbUpgrade()
+        {
+            Update.Table("Indexers").Set(new { ConfigContract = "AnimeTorrentsSettings" }).Where(new { Implementation = "AnimeTorrents" });
+        }
+    }
+}

src/NzbDrone.Core/Indexers/Definitions/AnimeTorrents.cs

@@ -1,14 +1,14 @@
 using System;
 using System.Collections.Generic;
-using System.Collections.Specialized;
 using System.Globalization;
 using System.Linq;
-using System.Net.Http;
 using System.Text.RegularExpressions;
 using System.Threading.Tasks;
 using AngleSharp.Html.Parser;
 using NLog;
+using NzbDrone.Common.Extensions;
 using NzbDrone.Common.Http;
+using NzbDrone.Core.Annotations;
 using NzbDrone.Core.Configuration;
 using NzbDrone.Core.Indexers.Exceptions;
 using NzbDrone.Core.Indexers.Settings;
@@ -19,15 +19,14 @@ using NzbDrone.Core.Parser.Model;
 
 namespace NzbDrone.Core.Indexers.Definitions
 {
-    public class AnimeTorrents : TorrentIndexerBase<UserPassTorrentBaseSettings>
+    public class AnimeTorrents : TorrentIndexerBase<AnimeTorrentsSettings>
     {
         public override string Name => "AnimeTorrents";
-
         public override string[] IndexerUrls => new[] { "https://animetorrents.me/" };
         public override string Description => "Definitive source for anime and manga";
-        private string LoginUrl => Settings.BaseUrl + "login.php";
         public override DownloadProtocol Protocol => DownloadProtocol.Torrent;
         public override IndexerPrivacy Privacy => IndexerPrivacy.Private;
+        public override TimeSpan RateLimit => TimeSpan.FromSeconds(4);
         public override IndexerCapabilities Capabilities => SetCapabilities();
 
         public AnimeTorrents(IIndexerHttpClient httpClient, IEventAggregator eventAggregator, IIndexerStatusService indexerStatusService, IConfigService configService, Logger logger)
@@ -37,7 +36,7 @@ namespace NzbDrone.Core.Indexers.Definitions
 
         public override IIndexerRequestGenerator GetRequestGenerator()
         {
-            return new AnimeTorrentsRequestGenerator { Settings = Settings, Capabilities = Capabilities };
+            return new AnimeTorrentsRequestGenerator(Settings, Capabilities);
         }
 
         public override IParseIndexerResponse GetParser()
@@ -49,36 +48,37 @@ namespace NzbDrone.Core.Indexers.Definitions
         {
             UpdateCookies(null, null);
 
-            var loginPage = await ExecuteAuth(new HttpRequest(LoginUrl));
+            var loginUrl = Settings.BaseUrl + "login.php";
 
-            var requestBuilder = new HttpRequestBuilder(LoginUrl)
+            var loginPage = await ExecuteAuth(new HttpRequest(loginUrl));
+
+            var requestBuilder = new HttpRequestBuilder(loginUrl)
             {
                 LogResponseContent = true,
-                AllowAutoRedirect = true,
+                AllowAutoRedirect = true
-                Method = HttpMethod.Post
             };
 
             var authLoginRequest = requestBuilder
+                .Post()
                 .SetCookies(loginPage.GetCookies())
                 .AddFormParameter("username", Settings.Username)
                 .AddFormParameter("password", Settings.Password)
                 .AddFormParameter("form", "login")
                 .AddFormParameter("rememberme[]", "1")
                 .SetHeader("Content-Type", "application/x-www-form-urlencoded")
+                .SetHeader("Referer", loginUrl)
                 .Build();
 
             var response = await ExecuteAuth(authLoginRequest);
 
-            if (response.Content != null && response.Content.Contains("logout.php"))
+            if (response.Content == null || !response.Content.Contains("logout.php"))
-            {
-                UpdateCookies(response.GetCookies(), DateTime.Now.AddDays(30));
-
-                _logger.Debug("AnimeTorrents authentication succeeded");
-            }
-            else
             {
                 throw new IndexerAuthException("AnimeTorrents authentication failed");
             }
+
+            UpdateCookies(response.GetCookies(), DateTime.Now.AddDays(30));
+
+            _logger.Debug("AnimeTorrents authentication succeeded");
         }
 
         protected override bool CheckIfLoginNeeded(HttpResponse httpResponse)
@@ -126,49 +126,25 @@ namespace NzbDrone.Core.Indexers.Definitions
 
     public class AnimeTorrentsRequestGenerator : IIndexerRequestGenerator
     {
-        public UserPassTorrentBaseSettings Settings { get; set; }
+        private readonly AnimeTorrentsSettings _settings;
-        public IndexerCapabilities Capabilities { get; set; }
+        private readonly IndexerCapabilities _capabilities;
 
-        private IEnumerable<IndexerRequest> GetPagedRequests(string term, int[] categories)
+        public AnimeTorrentsRequestGenerator(AnimeTorrentsSettings settings, IndexerCapabilities capabilities)
         {
-            var searchString = term;
+            _settings = settings;
-
+            _capabilities = capabilities;
-            //  replace any space, special char, etc. with % (wildcard)
-            var replaceRegex = new Regex("[^a-zA-Z0-9]+");
-            searchString = replaceRegex.Replace(searchString, "%");
-            var searchUrl = Settings.BaseUrl + "ajax/torrents_data.php";
-            var searchUrlReferer = Settings.BaseUrl + "torrents.php?cat=0&searchin=filename&search=";
-
-            var trackerCats = Capabilities.Categories.MapTorznabCapsToTrackers(categories) ?? new List<string>();
-
-            var queryCollection = new NameValueCollection
-            {
-                { "total", "146" }, // Not sure what this is about but its required!
-                { "cat", trackerCats.Count == 1 ? trackerCats.First() : "0" },
-                { "page", "1" },
-                { "searchin", "filename" },
-                { "search", searchString }
-            };
-
-            searchUrl += "?" + queryCollection.GetQueryString();
-
-            var extraHeaders = new NameValueCollection
-            {
-                { "X-Requested-With", "XMLHttpRequest" },
-                { "Referer", searchUrlReferer }
-            };
-
-            var request = new IndexerRequest(searchUrl, null);
-            request.HttpRequest.Headers.Add(extraHeaders);
-
-            yield return request;
         }
 
         public IndexerPageableRequestChain GetSearchRequests(MovieSearchCriteria searchCriteria)
         {
             var pageableRequests = new IndexerPageableRequestChain();
 
-            pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SanitizedSearchTerm), searchCriteria.Categories));
+            var searchTerm = $"{searchCriteria.SanitizedSearchTerm}";
+
+            foreach (var category in GetTrackerCategories(searchTerm, searchCriteria))
+            {
+                pageableRequests.Add(GetPagedRequests(searchTerm, category, searchCriteria));
+            }
 
             return pageableRequests;
         }
@@ -177,7 +153,12 @@ namespace NzbDrone.Core.Indexers.Definitions
         {
             var pageableRequests = new IndexerPageableRequestChain();
 
-            pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SanitizedSearchTerm), searchCriteria.Categories));
+            var searchTerm = $"{searchCriteria.SanitizedSearchTerm}";
+
+            foreach (var category in GetTrackerCategories(searchTerm, searchCriteria))
+            {
+                pageableRequests.Add(GetPagedRequests(searchTerm, category, searchCriteria));
+            }
 
             return pageableRequests;
         }
@@ -186,7 +167,12 @@ namespace NzbDrone.Core.Indexers.Definitions
         {
             var pageableRequests = new IndexerPageableRequestChain();
 
-            pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SanitizedSearchTerm), searchCriteria.Categories));
+            var searchTerm = $"{searchCriteria.SanitizedSearchTerm}";
+
+            foreach (var category in GetTrackerCategories(searchTerm, searchCriteria))
+            {
+                pageableRequests.Add(GetPagedRequests(searchTerm, category, searchCriteria));
+            }
 
             return pageableRequests;
         }
@@ -195,7 +181,12 @@ namespace NzbDrone.Core.Indexers.Definitions
         {
             var pageableRequests = new IndexerPageableRequestChain();
 
-            pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SanitizedSearchTerm), searchCriteria.Categories));
+            var searchTerm = $"{searchCriteria.SanitizedSearchTerm}";
+
+            foreach (var category in GetTrackerCategories(searchTerm, searchCriteria))
+            {
+                pageableRequests.Add(GetPagedRequests(searchTerm, category, searchCriteria));
+            }
 
             return pageableRequests;
         }
@@ -204,21 +195,76 @@ namespace NzbDrone.Core.Indexers.Definitions
         {
             var pageableRequests = new IndexerPageableRequestChain();
 
-            pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SanitizedSearchTerm), searchCriteria.Categories));
+            var searchTerm = $"{searchCriteria.SanitizedSearchTerm}";
+
+            foreach (var category in GetTrackerCategories(searchTerm, searchCriteria))
+            {
+                pageableRequests.Add(GetPagedRequests(searchTerm, category, searchCriteria));
+            }
 
             return pageableRequests;
         }
 
+        private IEnumerable<IndexerRequest> GetPagedRequests(string term, string category, SearchCriteriaBase searchCriteria)
+        {
+            var searchUrl = _settings.BaseUrl + "ajax/torrents_data.php";
+
+            // replace non-alphanumeric characters with % (wildcard)
+            var searchString = Regex.Replace(term.Trim(), "[^a-zA-Z0-9]+", "%");
+
+            var page = searchCriteria.Limit is > 0 && searchCriteria.Offset is > 0 ? (int)(searchCriteria.Offset / searchCriteria.Limit) + 1 : 1;
+
+            var refererUri = new HttpUri(_settings.BaseUrl)
+                .CombinePath("/torrents.php")
+                .AddQueryParam("cat", $"{category}");
+
+            if (_settings.DownloadableOnly)
+            {
+                refererUri = refererUri.AddQueryParam("dlable", "1");
+            }
+
+            var requestBuilder = new HttpRequestBuilder(searchUrl)
+                .AddQueryParam("total", "100") // Assuming the total number of pages
+                .AddQueryParam("cat", $"{category}")
+                .AddQueryParam("searchin", "filename")
+                .AddQueryParam("search", searchString)
+                .AddQueryParam("page", page)
+                .SetHeader("X-Requested-With", "XMLHttpRequest")
+                .SetHeader("Referer", refererUri.FullUri)
+                .Accept(HttpAccept.Html);
+
+            if (_settings.DownloadableOnly)
+            {
+                requestBuilder.AddQueryParam("dlable", "1");
+            }
+
+            yield return new IndexerRequest(requestBuilder.Build());
+        }
+
+        private IEnumerable<string> GetTrackerCategories(string term, SearchCriteriaBase searchCriteria)
+        {
+            var searchTerm = term.Trim();
+
+            var categoryMapping = _capabilities.Categories
+                .MapTorznabCapsToTrackers(searchCriteria.Categories)
+                .Distinct()
+                .ToList();
+
+            return searchTerm.IsNullOrWhiteSpace() && categoryMapping.Count == 2
+                ? categoryMapping
+                : new List<string> { categoryMapping.FirstIfSingleOrDefault("0") };
+        }
+
         public Func<IDictionary<string, string>> GetCookies { get; set; }
         public Action<IDictionary<string, string>, DateTime?> CookiesUpdater { get; set; }
     }
 
     public class AnimeTorrentsParser : IParseIndexerResponse
     {
-        private readonly UserPassTorrentBaseSettings _settings;
+        private readonly AnimeTorrentsSettings _settings;
         private readonly IndexerCapabilitiesCategories _categories;
 
-        public AnimeTorrentsParser(UserPassTorrentBaseSettings settings, IndexerCapabilitiesCategories categories)
+        public AnimeTorrentsParser(AnimeTorrentsSettings settings, IndexerCapabilitiesCategories categories)
         {
             _settings = settings;
             _categories = categories;
@@ -226,94 +272,88 @@ namespace NzbDrone.Core.Indexers.Definitions
 
         public IList<ReleaseInfo> ParseResponse(IndexerResponse indexerResponse)
         {
-            var torrentInfos = new List<TorrentInfo>();
+            var releaseInfos = new List<ReleaseInfo>();
 
             var parser = new HtmlParser();
             var dom = parser.ParseDocument(indexerResponse.Content);
 
-            var rows = dom.QuerySelectorAll("tr");
+            var rows = dom.QuerySelectorAll("table tr");
             foreach (var row in rows.Skip(1))
             {
-                var release = new TorrentInfo();
+                var downloadVolumeFactor = row.QuerySelector("img[alt=\"Gold Torrent\"]") != null ? 0 : row.QuerySelector("img[alt=\"Silver Torrent\"]") != null ? 0.5 : 1;
+
+                // skip non-freeleech results when freeleech only is set
+                if (_settings.FreeleechOnly && downloadVolumeFactor != 0)
+                {
+                    continue;
+                }
+
                 var qTitleLink = row.QuerySelector("td:nth-of-type(2) a:nth-of-type(1)");
-                release.Title = qTitleLink.TextContent.Trim();
+                var title = qTitleLink?.TextContent.Trim();
 
                 // If we search an get no results, we still get a table just with no info.
-                if (string.IsNullOrWhiteSpace(release.Title))
+                if (title.IsNullOrWhiteSpace())
                 {
                     break;
                 }
 
-                release.Guid = qTitleLink.GetAttribute("href");
+                var infoUrl = qTitleLink?.GetAttribute("href");
-                release.InfoUrl = release.Guid;
-
-                var dateString = row.QuerySelector("td:nth-of-type(5)").TextContent;
-                release.PublishDate = DateTime.ParseExact(dateString, "dd MMM yy", CultureInfo.InvariantCulture);
 
                 // newbie users don't see DL links
-                var qLink = row.QuerySelector("td:nth-of-type(3) a");
+                // use details link as placeholder
-                if (qLink != null)
+                // skipping the release prevents newbie users from adding the tracker (empty result)
-                {
+                var downloadUrl = row.QuerySelector("td:nth-of-type(3) a")?.GetAttribute("href") ?? infoUrl;
-                    release.DownloadUrl = qLink.GetAttribute("href");
-                }
-                else
-                {
-                    // use details link as placeholder
-                    // null causes errors during export to torznab
-                    // skipping the release prevents newbie users from adding the tracker (empty result)
-                    release.DownloadUrl = release.InfoUrl;
-                }
 
-                var sizeStr = row.QuerySelector("td:nth-of-type(6)").TextContent;
+                var connections = row.QuerySelector("td:nth-of-type(8)").TextContent.Trim().Split('/', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries);
-                release.Size = ParseUtil.GetBytes(sizeStr);
+                var seeders = ParseUtil.CoerceInt(connections[0]);
 
-                var connections = row.QuerySelector("td:nth-of-type(8)").TextContent.Trim().Split("/".ToCharArray(), StringSplitOptions.RemoveEmptyEntries);
+                var categoryLink = row.QuerySelector("td:nth-of-type(1) a")?.GetAttribute("href") ?? string.Empty;
+                var categoryId = ParseUtil.GetArgumentFromQueryString(categoryLink, "cat");
 
-                release.Seeders = ParseUtil.CoerceInt(connections[0].Trim());
+                var release = new TorrentInfo
-                release.Peers = ParseUtil.CoerceInt(connections[1].Trim()) + release.Seeders;
-                release.Grabs = ParseUtil.CoerceInt(connections[2].Trim());
-
-                var rCat = row.QuerySelector("td:nth-of-type(1) a").GetAttribute("href");
-                var rCatIdx = rCat.IndexOf("cat=");
-                if (rCatIdx > -1)
                 {
-                    rCat = rCat.Substring(rCatIdx + 4);
+                    Guid = infoUrl,
-                }
+                    InfoUrl = infoUrl,
-
+                    DownloadUrl = downloadUrl,
-                release.Categories = _categories.MapTrackerCatToNewznab(rCat);
+                    Title = title,
-
+                    Categories = _categories.MapTrackerCatToNewznab(categoryId),
-                if (row.QuerySelector("img[alt=\"Gold Torrent\"]") != null)
+                    PublishDate = DateTime.ParseExact(row.QuerySelector("td:nth-of-type(5)").TextContent, "dd MMM yy", CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal),
-                {
+                    Size = ParseUtil.GetBytes(row.QuerySelector("td:nth-of-type(6)").TextContent.Trim()),
-                    release.DownloadVolumeFactor = 0;
+                    Seeders = seeders,
-                }
+                    Peers = ParseUtil.CoerceInt(connections[1]) + seeders,
-                else if (row.QuerySelector("img[alt=\"Silver Torrent\"]") != null)
+                    Grabs = ParseUtil.CoerceInt(connections[2]),
-                {
+                    DownloadVolumeFactor = downloadVolumeFactor,
-                    release.DownloadVolumeFactor = 0.5;
+                    UploadVolumeFactor = 1,
-                }
+                    Genres = row.QuerySelectorAll("td:nth-of-type(2) a.tortags").Select(t => t.TextContent.Trim()).ToList()
-                else
+                };
-                {
-                    release.DownloadVolumeFactor = 1;
-                }
 
                 var uLFactorImg = row.QuerySelector("img[alt*=\"x Multiplier Torrent\"]");
                 if (uLFactorImg != null)
                 {
                     release.UploadVolumeFactor = ParseUtil.CoerceDouble(uLFactorImg.GetAttribute("alt").Split('x')[0]);
                 }
-                else
-                {
-                    release.UploadVolumeFactor = 1;
-                }
 
-                qTitleLink.Remove();
+                releaseInfos.Add(release);
-
-                //release.Description = row.QuerySelector("td:nth-of-type(2)").TextContent;
-                torrentInfos.Add(release);
             }
 
-            return torrentInfos.ToArray();
+            return releaseInfos.ToArray();
         }
 
         public Action<IDictionary<string, string>, DateTime?> CookiesUpdater { get; set; }
     }
+
+    public class AnimeTorrentsSettings : UserPassTorrentBaseSettings
+    {
+        public AnimeTorrentsSettings()
+        {
+            FreeleechOnly = false;
+            DownloadableOnly = false;
+        }
+
+        [FieldDefinition(4, Label = "Freeleech Only", Type = FieldType.Checkbox, HelpText = "Show freeleech torrents only")]
+        public bool FreeleechOnly { get; set; }
+
+        [FieldDefinition(5, Label = "Downloadable Only", Type = FieldType.Checkbox, HelpText = "Search downloadable torrents only (enable this only if your account class is Newbie)")]
+        public bool DownloadableOnly { get; set; }
+    }
 }