Merge remote-tracking branch 'origin/development' into development

bazarr/api/system/languages.py

@@ -4,7 +4,7 @@ from flask_restx import Resource, Namespace, reqparse
 from operator import itemgetter
 
 from app.database import TableHistory, TableHistoryMovie, TableSettingsLanguages
-from languages.get_languages import alpha2_from_alpha3, language_from_alpha2
+from languages.get_languages import alpha2_from_alpha3, language_from_alpha2, alpha3_from_alpha2
 
 from ..utils import authenticate, False_Keys
 
@@ -46,6 +46,7 @@ class Languages(Resource):
                     try:
                         languages_dicts.append({
                             'code2': code2,
+                            'code3': alpha3_from_alpha2(code2),
                             'name': language_from_alpha2(code2),
                             # Compatibility: Use false temporarily
                             'enabled': False
@@ -55,6 +56,7 @@ class Languages(Resource):
         else:
             languages_dicts = TableSettingsLanguages.select(TableSettingsLanguages.name,
                                                             TableSettingsLanguages.code2,
+                                                            TableSettingsLanguages.code3,
                                                             TableSettingsLanguages.enabled)\
                 .order_by(TableSettingsLanguages.name).dicts()
             languages_dicts = list(languages_dicts)

bazarr/app/config.py

@@ -83,7 +83,8 @@ defaults = {
         'default_und_audio_lang': '',
         'default_und_embedded_subtitles_lang': '',
         'parse_embedded_audio_track': 'False',
-        'skip_hashing': 'False'
+        'skip_hashing': 'False',
+        'language_equals': '[]',
     },
     'auth': {
         'type': 'None',
@@ -300,7 +301,8 @@ array_keys = ['excluded_tags',
               'excluded_series_types',
               'enabled_providers',
               'path_mappings',
-              'path_mappings_movie']
+              'path_mappings_movie',
+              'language_equals']
 
 str_keys = ['chmod']
 

bazarr/app/get_providers.py

@@ -20,6 +20,7 @@ from subliminal_patch.extensions import provider_registry
 
 from app.get_args import args
 from app.config import settings, get_array_from
+from languages.get_languages import CustomLanguage
 from app.event_handler import event_stream
 from utilities.binaries import get_binary
 from radarr.blacklist import blacklist_log_movie
@@ -115,6 +116,49 @@ def provider_pool():
     return subliminal_patch.core.SZProviderPool
 
 
+def _lang_from_str(content: str):
+    " Formats: es-MX en@hi es-MX@forced "
+    extra_info = content.split("@")
+    if len(extra_info) > 1:
+        kwargs = {extra_info[-1]: True}
+    else:
+        kwargs = {}
+
+    content = extra_info[0]
+
+    try:
+        code, country = content.split("-")
+    except ValueError:
+        lang = CustomLanguage.from_value(content)
+        if lang is not None:
+            lang = lang.subzero_language()
+            return lang.rebuild(lang, **kwargs)
+
+        code, country = content, None
+
+    return subliminal_patch.core.Language(code, country, **kwargs)
+
+
+def get_language_equals(settings_=None):
+    settings_ = settings_ or settings
+
+    equals = get_array_from(settings_.general.language_equals)
+    if not equals:
+        return []
+
+    items = []
+    for equal in equals:
+        try:
+            from_, to_ = equal.split(":")
+            from_, to_ = _lang_from_str(from_), _lang_from_str(to_)
+        except Exception as error:
+            logging.info("Invalid equal value: '%s' [%s]", equal, error)
+        else:
+            items.append((from_, to_))
+
+    return items
+
+
 def get_providers():
     providers_list = []
     existing_providers = provider_registry.names()

bazarr/subtitles/pool.py

@@ -8,7 +8,7 @@ from inspect import getfullargspec
 
 from radarr.blacklist import get_blacklist_movie
 from sonarr.blacklist import get_blacklist
-from app.get_providers import get_providers, get_providers_auth, provider_throttle, provider_pool
+from app.get_providers import get_providers, get_providers_auth, provider_throttle, provider_pool, get_language_equals
 
 from .utils import get_ban_list
 
@@ -19,10 +19,11 @@ def _init_pool(media_type, profile_id=None, providers=None):
     return pool(
         providers=providers or get_providers(),
         provider_configs=get_providers_auth(),
-        blacklist=get_blacklist() if media_type == 'series' else get_blacklist_movie(),
+        blacklist=get_blacklist() if media_type == "series" else get_blacklist_movie(),
         throttle_callback=provider_throttle,
         ban_list=get_ban_list(profile_id),
         language_hook=None,
+        language_equals=get_language_equals(),
     )
 
 
@@ -54,8 +55,19 @@ def _update_pool(media_type, profile_id=None):
     return pool.update(
         get_providers(),
         get_providers_auth(),
-        get_blacklist() if media_type == 'series' else get_blacklist_movie(),
+        get_blacklist() if media_type == "series" else get_blacklist_movie(),
         get_ban_list(profile_id),
+        get_language_equals(),
+    )
+
+
+def _pool_update(pool, media_type, profile_id=None):
+    return pool.update(
+        get_providers(),
+        get_providers_auth(),
+        get_blacklist() if media_type == "series" else get_blacklist_movie(),
+        get_ban_list(profile_id),
+        get_language_equals(),
     )
 
 

frontend/src/components/bazarr/LanguageSelector.tsx

@@ -0,0 +1,34 @@
+import { useLanguages } from "@/apis/hooks";
+import { Selector, SelectorProps } from "@/components/inputs";
+import { useSelectorOptions } from "@/utilities";
+import { FunctionComponent, useMemo } from "react";
+
+interface LanguageSelectorProps
+  extends Omit<SelectorProps<Language.Server>, "options" | "getkey"> {
+  enabled?: boolean;
+}
+
+const LanguageSelector: FunctionComponent<LanguageSelectorProps> = ({
+  enabled = false,
+  ...selector
+}) => {
+  const { data } = useLanguages();
+
+  const filteredData = useMemo(() => {
+    if (enabled) {
+      return data?.filter((value) => value.enabled);
+    } else {
+      return data;
+    }
+  }, [data, enabled]);
+
+  const options = useSelectorOptions(
+    filteredData ?? [],
+    (value) => value.name,
+    (value) => value.code3
+  );
+
+  return <Selector {...options} searchable {...selector}></Selector>;
+};
+
+export default LanguageSelector;

frontend/src/pages/Settings/Languages/equals.test.ts

@@ -0,0 +1,196 @@
+import {
+  decodeEqualData,
+  encodeEqualData,
+  LanguageEqualData,
+  LanguageEqualImmediateData,
+} from "@/pages/Settings/Languages/equals";
+import { describe, expect, it } from "vitest";
+
+describe("Equals Parser", () => {
+  it("should parse from string correctly", () => {
+    interface TestData {
+      text: string;
+      expected: LanguageEqualImmediateData;
+    }
+
+    function testParsedResult(
+      text: string,
+      expected: LanguageEqualImmediateData
+    ) {
+      const result = decodeEqualData(text);
+
+      if (result === undefined) {
+        expect(false, `Cannot parse '${text}' as language equal data`);
+        return;
+      }
+
+      expect(
+        result,
+        `${text} does not match with the expected equal data`
+      ).toStrictEqual(expected);
+    }
+
+    const testValues: TestData[] = [
+      {
+        text: "spa-MX:spa",
+        expected: {
+          source: {
+            content: "spa-MX",
+            hi: false,
+            forced: false,
+          },
+          target: {
+            content: "spa",
+            hi: false,
+            forced: false,
+          },
+        },
+      },
+      {
+        text: "zho@hi:zht",
+        expected: {
+          source: {
+            content: "zho",
+            hi: true,
+            forced: false,
+          },
+          target: {
+            content: "zht",
+            hi: false,
+            forced: false,
+          },
+        },
+      },
+      {
+        text: "es-MX@forced:es-MX",
+        expected: {
+          source: {
+            content: "es-MX",
+            hi: false,
+            forced: true,
+          },
+          target: {
+            content: "es-MX",
+            hi: false,
+            forced: false,
+          },
+        },
+      },
+      {
+        text: "en:en@hi",
+        expected: {
+          source: {
+            content: "en",
+            hi: false,
+            forced: false,
+          },
+          target: {
+            content: "en",
+            hi: true,
+            forced: false,
+          },
+        },
+      },
+    ];
+
+    testValues.forEach((data) => {
+      testParsedResult(data.text, data.expected);
+    });
+  });
+
+  it("should encode to string correctly", () => {
+    interface TestData {
+      source: LanguageEqualData;
+      expected: string;
+    }
+
+    const testValues: TestData[] = [
+      {
+        source: {
+          source: {
+            content: {
+              name: "Abkhazian",
+              code2: "ab",
+              code3: "abk",
+              enabled: false,
+            },
+            hi: false,
+            forced: false,
+          },
+          target: {
+            content: {
+              name: "Aragonese",
+              code2: "an",
+              code3: "arg",
+              enabled: false,
+            },
+            hi: false,
+            forced: false,
+          },
+        },
+        expected: "abk:arg",
+      },
+      {
+        source: {
+          source: {
+            content: {
+              name: "Abkhazian",
+              code2: "ab",
+              code3: "abk",
+              enabled: false,
+            },
+            hi: true,
+            forced: false,
+          },
+          target: {
+            content: {
+              name: "Aragonese",
+              code2: "an",
+              code3: "arg",
+              enabled: false,
+            },
+            hi: false,
+            forced: false,
+          },
+        },
+        expected: "abk@hi:arg",
+      },
+      {
+        source: {
+          source: {
+            content: {
+              name: "Abkhazian",
+              code2: "ab",
+              code3: "abk",
+              enabled: false,
+            },
+            hi: false,
+            forced: true,
+          },
+          target: {
+            content: {
+              name: "Aragonese",
+              code2: "an",
+              code3: "arg",
+              enabled: false,
+            },
+            hi: false,
+            forced: false,
+          },
+        },
+        expected: "abk@forced:arg",
+      },
+    ];
+
+    function testEncodeResult({ source, expected }: TestData) {
+      const encoded = encodeEqualData(source);
+
+      expect(
+        encoded,
+        `Encoded result '${encoded}' is not matched to '${expected}'`
+      ).toEqual(expected);
+    }
+
+    testValues.forEach(testEncodeResult);
+  });
+});

frontend/src/pages/Settings/Languages/equals.tsx

@@ -0,0 +1,365 @@
+import { useLanguages } from "@/apis/hooks";
+import { Action, SimpleTable } from "@/components";
+import LanguageSelector from "@/components/bazarr/LanguageSelector";
+import { languageEqualsKey } from "@/pages/Settings/keys";
+import { useFormActions } from "@/pages/Settings/utilities/FormValues";
+import { useSettingValue } from "@/pages/Settings/utilities/hooks";
+import { LOG } from "@/utilities/console";
+import { faEquals, faTrash } from "@fortawesome/free-solid-svg-icons";
+import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
+import { Button, Checkbox } from "@mantine/core";
+import { FunctionComponent, useCallback, useMemo } from "react";
+import { Column } from "react-table";
+
+interface GenericEqualTarget<T> {
+  content: T;
+  hi: boolean;
+  forced: boolean;
+}
+
+interface LanguageEqualGenericData<T> {
+  source: GenericEqualTarget<T>;
+  target: GenericEqualTarget<T>;
+}
+
+export type LanguageEqualImmediateData =
+  LanguageEqualGenericData<Language.CodeType>;
+
+export type LanguageEqualData = LanguageEqualGenericData<Language.Server>;
+
+function decodeEqualTarget(
+  text: string
+): GenericEqualTarget<Language.CodeType> | undefined {
+  const [code, decoration] = text.split("@");
+
+  if (code.length === 0) {
+    return undefined;
+  }
+
+  const forced = decoration === "forced";
+  const hi = decoration === "hi";
+
+  return {
+    content: code,
+    forced,
+    hi,
+  };
+}
+
+export function decodeEqualData(
+  text: string
+): LanguageEqualImmediateData | undefined {
+  const [first, second] = text.split(":");
+
+  const source = decodeEqualTarget(first);
+  const target = decodeEqualTarget(second);
+
+  if (source === undefined || target === undefined) {
+    return undefined;
+  }
+
+  return {
+    source,
+    target,
+  };
+}
+
+function encodeEqualTarget(data: GenericEqualTarget<Language.Server>): string {
+  let text = data.content.code3;
+  if (data.hi) {
+    text += "@hi";
+  } else if (data.forced) {
+    text += "@forced";
+  }
+
+  return text;
+}
+
+export function encodeEqualData(data: LanguageEqualData): string {
+  const source = encodeEqualTarget(data.source);
+  const target = encodeEqualTarget(data.target);
+
+  return `${source}:${target}`;
+}
+
+export function useLatestLanguageEquals(): LanguageEqualData[] {
+  const { data } = useLanguages();
+
+  const latest = useSettingValue<string[]>(languageEqualsKey);
+
+  return useMemo(
+    () =>
+      latest
+        ?.map(decodeEqualData)
+        .map((parsed) => {
+          if (parsed === undefined) {
+            return undefined;
+          }
+
+          const source = data?.find(
+            (value) => value.code3 === parsed.source.content
+          );
+          const target = data?.find(
+            (value) => value.code3 === parsed.target.content
+          );
+
+          if (source === undefined || target === undefined) {
+            return undefined;
+          }
+
+          return {
+            source: { ...parsed.source, content: source },
+            target: { ...parsed.target, content: target },
+          };
+        })
+        .filter((v): v is LanguageEqualData => v !== undefined) ?? [],
+    [data, latest]
+  );
+}
+
+interface EqualsTableProps {}
+
+const EqualsTable: FunctionComponent<EqualsTableProps> = () => {
+  const { data: languages } = useLanguages();
+  const canAdd = languages !== undefined;
+
+  const equals = useLatestLanguageEquals();
+
+  const { setValue } = useFormActions();
+
+  const setEquals = useCallback(
+    (values: LanguageEqualData[]) => {
+      const encodedValues = values.map(encodeEqualData);
+
+      LOG("info", "updating language equals data", values);
+      setValue(encodedValues, languageEqualsKey);
+    },
+    [setValue]
+  );
+
+  const add = useCallback(() => {
+    if (languages === undefined) {
+      return;
+    }
+
+    const enabled = languages.find((value) => value.enabled);
+
+    if (enabled === undefined) {
+      return;
+    }
+
+    const newValue: LanguageEqualData[] = [
+      ...equals,
+      {
+        source: {
+          content: enabled,
+          hi: false,
+          forced: false,
+        },
+        target: {
+          content: enabled,
+          hi: false,
+          forced: false,
+        },
+      },
+    ];
+
+    setEquals(newValue);
+  }, [equals, languages, setEquals]);
+
+  const update = useCallback(
+    (index: number, value: LanguageEqualData) => {
+      if (index < 0 || index >= equals.length) {
+        return;
+      }
+
+      const newValue: LanguageEqualData[] = [...equals];
+
+      newValue[index] = { ...value };
+      setEquals(newValue);
+    },
+    [equals, setEquals]
+  );
+
+  const remove = useCallback(
+    (index: number) => {
+      if (index < 0 || index >= equals.length) {
+        return;
+      }
+
+      const newValue: LanguageEqualData[] = [...equals];
+
+      newValue.splice(index, 1);
+
+      setEquals(newValue);
+    },
+    [equals, setEquals]
+  );
+
+  const columns = useMemo<Column<LanguageEqualData>[]>(
+    () => [
+      {
+        Header: "Source",
+        id: "source-lang",
+        accessor: "source",
+        Cell: ({ value: { content }, row }) => {
+          return (
+            <LanguageSelector
+              enabled
+              value={content}
+              onChange={(result) => {
+                if (result !== null) {
+                  update(row.index, {
+                    ...row.original,
+                    source: { ...row.original.source, content: result },
+                  });
+                }
+              }}
+            ></LanguageSelector>
+          );
+        },
+      },
+      {
+        id: "source-hi",
+        accessor: "source",
+        Cell: ({ value: { hi }, row }) => {
+          return (
+            <Checkbox
+              label="HI"
+              checked={hi}
+              onChange={({ currentTarget: { checked } }) => {
+                update(row.index, {
+                  ...row.original,
+                  source: {
+                    ...row.original.source,
+                    hi: checked,
+                    forced: checked ? false : row.original.source.forced,
+                  },
+                });
+              }}
+            ></Checkbox>
+          );
+        },
+      },
+      {
+        id: "source-forced",
+        accessor: "source",
+        Cell: ({ value: { forced }, row }) => {
+          return (
+            <Checkbox
+              label="Forced"
+              checked={forced}
+              onChange={({ currentTarget: { checked } }) => {
+                update(row.index, {
+                  ...row.original,
+                  source: {
+                    ...row.original.source,
+                    forced: checked,
+                    hi: checked ? false : row.original.source.hi,
+                  },
+                });
+              }}
+            ></Checkbox>
+          );
+        },
+      },
+      {
+        id: "equal-icon",
+        Cell: () => {
+          return <FontAwesomeIcon icon={faEquals} />;
+        },
+      },
+      {
+        Header: "Target",
+        id: "target-lang",
+        accessor: "target",
+        Cell: ({ value: { content }, row }) => {
+          return (
+            <LanguageSelector
+              enabled
+              value={content}
+              onChange={(result) => {
+                if (result !== null) {
+                  update(row.index, {
+                    ...row.original,
+                    target: { ...row.original.target, content: result },
+                  });
+                }
+              }}
+            ></LanguageSelector>
+          );
+        },
+      },
+      {
+        id: "target-hi",
+        accessor: "target",
+        Cell: ({ value: { hi }, row }) => {
+          return (
+            <Checkbox
+              label="HI"
+              checked={hi}
+              onChange={({ currentTarget: { checked } }) => {
+                update(row.index, {
+                  ...row.original,
+                  target: {
+                    ...row.original.target,
+                    hi: checked,
+                    forced: checked ? false : row.original.target.forced,
+                  },
+                });
+              }}
+            ></Checkbox>
+          );
+        },
+      },
+      {
+        id: "target-forced",
+        accessor: "target",
+        Cell: ({ value: { forced }, row }) => {
+          return (
+            <Checkbox
+              label="Forced"
+              checked={forced}
+              onChange={({ currentTarget: { checked } }) => {
+                update(row.index, {
+                  ...row.original,
+                  target: {
+                    ...row.original.target,
+                    forced: checked,
+                    hi: checked ? false : row.original.target.hi,
+                  },
+                });
+              }}
+            ></Checkbox>
+          );
+        },
+      },
+      {
+        id: "action",
+        accessor: "target",
+        Cell: ({ row }) => {
+          return (
+            <Action
+              label="Remove"
+              icon={faTrash}
+              color="red"
+              onClick={() => remove(row.index)}
+            ></Action>
+          );
+        },
+      },
+    ],
+    [remove, update]
+  );
+
+  return (
+    <>
+      <SimpleTable data={equals} columns={columns}></SimpleTable>
+      <Button fullWidth disabled={!canAdd} color="light" onClick={add}>
+        {canAdd ? "Add Equal" : "No Enabled Languages"}
+      </Button>
+    </>
+  );
+};
+
+export default EqualsTable;

frontend/src/pages/Settings/Languages/index.tsx

@@ -17,6 +17,7 @@ import {
 } from "../keys";
 import { useSettingValue } from "../utilities/hooks";
 import { LanguageSelector, ProfileSelector } from "./components";
+import EqualsTable from "./equals";
 import Table from "./table";
 
 export function useLatestEnabledLanguages() {
@@ -69,6 +70,13 @@ const SettingsLanguagesView: FunctionComponent = () => {
         ></LanguageSelector>
       </Section>
 
+      <Section header="Language Equals">
+        <Message>
+          Treat the following languages as equal across all providers.
+        </Message>
+        <EqualsTable></EqualsTable>
+      </Section>
+
       <Section header="Embedded Tracks Language">
         <Check
           label="Deep analyze media file to get audio tracks language."
@@ -91,7 +99,6 @@ const SettingsLanguagesView: FunctionComponent = () => {
             }}
           ></Selector>
         </CollapseBox>
-
         <Selector
           clearable
           settingKey={defaultUndEmbeddedSubtitlesLang}

frontend/src/pages/Settings/keys.ts

@@ -5,6 +5,8 @@ export const defaultUndEmbeddedSubtitlesLang =
 export const languageProfileKey = "languages-profiles";
 export const notificationsKey = "notifications-providers";
 
+export const languageEqualsKey = "settings-general-language_equals";
+
 export const pathMappingsKey = "settings-general-path_mappings";
 export const pathMappingsMovieKey = "settings-general-path_mappings_movie";
 

frontend/src/types/api.d.ts

@@ -12,6 +12,7 @@ declare namespace Language {
   type CodeType = string;
   interface Server {
     code2: CodeType;
+    code3: CodeType;
     name: string;
     enabled: boolean;
   }

frontend/src/types/settings.d.ts

@@ -25,6 +25,7 @@ interface Settings {
   titlovi: Settings.Titlovi;
   ktuvit: Settings.Ktuvit;
   notifications: Settings.Notifications;
+  language_equals: string[][];
 }
 
 declare namespace Settings {

frontend/src/utilities/languages.ts

@@ -42,3 +42,12 @@ export function useProfileItemsToLanguages(profile?: Language.Profile) {
     [data, profile?.items]
   );
 }
+
+export function useLanguageFromCode3(code3: string) {
+  const { data } = useLanguages();
+
+  return useMemo(
+    () => data?.find((value) => value.code3 === code3),
+    [data, code3]
+  );
+}

libs/subliminal_patch/core.py

@@ -142,9 +142,52 @@ class _Blacklist(list):
         return not blacklisted
 
 
+class _LanguageEquals(list):
+    """ An optional config field for the pool. It will treat a couple of languages as equal for
+    list-subtitles operations. It's optional; its methods won't do anything if an empy list
+    is set.
+
+    Example usage: [(language_instance, language_instance), ...]"""
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        for item in self:
+            if len(item) != 2 or not any(isinstance(i, Language) for i in item):
+                raise ValueError(f"Not a valid equal tuple: {item}")
+
+    def check_set(self, items: set):
+        """ Check a set of languages. For example, if the set is {Language('es')} and one of the
+        equals of the instance is (Language('es'), Language('es', 'MX')), the set will now have
+        to {Language('es'), Language('es', 'MX')}.
+
+        It will return a copy of the original set to avoid messing up outside its scope.
+
+        Note that hearing_impaired and forced language attributes are not yet tested.
+        """
+        to_add = []
+        for equals in self:
+            from_, to_ = equals
+            if from_ in items:
+                logger.debug("Adding %s to %s", to_, items)
+                to_add.append(to_)
+
+        new_items = items.copy()
+        new_items.update(to_add)
+        logger.debug("New set: %s", new_items)
+        return new_items
+
+    def update_subtitle(self, subtitle):
+        for equals in self:
+            from_, to_ = equals
+            if from_ == subtitle.language:
+                logger.debug("Updating language for %s (to %s)", subtitle, to_)
+                subtitle.language = to_
+                break
+
+
 class SZProviderPool(ProviderPool):
     def __init__(self, providers=None, provider_configs=None, blacklist=None, ban_list=None, throttle_callback=None,
-                 pre_download_hook=None, post_download_hook=None, language_hook=None):
+                 pre_download_hook=None, post_download_hook=None, language_hook=None, language_equals=None):
         #: Name of providers to use
         self.providers = set(providers or [])
 
@@ -159,6 +202,8 @@ class SZProviderPool(ProviderPool):
         #: Should be a dict of 2 lists of strings
         self.ban_list = _Banlist(**(ban_list or {'must_contain': [], 'must_not_contain': []}))
 
+        self.lang_equals = _LanguageEquals(language_equals or [])
+
         self.throttle_callback = throttle_callback
 
         self.pre_download_hook = pre_download_hook
@@ -174,7 +219,7 @@ class SZProviderPool(ProviderPool):
         self.provider_configs = _ProviderConfigs(self)
         self.provider_configs.update(provider_configs or {})
 
-    def update(self, providers, provider_configs, blacklist, ban_list):
+    def update(self, providers, provider_configs, blacklist, ban_list, language_equals=None):
         # Check if the pool was initialized enough hours ago
         self._check_lifetime()
 
@@ -211,6 +256,7 @@ class SZProviderPool(ProviderPool):
 
         self.blacklist = _Blacklist(blacklist or [])
         self.ban_list = _Banlist(**ban_list or {'must_contain': [], 'must_not_contain': []})
+        self.lang_equals = _LanguageEquals(language_equals or [])
 
         return updated
 
@@ -288,7 +334,7 @@ class SZProviderPool(ProviderPool):
             return []
 
         # check supported languages
-        provider_languages = provider_registry[provider].languages & use_languages
+        provider_languages = self.lang_equals.check_set(set(provider_registry[provider].languages)) & use_languages
         if not provider_languages:
             logger.info('Skipping provider %r: no language to search for', provider)
             return []
@@ -301,6 +347,8 @@ class SZProviderPool(ProviderPool):
             seen = []
             out = []
             for s in results:
+                self.lang_equals.update_subtitle(s)
+
                 if not self.blacklist.is_valid(provider, s):
                     continue
 
@@ -558,7 +606,7 @@ class SZProviderPool(ProviderPool):
                 continue
 
             # add the languages for this provider
-            languages.append({'provider': name, 'languages': provider_languages})
+            languages.append({'provider': name, 'languages': self.lang_equals.check_set(set(provider_languages))})
 
         return languages
 

tests/bazarr/app/test_get_providers.py

@@ -1,45 +0,0 @@
-import pytest
-
-import inspect
-
-from bazarr.app import get_providers
-
-
-def test_get_providers_auth():
-    for val in get_providers.get_providers_auth().values():
-        assert isinstance(val, dict)
-
-
-def test_get_providers_auth_with_provider_registry():
-    """Make sure all providers will be properly initialized with bazarr
-    configs"""
-    from subliminal_patch.extensions import provider_registry
-
-    auths = get_providers.get_providers_auth()
-    for key, val in auths.items():
-        provider = provider_registry[key]
-        sign = inspect.signature(provider.__init__)
-        for sub_key in val.keys():
-            if sub_key not in sign.parameters:
-                raise ValueError(f"'{sub_key}' parameter not present in {provider}")
-
-            assert sign.parameters[sub_key] is not None
-
-
-def test_get_providers_auth_embeddedsubtitles():
-    item = get_providers.get_providers_auth()["embeddedsubtitles"]
-    assert isinstance(item["included_codecs"], list)
-    assert isinstance(item["hi_fallback"], bool)
-    assert isinstance(item["cache_dir"], str)
-    assert isinstance(item["ffprobe_path"], str)
-    assert isinstance(item["ffmpeg_path"], str)
-    assert isinstance(item["timeout"], str)
-    assert isinstance(item["unknown_as_english"], bool)
-
-
-def test_get_providers_auth_karagarga():
-    item = get_providers.get_providers_auth()["karagarga"]
-    assert item["username"] is not None
-    assert item["password"] is not None
-    assert item["f_username"] is not None
-    assert item["f_password"] is not None

tests/bazarr/conftest.py

@@ -3,5 +3,6 @@ import logging
 
 os.environ["NO_CLI"] = "true"
 os.environ["SZ_USER_AGENT"] = "test"
+os.environ["BAZARR_VERSION"] = "test" # fixme
 
 logging.getLogger("rebulk").setLevel(logging.WARNING)

tests/bazarr/test_app_get_providers.py

@@ -0,0 +1,115 @@
+import inspect
+
+import pytest
+from subliminal_patch.core import Language
+
+from bazarr.app import get_providers
+
+
+def test_get_providers_auth():
+    for val in get_providers.get_providers_auth().values():
+        assert isinstance(val, dict)
+
+
+def test_get_providers_auth_with_provider_registry():
+    """Make sure all providers will be properly initialized with bazarr
+    configs"""
+    from subliminal_patch.extensions import provider_registry
+
+    auths = get_providers.get_providers_auth()
+    for key, val in auths.items():
+        provider = provider_registry[key]
+        sign = inspect.signature(provider.__init__)
+        for sub_key in val.keys():
+            if sub_key not in sign.parameters:
+                raise ValueError(f"'{sub_key}' parameter not present in {provider}")
+
+            assert sign.parameters[sub_key] is not None
+
+
+def test_get_providers_auth_embeddedsubtitles():
+    item = get_providers.get_providers_auth()["embeddedsubtitles"]
+    assert isinstance(item["included_codecs"], list)
+    assert isinstance(item["hi_fallback"], bool)
+    assert isinstance(item["cache_dir"], str)
+    assert isinstance(item["ffprobe_path"], str)
+    assert isinstance(item["ffmpeg_path"], str)
+    assert isinstance(item["timeout"], str)
+    assert isinstance(item["unknown_as_english"], bool)
+
+
+def test_get_providers_auth_karagarga():
+    item = get_providers.get_providers_auth()["karagarga"]
+    assert item["username"] is not None
+    assert item["password"] is not None
+    assert item["f_username"] is not None
+    assert item["f_password"] is not None
+
+
+def test_get_language_equals_default_settings():
+    assert isinstance(get_providers.get_language_equals(), list)
+
+
+def test_get_language_equals_injected_settings_invalid():
+    config = get_providers.settings
+    config.set("general", "language_equals", '["invalid"]')
+    assert not get_providers.get_language_equals(config)
+
+
+def test_get_language_equals_injected_settings_valid():
+    config = get_providers.settings
+    config.set("general", "language_equals", '["spa:spa-MX"]')
+
+    result = get_providers.get_language_equals(config)
+    assert result == [(Language("spa"), Language("spa", "MX"))]
+
+
+@pytest.mark.parametrize(
+    "config_value,expected",
+    [
+        ('["spa:spl"]', (Language("spa"), Language("spa", "MX"))),
+        ('["por:pob"]', (Language("por"), Language("por", "BR"))),
+        ('["zho:zht"]', (Language("zho"), Language("zho", "TW"))),
+    ],
+)
+def test_get_language_equals_injected_settings_custom_lang_alpha3(
+    config_value, expected
+):
+    config = get_providers.settings
+
+    config.set("general", "language_equals", config_value)
+
+    result = get_providers.get_language_equals(config)
+    assert result == [expected]
+
+
+def test_get_language_equals_injected_settings_multiple():
+    config = get_providers.settings
+
+    config.set(
+        "general",
+        "language_equals",
+        "['eng@hi:eng', 'spa:spl', 'spa@hi:spl', 'spl@hi:spl']",
+    )
+
+    result = get_providers.get_language_equals(config)
+    assert len(result) == 4
+
+
+def test_get_language_equals_injected_settings_valid_multiple():
+    config = get_providers.settings
+    config.set("general", "language_equals", '["spa:spa-MX", "spa-MX:spa"]')
+
+    result = get_providers.get_language_equals(config)
+    assert result == [
+        (Language("spa"), Language("spa", "MX")),
+        (Language("spa", "MX"), Language("spa")),
+    ]
+
+
+def test_get_language_equals_injected_settings_hi():
+    config = get_providers.settings
+    config.set("general", "language_equals", '["eng@hi:eng"]')
+
+    result = get_providers.get_language_equals(config)
+    assert result == [(Language("eng", hi=True), Language("eng"))]

tests/bazarr/test_subtitles_pool.py

@@ -0,0 +1,10 @@
+from bazarr.subtitles import pool
+
+
+def test_init_pool():
+    assert pool._init_pool("movie")
+
+
+def test_pool_update():
+    pool_ = pool._init_pool("movie")
+    assert pool._pool_update(pool_, "movie")

tests/subliminal_patch/test_core.py

@@ -70,3 +70,99 @@ def test_pool_update_discarded_providers_2(pool_instance):
 
     # Provider should not disappear from discarded providers
     assert pool_instance.discarded_providers == {"argenteam"}
+
+
+def test_language_equals_init():
+    assert core._LanguageEquals([(core.Language("spa"), core.Language("spa", "MX"))])
+
+
+def test_language_equals_init_invalid():
+    with pytest.raises(ValueError):
+        assert core._LanguageEquals([(core.Language("spa", "MX"),)])
+
+
+def test_language_equals_init_empty_list_gracefully():
+    assert core._LanguageEquals([]) == []
+
+
+@pytest.mark.parametrize(
+    "langs",
+    [
+        [(core.Language("spa"), core.Language("spa", "MX"))],
+        [(core.Language("por"), core.Language("por", "BR"))],
+        [(core.Language("zho"), core.Language("zho", "TW"))],
+    ],
+)
+def test_language_equals_check_set(langs):
+    equals = core._LanguageEquals(langs)
+    lang_set = {langs[0]}
+    assert equals.check_set(lang_set) == set(langs)
+
+
+def test_language_equals_check_set_do_nothing():
+    equals = core._LanguageEquals([(core.Language("eng"), core.Language("spa"))])
+    lang_set = {core.Language("spa")}
+    assert equals.check_set(lang_set) == {core.Language("spa")}
+
+
+def test_language_equals_check_set_do_nothing_w_forced():
+    equals = core._LanguageEquals(
+        [(core.Language("spa", forced=True), core.Language("spa", "MX"))]
+    )
+    lang_set = {core.Language("spa")}
+    assert equals.check_set(lang_set) == {core.Language("spa")}
+
+
+@pytest.fixture
+def language_equals_pool_intance():
+    equals = [(core.Language("spa"), core.Language("spa", "MX"))]
+    yield core.SZProviderPool({"subdivx"}, language_equals=equals)
+
+
+def test_language_equals_pool_intance_list_subtitles(
+    language_equals_pool_intance, movies
+):
+    subs = language_equals_pool_intance.list_subtitles(
+        movies["dune"], {core.Language("spa")}
+    )
+    assert subs
+    assert all(sub.language == core.Language("spa", "MX") for sub in subs)
+
+
+def test_language_equals_pool_intance_list_subtitles_reversed(movies):
+    equals = [(core.Language("spa", "MX"), core.Language("spa"))]
+    language_equals_pool_intance = core.SZProviderPool(
+        {"subdivx"}, language_equals=equals
+    )
+    subs = language_equals_pool_intance.list_subtitles(
+        movies["dune"], {core.Language("spa")}
+    )
+    assert subs
+    assert all(sub.language == core.Language("spa") for sub in subs)
+
+
+def test_language_equals_pool_intance_list_subtitles_empty_lang_equals(movies):
+    language_equals_pool_intance = core.SZProviderPool(
+        {"subdivx"}, language_equals=None
+    )
+    subs = language_equals_pool_intance.list_subtitles(
+        movies["dune"], {core.Language("spa")}
+    )
+    assert subs
+    assert not all(sub.language == core.Language("spa", "MX") for sub in subs)
+
+
+def test_language_equals_pool_intance_list_subtitles_return_nothing(movies):
+    equals = [
+        (core.Language("spa", "MX"), core.Language("eng")),
+        (core.Language("spa"), core.Language("eng")),
+    ]
+    language_equals_pool_intance = core.SZProviderPool(
+        {"subdivx"}, language_equals=equals
+    )
+    subs = language_equals_pool_intance.list_subtitles(
+        movies["dune"], {core.Language("spa")}
+    )
+    assert not language_equals_pool_intance.download_best_subtitles(
+        subs, movies["dune"], {core.Language("spa")}
+    )