From 8c11676d54f50f6b9a9c288672b2f89c202460f8 Mon Sep 17 00:00:00 2001 From: mkb79 Date: Sat, 9 Apr 2022 19:15:04 +0200 Subject: [PATCH 01/79] Allow httpx version 0.20 - 0.22 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d9280a8..9068615 100644 --- a/setup.py +++ b/setup.py @@ -49,7 +49,7 @@ setup( "audible==0.7.2", "click>=8", "colorama; platform_system=='Windows'", - "httpx==0.20.*", + "httpx>=0.20.*,<=0.22.*", "packaging", "Pillow", "tabulate", From 54a879c52e5abc02ecd5985eb2bf28b159a3230d Mon Sep 17 00:00:00 2001 From: mkb79 Date: Mon, 11 Apr 2022 07:08:03 +0200 Subject: [PATCH 02/79] Create cmd_listening-stats.py --- plugin_cmds/cmd_listening-stats.py | 82 ++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 plugin_cmds/cmd_listening-stats.py diff --git a/plugin_cmds/cmd_listening-stats.py b/plugin_cmds/cmd_listening-stats.py new file mode 100644 index 0000000..7d914a4 --- /dev/null +++ b/plugin_cmds/cmd_listening-stats.py @@ -0,0 +1,82 @@ +import asyncio +import json +import logging +import pathlib +from datetime import datetime + +import audible +import click +from audible_cli.config import pass_session + + +logger = logging.getLogger("audible_cli.cmds.cmd_listening-stats") + +current_year = datetime.now().year + + +def ms_to_hms(milliseconds): + seconds = (int) (milliseconds / 1000) % 60 + minutes = (int) ((milliseconds / (1000*60)) % 60) + hours = (int) ((milliseconds / (1000*60*60)) % 24) + return hours, minutes, seconds + + +async def _get_stats_year(client, year): + stats_year = {} + stats = await client.get( + "stats/aggregates", + monthly_listening_interval_duration="12", + monthly_listening_interval_start_date=f"{year}-01", + store="Audible" + ) + #iterate over each month + for stat in stats['aggregated_monthly_listening_stats']: + stats_year[stat["interval_identifier"]] = ms_to_hms(stat["aggregated_sum"]) + return stats_year + + +async def _listening_stats(auth, output, signup_year): + year_range = [y for y in range(signup_year, current_year+1)] + + async with audible.AsyncClient(auth=auth) as client: + + r = await asyncio.gather( + *[_get_stats_year(client, y) for y in year_range] + ) + + aggreated_stats = {} + for i in r: + for k, v in i.items(): + aggreated_stats[k] = v + + aggreated_stats = json.dumps(aggreated_stats, indent=4) + output.write_text(aggreated_stats) + + +@click.command("listening-stats") +@click.option( + "--output", "-o", + type=click.Path(path_type=pathlib.Path), + default=pathlib.Path().cwd() / "listening-stats.json", + show_default=True, + help="output file" +) +@click.option( + "--signup-year", "-s", + type=click.IntRange(1997, current_year), + default="2010", + show_default=True, + help="start year for collecting listening stats" +) +@pass_session +def cli(session, output, signup_year): + """get and analyse listening statistics""" + loop = asyncio.get_event_loop() + try: + loop.run_until_complete( + _listening_stats(session.auth, output, signup_year) + ) + finally: + loop.run_until_complete(loop.shutdown_asyncgens()) + loop.close() + From 88cbd94a86f7c32e863faf8a18bdb28b05c9884f Mon Sep 17 00:00:00 2001 From: mkb79 Date: Tue, 17 May 2022 08:50:35 +0200 Subject: [PATCH 03/79] Update README.md --- README.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index cf3132a..417bb0e 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ It depends on the following packages: * aiofiles * audible * click -* colorama (on windows machines) +* colorama (on Windows machines) * httpx * Pillow * tabulate @@ -30,7 +30,7 @@ pip install audible-cli ``` -or install it directly from github with +or install it directly from GitHub with ```shell @@ -44,7 +44,7 @@ pip install . If you don't want to install `Python` and `audible-cli` on your machine, you can find standalone exe files below or on the [releases](https://github.com/mkb79/audible-cli/releases) -page. At this moment Windows, Linux and MacOS are supported. +page (including beta releases). At this moment Windows, Linux and macOS are supported. ### Links @@ -82,7 +82,7 @@ pyinstaller --clean -D --hidden-import audible_cli -n audible -c pyi_entrypoint ### Hints -There are some limitations when using plugins. The binarys maybe does not contain +There are some limitations when using plugins. The binary maybe does not contain all the dependencies from your plugin script. ## Tab Completion @@ -103,7 +103,7 @@ as config dir. Otherwise, it will use a folder depending on the operating system. | OS | Path | -| --- | --- | +|----------|-------------------------------------------| | Windows | ``C:\Users\\AppData\Local\audible`` | | Unix | ``~/.audible`` | | Mac OS X | ``~/.audible`` | @@ -199,9 +199,9 @@ There are 6 different verbosity levels: - error - critical -By default the verbosity level is set to `info`. You can provide another level like so: `audible -v ...`. +By default, the verbosity level is set to `info`. You can provide another level like so: `audible -v ...`. -If you use the `download` sudcommand with the `--all` flag there will be a huge output. Best practise is to set the verbosity level to `error` with `audible -v error download --all ...` +If you use the `download` subcommand with the `--all` flag there will be a huge output. Best practise is to set the verbosity level to `error` with `audible -v error download --all ...` ## Plugins @@ -217,13 +217,13 @@ You can provide own subcommands and execute them with `audible SUBCOMMAND`. All plugin commands must be placed in the plugin folder. Every subcommand must have his own file. Every file have to be named ``cmd_{SUBCOMMAND}.py``. Each subcommand file must have a function called `cli` as entrypoint. -This function have to be decorated with ``@click.group(name="GROUP_NAME")`` or +This function has to be decorated with ``@click.group(name="GROUP_NAME")`` or ``@click.command(name="GROUP_NAME")``. Relative imports in the command files doesn't work. So you have to work with absolute imports. Please take care about this. If you have any issues with absolute imports please add your plugin path to the `PYTHONPATH` variable or -add this lines of code to the begining of your command script: +add this lines of code to the beginning of your command script: ```python import sys @@ -239,7 +239,7 @@ Examples can be found If you want to develop a complete plugin package for ``audible-cli`` you can do this on an easy way. You only need to register your sub-commands or -sub-groups to an entry-point in your setup.py that is loaded by the core +subgroups to an entry-point in your setup.py that is loaded by the core package. Example for a setup.py From 90707a88176bc32dbaa83711cc23c7fb27d99e8d Mon Sep 17 00:00:00 2001 From: mkb79 Date: Wed, 25 May 2022 14:56:41 +0200 Subject: [PATCH 04/79] v0.2 (#80) # Added - `--aax-fallback` option to `download` command to download books in aax format and fallback to aaxc, if the book is not available as aax - `--annotation` option to `download` command to get bookmarks and notes - `questionary` package to dependencies - `add` and `remove` subcommands to wishlist - `full_response_callback` to `utils` - `export_to_csv` to `utils` - `run_async` to `decorators` - `pass_client` to `decorators` - `profile_option` to `decorators` - `password_option` to `decorators` - `timeout_option` to `decorators` - `bunch_size_option` to `decorators` - `ConfigFile.get_profile_option` get the value for an option for a given profile - `Session.selected.profile` to get the profile name for the current session - `Session.get_auth_for_profile` to get an auth file for a given profile - `models.BaseItem.create_base_filename` to build a filename in given mode - `models.LibraryItem.get_annotations` to get annotations for a library item # Changed - bump `audible` to v0.8.2 to fix a bug in httpx - rework plugin examples in `plugin_cmds` - rename `config.Config` to `config.ConfigFile` - move `click_verbosity_logger` from `_logging` to `decorators` and rename it to `verbosity_option` - move `wrap_async` from `utils` to `decorators` - move `add_param_to_session` from `config` to `decorators` - move `pass_session` from `config` to `decorators` - `download` command let you now select items when using `--title` option # Fixed - the `library export` and `wishlist export` command will now export to `csv` correctly --- CHANGELOG.md | 38 +- README.md | 2 + plugin_cmds/cmd_get-annotations.py | 21 + plugin_cmds/cmd_goodreads-transform.py | 110 ++++ plugin_cmds/cmd_image-urls.py | 32 +- plugin_cmds/cmd_listening-stats.py | 57 +- plugin_cmds/cmd_remove-encryption.py | 155 +++-- setup.py | 7 +- src/audible_cli/_logging.py | 36 -- src/audible_cli/_version.py | 2 +- src/audible_cli/cli.py | 86 +-- src/audible_cli/cmds/__init__.py | 5 +- src/audible_cli/cmds/cmd_activation_bytes.py | 2 +- src/audible_cli/cmds/cmd_api.py | 6 +- src/audible_cli/cmds/cmd_download.py | 546 +++++++++--------- src/audible_cli/cmds/cmd_library.py | 272 ++++----- src/audible_cli/cmds/cmd_manage.py | 14 +- src/audible_cli/cmds/cmd_quickstart.py | 45 +- src/audible_cli/cmds/cmd_wishlist.py | 397 ++++++++----- src/audible_cli/config.py | 317 ++++++---- src/audible_cli/constants.py | 9 +- src/audible_cli/decorators.py | 238 ++++++++ src/audible_cli/exceptions.py | 4 + src/audible_cli/models.py | 140 +++-- src/audible_cli/utils.py | 39 +- utils/code_completion/README.md | 2 +- .../code_completion/audible-complete-bash.sh | 4 +- .../audible-complete-zsh-fish.sh | 4 +- utils/update_chapter_titles.py | 2 +- 29 files changed, 1587 insertions(+), 1005 deletions(-) create mode 100644 plugin_cmds/cmd_get-annotations.py create mode 100644 plugin_cmds/cmd_goodreads-transform.py create mode 100644 src/audible_cli/decorators.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 0da54f8..50beed5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,41 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## Unreleased -- +### Added + +- `--aax-fallback` option to `download` command to download books in aax format and fallback to aaxc, if the book is not available as aax +- `--annotation` option to `download` command to get bookmarks and notes +- `questionary` package to dependencies +- `add` and `remove` subcommands to wishlist +- `full_response_callback` to `utils` +- `export_to_csv` to `utils` +- `run_async` to `decorators` +- `pass_client` to `decorators` +- `profile_option` to `decorators` +- `password_option` to `decorators` +- `timeout_option` to `decorators` +- `bunch_size_option` to `decorators` +- `ConfigFile.get_profile_option` get the value for an option for a given profile +- `Session.selected.profile` to get the profile name for the current session +- `Session.get_auth_for_profile` to get an auth file for a given profile +- `models.BaseItem.create_base_filename` to build a filename in given mode +- `models.LibraryItem.get_annotations` to get annotations for a library item + +### Changed + +- bump `audible` to v0.8.2 to fix a bug in httpx +- rework plugin examples in `plugin_cmds` +- rename `config.Config` to `config.ConfigFile` +- move `click_verbosity_logger` from `_logging` to `decorators` and rename it to `verbosity_option` +- move `wrap_async` from `utils` to `decorators` +- move `add_param_to_session` from `config` to `decorators` +- move `pass_session` from `config` to `decorators` +- `download` command let you now select items when using `--title` option + +### Fixed + +- the `library export` and `wishlist export` command will now export to `csv` correctly +- ## [0.1.3] - 2022-03-27 @@ -25,7 +59,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Added - the `--version` option now checks if an update for `audible-cli` is available -- build macOS release in onedir mode +- build macOS releases in onedir mode ### Bugfix diff --git a/README.md b/README.md index 417bb0e..28fe808 100644 --- a/README.md +++ b/README.md @@ -188,6 +188,8 @@ At this time, there the following buildin subcommands: - `wishlist` - `export` - `list` + - `add` + - `remove` ## Verbosity option diff --git a/plugin_cmds/cmd_get-annotations.py b/plugin_cmds/cmd_get-annotations.py new file mode 100644 index 0000000..f8d2dcc --- /dev/null +++ b/plugin_cmds/cmd_get-annotations.py @@ -0,0 +1,21 @@ +import click + +from audible.exceptions import NotFoundError +from audible_cli.decorators import pass_client + + +@click.command("get-annotations") +@click.argument("asin") +@pass_client +async def cli(client, asin): + url = f"https://cde-ta-g7g.amazon.com/FionaCDEServiceEngine/sidecar" + params = { + "type": "AUDI", + "key": asin + } + try: + r = await client.get(url, params=params) + except NotFoundError: + click.echo(f"No annotations found for asin {asin}") + else: + click.echo(r) diff --git a/plugin_cmds/cmd_goodreads-transform.py b/plugin_cmds/cmd_goodreads-transform.py new file mode 100644 index 0000000..f5c0163 --- /dev/null +++ b/plugin_cmds/cmd_goodreads-transform.py @@ -0,0 +1,110 @@ +import logging +import pathlib +from datetime import datetime, timezone + +import click +from audible_cli.decorators import ( + bunch_size_option, + timeout_option, + pass_client, + pass_session +) +from audible_cli.models import Library +from audible_cli.utils import export_to_csv +from isbntools.app import isbn_from_words + + +logger = logging.getLogger("audible_cli.cmds.cmd_goodreads-transform") + + +@click.command("goodreads-transform") +@click.option( + "--output", "-o", + type=click.Path(path_type=pathlib.Path), + default=pathlib.Path().cwd() / "library.csv", + show_default=True, + help="output file" +) +@timeout_option +@bunch_size_option +@pass_session +@pass_client +async def cli(session, client, output): + """YOUR COMMAND DESCRIPTION""" + + logger.debug("fetching library") + bunch_size = session.params.get("bunch_size") + library = await Library.from_api_full_sync( + client, + response_groups=( + "product_details, contributors, is_finished, product_desc" + ), + bunch_size=bunch_size + ) + + logger.debug("prepare library") + library = _prepare_library_for_export(library) + + logger.debug("write data rows to file") + + headers = ("isbn", "Date Added", "Date Read", "Title") + export_to_csv( + file=output, + data=library, + headers=headers, + dialect="excel" + ) + + logger.info(f"File saved to {output}") + + +def _prepare_library_for_export(library): + prepared_library = [] + + isbn_counter = 0 + isbn_api_counter = 0 + isbn_no_result_counter = 0 + skipped_items = 0 + + for i in library: + title = i.title + authors = i.authors + if authors is not None: + authors = ", ".join([a["name"] for a in authors]) + is_finished = i.is_finished + + isbn = i.isbn + if isbn is None: + isbn_counter += 1 + isbn = isbn_from_words(f"{title} {authors}") or None + if isbn is None: + isbn_no_result_counter += 1 + else: + isbn_api_counter += 1 + + date_added = i.library_status + if date_added is not None: + date_added = date_added["date_added"] + date_added = datetime.strptime( + date_added, '%Y-%m-%dT%H:%M:%S.%fZ' + ).replace(tzinfo=timezone.utc).astimezone() + date_added = date_added.astimezone().date().isoformat() + + date_read = None + if is_finished: + date_read = date_added + + if isbn and date_read: + data_row = [isbn, date_added, date_read, title] + prepared_library.append(data_row) + else: + skipped_items += 1 + + logger.debug(f"ISBNs from API: {isbn_api_counter}") + logger.debug(f"ISBNs requested with isbntools: {isbn_counter}") + logger.debug(f"No result with isbntools: {isbn_no_result_counter}") + logger.debug( + f"title skipped from file due to no ISBN or title not read: " + f"{skipped_items}") + + return prepared_library diff --git a/plugin_cmds/cmd_image-urls.py b/plugin_cmds/cmd_image-urls.py index 40b6960..a261256 100644 --- a/plugin_cmds/cmd_image-urls.py +++ b/plugin_cmds/cmd_image-urls.py @@ -1,25 +1,19 @@ -import audible import click -from audible_cli.config import pass_session +from audible_cli.decorators import pass_client, timeout_option -@click.command("get-cover-urls") -@click.option( - "--asin", "-a", - multiple=False, - help="asin of the audiobook" -) -@pass_session -def cli(session, asin): - "Print out the image urls for different resolutions for a book" - with audible.Client(auth=session.auth) as client: - r = client.get( - f"catalog/products/{asin}", - response_groups="media", - image_sizes=("1215, 408, 360, 882, 315, 570, 252, " - "558, 900, 500") - ) +@click.command("image-urls") +@click.argument("asin") +@timeout_option() +@pass_client() +async def cli(client, asin): + """Print out the image urls for different resolutions for a book""" + r = await client.get( + f"catalog/products/{asin}", + response_groups="media", + image_sizes=( + "1215, 408, 360, 882, 315, 570, 252, 558, 900, 500") + ) images = r["product"]["product_images"] for res, url in images.items(): click.echo(f"Resolution {res}: {url}") - diff --git a/plugin_cmds/cmd_listening-stats.py b/plugin_cmds/cmd_listening-stats.py index 7d914a4..99a12cc 100644 --- a/plugin_cmds/cmd_listening-stats.py +++ b/plugin_cmds/cmd_listening-stats.py @@ -4,9 +4,8 @@ import logging import pathlib from datetime import datetime -import audible import click -from audible_cli.config import pass_session +from audible_cli.decorators import pass_client logger = logging.getLogger("audible_cli.cmds.cmd_listening-stats") @@ -15,10 +14,10 @@ current_year = datetime.now().year def ms_to_hms(milliseconds): - seconds = (int) (milliseconds / 1000) % 60 - minutes = (int) ((milliseconds / (1000*60)) % 60) - hours = (int) ((milliseconds / (1000*60*60)) % 24) - return hours, minutes, seconds + seconds = int((milliseconds / 1000) % 60) + minutes = int(((milliseconds / (1000*60)) % 60)) + hours = int(((milliseconds / (1000*60*60)) % 24)) + return {"hours": hours, "minutes": minutes, "seconds": seconds} async def _get_stats_year(client, year): @@ -29,30 +28,12 @@ async def _get_stats_year(client, year): monthly_listening_interval_start_date=f"{year}-01", store="Audible" ) - #iterate over each month + # iterate over each month for stat in stats['aggregated_monthly_listening_stats']: stats_year[stat["interval_identifier"]] = ms_to_hms(stat["aggregated_sum"]) return stats_year -async def _listening_stats(auth, output, signup_year): - year_range = [y for y in range(signup_year, current_year+1)] - - async with audible.AsyncClient(auth=auth) as client: - - r = await asyncio.gather( - *[_get_stats_year(client, y) for y in year_range] - ) - - aggreated_stats = {} - for i in r: - for k, v in i.items(): - aggreated_stats[k] = v - - aggreated_stats = json.dumps(aggreated_stats, indent=4) - output.write_text(aggreated_stats) - - @click.command("listening-stats") @click.option( "--output", "-o", @@ -68,15 +49,19 @@ async def _listening_stats(auth, output, signup_year): show_default=True, help="start year for collecting listening stats" ) -@pass_session -def cli(session, output, signup_year): +@pass_client +async def cli(client, output, signup_year): """get and analyse listening statistics""" - loop = asyncio.get_event_loop() - try: - loop.run_until_complete( - _listening_stats(session.auth, output, signup_year) - ) - finally: - loop.run_until_complete(loop.shutdown_asyncgens()) - loop.close() - + year_range = [y for y in range(signup_year, current_year+1)] + + r = await asyncio.gather( + *[_get_stats_year(client, y) for y in year_range] + ) + + aggregated_stats = {} + for i in r: + for k, v in i.items(): + aggregated_stats[k] = v + + aggregated_stats = json.dumps(aggregated_stats, indent=4) + output.write_text(aggregated_stats) diff --git a/plugin_cmds/cmd_remove-encryption.py b/plugin_cmds/cmd_remove-encryption.py index a4f848d..df79a21 100644 --- a/plugin_cmds/cmd_remove-encryption.py +++ b/plugin_cmds/cmd_remove-encryption.py @@ -2,7 +2,7 @@ This is a proof-of-concept and for testing purposes only. No error handling. Need further work. Some options does not work or options are missing. -Needs at least ffmpeg 4.1 with aaxc patch. +Needs at least ffmpeg 4.4 """ @@ -14,7 +14,7 @@ import subprocess from shutil import which import click -from audible_cli.config import pass_session +from audible_cli.decorators import pass_session from click import echo, secho @@ -49,6 +49,10 @@ class ApiMeta: return self._meta_parsed["content_metadata"]["chapter_info"][ "runtime_length_ms"] + def is_accurate(self): + return self._meta_parsed["content_metadata"]["chapter_info"][ + "is_accurate"] + class FFMeta: SECTION = re.compile(r"\[(?P
[^]]+)\]") @@ -107,7 +111,8 @@ class FFMeta: self._write_section(fp, section, self._ffmeta_parsed[section], d) - def _write_section(self, fp, section_name, section_items, delimiter): + @staticmethod + def _write_section(fp, section_name, section_items, delimiter): """Write a single section to the specified `fp`.""" if section_name is not None: fp.write(f"[{section_name}]\n") @@ -122,6 +127,10 @@ class FFMeta: if not isinstance(api_meta, ApiMeta): api_meta = ApiMeta(api_meta) + if not api_meta.is_accurate(): + echo("Metadata from API is not accurate. Skip.") + return + # assert api_meta.count_chapters() == self.count_chapters() echo(f"Found {self.count_chapters()} chapters to prepare.") @@ -170,45 +179,54 @@ class FFMeta: self._ffmeta_parsed["CHAPTER"] = new_chapters -def decrypt_aax(files, session): +def decrypt_aax(files, activation_bytes, rebuild_chapters): for file in files: outfile = file.with_suffix(".m4b") metafile = file.with_suffix(".meta") metafile_new = file.with_suffix(".new.meta") - # apimeta = CHAPTERFILE + base_filename = file.stem.rsplit("-")[0] + chapters = file.with_name(base_filename + "-chapters").with_suffix(".json") + apimeta = json.loads(chapters.read_text()) if outfile.exists(): secho(f"file {outfile} already exists Skip.", fg="blue") continue - - ab = session.auth.activation_bytes - - cmd = ["ffmpeg", - "-activation_bytes", ab, - "-i", str(file), - "-f", "ffmetadata", - str(metafile)] - subprocess.check_output(cmd, universal_newlines=True) - - ffmeta_class = FFMeta(metafile) - #ffmeta_class.update_chapters_from_api_meta(apimeta) - ffmeta_class.write(metafile_new) - click.echo("Replaced all titles.") - - cmd = ["ffmpeg", - "-activation_bytes", ab, - "-i", str(file), - "-i", str(metafile_new), - "-map_metadata", "0", - "-map_chapters", "1", - "-c", "copy", - str(outfile)] - subprocess.check_output(cmd, universal_newlines=True) - metafile.unlink() - metafile_new.unlink() + + if rebuild_chapters and apimeta["content_metadata"]["chapter_info"][ + "is_accurate"]: + cmd = ["ffmpeg", + "-activation_bytes", activation_bytes, + "-i", str(file), + "-f", "ffmetadata", + str(metafile)] + subprocess.check_output(cmd, universal_newlines=True) + + ffmeta_class = FFMeta(metafile) + ffmeta_class.update_chapters_from_api_meta(apimeta) + ffmeta_class.write(metafile_new) + click.echo("Replaced all titles.") + + cmd = ["ffmpeg", + "-activation_bytes", activation_bytes, + "-i", str(file), + "-i", str(metafile_new), + "-map_metadata", "0", + "-map_chapters", "1", + "-c", "copy", + str(outfile)] + subprocess.check_output(cmd, universal_newlines=True) + metafile.unlink() + metafile_new.unlink() + else: + cmd = ["ffmpeg", + "-activation_bytes", activation_bytes, + "-i", str(file), + "-c", "copy", + str(outfile)] + subprocess.check_output(cmd, universal_newlines=True) -def decrypt_aaxc(files, session): +def decrypt_aaxc(files, rebuild_chapters): for file in files: metafile = file.with_suffix(".meta") metafile_new = file.with_suffix(".new.meta") @@ -223,32 +241,42 @@ def decrypt_aaxc(files, session): apimeta = voucher["content_license"] audible_key = apimeta["license_response"]["key"] audible_iv = apimeta["license_response"]["iv"] + + if rebuild_chapters and apimeta["content_metadata"]["chapter_info"][ + "is_accurate"]: + cmd = ["ffmpeg", + "-audible_key", audible_key, + "-audible_iv", audible_iv, + "-i", str(file), + "-f", "ffmetadata", + str(metafile)] + subprocess.check_output(cmd, universal_newlines=True) - cmd = ["ffmpeg", - "-audible_key", audible_key, - "-audible_iv", audible_iv, - "-i", str(file), - "-f", "ffmetadata", - str(metafile)] - subprocess.check_output(cmd, universal_newlines=True) + ffmeta_class = FFMeta(metafile) + ffmeta_class.update_chapters_from_api_meta(apimeta) + ffmeta_class.write(metafile_new) + click.echo("Replaced all titles.") - ffmeta_class = FFMeta(metafile) - ffmeta_class.update_chapters_from_api_meta(apimeta) - ffmeta_class.write(metafile_new) - click.echo("Replaced all titles.") - - cmd = ["ffmpeg", - "-audible_key", audible_key, - "-audible_iv", audible_iv, - "-i", str(file), - "-i", str(metafile_new), - "-map_metadata", "0", - "-map_chapters", "1", - "-c", "copy", - str(outfile)] - subprocess.check_output(cmd, universal_newlines=True) - metafile.unlink() - metafile_new.unlink() + cmd = ["ffmpeg", + "-audible_key", audible_key, + "-audible_iv", audible_iv, + "-i", str(file), + "-i", str(metafile_new), + "-map_metadata", "0", + "-map_chapters", "1", + "-c", "copy", + str(outfile)] + subprocess.check_output(cmd, universal_newlines=True) + metafile.unlink() + metafile_new.unlink() + else: + cmd = ["ffmpeg", + "-audible_key", audible_key, + "-audible_iv", audible_iv, + "-i", str(file), + "-c", "copy", + str(outfile)] + subprocess.check_output(cmd, universal_newlines=True) CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"]) @@ -270,21 +298,25 @@ CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"]) is_flag=True, help="overwrite existing files" ) +@click.option( + "--rebuild-chapters", + is_flag=True, + help="Rebuild chapters from chapter file" +) @pass_session def cli(session, **options): if not which("ffmpeg"): ctx = click.get_current_context() ctx.fail("ffmpeg not found") + rebuild_chapters = options.get("rebuild_chapters") + jobs = {"aaxc": [], "aax":[]} if options.get("all"): cwd = pathlib.Path.cwd() jobs["aaxc"].extend(list(cwd.glob('*.aaxc'))) jobs["aax"].extend(list(cwd.glob('*.aax'))) - for suffix in jobs: - for i in jobs[suffix]: - i = i.resolve() else: for file in options.get("input"): @@ -296,6 +328,5 @@ def cli(session, **options): else: secho(f"file suffix {file.suffix} not supported", fg="red") - decrypt_aaxc(jobs["aaxc"], session) - decrypt_aax(jobs["aax"], session) - + decrypt_aaxc(jobs["aaxc"], rebuild_chapters) + decrypt_aax(jobs["aax"], session.auth.activation_bytes, rebuild_chapters) diff --git a/setup.py b/setup.py index 9068615..f7848f4 100644 --- a/setup.py +++ b/setup.py @@ -46,15 +46,16 @@ setup( ], install_requires=[ "aiofiles", - "audible==0.7.2", + "audible>=0.8.2", "click>=8", "colorama; platform_system=='Windows'", - "httpx>=0.20.*,<=0.22.*", + "httpx>=0.20.0,<0.24.0", "packaging", "Pillow", "tabulate", "toml", - "tqdm" + "tqdm", + "questionary" ], extras_require={ 'pyi': [ diff --git a/src/audible_cli/_logging.py b/src/audible_cli/_logging.py index dcd5bcf..82cbf18 100644 --- a/src/audible_cli/_logging.py +++ b/src/audible_cli/_logging.py @@ -73,42 +73,6 @@ log_helper = AudibleCliLogHelper() # copied from https://github.com/Toilal/click-logging - -def click_verbosity_option(logger=None, *names, **kwargs): - """A decorator that adds a `--verbosity, -v` option to the decorated - command. - Name can be configured through ``*names``. Keyword arguments are passed to - the underlying ``click.option`` decorator. - """ - - if not names: - names = ["--verbosity", "-v"] - - kwargs.setdefault("default", "INFO") - kwargs.setdefault("metavar", "LVL") - kwargs.setdefault("expose_value", False) - kwargs.setdefault( - "help", "Either CRITICAL, ERROR, WARNING, " - "INFO or DEBUG. [default: INFO]" - ) - kwargs.setdefault("is_eager", True) - - logger = _normalize_logger(logger) - - def decorator(f): - def _set_level(ctx, param, value): - x = getattr(logging, value.upper(), None) - if x is None: - raise click.BadParameter( - f"Must be CRITICAL, ERROR, WARNING, INFO or DEBUG, " - f"not {value}" - ) - logger.setLevel(x) - - return click.option(*names, callback=_set_level, **kwargs)(f) - return decorator - - class ColorFormatter(logging.Formatter): def __init__(self, style_kwargs): self.style_kwargs = style_kwargs diff --git a/src/audible_cli/_version.py b/src/audible_cli/_version.py index 93751c9..0590716 100644 --- a/src/audible_cli/_version.py +++ b/src/audible_cli/_version.py @@ -1,7 +1,7 @@ __title__ = "audible-cli" __description__ = "Command line interface (cli) for the audible package." __url__ = "https://github.com/mkb79/audible-cli" -__version__ = "0.1.3" +__version__ = "0.2.b1" __author__ = "mkb79" __author_email__ = "mkb79@hackitall.de" __license__ = "AGPL" diff --git a/src/audible_cli/cli.py b/src/audible_cli/cli.py index 7e1a265..4bb75aa 100644 --- a/src/audible_cli/cli.py +++ b/src/audible_cli/cli.py @@ -3,18 +3,19 @@ import sys from pkg_resources import iter_entry_points import click -import httpx -from packaging.version import parse from .cmds import build_in_cmds, cmd_quickstart -from .config import ( - get_plugin_dir, - add_param_to_session -) +from .config import get_plugin_dir from .constants import PLUGIN_ENTRY_POINT +from .decorators import ( + password_option, + profile_option, + verbosity_option, + version_option +) from .exceptions import AudibleCliException -from ._logging import click_basic_config, click_verbosity_option -from . import __version__, plugins +from ._logging import click_basic_config +from . import plugins logger = logging.getLogger("audible_cli") @@ -23,77 +24,22 @@ click_basic_config(logger) CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"]) -def version_option(**kwargs): - def callback(ctx, param, value): - if not value or ctx.resilient_parsing: - return - - message = f"audible-cli, version {__version__}" - click.echo(message, color=ctx.color, nl=False) - - url = "https://api.github.com/repos/mkb79/audible-cli/releases/latest" - headers = {"Accept": "application/vnd.github.v3+json"} - logger.debug(f"Requesting Github API for latest release information") - try: - response = httpx.get(url, headers=headers, follow_redirects=True) - response.raise_for_status() - except Exception as e: - logger.error(e) - click.Abort() - - content = response.json() - - current_version = parse(__version__) - latest_version = parse(content["tag_name"]) - - html_url = content["html_url"] - if latest_version > current_version: - click.echo( - f" (update available)\nVisit {html_url} " - f"for information about the new release.", - color=ctx.color - ) - else: - click.echo(" (up-to-date)", color=ctx.color) - - ctx.exit() - - kwargs.setdefault("is_flag", True) - kwargs.setdefault("expose_value", False) - kwargs.setdefault("is_eager", True) - kwargs.setdefault("help", "Show the version and exit.") - kwargs["callback"] = callback - return click.option("--version", **kwargs) - - @plugins.from_folder(get_plugin_dir()) @plugins.from_entry_point(iter_entry_points(PLUGIN_ENTRY_POINT)) -@build_in_cmds() +@build_in_cmds @click.group(context_settings=CONTEXT_SETTINGS) -@click.option( - "--profile", - "-P", - callback=add_param_to_session, - expose_value=False, - help="The profile to use instead primary profile (case sensitive!)." -) -@click.option( - "--password", - "-p", - callback=add_param_to_session, - expose_value=False, - help="The password for the profile auth file." -) -@version_option() -@click_verbosity_option(logger) +@profile_option +@password_option +@version_option +@verbosity_option(cli_logger=logger) def cli(): """Entrypoint for all other subcommands and groups.""" @click.command(context_settings=CONTEXT_SETTINGS) @click.pass_context -@version_option() -@click_verbosity_option(logger) +@version_option +@verbosity_option(cli_logger=logger) def quickstart(ctx): """Entrypoint for the quickstart command""" try: diff --git a/src/audible_cli/cmds/__init__.py b/src/audible_cli/cmds/__init__.py index f212520..6123aef 100644 --- a/src/audible_cli/cmds/__init__.py +++ b/src/audible_cli/cmds/__init__.py @@ -21,7 +21,7 @@ cli_cmds = [ ] -def build_in_cmds(): +def build_in_cmds(func=None): """ A decorator to register build-in CLI commands to an instance of `click.Group()`. @@ -42,4 +42,7 @@ def build_in_cmds(): return group + if callable(func): + return decorator(func) + return decorator diff --git a/src/audible_cli/cmds/cmd_activation_bytes.py b/src/audible_cli/cmds/cmd_activation_bytes.py index 24558c1..c03fd15 100644 --- a/src/audible_cli/cmds/cmd_activation_bytes.py +++ b/src/audible_cli/cmds/cmd_activation_bytes.py @@ -6,7 +6,7 @@ from audible.activation_bytes import ( fetch_activation_sign_auth ) -from ..config import pass_session +from ..decorators import pass_session logger = logging.getLogger("audible_cli.cmds.cmd_activation_bytes") diff --git a/src/audible_cli/cmds/cmd_api.py b/src/audible_cli/cmds/cmd_api.py index 054527d..95cfc9c 100644 --- a/src/audible_cli/cmds/cmd_api.py +++ b/src/audible_cli/cmds/cmd_api.py @@ -6,7 +6,7 @@ import sys import click from audible import Client -from ..config import pass_session +from ..decorators import pass_session logger = logging.getLogger("audible_cli.cmds.cmd_api") @@ -63,7 +63,7 @@ logger = logging.getLogger("audible_cli.cmds.cmd_api") @pass_session def cli(session, **options): """Send requests to an Audible API endpoint - + Take a look at https://audible.readthedocs.io/en/latest/misc/external_api.html for known endpoints and parameters. @@ -96,7 +96,7 @@ def cli(session, **options): with Client(auth=auth, country_code=country_code) as client: r = client._request(method, endpoint, params=params, json=body) except Exception as e: - logger.error(e) + logger.error(e) sys.exit(1) if output_format == "json": diff --git a/src/audible_cli/cmds/cmd_download.py b/src/audible_cli/cmds/cmd_download.py index 979467b..6c2f4c9 100644 --- a/src/audible_cli/cmds/cmd_download.py +++ b/src/audible_cli/cmds/cmd_download.py @@ -3,80 +3,38 @@ import asyncio.log import asyncio.sslproto import json import pathlib -import ssl import logging -import sys -import unicodedata import aiofiles -import audible import click import httpx +import questionary +from audible.exceptions import NotFoundError from click import echo -from tabulate import tabulate -from ..config import pass_session -from ..exceptions import DirectoryDoesNotExists, NotFoundError +from ..decorators import ( + bunch_size_option, + timeout_option, + pass_client, + pass_session +) +from ..exceptions import DirectoryDoesNotExists, NotDownloadableAsAAX from ..models import Library from ..utils import Downloader logger = logging.getLogger("audible_cli.cmds.cmd_download") -SSL_PROTOCOLS = (asyncio.sslproto.SSLProtocol,) - - -def ignore_httpx_ssl_eror(loop): - """Ignore aiohttp #3535 / cpython #13548 issue with SSL data after close - - There is an issue in Python 3.7 up to 3.7.3 that over-reports a - ssl.SSLError fatal error (ssl.SSLError: [SSL: KRB5_S_INIT] application data - after close notify (_ssl.c:2609)) after we are already done with the - connection. See GitHub issues aio-libs/aiohttp#3535 and - python/cpython#13548. - - Given a loop, this sets up an exception handler that ignores this specific - exception, but passes everything else on to the previous exception handler - this one replaces. - - Checks for fixed Python versions, disabling itself when running on 3.7.4+ - or 3.8. - - """ - if sys.version_info >= (3, 7, 4): - return - - orig_handler = loop.get_exception_handler() - - def ignore_ssl_error(context): - if context.get("message") in { - "SSL error in data received", - "Fatal error on transport", - }: - # validate we have the right exception, transport and protocol - exception = context.get("exception") - protocol = context.get("protocol") - if ( - isinstance(exception, ssl.SSLError) - and exception.reason == "KRB5_S_INIT" - and isinstance(protocol, SSL_PROTOCOLS) - ): - if loop.get_debug(): - asyncio.log.logger.debug( - "Ignoring httpx SSL KRB5_S_INIT error") - return - if orig_handler is not None: - orig_handler(loop, context) - else: - loop.default_exception_handler(context) - - loop.set_exception_handler(ignore_ssl_error) +CLIENT_HEADERS = { + "User-Agent": "Audible/671 CFNetwork/1240.0.4 Darwin/20.6.0" +} class DownloadCounter: def __init__(self): self._aax: int = 0 self._aaxc: int = 0 + self._annotation: int = 0 self._chapter: int = 0 self._cover: int = 0 self._pdf: int = 0 @@ -99,6 +57,14 @@ class DownloadCounter: self._aaxc += 1 logger.debug(f"Currently downloaded aaxc files: {self.aaxc}") + @property + def annotation(self): + return self._annotation + + def count_annotation(self): + self._annotation += 1 + logger.debug(f"Currently downloaded annotations: {self.annotation}") + @property def chapter(self): return self._chapter @@ -143,6 +109,7 @@ class DownloadCounter: return { "aax": self.aax, "aaxc": self.aaxc, + "annotation": self.annotation, "chapter": self.chapter, "cover": self.cover, "pdf": self.pdf, @@ -161,22 +128,6 @@ class DownloadCounter: counter = DownloadCounter() -def create_base_filename(item, mode): - if "ascii" in mode: - base_filename = item.full_title_slugify - - elif "unicode" in mode: - base_filename = unicodedata.normalize("NFKD", item.full_title) - - else: - base_filename = item.asin - - if "asin" in mode: - base_filename = item.asin + "_" + base_filename - - return base_filename - - async def download_cover( client, output_dir, base_filename, item, res, overwrite_existing ): @@ -234,8 +185,8 @@ async def download_chapters( try: metadata = await item.get_content_metadata(quality) except NotFoundError: - logger.error( - f"Can't get chapters for {item.full_title}. Skip item." + logger.info( + f"No chapters found for {item.full_title}." ) return metadata = json.dumps(metadata, indent=4) @@ -245,11 +196,54 @@ async def download_chapters( counter.count_chapter() +async def download_annotations( + output_dir, base_filename, item, overwrite_existing +): + if not output_dir.is_dir(): + raise DirectoryDoesNotExists(output_dir) + + filename = base_filename + "-annotations.json" + file = output_dir / filename + if file.exists() and not overwrite_existing: + logger.info( + f"File {file} already exists. Skip saving annotations" + ) + return True + + try: + annotation = await item.get_annotations() + except NotFoundError: + logger.info( + f"No annotations found for {item.full_title}." + ) + return + annotation = json.dumps(annotation, indent=4) + async with aiofiles.open(file, "w") as f: + await f.write(annotation) + logger.info(f"Annotation file saved to {file}.") + counter.count_annotation() + + async def download_aax( - client, output_dir, base_filename, item, quality, overwrite_existing + client, output_dir, base_filename, item, quality, overwrite_existing, + aax_fallback ): # url, codec = await item.get_aax_url(quality) - url, codec = await item.get_aax_url_old(quality) + try: + url, codec = await item.get_aax_url_old(quality) + except NotDownloadableAsAAX: + if aax_fallback: + logger.info(f"Fallback to aaxc for {item.full_title}") + return await download_aaxc( + client=client, + output_dir=output_dir, + base_filename=base_filename, + item=item, + quality=quality, + overwrite_existing=overwrite_existing + ) + raise + filename = base_filename + f"-{codec}.aax" filepath = output_dir / filename dl = Downloader( @@ -346,13 +340,16 @@ async def consume(queue): await item except Exception as e: logger.error(e) - queue.task_done() + raise + finally: + queue.task_done() def queue_job( queue, get_cover, get_pdf, + get_annotation, get_chapters, get_aax, get_aaxc, @@ -362,9 +359,10 @@ def queue_job( item, cover_size, quality, - overwrite_existing + overwrite_existing, + aax_fallback ): - base_filename = create_base_filename(item=item, mode=filename_mode) + base_filename = item.create_base_filename(filename_mode) if get_cover: queue.put_nowait( @@ -400,6 +398,16 @@ def queue_job( ) ) + if get_annotation: + queue.put_nowait( + download_annotations( + output_dir=output_dir, + base_filename=base_filename, + item=item, + overwrite_existing=overwrite_existing + ) + ) + if get_aax: queue.put_nowait( download_aax( @@ -408,7 +416,8 @@ def queue_job( base_filename=base_filename, item=item, quality=quality, - overwrite_existing=overwrite_existing + overwrite_existing=overwrite_existing, + aax_fallback=aax_fallback ) ) @@ -425,158 +434,23 @@ def queue_job( ) -async def main(config, auth, **params): - output_dir = pathlib.Path(params.get("output_dir")).resolve() +def display_counter(): + if counter.has_downloads(): + echo("The download ended with the following result:") + for k, v in counter.as_dict().items(): + if v == 0: + continue - # which item(s) to download - get_all = params.get("all") is True - asins = params.get("asin") - titles = params.get("title") - if get_all and (asins or titles): - logger.error(f"Do not mix *asin* or *title* option with *all* option.") - click.Abort() - - # what to download - get_aax = params.get("aax") - get_aaxc = params.get("aaxc") - get_chapters = params.get("chapter") - get_cover = params.get("cover") - get_pdf = params.get("pdf") - if not any([get_aax, get_aaxc, get_chapters, get_cover, get_pdf]): - logger.error("Please select an option what you want download.") - click.Abort() - - # additional options - sim_jobs = params.get("jobs") - quality = params.get("quality") - cover_size = params.get("cover_size") - overwrite_existing = params.get("overwrite") - ignore_errors = params.get("ignore_errors") - no_confirm = params.get("no_confirm") - resolve_podcats = params.get("resolve_podcasts") - ignore_podcasts = params.get("ignore_podcasts") - bunch_size = params.get("bunch_size") - timeout = params.get("timeout") - if timeout == 0: - timeout = None - - filename_mode = params.get("filename_mode") - if filename_mode == "config": - filename_mode = config.profile_config.get("filename_mode") or \ - config.app_config.get("filename_mode") or \ - "ascii" - - headers = { - "User-Agent": "Audible/671 CFNetwork/1240.0.4 Darwin/20.6.0" - } - client = httpx.AsyncClient(auth=auth, timeout=timeout, headers=headers) - api_client = audible.AsyncClient(auth, timeout=timeout) - - async with client, api_client: - # fetch the user library - library = await Library.from_api_full_sync( - api_client, - image_sizes="1215, 408, 360, 882, 315, 570, 252, 558, 900, 500", - bunch_size=bunch_size - ) - - if resolve_podcats: - await library.resolve_podcats() - - # collect jobs - jobs = [] - - if get_all: - asins = [] - titles = [] - for i in library: - jobs.append(i.asin) - - for asin in asins: - if library.has_asin(asin): - jobs.append(asin) - else: - if not ignore_errors: - logger.error(f"Asin {asin} not found in library.") - click.Abort() - logger.error( - f"Skip asin {asin}: Not found in library" - ) - - for title in titles: - match = library.search_item_by_title(title) - full_match = [i for i in match if i[1] == 100] - - if full_match or match: - echo(f"\nFound the following matches for '{title}'") - table_data = [[i[1], i[0].full_title, i[0].asin] - for i in full_match or match] - head = ["% match", "title", "asin"] - table = tabulate( - table_data, head, tablefmt="pretty", - colalign=("center", "left", "center")) - echo(table) - - if no_confirm or click.confirm( - "Proceed with this audiobook(s)", - default=True - ): - jobs.extend([i[0].asin for i in full_match or match]) - - else: - logger.error( - f"Skip title {title}: Not found in library" - ) - - queue = asyncio.Queue() - - for job in jobs: - item = library.get_item_by_asin(job) - items = [item] - odir = pathlib.Path(output_dir) - - if not ignore_podcasts and item.is_parent_podcast(): - items.remove(item) - if item._children is None: - await item.get_child_items() - - for i in item._children: - if i.asin not in jobs: - items.append(i) - - podcast_dir = create_base_filename(item, filename_mode) - odir = output_dir / podcast_dir - if not odir.is_dir(): - odir.mkdir(parents=True) - - for item in items: - queue_job( - queue=queue, - get_cover=get_cover, - get_pdf=get_pdf, - get_chapters=get_chapters, - get_aax=get_aax, - get_aaxc=get_aaxc, - client=client, - output_dir=odir, - filename_mode=filename_mode, - item=item, - cover_size=cover_size, - quality=quality, - overwrite_existing=overwrite_existing - ) - - # schedule the consumer - consumers = [ - asyncio.ensure_future(consume(queue)) for _ in range(sim_jobs) - ] - - # wait until the consumer has processed all items - await queue.join() - - # the consumer is still awaiting an item, cancel it - for consumer in consumers: - consumer.cancel() + if k == "voucher_saved": + k = "voucher" + elif k == "voucher": + diff = v - counter.voucher_saved + if diff > 0: + echo(f"Unsaved voucher: {diff}") + continue + echo(f"New {k} files: {v}") + else: + echo("No new files downloaded.") @click.command("download") @@ -611,6 +485,11 @@ async def main(config, auth, **params): is_flag=True, help="Download book in aaxc format incl. voucher file" ) +@click.option( + "--aax-fallback", + is_flag=True, + help="Download book in aax format and fallback to aaxc, if former is not supported." +) @click.option( "--quality", "-q", default="best", @@ -640,6 +519,11 @@ async def main(config, auth, **params): is_flag=True, help="saves chapter metadata as JSON file" ) +@click.option( + "--annotation", + is_flag=True, + help="saves the annotations (e.g. bookmarks, notes) as JSON file" +) @click.option( "--no-confirm", "-y", is_flag=True, @@ -670,14 +554,7 @@ async def main(config, auth, **params): default="config", help="Filename mode to use. [default: config]" ) -@click.option( - "--timeout", - type=click.INT, - default=10, - show_default=True, - help="Increase the timeout time if you got any TimeoutErrors. " - "Set to 0 to disable timeout." -) +@timeout_option @click.option( "--resolve-podcasts", is_flag=True, @@ -688,41 +565,166 @@ async def main(config, auth, **params): is_flag=True, help="Ignore a podcast if it have episodes" ) -@click.option( - "--bunch-size", - type=click.IntRange(10, 1000), - default=1000, - show_default=True, - help="How many library items should be requested per request. A lower " - "size results in more requests to get the full library. A higher " - "size can result in a TimeOutError on low internet connections." -) +@bunch_size_option @pass_session -def cli(session, **params): +@pass_client(headers=CLIENT_HEADERS) +async def cli(session, api_client, **params): """download audiobook(s) from library""" - loop = asyncio.get_event_loop() - ignore_httpx_ssl_eror(loop) - auth = session.auth - config = session.config - try: - loop.run_until_complete(main(config, auth, **params)) - finally: - loop.run_until_complete(loop.shutdown_asyncgens()) - loop.close() + client = api_client.session + output_dir = pathlib.Path(params.get("output_dir")).resolve() - if counter.has_downloads(): - echo("The download ended with the following result:") - for k, v in counter.as_dict().items(): - if v == 0: - continue - - if k == "voucher_saved": - k = "voucher" - elif k == "voucher": - diff = v - counter.voucher_saved - if diff > 0: - echo(f"Unsaved voucher: {diff}") - continue - echo(f"New {k} files: {v}") + # which item(s) to download + get_all = params.get("all") is True + asins = params.get("asin") + titles = params.get("title") + if get_all and (asins or titles): + logger.error(f"Do not mix *asin* or *title* option with *all* option.") + click.Abort() + + # what to download + get_aax = params.get("aax") + get_aaxc = params.get("aaxc") + aax_fallback = params.get("aax_fallback") + if aax_fallback: + if get_aax: + logger.info("Using --aax is redundant and can be left when using --aax-fallback") + get_aax = True + if get_aaxc: + logger.warning("Do not mix --aaxc with --aax-fallback option.") + get_annotation = params.get("annotation") + get_chapters = params.get("chapter") + get_cover = params.get("cover") + get_pdf = params.get("pdf") + if not any( + [get_aax, get_aaxc, get_annotation, get_chapters, get_cover, get_pdf] + ): + logger.error("Please select an option what you want download.") + click.Abort() + + # additional options + sim_jobs = params.get("jobs") + quality = params.get("quality") + cover_size = params.get("cover_size") + overwrite_existing = params.get("overwrite") + ignore_errors = params.get("ignore_errors") + no_confirm = params.get("no_confirm") + resolve_podcats = params.get("resolve_podcasts") + ignore_podcasts = params.get("ignore_podcasts") + bunch_size = session.params.get("bunch_size") + + filename_mode = params.get("filename_mode") + if filename_mode == "config": + filename_mode = session.config.get_profile_option( + session.selected_profile, "filename_mode") or "ascii" + + # fetch the user library + library = await Library.from_api_full_sync( + api_client, + image_sizes="1215, 408, 360, 882, 315, 570, 252, 558, 900, 500", + bunch_size=bunch_size + ) + + if resolve_podcats: + await library.resolve_podcats() + + # collect jobs + jobs = [] + + if get_all: + asins = [] + titles = [] + for i in library: + jobs.append(i.asin) + + for asin in asins: + if library.has_asin(asin): + jobs.append(asin) else: - echo("No new files downloaded.") + if not ignore_errors: + logger.error(f"Asin {asin} not found in library.") + click.Abort() + logger.error( + f"Skip asin {asin}: Not found in library" + ) + + for title in titles: + match = library.search_item_by_title(title) + full_match = [i for i in match if i[1] == 100] + + if match: + if no_confirm: + [jobs.append(i[0].asin) for i in full_match or match] + else: + choices = [] + for i in full_match or match: + a = i[0].asin + t = i[0].full_title + c = questionary.Choice(title=f"{a} # {t}", value=a) + choices.append(c) + + answer = await questionary.checkbox( + f"Found the following matches for '{title}'. Which you want to download?", + choices=choices + ).unsafe_ask_async() + if answer is not None: + [jobs.append(i) for i in answer] + + else: + logger.error( + f"Skip title {title}: Not found in library" + ) + + queue = asyncio.Queue() + for job in jobs: + item = library.get_item_by_asin(job) + items = [item] + odir = pathlib.Path(output_dir) + + if not ignore_podcasts and item.is_parent_podcast(): + items.remove(item) + if item._children is None: + await item.get_child_items() + + for i in item._children: + if i.asin not in jobs: + items.append(i) + + podcast_dir = item.create_base_filename(filename_mode) + odir = output_dir / podcast_dir + if not odir.is_dir(): + odir.mkdir(parents=True) + + for item in items: + queue_job( + queue=queue, + get_cover=get_cover, + get_pdf=get_pdf, + get_annotation=get_annotation, + get_chapters=get_chapters, + get_aax=get_aax, + get_aaxc=get_aaxc, + client=client, + output_dir=odir, + filename_mode=filename_mode, + item=item, + cover_size=cover_size, + quality=quality, + overwrite_existing=overwrite_existing, + aax_fallback=aax_fallback + ) + + try: + # schedule the consumer + consumers = [ + asyncio.ensure_future(consume(queue)) for _ in range(sim_jobs) + ] + # wait until the consumer has processed all items + await queue.join() + + finally: + # the consumer is still awaiting an item, cancel it + for consumer in consumers: + consumer.cancel() + + await asyncio.gather(*consumers, return_exceptions=True) + display_counter() diff --git a/src/audible_cli/cmds/cmd_library.py b/src/audible_cli/cmds/cmd_library.py index d11f106..3dc4172 100644 --- a/src/audible_cli/cmds/cmd_library.py +++ b/src/audible_cli/cmds/cmd_library.py @@ -1,15 +1,19 @@ import asyncio -import csv import json import pathlib -from typing import Union -import audible import click from click import echo -from ..config import pass_session +from ..decorators import ( + bunch_size_option, + timeout_option, + pass_client, + pass_session, + wrap_async +) from ..models import Library +from ..utils import export_to_csv @click.group("library") @@ -17,65 +21,53 @@ def cli(): """interact with library""" -async def _get_library(auth, **params): - timeout = params.get("timeout") - if timeout == 0: - timeout = None +async def _get_library(session, client): + bunch_size = session.params.get("bunch_size") - bunch_size = params.get("bunch_size") - - async with audible.AsyncClient(auth, timeout=timeout) as client: - library = await Library.from_api_full_sync( - client, - response_groups=( - "contributors, media, price, product_attrs, product_desc, " - "product_extended_attrs, product_plan_details, product_plans, " - "rating, sample, sku, series, reviews, ws4v, origin, " - "relationships, review_attrs, categories, badge_types, " - "category_ladders, claim_code_url, is_downloaded, " - "is_finished, is_returnable, origin_asin, pdf_url, " - "percent_complete, provided_review" - ), - bunch_size=bunch_size - ) - return library - - -async def _list_library(auth, **params): - library = await _get_library(auth, **params) - - books = [] - - for item in library: - asin = item.asin - authors = ", ".join( - sorted(a["name"] for a in item.authors) if item.authors else "" - ) - series = ", ".join( - sorted(s["title"] for s in item.series) if item.series else "" - ) - title = item.title - books.append((asin, authors, series, title)) - - for asin, authors, series, title in sorted(books): - fields = [asin] - if authors: - fields.append(authors) - if series: - fields.append(series) - fields.append(title) - echo(": ".join(fields)) - - -def _prepare_library_for_export(library: Library): - keys_with_raw_values = ( - "asin", "title", "subtitle", "runtime_length_min", "is_finished", - "percent_complete", "release_date" + return await Library.from_api_full_sync( + client, + response_groups=( + "contributors, media, price, product_attrs, product_desc, " + "product_extended_attrs, product_plan_details, product_plans, " + "rating, sample, sku, series, reviews, ws4v, origin, " + "relationships, review_attrs, categories, badge_types, " + "category_ladders, claim_code_url, is_downloaded, " + "is_finished, is_returnable, origin_asin, pdf_url, " + "percent_complete, provided_review" + ), + bunch_size=bunch_size ) - prepared_library = [] - for item in library: +@cli.command("export") +@click.option( + "--output", "-o", + type=click.Path(path_type=pathlib.Path), + default=pathlib.Path().cwd() / r"library.{format}", + show_default=True, + help="output file" +) +@timeout_option +@click.option( + "--format", "-f", + type=click.Choice(["tsv", "csv", "json"]), + default="tsv", + show_default=True, + help="Output format" +) +@bunch_size_option +@click.option( + "--resolve-podcasts", + is_flag=True, + help="Resolve podcasts to show all episodes" +) +@pass_session +@pass_client +async def export_library(session, client, **params): + """export library""" + + @wrap_async + def _prepare_item(item): data_row = {} for key in item: v = getattr(item, key) @@ -105,128 +97,88 @@ def _prepare_library_for_export(library: Library): genres.append(ladder["name"]) data_row["genres"] = ", ".join(genres) - prepared_library.append(data_row) + return data_row - prepared_library.sort(key=lambda x: x["asin"]) - - return prepared_library - - -def _export_to_csv( - file: pathlib.Path, - data: list, - headers: Union[list, tuple], - dialect: str -): - with file.open("w", encoding="utf-8", newline="") as f: - writer = csv.DictWriter(f, fieldnames=headers, dialect=dialect) - writer.writeheader() - - for i in data: - writer.writerow(i) - - -async def _export_library(auth, **params): output_format = params.get("format") output_filename: pathlib.Path = params.get("output") if output_filename.suffix == r".{format}": suffix = "." + output_format output_filename = output_filename.with_suffix(suffix) - library = await _get_library(auth, **params) + library = await _get_library(session, client) + if params.get("resolve_podcasts"): + await library.resolve_podcats() - prepared_library = _prepare_library_for_export(library) - - headers = ( - "asin", "title", "subtitle", "authors", "narrators", "series_title", - "series_sequence", "genres", "runtime_length_min", "is_finished", - "percent_complete", "rating", "num_ratings", "date_added", - "release_date", "cover_url" + keys_with_raw_values = ( + "asin", "title", "subtitle", "runtime_length_min", "is_finished", + "percent_complete", "release_date" ) + prepared_library = await asyncio.gather( + *[_prepare_item(i) for i in library] + ) + prepared_library.sort(key=lambda x: x["asin"]) + if output_format in ("tsv", "csv"): - if output_format == csv: + if output_format == "csv": dialect = "excel" else: dialect = "excel-tab" - _export_to_csv(output_filename, prepared_library, headers, dialect) - if output_format == "json": + headers = ( + "asin", "title", "subtitle", "authors", "narrators", "series_title", + "series_sequence", "genres", "runtime_length_min", "is_finished", + "percent_complete", "rating", "num_ratings", "date_added", + "release_date", "cover_url" + ) + + export_to_csv(output_filename, prepared_library, headers, dialect) + + elif output_format == "json": data = json.dumps(prepared_library, indent=4) output_filename.write_text(data) -@cli.command("export") -@click.option( - "--output", "-o", - type=click.Path(path_type=pathlib.Path), - default=pathlib.Path().cwd() / r"library.{format}", - show_default=True, - help="output file" -) -@click.option( - "--timeout", "-t", - type=click.INT, - default=10, - show_default=True, - help=( - "Increase the timeout time if you got any TimeoutErrors. " - "Set to 0 to disable timeout." - ) -) -@click.option( - "--format", "-f", - type=click.Choice(["tsv", "csv", "json"]), - default="tsv", - show_default=True, - help="Output format" -) -@click.option( - "--bunch-size", - type=click.IntRange(10, 1000), - default=1000, - show_default=True, - help="How many library items should be requested per request. A lower " - "size results in more requests to get the full library. A higher " - "size can result in a TimeOutError on low internet connections." -) -@pass_session -def export_library(session, **params): - """export library""" - loop = asyncio.get_event_loop() - try: - loop.run_until_complete(_export_library(session.auth, **params)) - finally: - loop.run_until_complete(loop.shutdown_asyncgens()) - loop.close() - - @cli.command("list") +@timeout_option +@bunch_size_option @click.option( - "--timeout", "-t", - type=click.INT, - default=10, - show_default=True, - help=( - "Increase the timeout time if you got any TimeoutErrors. " - "Set to 0 to disable timeout." - ) -) -@click.option( - "--bunch-size", - type=click.IntRange(10, 1000), - default=1000, - show_default=True, - help="How many library items should be requested per request. A lower " - "size results in more requests to get the full library. A higher " - "size can result in a TimeOutError on low internet connections." + "--resolve-podcasts", + is_flag=True, + help="Resolve podcasts to show all episodes" ) @pass_session -def list_library(session, **params): +@pass_client +async def list_library(session, client, resolve_podcasts=False): """list titles in library""" - loop = asyncio.get_event_loop() - try: - loop.run_until_complete(_list_library(session.auth, **params)) - finally: - loop.run_until_complete(loop.shutdown_asyncgens()) - loop.close() + + @wrap_async + def _prepare_item(item): + fields = [item.asin] + + authors = ", ".join( + sorted(a["name"] for a in item.authors) if item.authors else "" + ) + if authors: + fields.append(authors) + + series = ", ".join( + sorted(s["title"] for s in item.series) if item.series else "" + ) + if series: + fields.append(series) + + fields.append(item.title) + return ": ".join(fields) + + library = await _get_library(session, client) + + if resolve_podcasts: + await library.resolve_podcats() + + books = await asyncio.gather( + *[_prepare_item(i) for i in library] + ) + + for i in sorted(books): + echo(i) diff --git a/src/audible_cli/cmds/cmd_manage.py b/src/audible_cli/cmds/cmd_manage.py index e132a43..03e4259 100644 --- a/src/audible_cli/cmds/cmd_manage.py +++ b/src/audible_cli/cmds/cmd_manage.py @@ -6,7 +6,7 @@ from audible import Authenticator from click import echo, secho from tabulate import tabulate -from ..config import pass_session +from ..decorators import pass_session from ..utils import build_auth_file @@ -45,13 +45,13 @@ def config_editor(session): def list_profiles(session): """List all profiles in the config file""" head = ["P", "Profile", "auth file", "cc"] - profiles = session.config.data.get("profile") + config = session.config + profiles = config.data.get("profile") data = [] for profile in profiles: - p = profiles.get(profile) - auth_file = p.get("auth_file") - country_code = p.get("country_code") + auth_file = config.get_profile_option(profile, "auth_file") + country_code = config.get_profile_option(profile, "country_code") is_primary = profile == session.config.primary_profile data.append( ["*" if is_primary else "", profile, auth_file, country_code]) @@ -92,7 +92,7 @@ def list_profiles(session): def add_profile(ctx, session, profile, country_code, auth_file, is_primary): """Adds a profile to config file""" if not (session.config.dirname / auth_file).exists(): - logger.error("Auth file doesn't exists.") + logger.error("Auth file doesn't exists") raise click.Abort() session.config.add_profile( @@ -167,7 +167,7 @@ def check_if_auth_file_not_exists(session, ctx, param, value): @click.option( "--external-login", is_flag=True, - help="Authenticate using a webbrowser." + help="Authenticate using a web browser." ) @click.option( "--with-username", diff --git a/src/audible_cli/cmds/cmd_quickstart.py b/src/audible_cli/cmds/cmd_quickstart.py index e8eee9e..a5bd8d3 100644 --- a/src/audible_cli/cmds/cmd_quickstart.py +++ b/src/audible_cli/cmds/cmd_quickstart.py @@ -1,13 +1,15 @@ import logging +import pathlib import sys -import audible import click from click import echo, secho, prompt from tabulate import tabulate -from ..config import Config, pass_session +from .. import __version__ +from ..config import ConfigFile from ..constants import CONFIG_FILE, DEFAULT_AUTH_FILE_EXTENSION +from ..decorators import pass_session from ..utils import build_auth_file @@ -31,10 +33,10 @@ def tabulate_summary(d: dict) -> str: return tabulate(data, head, tablefmt="pretty", colalign=("left", "left")) -def ask_user(config: Config): +def ask_user(config: ConfigFile): d = {} welcome_message = ( - f"Welcome to the audible {audible.__version__} quickstart utility.") + f"\nWelcome to the audible-cli {__version__} quickstart utility.") secho(welcome_message, bold=True) secho(len(welcome_message) * "=", bold=True) @@ -50,11 +52,11 @@ config dir. If the auth file doesn't exists, it will be created. In this case, an authentication to the audible server is necessary to register a new device. """ echo() - secho(intro, bold=True) + secho(intro) path = config.dirname.absolute() secho("Selected dir to proceed with:", bold=True) - echo(path.absolute()) + echo(path) echo() echo("Please enter values for the following settings (just press Enter " @@ -137,17 +139,14 @@ an authentication to the audible server is necessary to register a new device. @click.command("quickstart") -@click.pass_context @pass_session -def cli(session, ctx): - """Quicksetup audible""" - session._config = Config() - config = session.config - config._config_file = session.app_dir / CONFIG_FILE - if config.file_exists(): - m = f"Config file {config.filename} already exists. Quickstart will " \ +def cli(session): + """Quick setup audible""" + config_file: pathlib.Path = session.app_dir / CONFIG_FILE + config = ConfigFile(config_file, file_exists=False) + if config_file.is_file(): + m = f"Config file {config_file} already exists. Quickstart will " \ f"not overwrite existing files." - logger.error(m) raise click.Abort() @@ -157,16 +156,9 @@ def cli(session, ctx): echo(tabulate_summary(d)) click.confirm("Do you want to continue?", abort=True) - config.add_profile( - name=d.get("profile_name"), - auth_file=d.get("auth_file"), - country_code=d.get("country_code"), - is_primary=True, - write_config=False) - if "use_existing_auth_file" not in d: build_auth_file( - filename=config.dirname / d.get("auth_file"), + filename=session.app_dir / d.get("auth_file"), username=d.get("audible_username"), password=d.get("audible_password"), country_code=d.get("country_code"), @@ -175,4 +167,9 @@ def cli(session, ctx): with_username=d.get("with_username") ) - config.write_config() + config.add_profile( + name=d.get("profile_name"), + auth_file=d.get("auth_file"), + country_code=d.get("country_code"), + is_primary=True, + ) diff --git a/src/audible_cli/cmds/cmd_wishlist.py b/src/audible_cli/cmds/cmd_wishlist.py index 87eb8bd..a4351d2 100644 --- a/src/audible_cli/cmds/cmd_wishlist.py +++ b/src/audible_cli/cmds/cmd_wishlist.py @@ -1,70 +1,65 @@ import asyncio -import csv import json +import logging import pathlib -from typing import Union -import audible import click +import httpx +import questionary from click import echo -from ..config import pass_session -from ..models import Wishlist +from ..decorators import timeout_option, pass_client, wrap_async +from ..models import Catalog, Wishlist +from ..utils import export_to_csv -async def _get_wishlist(auth, **params): - timeout = params.get("timeout") - if timeout == 0: - timeout = None +logger = logging.getLogger("audible_cli.cmds.cmd_wishlist") - async with audible.AsyncClient(auth, timeout=timeout) as client: - wishlist = await Wishlist.from_api( - client, - response_groups=( - "contributors, media, price, product_attrs, product_desc, " - "product_extended_attrs, product_plan_details, product_plans, " - "rating, sample, sku, series, reviews, review_attrs, ws4v, " - "customer_rights, categories, category_ladders, claim_code_url" - ) +# audible api raises a 500 status error when to many requests +# where made to wishlist endpoint in short time +limits = httpx.Limits(max_keepalive_connections=1, max_connections=1) + + +async def _get_wishlist(client): + wishlist = await Wishlist.from_api( + client, + response_groups=( + "contributors, media, price, product_attrs, product_desc, " + "product_extended_attrs, product_plan_details, product_plans, " + "rating, sample, sku, series, reviews, review_attrs, ws4v, " + "customer_rights, categories, category_ladders, claim_code_url" ) + ) return wishlist -async def _list_wishlist(auth, **params): - wishlist = await _get_wishlist(auth, **params) - - books = [] - - for item in wishlist: - asin = item.asin - authors = ", ".join( - sorted(a["name"] for a in item.authors) if item.authors else "" - ) - series = ", ".join( - sorted(s["title"] for s in item.series) if item.series else "" - ) - title = item.title - books.append((asin, authors, series, title)) - - for asin, authors, series, title in sorted(books): - fields = [asin] - if authors: - fields.append(authors) - if series: - fields.append(series) - fields.append(title) - echo(": ".join(fields)) +@click.group("wishlist") +def cli(): + """interact with wishlist""" -def _prepare_wishlist_for_export(wishlist: dict): - keys_with_raw_values = ( - "asin", "title", "subtitle", "runtime_length_min", "is_finished", - "percent_complete", "release_date" - ) +@cli.command("export") +@click.option( + "--output", "-o", + type=click.Path(), + default=pathlib.Path().cwd() / r"wishlist.{format}", + show_default=True, + help="output file" +) +@timeout_option +@click.option( + "--format", "-f", + type=click.Choice(["tsv", "csv", "json"]), + default="tsv", + show_default=True, + help="Output format" +) +@pass_client +async def export_wishlist(client, **params): + """export wishlist""" - prepared_wishlist = [] - - for item in wishlist: + @wrap_async + def _prepare_item(item): data_row = {} for key in item: v = getattr(item, key) @@ -93,116 +88,234 @@ def _prepare_wishlist_for_export(wishlist: dict): for ladder in genre["ladder"]: genres.append(ladder["name"]) data_row["genres"] = ", ".join(genres) + return data_row - prepared_wishlist.append(data_row) - - prepared_wishlist.sort(key=lambda x: x["asin"]) - - return prepared_wishlist - - -def _export_to_csv( - file: pathlib.Path, - data: list, - headers: Union[list, tuple], - dialect: str -): - with file.open("w", encoding="utf-8", newline="") as f: - writer = csv.DictWriter(f, fieldnames=headers, dialect=dialect) - writer.writeheader() - - for i in data: - writer.writerow(i) - - -async def _export_wishlist(auth, **params): output_format = params.get("format") output_filename: pathlib.Path = params.get("output") if output_filename.suffix == r".{format}": suffix = "." + output_format output_filename = output_filename.with_suffix(suffix) - wishlist = await _get_wishlist(auth, **params) + wishlist = await _get_wishlist(client) - prepared_wishlist = _prepare_wishlist_for_export(wishlist) - - headers = ( - "asin", "title", "subtitle", "authors", "narrators", "series_title", - "series_sequence", "genres", "runtime_length_min", "is_finished", - "percent_complete", "rating", "num_ratings", "date_added", - "release_date", "cover_url" + keys_with_raw_values = ( + "asin", "title", "subtitle", "runtime_length_min", "is_finished", + "percent_complete", "release_date" ) + prepared_wishlist = await asyncio.gather( + *[_prepare_item(i) for i in wishlist] + ) + prepared_wishlist.sort(key=lambda x: x["asin"]) + if output_format in ("tsv", "csv"): - if output_format == csv: + if output_format == "csv": dialect = "excel" else: dialect = "excel-tab" - _export_to_csv(output_filename, prepared_wishlist, headers, dialect) - if output_format == "json": + headers = ( + "asin", "title", "subtitle", "authors", "narrators", "series_title", + "series_sequence", "genres", "runtime_length_min", "is_finished", + "percent_complete", "rating", "num_ratings", "date_added", + "release_date", "cover_url" + ) + + export_to_csv( + output_filename, prepared_wishlist, headers, dialect + ) + + elif output_format == "json": data = json.dumps(prepared_wishlist, indent=4) output_filename.write_text(data) -@click.group("wishlist") -def cli(): - """interact with wishlist""" - - -@cli.command("export") -@click.option( - "--output", "-o", - type=click.Path(), - default=pathlib.Path().cwd() / r"wishlist.{format}", - show_default=True, - help="output file" -) -@click.option( - "--timeout", "-t", - type=click.INT, - default=10, - show_default=True, - help=( - "Increase the timeout time if you got any TimeoutErrors. " - "Set to 0 to disable timeout." - ) -) -@click.option( - "--format", "-f", - type=click.Choice(["tsv", "csv", "json"]), - default="tsv", - show_default=True, - help="Output format" -) -@pass_session -def export_library(session, **params): - """export wishlist""" - loop = asyncio.get_event_loop() - try: - loop.run_until_complete(_export_wishlist(session.auth, **params)) - finally: - loop.run_until_complete(loop.shutdown_asyncgens()) - loop.close() - - @cli.command("list") -@click.option( - "--timeout", "-t", - type=click.INT, - default=10, - show_default=True, - help=( - "Increase the timeout time if you got any TimeoutErrors. " - "Set to 0 to disable timeout." - ) -) -@pass_session -def list_library(session, **params): +@timeout_option +@pass_client +async def list_wishlist(client): """list titles in wishlist""" - loop = asyncio.get_event_loop() - try: - loop.run_until_complete(_list_wishlist(session.auth, **params)) - finally: - loop.run_until_complete(loop.shutdown_asyncgens()) - loop.close() + + @wrap_async + def _prepare_item(item): + fields = [item.asin] + + authors = ", ".join( + sorted(a["name"] for a in item.authors) if item.authors else "" + ) + if authors: + fields.append(authors) + + series = ", ".join( + sorted(s["title"] for s in item.series) if item.series else "" + ) + if series: + fields.append(series) + + fields.append(item.title) + return ": ".join(fields) + + wishlist = await _get_wishlist(client) + + books = await asyncio.gather( + *[_prepare_item(i) for i in wishlist] + ) + + for i in sorted(books): + echo(i) + + +@cli.command("add") +@click.option( + "--asin", "-a", + multiple=True, + help="asin of the audiobook" +) +@click.option( + "--title", "-t", + multiple=True, + help="tile of the audiobook (partial search)" +) +@timeout_option +@pass_client(limits=limits) +async def add_wishlist(client, asin, title): + """add asin(s) to wishlist + + Run the command without any option for interactive mode. + """ + + async def add_asin(asin): + body = {"asin": asin} + r = await client.post("wishlist", body=body) + return r + + asin = list(asin) + title = list(title) + + if not asin and not title: + q = await questionary.select( + "Do you want to add an item by asin or title?", + choices=[ + questionary.Choice(title="by title", value="title"), + questionary.Choice(title="by asin", value="asin") + ] + ).unsafe_ask_async() + + if q == 'asin': + q = await questionary.text("Please enter the asin").unsafe_ask_async() + asin.append(q) + else: + q = await questionary.text("Please enter the title").unsafe_ask_async() + title.append(q) + + for t in title: + catalog = await Catalog.from_api( + client, + title=t, + num_results=50 + ) + + match = catalog.search_item_by_title(t) + full_match = [i for i in match if i[1] == 100] + + if match: + choices = [] + for i in full_match or match: + c = questionary.Choice(title=i[0].full_title, value=i[0].asin) + choices.append(c) + + answer = await questionary.checkbox( + f"Found the following matches for '{t}'. Which you want to add?", + choices=choices + ).unsafe_ask_async() + + if answer is not None: + [asin.append(i) for i in answer] + else: + logger.error( + f"Skip title {t}: Not found in library" + ) + + jobs = [add_asin(a) for a in asin] + await asyncio.gather(*jobs) + + wishlist = await _get_wishlist(client) + for a in asin: + if wishlist.has_asin(a): + item = wishlist.get_item_by_asin(a) + logger.info(f"{a} ({item.full_title}) added to wishlist") + else: + logger.error(f"{a} was not added to wishlist") + + +@cli.command("remove") +@click.option( + "--asin", "-a", + multiple=True, + help="asin of the audiobook" +) +@click.option( + "--title", "-t", + multiple=True, + help="tile of the audiobook (partial search)" +) +@timeout_option +@pass_client(limits=limits) +async def remove_wishlist(client, asin, title): + """remove asin(s) from wishlist + + Run the command without any option for interactive mode. + """ + + async def remove_asin(rasin): + r = await client.delete(f"wishlist/{rasin}") + item = wishlist.get_item_by_asin(rasin) + logger.info(f"{rasin} ({item.full_title}) removed from wishlist") + return r + + asin = list(asin) + wishlist = await _get_wishlist(client) + + if not asin and not title: + # interactive mode + choices = [] + for i in wishlist: + c = questionary.Choice(title=i.full_title, value=i.asin) + choices.append(c) + + asin = await questionary.checkbox( + "Select item(s) which you want to remove from whishlist", + choices=choices + ).unsafe_ask_async() + + for t in title: + match = wishlist.search_item_by_title(t) + full_match = [i for i in match if i[1] == 100] + + if match: + choices = [] + for i in full_match or match: + c = questionary.Choice(title=i[0].full_title, value=i[0].asin) + choices.append(c) + + answer = await questionary.checkbox( + f"Found the following matches for '{t}'. Which you want to remove?", + choices=choices + ).unsafe_ask_async() + + if answer is not None: + [asin.append(i) for i in answer] + else: + logger.error( + f"Skip title {t}: Not found in library" + ) + + if asin: + jobs = [] + for a in asin: + if wishlist.has_asin(a): + jobs.append(remove_asin(a)) + else: + logger.error(f"{a} not in wishlist") + + await asyncio.gather(*jobs) diff --git a/src/audible_cli/config.py b/src/audible_cli/config.py index c716f17..e9a778c 100644 --- a/src/audible_cli/config.py +++ b/src/audible_cli/config.py @@ -3,9 +3,10 @@ import os import pathlib from typing import Any, Dict, Optional, Union +import audible import click import toml -from audible import Authenticator +from audible import AsyncClient, Authenticator from audible.exceptions import FileEncryptionError from . import __version__ @@ -22,51 +23,116 @@ from .exceptions import AudibleCliException, ProfileAlreadyExists logger = logging.getLogger("audible_cli.config") -class Config: - """Holds the config file data and environment.""" +class ConfigFile: + """Presents an audible-cli configuration file - def __init__(self) -> None: - self._config_file: Optional[pathlib.Path] = None - self._config_data: Dict[str, Union[str, Dict]] = DEFAULT_CONFIG_DATA - self._current_profile: Optional[str] = None - self._is_read: bool = False + Instantiate a :class:`~audible_cli.config.ConfigFile` will load the file + content by default. To create a new config file, the ``file_exists`` + argument must be set to ``False``. + + Audible-cli configuration files are written in the toml markup language. + It has a main section named `APP` and sections for each profile named + `profile.`. + + Args: + filename: The file path to the config file + file_exists: If ``True``, the file must exist and the file content + is loaded. + """ + + def __init__( + self, + filename: Union[str, pathlib.Path], + file_exists: bool = True + ) -> None: + filename = pathlib.Path(filename).resolve() + config_data = DEFAULT_CONFIG_DATA.copy() + file_data = {} + + if file_exists: + if not filename.is_file(): + raise AudibleCliException( + f"Config file {click.format_filename(filename)} " + f"does not exists" + ) + file_data = toml.load(filename) + logger.debug( + f"Config loaded from " + f"{click.format_filename(filename, shorten=True)}" + ) + + config_data.update(file_data) + + self._config_file = filename + self._config_data = config_data @property - def filename(self) -> Optional[pathlib.Path]: + def filename(self) -> pathlib.Path: + """Returns the path to the config file""" return self._config_file - def file_exists(self) -> bool: - return self.filename.exists() - @property def dirname(self) -> pathlib.Path: + """Returns the path to the config file directory""" return self.filename.parent - def dir_exists(self) -> bool: - return self.dirname.exists() - - @property - def is_read(self) -> bool: - return self._is_read - @property def data(self) -> Dict[str, Union[str, Dict]]: + """Returns the configuration data""" return self._config_data @property def app_config(self) -> Dict[str, str]: - return self.data.get("APP", {}) - - @property - def profile_config(self) -> Dict[str, str]: - return self.data["profile"][self._current_profile] - - @property - def primary_profile(self) -> Optional[str]: - return self.app_config.get("primary_profile") + """Returns the configuration data for the APP section""" + return self.data["APP"] def has_profile(self, name: str) -> bool: - return name in self.data.get("profile", {}) + """Check if a profile with this name are in the configuration data + + Args: + name: The name of the profile + """ + return name in self.data["profile"] + + def get_profile(self, name: str) -> Dict[str, str]: + """Returns the configuration data for these profile name + + Args: + name: The name of the profile + """ + if not self.has_profile(name): + raise AudibleCliException(f"Profile {name} does not exists") + return self.data["profile"][name] + + @property + def primary_profile(self) -> str: + if "primary_profile" not in self.app_config: + raise AudibleCliException("No primary profile set in config") + return self.app_config["primary_profile"] + + def get_profile_option( + self, + profile: str, + option: str, + default: Optional[str] = None + ) -> str: + """Returns the value for an option for the given profile. + + Looks first, if an option is in the ``profile`` section. If not, it + searches for the option in the ``APP`` section. If not found, it + returns the ``default``. + + Args: + profile: The name of the profile + option: The name of the option to search for + default: The default value to return, if the option is not found + """ + profile = self.get_profile(profile) + if option in profile: + return profile[option] + if option in self.app_config: + return self.app_config[option] + return default def add_profile( self, @@ -74,12 +140,22 @@ class Config: auth_file: Union[str, pathlib.Path], country_code: str, is_primary: bool = False, - abort_on_existing_profile: bool = True, write_config: bool = True, **additional_options ) -> None: + """Adds a new profile to the config - if self.has_profile(name) and abort_on_existing_profile: + Args: + name: The name of the profile + auth_file: The name of the auth_file + country_code: The country code of the marketplace to use with + this profile + is_primary: If ``True``, this profile is set as primary in the + ``APP`` section + write_config: If ``True``, save the config to file + """ + + if self.has_profile(name): raise ProfileAlreadyExists(name) profile_data = { @@ -92,31 +168,41 @@ class Config: if is_primary: self.data["APP"]["primary_profile"] = name + logger.info(f"Profile {name} added to config") + if write_config: self.write_config() - def delete_profile(self, name: str) -> None: + def delete_profile(self, name: str, write_config: bool = True) -> None: + """Deletes a profile from config + + Args: + name: The name of the profile + write_config: If ``True``, save the config to file + + Note: + Does not delete the auth file. + """ + if not self.has_profile(name): + raise AudibleCliException(f"Profile {name} does not exists") + del self.data["profile"][name] - def read_config( - self, - filename: Optional[Union[str, pathlib.Path]] = None - ) -> None: - f = pathlib.Path(filename or self.filename).resolve() + logger.info(f"Profile {name} removed from config") - try: - self.data.update(toml.load(f)) - except FileNotFoundError: - message = f"Config file {click.format_filename(f)} not found" - raise AudibleCliException(message) - - self._config_file = f - self._is_read = True + if write_config: + self.write_config() def write_config( self, filename: Optional[Union[str, pathlib.Path]] = None ) -> None: + """Write the config data to file + + Args: + filename: If not ``None`` the config is written to these file path + instead of ``self.filename`` + """ f = pathlib.Path(filename or self.filename).resolve() if not f.parent.is_dir(): @@ -124,78 +210,99 @@ class Config: toml.dump(self.data, f.open("w")) + click_f = click.format_filename(f, shorten=True) + logger.info(f"Config written to {click_f}") + class Session: - """Holds the settings for the current session.""" + """Holds the settings for the current session""" def __init__(self) -> None: - self._auth: Optional[Authenticator] = None - self._config: Optional[Config] = None + self._auths: Dict[str, Authenticator] = {} + self._config: Optional[CONFIG_FILE] = None self._params: Dict[str, Any] = {} - self._app_dir = get_app_dir() - self._plugin_dir = get_plugin_dir() + self._app_dir: pathlib.Path = get_app_dir() + self._plugin_dir: pathlib.Path = get_plugin_dir() + logger.debug(f"Audible-cli version: {__version__}") logger.debug(f"App dir: {click.format_filename(self.app_dir)}") logger.debug(f"Plugin dir: {click.format_filename(self.plugin_dir)}") @property def params(self): + """Returns the parameter of the session + + Parameter are usually added using the ``add_param_to_session`` + callback on a click option. This way an option from a parent command + can be accessed from his subcommands. + """ return self._params @property def app_dir(self): + """Returns the path of the app dir""" return self._app_dir @property def plugin_dir(self): + """Returns the path of the plugin dir""" return self._plugin_dir @property def config(self): + """Returns the ConfigFile for this session""" if self._config is None: conf_file = self.app_dir / CONFIG_FILE - self._config = Config() - logger.debug( - f"Load config from file: " - f"{click.format_filename(conf_file, shorten=True)}" - ) - self._config.read_config(conf_file) - - name = self.params.get("profile") or self.config.primary_profile - logger.debug(f"Selected profile: {name}") - - if name is None: - message = ( - "No profile provided and primary profile not set " - "properly in config." - ) - try: - ctx = click.get_current_context() - ctx.fail(message) - except RuntimeError: - raise KeyError(message) - - if not self.config.has_profile(name): - message = "Provided profile not found in config." - try: - ctx = click.get_current_context() - ctx.fail(message) - except RuntimeError: - raise UserWarning(message) - - self.config._current_profile = name + self._config = ConfigFile(conf_file) return self._config - def _set_auth(self): - profile = self.config.profile_config - auth_file = self.config.dirname / profile["auth_file"] - country_code = profile["country_code"] - password = self.params.get("password") + @property + def selected_profile(self): + """Returns the selected config profile name for this session + + The `profile` to use must be set using the ``add_param_to_session`` + callback of a click option. Otherwise, the primary profile from the + config is used. + """ + profile = self.params.get("profile") or self.config.primary_profile + if profile is None: + message = ( + "No profile provided and primary profile not set " + "properly in config." + ) + raise AudibleCliException(message) + return profile + + def get_auth_for_profile( + self, + profile: str, + password: Optional[str] = None + ) -> audible.Authenticator: + """Returns an Authenticator for a profile + + If an Authenticator for this profile is already loaded, it will + return the Authenticator without reloading it. This way a session can + hold multiple Authenticators for different profiles. Commands can use + this to make API requests for more than one profile. + + Args: + profile: The name of the profile + password: The password of the auth file + """ + if profile in self._auths: + return self._auths[profile] + + if not self.config.has_profile(profile): + message = "Provided profile not found in config." + raise AudibleCliException(message) + + auth_file = self.config.get_profile_option(profile, "auth_file") + country_code = self.config.get_profile_option(profile, "country_code") while True: try: - self._auth = Authenticator.from_file( - filename=auth_file, + auth = Authenticator.from_file( + filename=self.config.dirname / auth_file, password=password, locale=country_code) break @@ -204,20 +311,39 @@ class Session: "Auth file is encrypted but no/wrong password is provided" ) password = click.prompt( - "Please enter the password (or enter to exit)", + "Please enter the auth-file password (or enter to exit)", hide_input=True, default="") if len(password) == 0: raise click.Abort() + click_f = click.format_filename(auth_file, shorten=True) + logger.debug(f"Auth file {click_f} for profile {profile} loaded.") + + self._auths[profile] = auth + return auth + @property def auth(self): - if self._auth is None: - self._set_auth() - return self._auth + """Returns the Authenticator for the selected profile""" + profile = self.selected_profile + password = self.params.get("password") + return self.get_auth_for_profile(profile, password) + def get_client_for_profile( + self, + profile: str, + password: Optional[str] = None, + **kwargs + ) -> AsyncClient: + auth = self.get_auth_for_profile(profile, password) + kwargs.setdefault("timeout", self.params.get("timeout", 5)) + return AsyncClient(auth=auth, **kwargs) -pass_session = click.make_pass_decorator(Session, ensure=True) + def get_client(self, **kwargs) -> AsyncClient: + profile = self.selected_profile + password = self.params.get("password") + return self.get_client_for_profile(profile, password, **kwargs) def get_app_dir() -> pathlib.Path: @@ -230,10 +356,3 @@ def get_app_dir() -> pathlib.Path: def get_plugin_dir() -> pathlib.Path: plugin_dir = os.getenv(PLUGIN_DIR_ENV) or (get_app_dir() / PLUGIN_PATH) return pathlib.Path(plugin_dir).resolve() - - -def add_param_to_session(ctx: click.Context, param, value): - """Add a parameter to :class:`Session` `param` attribute""" - session = ctx.ensure_object(Session) - session.params[param.name] = value - return value diff --git a/src/audible_cli/constants.py b/src/audible_cli/constants.py index aec325c..b9954a6 100644 --- a/src/audible_cli/constants.py +++ b/src/audible_cli/constants.py @@ -1,3 +1,6 @@ +from typing import Dict + + APP_NAME: str = "Audible" CONFIG_FILE: str = "config.toml" CONFIG_DIR_ENV: str = "AUDIBLE_CONFIG_DIR" @@ -6,10 +9,10 @@ PLUGIN_DIR_ENV: str = "AUDIBLE_PLUGIN_DIR" PLUGIN_ENTRY_POINT: str = "audible.cli_plugins" DEFAULT_AUTH_FILE_EXTENSION: str = "json" DEFAULT_AUTH_FILE_ENCRYPTION: str = "json" -DEFAULT_CONFIG_DATA = { +DEFAULT_CONFIG_DATA: Dict[str, str] = { "title": "Audible Config File", "APP": {}, "profile": {} } -CODEC_HIGH_QUALITY = "AAX_44_128" -CODEC_NORMAL_QUALITY = "AAX_44_64" +CODEC_HIGH_QUALITY: str = "AAX_44_128" +CODEC_NORMAL_QUALITY: str = "AAX_44_64" diff --git a/src/audible_cli/decorators.py b/src/audible_cli/decorators.py new file mode 100644 index 0000000..9e39c3a --- /dev/null +++ b/src/audible_cli/decorators.py @@ -0,0 +1,238 @@ +import asyncio +import logging +from functools import partial, wraps + +import click +import httpx +from packaging.version import parse + +from .config import Session +from ._logging import _normalize_logger +from . import __version__ + + +logger = logging.getLogger("audible_cli.options") + +pass_session = click.make_pass_decorator(Session, ensure=True) + + +def run_async(f): + @wraps(f) + def wrapper(*args, **kwargs): + if hasattr(asyncio, "run"): + logger.debug("Using asyncio.run ...") + return asyncio.run(f(*args, ** kwargs)) + else: + logger.debug("Using asyncio.run_until_complete ...") + loop = asyncio.get_event_loop() + + if loop.is_closed(): + loop = asyncio.new_event_loop() + + try: + return loop.run_until_complete(f(*args, ** kwargs)) + finally: + loop.run_until_complete(loop.shutdown_asyncgens()) + loop.close() + return wrapper + + +def wrap_async(f): + """Wrap a synchronous function and runs them in an executor""" + + @wraps(f) + async def wrapper(*args, loop=None, executor=None, **kwargs): + if loop is None: + loop = asyncio.get_event_loop() + + partial_func = partial(f, *args, **kwargs) + return await loop.run_in_executor(executor, partial_func) + + return wrapper + + +def pass_client(func=None, **client_kwargs): + def coro(f): + @wraps(f) + @pass_session + @run_async + async def wrapper(session, *args, **kwargs): + client = session.get_client(**client_kwargs) + async with client.session: + return await f(*args, client, **kwargs) + return wrapper + + if callable(func): + return coro(func) + + return coro + + +def add_param_to_session(ctx: click.Context, param, value): + """Add a parameter to :class:`Session` `param` attribute + + This is usually used as a callback for a click option + """ + session = ctx.ensure_object(Session) + session.params[param.name] = value + return value + + +def version_option(func=None, **kwargs): + def callback(ctx, param, value): + if not value or ctx.resilient_parsing: + return + + message = f"audible-cli, version {__version__}" + click.echo(message, color=ctx.color, nl=False) + + url = "https://api.github.com/repos/mkb79/audible-cli/releases/latest" + headers = {"Accept": "application/vnd.github.v3+json"} + logger.debug(f"Requesting Github API for latest release information") + try: + response = httpx.get(url, headers=headers, follow_redirects=True) + response.raise_for_status() + except Exception as e: + logger.error(e) + click.Abort() + + content = response.json() + + current_version = parse(__version__) + latest_version = parse(content["tag_name"]) + + html_url = content["html_url"] + if latest_version > current_version: + click.echo( + f" (update available)\nVisit {html_url} " + f"for information about the new release.", + color=ctx.color + ) + else: + click.echo(" (up-to-date)", color=ctx.color) + + ctx.exit() + + kwargs.setdefault("is_flag", True) + kwargs.setdefault("expose_value", False) + kwargs.setdefault("is_eager", True) + kwargs.setdefault("help", "Show the version and exit.") + kwargs["callback"] = callback + + option = click.option("--version", **kwargs) + + if callable(func): + return option(func) + + return option + + +def profile_option(func=None, **kwargs): + kwargs.setdefault("callback", add_param_to_session) + kwargs.setdefault("expose_value", False) + kwargs.setdefault( + "help", + "The profile to use instead primary profile (case sensitive!)." + ) + + option = click.option("--profile", "-P", **kwargs) + + if callable(func): + return option(func) + + return option + + +def password_option(func=None, **kwargs): + kwargs.setdefault("callback", add_param_to_session) + kwargs.setdefault("expose_value", False) + kwargs.setdefault("help", "The password for the profile auth file.") + + option = click.option("--password", "-p", **kwargs) + + if callable(func): + return option(func) + + return option + + +def verbosity_option(func=None, *, cli_logger=None, **kwargs): + """A decorator that adds a `--verbosity, -v` option to the decorated + command. + Keyword arguments are passed to + the underlying ``click.option`` decorator. + """ + def callback(ctx, param, value): + x = getattr(logging, value.upper(), None) + if x is None: + raise click.BadParameter( + f"Must be CRITICAL, ERROR, WARNING, INFO or DEBUG, " + f"not {value}" + ) + cli_logger.setLevel(x) + + kwargs.setdefault("default", "INFO") + kwargs.setdefault("metavar", "LVL") + kwargs.setdefault("expose_value", False) + kwargs.setdefault( + "help", "Either CRITICAL, ERROR, WARNING, " + "INFO or DEBUG. [default: INFO]" + ) + kwargs.setdefault("is_eager", True) + kwargs.setdefault("callback", callback) + + cli_logger = _normalize_logger(cli_logger) + + option = click.option("--verbosity", "-v", **kwargs) + + if callable(func): + return option(func) + + return option + + +def timeout_option(func=None, **kwargs): + def callback(ctx: click.Context, param, value): + if value == 0: + value = None + session = ctx.ensure_object(Session) + session.params[param.name] = value + return value + + kwargs.setdefault("type", click.INT) + kwargs.setdefault("default", 10) + kwargs.setdefault("show_default", True) + kwargs.setdefault( + "help", ("Increase the timeout time if you got any TimeoutErrors. " + "Set to 0 to disable timeout.") + ) + kwargs.setdefault("callback", callback) + kwargs.setdefault("expose_value", False) + + option = click.option("--timeout", **kwargs) + + if callable(func): + return option(func) + + return option + + +def bunch_size_option(func=None, **kwargs): + kwargs.setdefault("type", click.IntRange(10, 1000)) + kwargs.setdefault("default", 1000) + kwargs.setdefault("show_default", True) + kwargs.setdefault( + "help", ("How many library items should be requested per request. A " + "lower size results in more requests to get the full library. " + "A higher size can result in a TimeOutError on low internet " + "connections.") + ) + kwargs.setdefault("callback", add_param_to_session) + kwargs.setdefault("expose_value", False) + + option = click.option("--bunch-size", **kwargs) + + if callable(func): + return option(func) + + return option diff --git a/src/audible_cli/exceptions.py b/src/audible_cli/exceptions.py index 0a2f665..6d06840 100644 --- a/src/audible_cli/exceptions.py +++ b/src/audible_cli/exceptions.py @@ -9,6 +9,10 @@ class NotFoundError(AudibleCliException): """Raised if an item is not found""" +class NotDownloadableAsAAX(AudibleCliException): + """Raised if an item is not downloadable in aax format""" + + class FileDoesNotExists(AudibleCliException): """Raised if a file does not exist""" diff --git a/src/audible_cli/models.py b/src/audible_cli/models.py index 0bebce1..75bb646 100644 --- a/src/audible_cli/models.py +++ b/src/audible_cli/models.py @@ -2,16 +2,17 @@ import asyncio import logging import string import unicodedata +from math import ceil from typing import List, Optional, Union import audible import httpx from audible.aescipher import decrypt_voucher_from_licenserequest - +from audible.client import convert_response_content from .constants import CODEC_HIGH_QUALITY, CODEC_NORMAL_QUALITY -from .exceptions import AudibleCliException -from .utils import LongestSubString +from .exceptions import AudibleCliException, NotDownloadableAsAAX +from .utils import full_response_callback, LongestSubString logger = logging.getLogger("audible_cli.models") @@ -72,6 +73,27 @@ class BaseItem: return slug_title + def create_base_filename(self, mode: str): + supported_modes = ("ascii", "asin_ascii", "unicode", "asin_unicode") + if mode not in supported_modes: + raise AudibleCliException( + f"Unsupported mode {mode} for name creation" + ) + + if "ascii" in mode: + base_filename = self.full_title_slugify + + elif "unicode" in mode: + base_filename = unicodedata.normalize("NFKD", self.full_title) + + else: + base_filename = self.asin + + if "asin" in mode: + base_filename = self.asin + "_" + base_filename + + return base_filename + def substring_in_title_accuracy(self, substring): match = LongestSubString(substring, self.full_title) return round(match.percentage, 2) @@ -153,7 +175,7 @@ class LibraryItem(BaseItem): """ # Only items with content_delivery_type - # MultiPartBook or Periodical have child elemts + # MultiPartBook or Periodical have child elements if not self.has_children: return @@ -189,23 +211,22 @@ class LibraryItem(BaseItem): def is_downloadable(self): # customer_rights must be in response_groups if self.customer_rights is not None: - if not self.customer_rights["is_consumable_offline"]: - return False - else: + if self.customer_rights["is_consumable_offline"]: return True + return False async def get_aax_url_old(self, quality: str = "high"): if not self.is_downloadable(): raise AudibleCliException( - f"{self.full_title} is not downloadable. Skip item." + f"{self.full_title} is not downloadable." ) codec, codec_name = self._get_codec(quality) - if codec is None: - raise AudibleCliException( + if codec is None or self.is_ayce: + raise NotDownloadableAsAAX( f"{self.full_title} is not downloadable in AAX format" ) - + url = ( "https://cde-ta-g7g.amazon.com/FionaCDEServiceEngine/" "FSDownloadContent" @@ -238,8 +259,8 @@ class LibraryItem(BaseItem): ) codec, codec_name = self._get_codec(quality) - if codec is None: - raise AudibleCliException( + if codec is None or self.is_ayce: + raise NotDownloadableAsAAX( f"{self.full_title} is not downloadable in AAX format" ) @@ -252,6 +273,11 @@ class LibraryItem(BaseItem): return httpx.URL(url, params=params), codec_name async def get_aaxc_url(self, quality: str = "high"): + if not self.is_downloadable(): + raise AudibleCliException( + f"{self.full_title} is not downloadable." + ) + assert quality in ("best", "high", "normal",) body = { @@ -292,6 +318,17 @@ class LibraryItem(BaseItem): return metadata + async def get_annotations(self): + url = f"https://cde-ta-g7g.amazon.com/FionaCDEServiceEngine/sidecar" + params = { + "type": "AUDI", + "key": self.asin + } + + annotations = await self._client.get(url, params=params) + + return annotations + class WishlistItem(BaseItem): pass @@ -315,9 +352,13 @@ class BaseList: def _prepare_data(self, data: Union[dict, list]) -> Union[dict, list]: return data + @property + def data(self): + return self._data + def get_item_by_asin(self, asin): try: - return next(i for i in self._data if asin in i.asin) + return next(i for i in self._data if asin == i.asin) except StopIteration: return None @@ -354,6 +395,7 @@ class Library(BaseList): async def from_api( cls, api_client: audible.AsyncClient, + include_total_count_header: bool = False, **request_params ): if "response_groups" not in request_params: @@ -369,8 +411,18 @@ class Library(BaseList): "periodicals, provided_review, product_details" ) - resp = await api_client.get("library", **request_params) - return cls(resp, api_client=api_client) + resp: httpx.Response = await api_client.get( + "library", + response_callback=full_response_callback, + **request_params + ) + resp_content = convert_response_content(resp) + total_count_header = resp.headers.get("total-count") + cls_instance = cls(resp_content, api_client=api_client) + + if include_total_count_header: + return cls_instance, total_count_header + return cls_instance @classmethod async def from_api_full_sync( @@ -379,33 +431,42 @@ class Library(BaseList): bunch_size: int = 1000, **request_params ) -> "Library": - request_params["page"] = 1 + request_params.pop("page", None) request_params["num_results"] = bunch_size - library = [] - while True: - resp = await cls.from_api(api_client, params=request_params) - items = resp._data - len_items = len(items) - library.extend(items) - if len_items < bunch_size: - break - request_params["page"] += 1 + library, total_count = await cls.from_api( + api_client, + page=1, + params=request_params, + include_total_count_header=True, + ) + pages = ceil(int(total_count) / bunch_size) + if pages == 1: + return library - resp._data = library - return resp + additional_pages = [] + for page in range(2, pages+1): + additional_pages.append( + cls.from_api( + api_client, + page=page, + params=request_params, + ) + ) + + additional_pages = await asyncio.gather(*additional_pages) + + for p in additional_pages: + library.data.extend(p.data) + + return library async def resolve_podcats(self): - podcasts = [] - for i in self: - if i.is_parent_podcast(): - podcasts.append(i) - podcast_items = await asyncio.gather( - *[i.get_child_items() for i in podcasts] + *[i.get_child_items() for i in self if i.is_parent_podcast()] ) for i in podcast_items: - self._data.extend(i._data) + self.data.extend(i.data) class Catalog(BaseList): @@ -465,16 +526,11 @@ class Catalog(BaseList): return cls(resp, api_client=api_client) async def resolve_podcats(self): - podcasts = [] - for i in self: - if i.is_parent_podcast(): - podcasts.append(i) - podcast_items = await asyncio.gather( - *[i.get_child_items() for i in podcasts] + *[i.get_child_items() for i in self if i.is_parent_podcast()] ) for i in podcast_items: - self._data.extend(i._data) + self.data.extend(i.data) class Wishlist(BaseList): diff --git a/src/audible_cli/utils.py b/src/audible_cli/utils.py index e107ae6..3a5425d 100644 --- a/src/audible_cli/utils.py +++ b/src/audible_cli/utils.py @@ -1,9 +1,8 @@ -import asyncio +import csv import io import logging import pathlib from difflib import SequenceMatcher -from functools import partial, wraps from typing import List, Optional, Union import aiofiles @@ -12,6 +11,7 @@ import httpx import tqdm from PIL import Image from audible import Authenticator +from audible.client import raise_for_status from audible.login import default_login_url_callback from click import echo, secho, prompt @@ -32,7 +32,7 @@ def prompt_captcha_callback(captcha_url: str) -> str: img.show() else: echo( - "Please open the following url with a webbrowser " + "Please open the following url with a web browser " "to get the captcha:" ) echo(captcha_url) @@ -60,6 +60,11 @@ def prompt_external_callback(url: str) -> str: return default_login_url_callback(url) +def full_response_callback(resp: httpx.Response) -> httpx.Response: + raise_for_status(resp) + return resp + + def build_auth_file( filename: Union[str, pathlib.Path], username: Optional[str], @@ -142,17 +147,6 @@ def asin_in_library(asin, library): return False -def wrap_async(func): - @wraps(func) - async def run(*args, loop=None, executor=None, **kwargs): - if loop is None: - loop = asyncio.get_event_loop() - pfunc = partial(func, *args, **kwargs) - return await loop.run_in_executor(executor, pfunc) - - return run - - class DummyProgressBar: def __enter__(self): return self @@ -256,8 +250,7 @@ class Downloader: file.rename(file.with_suffix(f"{file.suffix}.old.{i}")) tmp_file.rename(file) logger.info( - f"File {self._file} downloaded to {self._file.parent} " - f"in {elapsed}." + f"File {self._file} downloaded in {elapsed}." ) return True @@ -300,3 +293,17 @@ class Downloader: await self._load() finally: self._remove_tmp_file() + + +def export_to_csv( + file: pathlib.Path, + data: list, + headers: Union[list, tuple], + dialect: str +) -> None: + with file.open("w", encoding="utf-8", newline="") as f: + writer = csv.DictWriter(f, fieldnames=headers, dialect=dialect) + writer.writeheader() + + for i in data: + writer.writerow(i) diff --git a/utils/code_completion/README.md b/utils/code_completion/README.md index 53025cc..0feda2c 100644 --- a/utils/code_completion/README.md +++ b/utils/code_completion/README.md @@ -5,5 +5,5 @@ Tab completion can be provided for commands, options and choice values. Bash, Zsh and Fish are supported. Simply copy the activation script for your shell from this folder to your machine. -Read [here](https://click.palletsprojects.com/en/7.x/bashcomplete/#activation-script) +Read [here](https://click.palletsprojects.com/en/8.0.x/shell-completion/) how-to activate the script in your shell. diff --git a/utils/code_completion/audible-complete-bash.sh b/utils/code_completion/audible-complete-bash.sh index 14aae91..06725a9 100644 --- a/utils/code_completion/audible-complete-bash.sh +++ b/utils/code_completion/audible-complete-bash.sh @@ -1,2 +1,2 @@ -_AUDIBLE_COMPLETE=source_bash audible -_AUDIBLE_QUICKSTART_COMPLETE=source_bash audible-quickstart +_AUDIBLE_COMPLETE=bash_source audible +_AUDIBLE_QUICKSTART_COMPLETE=bash_source audible-quickstart diff --git a/utils/code_completion/audible-complete-zsh-fish.sh b/utils/code_completion/audible-complete-zsh-fish.sh index bc42490..edb0812 100644 --- a/utils/code_completion/audible-complete-zsh-fish.sh +++ b/utils/code_completion/audible-complete-zsh-fish.sh @@ -1,2 +1,2 @@ -_AUDIBLE_COMPLETE=source_zsh audible -_AUDIBLE_QUICKSTART_COMPLETE=source_zsh audible-quickstart +_AUDIBLE_COMPLETE=zsh_source audible +_AUDIBLE_QUICKSTART_COMPLETE=zsh_source audible-quickstart diff --git a/utils/update_chapter_titles.py b/utils/update_chapter_titles.py index 15c7575..ba6da27 100644 --- a/utils/update_chapter_titles.py +++ b/utils/update_chapter_titles.py @@ -1,6 +1,6 @@ """ This script replaces the chapter titles from a ffmetadata file with the one -extracted from a api metadata/voucher file +extracted from an API metadata/voucher file Example: From 6061615b23573748c13107739389beb465d5f38b Mon Sep 17 00:00:00 2001 From: mkb79 Date: Wed, 1 Jun 2022 16:08:26 +0200 Subject: [PATCH 05/79] release v0.2.0 --- CHANGELOG.md | 4 ++++ src/audible_cli/_version.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 50beed5..bcf6001 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## Unreleased +- + +## [0.2.0] - 2022-06-01 + ### Added - `--aax-fallback` option to `download` command to download books in aax format and fallback to aaxc, if the book is not available as aax diff --git a/src/audible_cli/_version.py b/src/audible_cli/_version.py index 0590716..45136b1 100644 --- a/src/audible_cli/_version.py +++ b/src/audible_cli/_version.py @@ -1,7 +1,7 @@ __title__ = "audible-cli" __description__ = "Command line interface (cli) for the audible package." __url__ = "https://github.com/mkb79/audible-cli" -__version__ = "0.2.b1" +__version__ = "0.2.0" __author__ = "mkb79" __author_email__ = "mkb79@hackitall.de" __license__ = "AGPL" From 755240d13226d619195d83899587bb73c3248960 Mon Sep 17 00:00:00 2001 From: mkb79 Date: Sun, 5 Jun 2022 21:19:48 +0200 Subject: [PATCH 06/79] licenserequest (voucher) will not request chapter by default #92 --- CHANGELOG.md | 8 +++++++- src/audible_cli/models.py | 33 +++++++++++++++++++++++---------- 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bcf6001..9439b45 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## Unreleased -- +### Added + +- moved licenserequest part from `models.LibraryItem.get_aaxc_url` to its own `models.LibraryItem.get_license` function + +### Changed + +- by default a licenserequest (voucher) will not include chapter information for now ## [0.2.0] - 2022-06-01 diff --git a/src/audible_cli/models.py b/src/audible_cli/models.py index 75bb646..370c415 100644 --- a/src/audible_cli/models.py +++ b/src/audible_cli/models.py @@ -272,36 +272,49 @@ class LibraryItem(BaseItem): } return httpx.URL(url, params=params), codec_name - async def get_aaxc_url(self, quality: str = "high"): + async def get_aaxc_url( + self, + quality: str = "high", + license_response_groups: Optional[str] = None + ): if not self.is_downloadable(): raise AudibleCliException( f"{self.full_title} is not downloadable." ) + lr = await self.get_license(quality, license_response_groups) + + content_metadata = lr["content_license"]["content_metadata"] + url = httpx.URL(content_metadata["content_url"]["offline_url"]) + codec = content_metadata["content_reference"]["content_format"] + + return url, codec, lr + + async def get_license( + self, + quality: str = "high", + response_groups: Optional[str] = None + ): assert quality in ("best", "high", "normal",) + if response_groups is None: + response_groups = "last_position_heard, pdf_url, content_reference" + body = { "supported_drm_types": ["Mpeg", "Adrm"], "quality": "Extreme" if quality in ("best", "high") else "Normal", "consumption_type": "Download", - "response_groups": ( - "last_position_heard, pdf_url, content_reference, chapter_info" - ) + "response_groups": response_groups } lr = await self._client.post( f"content/{self.asin}/licenserequest", body=body ) - - content_metadata = lr["content_license"]["content_metadata"] - url = httpx.URL(content_metadata["content_url"]["offline_url"]) - codec = content_metadata["content_reference"]["content_format"] - voucher = decrypt_voucher_from_licenserequest(self._client.auth, lr) lr["content_license"]["license_response"] = voucher - return url, codec, lr + return lr async def get_content_metadata(self, quality: str = "high"): assert quality in ("best", "high", "normal",) From a785ff50b967b16b2f1f14d71d72c03923efea42 Mon Sep 17 00:00:00 2001 From: Johan Pretorius Date: Sun, 19 Jun 2022 08:19:09 +0200 Subject: [PATCH 07/79] Added explanation in README.md for creating a second profile (#94) --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 28fe808..324501d 100644 --- a/README.md +++ b/README.md @@ -162,6 +162,14 @@ Use the `audible-quickstart` or `audible quickstart` command in your shell to create your first config, profile and auth file. `audible-quickstart` runs on the interactive mode, so you have to answer multiple questions to finish. +If you have used `audible quickstart` and want to add a second profile, you need to first create a new authfile and then update your config.toml file. + +So the correct order is: + + 1. add a new auth file using your second account using `audible manage auth-file add` + 2. add a new profile to your config and use the second auth file using `audible manage profile add` + + ## Commands Call `audible -h` to show the help and a list of all available subcommands. You can show the help for each subcommand like so: `audible -h`. If a subcommand has another subcommands, you csn do it the same way. From 57694609830ae984707f1df14e44491a0149e509 Mon Sep 17 00:00:00 2001 From: Billie Thompson <133327+PurpleBooth@users.noreply.github.com> Date: Wed, 22 Jun 2022 05:38:41 +0100 Subject: [PATCH 08/79] Allow book tiltes with hyphens (#96) Currently we take the first value before the hyphen, unfortunately books sometimes have hyphens in the titles meaning that the command will fail. A simple fix for this is to limit the number of splits that we do once we have found the end delimiter. --- plugin_cmds/cmd_remove-encryption.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin_cmds/cmd_remove-encryption.py b/plugin_cmds/cmd_remove-encryption.py index df79a21..2cc06dc 100644 --- a/plugin_cmds/cmd_remove-encryption.py +++ b/plugin_cmds/cmd_remove-encryption.py @@ -184,7 +184,7 @@ def decrypt_aax(files, activation_bytes, rebuild_chapters): outfile = file.with_suffix(".m4b") metafile = file.with_suffix(".meta") metafile_new = file.with_suffix(".new.meta") - base_filename = file.stem.rsplit("-")[0] + base_filename = file.stem.rsplit("-", 1)[0] chapters = file.with_name(base_filename + "-chapters").with_suffix(".json") apimeta = json.loads(chapters.read_text()) From 0fef098bd7ff63d3d34685027bf359d1224decee Mon Sep 17 00:00:00 2001 From: Billie Thompson <133327+PurpleBooth@users.noreply.github.com> Date: Wed, 22 Jun 2022 05:45:22 +0100 Subject: [PATCH 09/79] Currently paths with dots will break the decrypt this correct that (#97) This problem is caused by "with_suffix" replacing the suffix and not just adding a new suffix on the end. We can fix this by adding the "json" to the with_name, so it doesn't remove everything after the first ".". Co-authored-by: mkb79 --- plugin_cmds/cmd_remove-encryption.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin_cmds/cmd_remove-encryption.py b/plugin_cmds/cmd_remove-encryption.py index 2cc06dc..cc359e2 100644 --- a/plugin_cmds/cmd_remove-encryption.py +++ b/plugin_cmds/cmd_remove-encryption.py @@ -185,7 +185,7 @@ def decrypt_aax(files, activation_bytes, rebuild_chapters): metafile = file.with_suffix(".meta") metafile_new = file.with_suffix(".new.meta") base_filename = file.stem.rsplit("-", 1)[0] - chapters = file.with_name(base_filename + "-chapters").with_suffix(".json") + chapters = file.with_name(base_filename + "-chapters.json") apimeta = json.loads(chapters.read_text()) if outfile.exists(): From 8582396b03e41867d82a215262757712e1067f5d Mon Sep 17 00:00:00 2001 From: Billie Thompson <133327+PurpleBooth@users.noreply.github.com> Date: Thu, 23 Jun 2022 12:03:04 +0100 Subject: [PATCH 10/79] Support nested chapters (#102) Relates-to: #99 --- plugin_cmds/cmd_remove-encryption.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/plugin_cmds/cmd_remove-encryption.py b/plugin_cmds/cmd_remove-encryption.py index cc359e2..6e1932f 100644 --- a/plugin_cmds/cmd_remove-encryption.py +++ b/plugin_cmds/cmd_remove-encryption.py @@ -1,6 +1,6 @@ """ This is a proof-of-concept and for testing purposes only. No error handling. -Need further work. Some options does not work or options are missing. +Need further work. Some options do not work or options are missing. Needs at least ffmpeg 4.4 """ @@ -11,6 +11,7 @@ import operator import pathlib import re import subprocess +from functools import reduce from shutil import which import click @@ -34,8 +35,13 @@ class ApiMeta: return len(self.get_chapters()) def get_chapters(self): - return self._meta_parsed["content_metadata"]["chapter_info"][ - "chapters"] + def extract_chapters(initial, current): + if "chapters" in current: + return initial + [current] + current['chapters'] + else: + return initial + [current] + + return list(reduce(extract_chapters, self._meta_parsed["content_metadata"]["chapter_info"]["chapters"], [])) def get_intro_duration_ms(self): return self._meta_parsed["content_metadata"]["chapter_info"][ From 3e3c679e69f59a94d39e871c305e3efc4fbc79c6 Mon Sep 17 00:00:00 2001 From: Billie Thompson <133327+PurpleBooth@users.noreply.github.com> Date: Thu, 23 Jun 2022 12:04:06 +0100 Subject: [PATCH 11/79] If the is no title fallback to an empty string (#98) in some instances full title can be None, and unicodedata.normalize requires a string. A better approach might be to work out why title is None, but this fixes that issue for now. --- src/audible_cli/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/audible_cli/models.py b/src/audible_cli/models.py index 75bb646..ad53ec4 100644 --- a/src/audible_cli/models.py +++ b/src/audible_cli/models.py @@ -61,7 +61,7 @@ class BaseItem: @property def full_title_slugify(self): valid_chars = "-_.() " + string.ascii_letters + string.digits - cleaned_title = unicodedata.normalize("NFKD", self.full_title) + cleaned_title = unicodedata.normalize("NFKD", self.full_title or "") cleaned_title = cleaned_title.encode("ASCII", "ignore") cleaned_title = cleaned_title.replace(b" ", b"_") slug_title = "".join( @@ -84,7 +84,7 @@ class BaseItem: base_filename = self.full_title_slugify elif "unicode" in mode: - base_filename = unicodedata.normalize("NFKD", self.full_title) + base_filename = unicodedata.normalize("NFKD", self.full_title or "") else: base_filename = self.asin From 289a5ce8d8e51f90c6593f6bf67cbaf907223649 Mon Sep 17 00:00:00 2001 From: mkb79 Date: Fri, 1 Jul 2022 07:35:04 +0200 Subject: [PATCH 12/79] Fix: download command continues execution after error #104 --- src/audible_cli/cmds/cmd_download.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/audible_cli/cmds/cmd_download.py b/src/audible_cli/cmds/cmd_download.py index 6c2f4c9..d0b4ac1 100644 --- a/src/audible_cli/cmds/cmd_download.py +++ b/src/audible_cli/cmds/cmd_download.py @@ -599,7 +599,7 @@ async def cli(session, api_client, **params): [get_aax, get_aaxc, get_annotation, get_chapters, get_cover, get_pdf] ): logger.error("Please select an option what you want download.") - click.Abort() + raise click.Abort() # additional options sim_jobs = params.get("jobs") From c29d0fa0b80261ee70c578dcd8db2385db5a8e3a Mon Sep 17 00:00:00 2001 From: mkb79 Date: Tue, 19 Jul 2022 21:25:07 +0200 Subject: [PATCH 13/79] Fix: set quality to `High` when make a licenserequest Due to a change to the Audible API, a licenserequest with the quality `Extreme` no longer works. Changing quality to `High` as a workaround. --- src/audible_cli/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/audible_cli/models.py b/src/audible_cli/models.py index 370c415..a54db85 100644 --- a/src/audible_cli/models.py +++ b/src/audible_cli/models.py @@ -302,7 +302,7 @@ class LibraryItem(BaseItem): body = { "supported_drm_types": ["Mpeg", "Adrm"], - "quality": "Extreme" if quality in ("best", "high") else "Normal", + "quality": "High" if quality in ("best", "high") else "Normal", "consumption_type": "Download", "response_groups": response_groups } From 4100e9a02fe6711c3d20bd18ac0e03240cbd791c Mon Sep 17 00:00:00 2001 From: mkb79 Date: Thu, 21 Jul 2022 06:51:28 +0200 Subject: [PATCH 14/79] Fix: requesting chapter does not work anymore #108 --- src/audible_cli/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/audible_cli/models.py b/src/audible_cli/models.py index ec5e79d..0b6033c 100644 --- a/src/audible_cli/models.py +++ b/src/audible_cli/models.py @@ -323,7 +323,7 @@ class LibraryItem(BaseItem): params = { "response_groups": "last_position_heard, content_reference, " "chapter_info", - "quality": "Extreme" if quality in ("best", "high") else "Normal", + "quality": "High" if quality in ("best", "high") else "Normal", "drm_type": "Adrm" } From 709baa3b7a0147039b404565d2bf04197df5d2d1 Mon Sep 17 00:00:00 2001 From: mkb79 Date: Thu, 21 Jul 2022 14:59:56 +0200 Subject: [PATCH 15/79] rework `cmd_remove-encryption` plugin command - decrypting aaxc files: now looks for chapter information in the chapter file if they are not in the voucher file - adding `-r` shortcut to `--rebuild-chapters` flag - adding `--separate-intro-outro, -s` flag - adding `--ignore-missing-chapters, -t` flag - to reach the same behave like before this update, provide `-rst` flag - the ffmpeg command now uses `-v quiet` and `-stats` flag to reduce output --- plugin_cmds/cmd_remove-encryption.py | 246 ++++++++++++++++++++------- 1 file changed, 180 insertions(+), 66 deletions(-) diff --git a/plugin_cmds/cmd_remove-encryption.py b/plugin_cmds/cmd_remove-encryption.py index 6e1932f..f16fa10 100644 --- a/plugin_cmds/cmd_remove-encryption.py +++ b/plugin_cmds/cmd_remove-encryption.py @@ -185,105 +185,193 @@ class FFMeta: self._ffmeta_parsed["CHAPTER"] = new_chapters -def decrypt_aax(files, activation_bytes, rebuild_chapters): +def decrypt_aax( + files, activation_bytes, rebuild_chapters, ignore_missing_chapters, + separate_intro_outro +): for file in files: outfile = file.with_suffix(".m4b") - metafile = file.with_suffix(".meta") - metafile_new = file.with_suffix(".new.meta") - base_filename = file.stem.rsplit("-", 1)[0] - chapters = file.with_name(base_filename + "-chapters.json") - apimeta = json.loads(chapters.read_text()) if outfile.exists(): - secho(f"file {outfile} already exists Skip.", fg="blue") + secho(f"Skip {file.name}: already decrypted", fg="blue") continue - if rebuild_chapters and apimeta["content_metadata"]["chapter_info"][ - "is_accurate"]: - cmd = ["ffmpeg", - "-activation_bytes", activation_bytes, - "-i", str(file), - "-f", "ffmetadata", - str(metafile)] + can_rebuild_chapters = False + if rebuild_chapters: + metafile = file.with_suffix(".meta") + metafile_new = file.with_suffix(".new.meta") + base_filename = file.stem.rsplit("-", 1)[0] + chapter_file = file.with_name(base_filename + "-chapters.json") + + has_chapters = False + try: + content_metadata = json.loads(chapter_file.read_text()) + except: + secho(f"No chapter data found for {file.name}", fg="red") + else: + echo(f"Using chapters from {chapter_file.name}") + has_chapters = True + + if has_chapters: + if not content_metadata["content_metadata"]["chapter_info"][ + "is_accurate"]: + secho(f"Chapter data are not accurate", fg="red") + else: + can_rebuild_chapters = True + + if rebuild_chapters and not can_rebuild_chapters and not ignore_missing_chapters: + secho(f"Skip {file.name}: chapter data can not be rebuild", fg="red") + continue + + if can_rebuild_chapters: + cmd = [ + "ffmpeg", + "-v", "quiet", + "-stats", + "-activation_bytes", activation_bytes, + "-i", str(file), + "-f", "ffmetadata", + str(metafile) + ] subprocess.check_output(cmd, universal_newlines=True) ffmeta_class = FFMeta(metafile) - ffmeta_class.update_chapters_from_api_meta(apimeta) + ffmeta_class.update_chapters_from_api_meta( + content_metadata, separate_intro_outro + ) ffmeta_class.write(metafile_new) click.echo("Replaced all titles.") - cmd = ["ffmpeg", - "-activation_bytes", activation_bytes, - "-i", str(file), - "-i", str(metafile_new), - "-map_metadata", "0", - "-map_chapters", "1", - "-c", "copy", - str(outfile)] + cmd = [ + "ffmpeg", + "-v", "quiet", + "-stats", + "-activation_bytes", activation_bytes, + "-i", str(file), + "-i", str(metafile_new), + "-map_metadata", "0", + "-map_chapters", "1", + "-c", "copy", + str(outfile) + ] subprocess.check_output(cmd, universal_newlines=True) metafile.unlink() metafile_new.unlink() else: - cmd = ["ffmpeg", - "-activation_bytes", activation_bytes, - "-i", str(file), - "-c", "copy", - str(outfile)] + cmd = [ + "ffmpeg", + "-v", "quiet", + "-stats", + "-activation_bytes", activation_bytes, + "-i", str(file), + "-c", "copy", + str(outfile) + ] subprocess.check_output(cmd, universal_newlines=True) + echo(f"File decryption successful: {outfile.name}") -def decrypt_aaxc(files, rebuild_chapters): + +def decrypt_aaxc( + files, rebuild_chapters, ignore_missing_chapters, separate_intro_outro +): for file in files: - metafile = file.with_suffix(".meta") - metafile_new = file.with_suffix(".new.meta") - voucher = file.with_suffix(".voucher") - voucher = json.loads(voucher.read_text()) outfile = file.with_suffix(".m4b") if outfile.exists(): - secho(f"file {outfile} already exists Skip.", fg="blue") + secho(f"Skip {file.name}: already decrypted", fg="blue") continue - - apimeta = voucher["content_license"] - audible_key = apimeta["license_response"]["key"] - audible_iv = apimeta["license_response"]["iv"] - if rebuild_chapters and apimeta["content_metadata"]["chapter_info"][ - "is_accurate"]: - cmd = ["ffmpeg", - "-audible_key", audible_key, - "-audible_iv", audible_iv, - "-i", str(file), - "-f", "ffmetadata", - str(metafile)] + voucher_file = file.with_suffix(".voucher") + voucher = json.loads(voucher_file.read_text()) + voucher = voucher["content_license"] + audible_key = voucher["license_response"]["key"] + audible_iv = voucher["license_response"]["iv"] + + can_rebuild_chapters = False + if rebuild_chapters: + metafile = file.with_suffix(".meta") + metafile_new = file.with_suffix(".new.meta") + + has_chapters = False + if "chapter_info" in voucher.get("content_metadata", {}): + content_metadata = voucher + echo(f"Using chapters from {voucher_file}") + has_chapters = True + else: + base_filename = file.stem.rsplit("-", 1)[0] + chapter_file = file.with_name(base_filename + "-chapters.json") + + try: + content_metadata = json.loads(chapter_file.read_text()) + except: + secho(f"No chapter data found for {file.name}", fg="red") + else: + echo(f"Using chapters from {chapter_file.name}") + has_chapters = True + + if has_chapters: + if not content_metadata["content_metadata"]["chapter_info"][ + "is_accurate"]: + secho(f"Chapter data are not accurate", fg="red") + else: + can_rebuild_chapters = True + + if rebuild_chapters and not can_rebuild_chapters and not ignore_missing_chapters: + secho(f"Skip {file.name}: chapter data can not be rebuild", fg="red") + continue + + if can_rebuild_chapters: + cmd = [ + "ffmpeg", + "-v", "quiet", + "-stats", + "-audible_key", audible_key, + "-audible_iv", audible_iv, + "-i", str(file), + "-f", "ffmetadata", + str(metafile) + ] subprocess.check_output(cmd, universal_newlines=True) ffmeta_class = FFMeta(metafile) - ffmeta_class.update_chapters_from_api_meta(apimeta) + ffmeta_class.update_chapters_from_api_meta( + content_metadata, separate_intro_outro + ) ffmeta_class.write(metafile_new) click.echo("Replaced all titles.") - cmd = ["ffmpeg", - "-audible_key", audible_key, - "-audible_iv", audible_iv, - "-i", str(file), - "-i", str(metafile_new), - "-map_metadata", "0", - "-map_chapters", "1", - "-c", "copy", - str(outfile)] + cmd = [ + "ffmpeg", + "-v", "quiet", + "-stats", + "-audible_key", audible_key, + "-audible_iv", audible_iv, + "-i", str(file), + "-i", str(metafile_new), + "-map_metadata", "0", + "-map_chapters", "1", + "-c", "copy", + str(outfile) + ] subprocess.check_output(cmd, universal_newlines=True) metafile.unlink() metafile_new.unlink() else: - cmd = ["ffmpeg", - "-audible_key", audible_key, - "-audible_iv", audible_iv, - "-i", str(file), - "-c", "copy", - str(outfile)] + cmd = [ + "ffmpeg", + "-v", "quiet", + "-stats", + "-audible_key", audible_key, + "-audible_iv", audible_iv, + "-i", str(file), + "-c", "copy", + str(outfile) + ] subprocess.check_output(cmd, universal_newlines=True) + echo(f"File decryption successful: {outfile.name}") + CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"]) @@ -295,7 +383,7 @@ CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"]) multiple=True, help="Input file") @click.option( - "--all", + "--all", "-a", is_flag=True, help="convert all files in folder" ) @@ -305,10 +393,23 @@ CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"]) help="overwrite existing files" ) @click.option( - "--rebuild-chapters", + "--rebuild-chapters", "-r", is_flag=True, help="Rebuild chapters from chapter file" ) +@click.option( + "--separate-intro-outro", "-s", + is_flag=True, + help="Separate Audible Brand Intro and Outro to own Chapter. Only use with `--rebuild-chapters`." +) +@click.option( + "--ignore-missing-chapters", "-t", + is_flag=True, + help=( + "Decrypt without rebuilding chapters when chapters are not present. " + "Otherwise an item is skipped when this option is not provided. Only use with `--rebuild-chapters`." + ) +) @pass_session def cli(session, **options): if not which("ffmpeg"): @@ -316,6 +417,8 @@ def cli(session, **options): ctx.fail("ffmpeg not found") rebuild_chapters = options.get("rebuild_chapters") + ignore_missing_chapters = options.get("ignore_missing_chapters") + separate_intro_outro = options.get("separate_intro_outro") jobs = {"aaxc": [], "aax":[]} @@ -334,5 +437,16 @@ def cli(session, **options): else: secho(f"file suffix {file.suffix} not supported", fg="red") - decrypt_aaxc(jobs["aaxc"], rebuild_chapters) - decrypt_aax(jobs["aax"], session.auth.activation_bytes, rebuild_chapters) + decrypt_aaxc( + jobs["aaxc"], + rebuild_chapters, + ignore_missing_chapters, + separate_intro_outro + ) + + decrypt_aax( + jobs["aax"], + session.auth.activation_bytes, rebuild_chapters, + ignore_missing_chapters, + separate_intro_outro + ) From fa29c69f242205249ebeee80a6e25411a15fc734 Mon Sep 17 00:00:00 2001 From: Isaac Lyons Date: Sun, 24 Jul 2022 20:57:27 -0400 Subject: [PATCH 16/79] Added extended_product_description to keys with raw values. --- src/audible_cli/cmds/cmd_library.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/audible_cli/cmds/cmd_library.py b/src/audible_cli/cmds/cmd_library.py index 3dc4172..2da2130 100644 --- a/src/audible_cli/cmds/cmd_library.py +++ b/src/audible_cli/cmds/cmd_library.py @@ -110,7 +110,7 @@ async def export_library(session, client, **params): await library.resolve_podcats() keys_with_raw_values = ( - "asin", "title", "subtitle", "runtime_length_min", "is_finished", + "asin", "title", "subtitle", "extended_product_description", "runtime_length_min", "is_finished", "percent_complete", "release_date" ) @@ -126,7 +126,7 @@ async def export_library(session, client, **params): dialect = "excel-tab" headers = ( - "asin", "title", "subtitle", "authors", "narrators", "series_title", + "asin", "title", "subtitle", "extended_product_description", "authors", "narrators", "series_title", "series_sequence", "genres", "runtime_length_min", "is_finished", "percent_complete", "rating", "num_ratings", "date_added", "release_date", "cover_url" From d75f397219b7384b1217c860b1e11158f35b8f21 Mon Sep 17 00:00:00 2001 From: mkb79 Date: Mon, 25 Jul 2022 13:28:06 +0200 Subject: [PATCH 17/79] update CHANGELOG.md --- CHANGELOG.md | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9439b45..6b3c997 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,11 +8,26 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Added -- moved licenserequest part from `models.LibraryItem.get_aaxc_url` to its own `models.LibraryItem.get_license` function +- `library` command now outputs the `extended_product_description` field +- ### Changed -- by default a licenserequest (voucher) will not include chapter information for now +- by default a licenserequest (voucher) will not include chapter information by default +- moved licenserequest part from `models.LibraryItem.get_aaxc_url` to its own `models.LibraryItem.get_license` function +- allow book tiltes with hyphens # 96 +- if there is no title fallback to an empty string #98 + +### Fixed + +- `Extreme` quality is not supported by the Audible API anymore #107 +- download command continued execution after error #104 +- Currently paths with dots will break the decryption #97 + +### Misc + +- reworked `cmd_remove-encryption` plugin command (e.g. support nested chapters, use chapter file for aaxc files) +- added explanation in README.md for creating a second profile ## [0.2.0] - 2022-06-01 From 8adeb17776dc5a8030a1cb55ca0c18f79cf59f49 Mon Sep 17 00:00:00 2001 From: mkb79 Date: Tue, 26 Jul 2022 09:48:23 +0200 Subject: [PATCH 18/79] reduce `response_groups` for the download command (#109) --- src/audible_cli/cmds/cmd_download.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/audible_cli/cmds/cmd_download.py b/src/audible_cli/cmds/cmd_download.py index d0b4ac1..9ecf396 100644 --- a/src/audible_cli/cmds/cmd_download.py +++ b/src/audible_cli/cmds/cmd_download.py @@ -621,7 +621,11 @@ async def cli(session, api_client, **params): library = await Library.from_api_full_sync( api_client, image_sizes="1215, 408, 360, 882, 315, 570, 252, 558, 900, 500", - bunch_size=bunch_size + bunch_size=bunch_size, + response_groups=( + "product_desc, media, product_attrs, relationships, " + "series, customer_rights" + ) ) if resolve_podcats: From 4bfe54a23fb8ff9908df9937a350cbd4b81a558f Mon Sep 17 00:00:00 2001 From: mkb79 Date: Tue, 26 Jul 2022 09:48:37 +0200 Subject: [PATCH 19/79] update CHANGELOG.md --- CHANGELOG.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b3c997..b264892 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,20 +9,20 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Added - `library` command now outputs the `extended_product_description` field -- ### Changed - by default a licenserequest (voucher) will not include chapter information by default - moved licenserequest part from `models.LibraryItem.get_aaxc_url` to its own `models.LibraryItem.get_license` function -- allow book tiltes with hyphens # 96 -- if there is no title fallback to an empty string #98 +- allow book tiltes with hyphens (#96) +- if there is no title fallback to an empty string (#98) +- reduce `response_groups` for the download command to speed up fetching the library (#109) ### Fixed -- `Extreme` quality is not supported by the Audible API anymore #107 -- download command continued execution after error #104 -- Currently paths with dots will break the decryption #97 +- `Extreme` quality is not supported by the Audible API anymore (#107) +- download command continued execution after error (#104) +- Currently paths with dots will break the decryption (#97) ### Misc From 35d795ffebddef8eccc2db2276e0603d523b3337 Mon Sep 17 00:00:00 2001 From: mkb79 Date: Wed, 27 Jul 2022 04:56:55 +0200 Subject: [PATCH 20/79] fix `models.Library.from_api_full_sync` --- src/audible_cli/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/audible_cli/models.py b/src/audible_cli/models.py index 0b6033c..d790d81 100644 --- a/src/audible_cli/models.py +++ b/src/audible_cli/models.py @@ -450,8 +450,8 @@ class Library(BaseList): library, total_count = await cls.from_api( api_client, page=1, - params=request_params, include_total_count_header=True, + **request_params ) pages = ceil(int(total_count) / bunch_size) if pages == 1: @@ -463,7 +463,7 @@ class Library(BaseList): cls.from_api( api_client, page=page, - params=request_params, + **request_params ) ) From e9f687029587a25691ef42a9951b16876f643726 Mon Sep 17 00:00:00 2001 From: mkb79 Date: Wed, 27 Jul 2022 04:57:09 +0200 Subject: [PATCH 21/79] update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b264892..ef1ca43 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - `Extreme` quality is not supported by the Audible API anymore (#107) - download command continued execution after error (#104) - Currently paths with dots will break the decryption (#97) +- `models.Library.from_api_full_sync` called `models.Library.from_api` with incorrect keyword arguments ### Misc From 94e2d9a713973393c2c13d3802e7c82b92e6e752 Mon Sep 17 00:00:00 2001 From: mkb79 Date: Fri, 29 Jul 2022 16:55:10 +0200 Subject: [PATCH 22/79] Release v0.2.1 --- CHANGELOG.md | 4 ++++ src/audible_cli/_version.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ef1ca43..c2ec79c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## Unreleased +- + +## [0.2.1] - 2022-07-29 + ### Added - `library` command now outputs the `extended_product_description` field diff --git a/src/audible_cli/_version.py b/src/audible_cli/_version.py index 45136b1..307b474 100644 --- a/src/audible_cli/_version.py +++ b/src/audible_cli/_version.py @@ -1,7 +1,7 @@ __title__ = "audible-cli" __description__ = "Command line interface (cli) for the audible package." __url__ = "https://github.com/mkb79/audible-cli" -__version__ = "0.2.0" +__version__ = "0.2.1" __author__ = "mkb79" __author_email__ = "mkb79@hackitall.de" __license__ = "AGPL" From 0ae303f181055bd93984404fd0ec4cf47a841613 Mon Sep 17 00:00:00 2001 From: mkb79 Date: Mon, 8 Aug 2022 18:10:59 +0200 Subject: [PATCH 23/79] Bugfix: PDFs could not be downloaded using the download command (#112) --- CHANGELOG.md | 4 +++- src/audible_cli/cmds/cmd_download.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c2ec79c..aa42874 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## Unreleased -- +### Bugfix + +- PDFs could not be found using the download command (#112) ## [0.2.1] - 2022-07-29 diff --git a/src/audible_cli/cmds/cmd_download.py b/src/audible_cli/cmds/cmd_download.py index 9ecf396..e50049c 100644 --- a/src/audible_cli/cmds/cmd_download.py +++ b/src/audible_cli/cmds/cmd_download.py @@ -624,7 +624,7 @@ async def cli(session, api_client, **params): bunch_size=bunch_size, response_groups=( "product_desc, media, product_attrs, relationships, " - "series, customer_rights" + "series, customer_rights, pdf_url" ) ) From e6808c33dd797f3dfec56dab37ab69121cce19e2 Mon Sep 17 00:00:00 2001 From: mkb79 Date: Tue, 9 Aug 2022 08:41:18 +0200 Subject: [PATCH 24/79] release v0.2.2 --- CHANGELOG.md | 4 ++++ src/audible_cli/_version.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aa42874..49bcd26 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## Unreleased +- + +## [0.2.2] - 2022-08-09 + ### Bugfix - PDFs could not be found using the download command (#112) diff --git a/src/audible_cli/_version.py b/src/audible_cli/_version.py index 307b474..fbfd06d 100644 --- a/src/audible_cli/_version.py +++ b/src/audible_cli/_version.py @@ -1,7 +1,7 @@ __title__ = "audible-cli" __description__ = "Command line interface (cli) for the audible package." __url__ = "https://github.com/mkb79/audible-cli" -__version__ = "0.2.1" +__version__ = "0.2.2" __author__ = "mkb79" __author_email__ = "mkb79@hackitall.de" __license__ = "AGPL" From 09c0b00d69adaf936b25d924520f41a3285449de Mon Sep 17 00:00:00 2001 From: mkb79 Date: Wed, 31 Aug 2022 08:30:58 +0200 Subject: [PATCH 25/79] better error handling for license requests --- CHANGELOG.md | 4 +++- src/audible_cli/exceptions.py | 12 ++++++++++++ src/audible_cli/models.py | 30 +++++++++++++++++++++++++++--- 3 files changed, 42 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 49bcd26..9f88858 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## Unreleased -- +### Added + +- better error handling for license requests ## [0.2.2] - 2022-08-09 diff --git a/src/audible_cli/exceptions.py b/src/audible_cli/exceptions.py index 6d06840..2a3a997 100644 --- a/src/audible_cli/exceptions.py +++ b/src/audible_cli/exceptions.py @@ -41,3 +41,15 @@ class ProfileAlreadyExists(AudibleCliException): def __init__(self, name): message = f"Profile {name} already exist" super().__init__(message) + + +class LicenseDenied(AudibleCliException): + """Raised if a license request is not granted""" + + +class NoDownloadUrl(AudibleCliException): + """Raised if a license response does not contain a download url""" + + def __init__(self, asin): + message = f"License response for {asin} does not contain a download url" + super().__init__(message) diff --git a/src/audible_cli/models.py b/src/audible_cli/models.py index d790d81..1cd72bd 100644 --- a/src/audible_cli/models.py +++ b/src/audible_cli/models.py @@ -11,7 +11,12 @@ from audible.aescipher import decrypt_voucher_from_licenserequest from audible.client import convert_response_content from .constants import CODEC_HIGH_QUALITY, CODEC_NORMAL_QUALITY -from .exceptions import AudibleCliException, NotDownloadableAsAAX +from .exceptions import ( + AudibleCliException, + LicenseDenied, + NoDownloadUrl, + NotDownloadableAsAAX +) from .utils import full_response_callback, LongestSubString @@ -311,8 +316,27 @@ class LibraryItem(BaseItem): f"content/{self.asin}/licenserequest", body=body ) - voucher = decrypt_voucher_from_licenserequest(self._client.auth, lr) - lr["content_license"]["license_response"] = voucher + + if lr["content_license"]["status_code"] == "Denied": + msg = lr["content_license"]["message"] + raise LicenseDenied(msg) + + content_url = lr["content_license"]["content_metadata"]\ + .get("content_url", {}).get("offline_url") + if content_url is None: + raise NoDownloadUrl(self.asin) + + if "license_response" in lr["content_license"]: + try: + voucher = decrypt_voucher_from_licenserequest( + self._client.auth, lr + ) + except Exception: + logger.error(f"Decrypting voucher for {self.asin} failed") + else: + lr["content_license"]["license_response"] = voucher + else: + logger.error(f"No voucher for {self.asin} found") return lr From 4787794588a556de0cd68d9095280b3431dee1ac Mon Sep 17 00:00:00 2001 From: mkb79 Date: Wed, 31 Aug 2022 08:52:55 +0200 Subject: [PATCH 26/79] extend error handling for license denied responses --- src/audible_cli/models.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/audible_cli/models.py b/src/audible_cli/models.py index 1cd72bd..1b689a1 100644 --- a/src/audible_cli/models.py +++ b/src/audible_cli/models.py @@ -318,6 +318,17 @@ class LibraryItem(BaseItem): ) if lr["content_license"]["status_code"] == "Denied": + if "license_denial_reasons" in lr["content_license"]: + for reason in lr["content_license"]["license_denial_reasons"]: + message = reason.get("message", "UNKNOWN") + rejection_reason = reason.get("rejectionReason", "UNKNOWN") + validation_type = reason.get("validationType", "UNKNOWN") + logger.error( + f"License denied message for {self.asin}: {message}." + f"Reason: {rejection_reason}." + f"Type: {validation_type}" + ) + msg = lr["content_license"]["message"] raise LicenseDenied(msg) From 1c201b359607fa00555a4a78673a8e7c16e0ba4d Mon Sep 17 00:00:00 2001 From: mkb79 Date: Wed, 31 Aug 2022 16:18:20 +0200 Subject: [PATCH 27/79] make some license response error messages to debug messages --- src/audible_cli/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/audible_cli/models.py b/src/audible_cli/models.py index 1b689a1..59d89b4 100644 --- a/src/audible_cli/models.py +++ b/src/audible_cli/models.py @@ -323,7 +323,7 @@ class LibraryItem(BaseItem): message = reason.get("message", "UNKNOWN") rejection_reason = reason.get("rejectionReason", "UNKNOWN") validation_type = reason.get("validationType", "UNKNOWN") - logger.error( + logger.debug( f"License denied message for {self.asin}: {message}." f"Reason: {rejection_reason}." f"Type: {validation_type}" From c53e4d2126ef0bc3b8506e5689a39ed9c1a0a7e6 Mon Sep 17 00:00:00 2001 From: Isaac Lyons <51010664+snowskeleton@users.noreply.github.com> Date: Wed, 31 Aug 2022 14:51:55 -0400 Subject: [PATCH 28/79] Add start-date and end-date option to download command --- src/audible_cli/cmds/cmd_download.py | 80 ++++++++++++++++++++-------- src/audible_cli/cmds/cmd_library.py | 4 +- 2 files changed, 61 insertions(+), 23 deletions(-) diff --git a/src/audible_cli/cmds/cmd_download.py b/src/audible_cli/cmds/cmd_download.py index e50049c..b7e1fd8 100644 --- a/src/audible_cli/cmds/cmd_download.py +++ b/src/audible_cli/cmds/cmd_download.py @@ -4,6 +4,7 @@ import asyncio.sslproto import json import pathlib import logging +from datetime import datetime import aiofiles import click @@ -28,6 +29,12 @@ logger = logging.getLogger("audible_cli.cmds.cmd_download") CLIENT_HEADERS = { "User-Agent": "Audible/671 CFNetwork/1240.0.4 Darwin/20.6.0" } +datetime_type = click.DateTime([ + "%Y-%m-%d", + "%Y-%m-%dT%H:%M:%S", + "%Y-%m-%d %H:%M:%S", + "%Y-%m-%dT%H:%M:%S.%fZ" +]) class DownloadCounter: @@ -269,7 +276,7 @@ async def download_aaxc( filepath = pathlib.Path( output_dir) / f"{base_filename}-{codec}.aaxc" lr_file = filepath.with_suffix(".voucher") - + if lr_file.is_file(): if filepath.is_file(): logger.info( @@ -524,6 +531,18 @@ def display_counter(): is_flag=True, help="saves the annotations (e.g. bookmarks, notes) as JSON file" ) +@click.option( + "--start-date", + type=datetime_type, + default="1970-1-1", + help="Only considers books added to library on or after this UTC date." +) +@click.option( + "--end-date", + type=datetime_type, + default=datetime.utcnow(), + help="Only considers books added to library on or before this UTC date." +) @click.option( "--no-confirm", "-y", is_flag=True, @@ -587,7 +606,9 @@ async def cli(session, api_client, **params): aax_fallback = params.get("aax_fallback") if aax_fallback: if get_aax: - logger.info("Using --aax is redundant and can be left when using --aax-fallback") + logger.info( + "Using --aax is redundant and can be left when using --aax-fallback" + ) get_aax = True if get_aaxc: logger.warning("Do not mix --aaxc with --aax-fallback option.") @@ -612,6 +633,17 @@ async def cli(session, api_client, **params): ignore_podcasts = params.get("ignore_podcasts") bunch_size = session.params.get("bunch_size") + start_date = params.get("start_date") + purchased_after = start_date.strftime('%Y-%m-%dT%H:%M:%S.%fZ') + end_date = params.get("end_date") + if start_date > end_date: + logger.error("start date must be before or equal the end date") + raise click.Abort() + + logger.info(f"Selected start date: {purchased_after}") + logger.info( + f"Selected end date: {end_date.strftime('%Y-%m-%dT%H:%M:%S.%fZ')}") + filename_mode = params.get("filename_mode") if filename_mode == "config": filename_mode = session.config.get_profile_option( @@ -625,7 +657,9 @@ async def cli(session, api_client, **params): response_groups=( "product_desc, media, product_attrs, relationships, " "series, customer_rights, pdf_url" - ) + ), + purchased_after=purchased_after, + status="Active", ) if resolve_podcats: @@ -672,7 +706,7 @@ async def cli(session, api_client, **params): ).unsafe_ask_async() if answer is not None: [jobs.append(i) for i in answer] - + else: logger.error( f"Skip title {title}: Not found in library" @@ -699,23 +733,27 @@ async def cli(session, api_client, **params): odir.mkdir(parents=True) for item in items: - queue_job( - queue=queue, - get_cover=get_cover, - get_pdf=get_pdf, - get_annotation=get_annotation, - get_chapters=get_chapters, - get_aax=get_aax, - get_aaxc=get_aaxc, - client=client, - output_dir=odir, - filename_mode=filename_mode, - item=item, - cover_size=cover_size, - quality=quality, - overwrite_existing=overwrite_existing, - aax_fallback=aax_fallback + purchase_date = datetime_type.convert( + item.purchase_date, None, None ) + if start_date <= purchase_date <= end_date: + queue_job( + queue=queue, + get_cover=get_cover, + get_pdf=get_pdf, + get_annotation=get_annotation, + get_chapters=get_chapters, + get_aax=get_aax, + get_aaxc=get_aaxc, + client=client, + output_dir=odir, + filename_mode=filename_mode, + item=item, + cover_size=cover_size, + quality=quality, + overwrite_existing=overwrite_existing, + aax_fallback=aax_fallback + ) try: # schedule the consumer @@ -729,6 +767,6 @@ async def cli(session, api_client, **params): # the consumer is still awaiting an item, cancel it for consumer in consumers: consumer.cancel() - + await asyncio.gather(*consumers, return_exceptions=True) display_counter() diff --git a/src/audible_cli/cmds/cmd_library.py b/src/audible_cli/cmds/cmd_library.py index 2da2130..6e2c26f 100644 --- a/src/audible_cli/cmds/cmd_library.py +++ b/src/audible_cli/cmds/cmd_library.py @@ -111,7 +111,7 @@ async def export_library(session, client, **params): keys_with_raw_values = ( "asin", "title", "subtitle", "extended_product_description", "runtime_length_min", "is_finished", - "percent_complete", "release_date" + "percent_complete", "release_date", "purchase_date" ) prepared_library = await asyncio.gather( @@ -129,7 +129,7 @@ async def export_library(session, client, **params): "asin", "title", "subtitle", "extended_product_description", "authors", "narrators", "series_title", "series_sequence", "genres", "runtime_length_min", "is_finished", "percent_complete", "rating", "num_ratings", "date_added", - "release_date", "cover_url" + "release_date", "cover_url", "purchase_date" ) export_to_csv(output_filename, prepared_library, headers, dialect) From 5390a4fea11a5841df38c91ee3332fc4816b6d1f Mon Sep 17 00:00:00 2001 From: mkb79 Date: Wed, 31 Aug 2022 20:55:52 +0200 Subject: [PATCH 29/79] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f88858..dd6f55e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Added +- `start-date` and `end-date` option to download command - better error handling for license requests ## [0.2.2] - 2022-08-09 From cf7d6c02cf80791febc7db03ddcf0984cad70334 Mon Sep 17 00:00:00 2001 From: mkb79 Date: Thu, 1 Sep 2022 17:49:30 +0200 Subject: [PATCH 30/79] add start-date and end-date option to library export and list command (#116) --- src/audible_cli/cmds/cmd_library.py | 66 +++++++++++++++++++++++++---- 1 file changed, 58 insertions(+), 8 deletions(-) diff --git a/src/audible_cli/cmds/cmd_library.py b/src/audible_cli/cmds/cmd_library.py index 6e2c26f..d76d1bd 100644 --- a/src/audible_cli/cmds/cmd_library.py +++ b/src/audible_cli/cmds/cmd_library.py @@ -1,6 +1,7 @@ import asyncio import json import pathlib +from datetime import datetime import click from click import echo @@ -16,12 +17,20 @@ from ..models import Library from ..utils import export_to_csv +datetime_type = click.DateTime([ + "%Y-%m-%d", + "%Y-%m-%dT%H:%M:%S", + "%Y-%m-%d %H:%M:%S", + "%Y-%m-%dT%H:%M:%S.%fZ" +]) + + @click.group("library") def cli(): """interact with library""" -async def _get_library(session, client): +async def _get_library(session, client, purchased_after: str): bunch_size = session.params.get("bunch_size") return await Library.from_api_full_sync( @@ -35,7 +44,8 @@ async def _get_library(session, client): "is_finished, is_returnable, origin_asin, pdf_url, " "percent_complete, provided_review" ), - bunch_size=bunch_size + bunch_size=bunch_size, + purchased_after=purchased_after ) @@ -61,6 +71,18 @@ async def _get_library(session, client): is_flag=True, help="Resolve podcasts to show all episodes" ) +@click.option( + "--start-date", + type=datetime_type, + default="1970-1-1", + help="Only considers books added to library on or after this UTC date." +) +@click.option( + "--end-date", + type=datetime_type, + default=datetime.utcnow(), + help="Only considers books added to library on or before this UTC date." +) @pass_session @pass_client async def export_library(session, client, **params): @@ -68,6 +90,12 @@ async def export_library(session, client, **params): @wrap_async def _prepare_item(item): + purchase_date = datetime_type.convert( + item.purchase_date, None, None + ) + if not start_date <= purchase_date <= end_date: + return None + data_row = {} for key in item: v = getattr(item, key) @@ -105,7 +133,11 @@ async def export_library(session, client, **params): suffix = "." + output_format output_filename = output_filename.with_suffix(suffix) - library = await _get_library(session, client) + start_date = params.get("start_date") + purchased_after = start_date.strftime('%Y-%m-%dT%H:%M:%S.%fZ') + end_date = params.get("end_date") + + library = await _get_library(session, client, purchased_after) if params.get("resolve_podcasts"): await library.resolve_podcats() @@ -117,6 +149,7 @@ async def export_library(session, client, **params): prepared_library = await asyncio.gather( *[_prepare_item(i) for i in library] ) + prepared_library = [i for i in prepared_library if i is not None] prepared_library.sort(key=lambda x: x["asin"]) if output_format in ("tsv", "csv"): @@ -147,13 +180,31 @@ async def export_library(session, client, **params): is_flag=True, help="Resolve podcasts to show all episodes" ) +@click.option( + "--start-date", + type=datetime_type, + default="1970-1-1", + help="Only considers books added to library on or after this UTC date." +) +@click.option( + "--end-date", + type=datetime_type, + default=datetime.utcnow(), + help="Only considers books added to library on or before this UTC date." +) @pass_session @pass_client -async def list_library(session, client, resolve_podcasts=False): +async def list_library(session, client, resolve_podcasts, start_date, end_date): """list titles in library""" @wrap_async def _prepare_item(item): + purchase_date = datetime_type.convert( + item.purchase_date, None, None + ) + if not start_date <= purchase_date <= end_date: + return "" + fields = [item.asin] authors = ", ".join( @@ -171,7 +222,8 @@ async def list_library(session, client, resolve_podcasts=False): fields.append(item.title) return ": ".join(fields) - library = await _get_library(session, client) + purchased_after = start_date.strftime('%Y-%m-%dT%H:%M:%S.%fZ') + library = await _get_library(session, client, purchased_after) if resolve_podcasts: await library.resolve_podcats() @@ -179,6 +231,4 @@ async def list_library(session, client, resolve_podcasts=False): books = await asyncio.gather( *[_prepare_item(i) for i in library] ) - - for i in sorted(books): - echo(i) + [echo(i) for i in sorted(books) if len(i) > 0] From 107fc75f36eb6b08106082b78371680ce95e6170 Mon Sep 17 00:00:00 2001 From: mkb79 Date: Thu, 1 Sep 2022 17:53:10 +0200 Subject: [PATCH 31/79] Update CHANGELOG.md --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dd6f55e..2503a93 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Added -- `start-date` and `end-date` option to download command +- `start-date` and `end-date` option to `download` command +- `start-date` and `end-date` option to `library export` and `library list` command - better error handling for license requests ## [0.2.2] - 2022-08-09 From 0924df43b070fb52a524d35f862782d505617b43 Mon Sep 17 00:00:00 2001 From: mkb79 Date: Tue, 6 Sep 2022 21:13:16 +0200 Subject: [PATCH 32/79] Extend download command and other optimizations - check a download link before reuse them - `--ignore-errors` flag of the download command will now continue if on item failed to download - add the `--start-date` and `--end-date` option to the `library list` and `library export` command - make sure an item is published before downloading the aax, aaxc or pdf file --- CHANGELOG.md | 2 + src/audible_cli/cmds/cmd_download.py | 92 ++++++++++++++++++++++------ src/audible_cli/exceptions.py | 39 ++++++++++++ src/audible_cli/models.py | 50 ++++++++++++--- 4 files changed, 155 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2503a93..4c407e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - `start-date` and `end-date` option to `download` command - `start-date` and `end-date` option to `library export` and `library list` command - better error handling for license requests +- verify that a download link is valid +- extend `ignore-errors` flag of the download command ## [0.2.2] - 2022-08-09 diff --git a/src/audible_cli/cmds/cmd_download.py b/src/audible_cli/cmds/cmd_download.py index b7e1fd8..fc3ed34 100644 --- a/src/audible_cli/cmds/cmd_download.py +++ b/src/audible_cli/cmds/cmd_download.py @@ -19,7 +19,13 @@ from ..decorators import ( pass_client, pass_session ) -from ..exceptions import DirectoryDoesNotExists, NotDownloadableAsAAX +from ..exceptions import ( + AudibleCliException, + DirectoryDoesNotExists, + DownloadUrlExpired, + NotDownloadableAsAAX, + VoucherNeedRefresh +) from ..models import Library from ..utils import Downloader @@ -33,7 +39,8 @@ datetime_type = click.DateTime([ "%Y-%m-%d", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%d %H:%M:%S", - "%Y-%m-%dT%H:%M:%S.%fZ" + "%Y-%m-%dT%H:%M:%S.%fZ", + "%Y-%m-%dT%H:%M:%SZ" ]) @@ -263,6 +270,56 @@ async def download_aax( counter.count_aax() +async def _reuse_voucher(lr_file, item): + logger.info(f"Loading data from voucher file {lr_file}.") + async with aiofiles.open(lr_file, "r") as f: + lr = await f.read() + lr = json.loads(lr) + content_license = lr["content_license"] + + assert content_license["status_code"] == "Granted", "License not granted" + + # try to get the user id + user_id = None + if item._client is not None: + auth = item._client.auth + if auth.customer_info is not None: + user_id = auth.customer_info.get("user_id") + + # Verification of allowed user + if user_id is None: + logger.debug("No user id found. Skip user verification.") + else: + if "allowed_users" in content_license: + allowed_users = content_license["allowed_users"] + if allowed_users and user_id not in allowed_users: + # Don't proceed here to prevent overwriting voucher file + msg = f"The current user is not entitled to use the voucher {lr_file}." + raise AudibleCliException(msg) + else: + logger.debug(f"{lr_file} does not contain allowed users key.") + + # Verification of voucher validity + if "refresh_date" in content_license: + refresh_date = content_license["refresh_date"] + refresh_date = datetime_type.convert(refresh_date, None, None) + if refresh_date < datetime.utcnow(): + raise VoucherNeedRefresh(lr_file) + + content_metadata = content_license["content_metadata"] + url = httpx.URL(content_metadata["content_url"]["offline_url"]) + codec = content_metadata["content_reference"]["content_format"] + + expires = url.params.get("Expires") + if expires: + expires = datetime.utcfromtimestamp(int(expires)) + now = datetime.utcnow() + if expires < now: + raise DownloadUrlExpired(lr_file) + + return lr, url, codec + + async def download_aaxc( client, output_dir, base_filename, item, quality, overwrite_existing @@ -286,21 +343,17 @@ async def download_aaxc( f"File {filepath} already exists. Skip download." ) return - else: - logger.info( - f"Loading data from voucher file {lr_file}." - ) - async with aiofiles.open(lr_file, "r") as f: - lr = await f.read() - lr = json.loads(lr) - content_metadata = lr["content_license"][ - "content_metadata"] - url = httpx.URL( - content_metadata["content_url"]["offline_url"]) - codec = content_metadata["content_reference"][ - "content_format"] - if url is None or codec is None or lr is None: + try: + lr, url, codec = await _reuse_voucher(lr_file, item) + except DownloadUrlExpired: + logger.debug(f"Download url in {lr_file} is expired. Refreshing license.") + overwrite_existing = True + except VoucherNeedRefresh: + logger.debug(f"Refresh date for voucher {lr_file} reached. Refreshing license.") + overwrite_existing = True + + if lr is None or url is None or codec is None: url, codec, lr = await item.get_aaxc_url(quality) counter.count_voucher() @@ -340,14 +393,15 @@ async def download_aaxc( counter.count_aaxc() -async def consume(queue): +async def consume(queue, ignore_errors): while True: item = await queue.get() try: await item except Exception as e: logger.error(e) - raise + if not ignore_errors: + raise finally: queue.task_done() @@ -758,7 +812,7 @@ async def cli(session, api_client, **params): try: # schedule the consumer consumers = [ - asyncio.ensure_future(consume(queue)) for _ in range(sim_jobs) + asyncio.ensure_future(consume(queue, ignore_errors)) for _ in range(sim_jobs) ] # wait until the consumer has processed all items await queue.join() diff --git a/src/audible_cli/exceptions.py b/src/audible_cli/exceptions.py index 2a3a997..de865cd 100644 --- a/src/audible_cli/exceptions.py +++ b/src/audible_cli/exceptions.py @@ -1,3 +1,4 @@ +from datetime import datetime from pathlib import Path @@ -53,3 +54,41 @@ class NoDownloadUrl(AudibleCliException): def __init__(self, asin): message = f"License response for {asin} does not contain a download url" super().__init__(message) + + +class DownloadUrlExpired(AudibleCliException): + """Raised if a download url is expired""" + + def __init__(self, lr_file): + message = f"Download url in {lr_file} is expired." + super().__init__(message) + + +class VoucherNeedRefresh(AudibleCliException): + """Raised if a voucher reached his refresh date""" + + def __init__(self, lr_file): + message = f"Refresh date for voucher {lr_file} reached." + super().__init__(message) + + +class ItemNotPublished(AudibleCliException): + """Raised if a voucher reached his refresh date""" + + def __init__(self, asin: str, pub_date): + pub_date = datetime.strptime(pub_date, "%Y-%m-%dT%H:%M:%SZ") + now = datetime.utcnow() + published_in = pub_date - now + + pub_str = "" + if published_in.days > 0: + pub_str += f"{published_in.days} days, " + + seconds = published_in.seconds + hours, remainder = divmod(seconds, 3600) + minutes, seconds = divmod(remainder, 60) + hms = "{:02}h:{:02}m:{:02}s".format(int(hours), int(minutes), int(seconds)) + pub_str += hms + + message = f"{asin} is not published. It will be available in {pub_str}" + super().__init__(message) diff --git a/src/audible_cli/models.py b/src/audible_cli/models.py index 59d89b4..99f2a3b 100644 --- a/src/audible_cli/models.py +++ b/src/audible_cli/models.py @@ -1,7 +1,9 @@ import asyncio import logging +import secrets import string import unicodedata +from datetime import datetime from math import ceil from typing import List, Optional, Union @@ -15,7 +17,8 @@ from .exceptions import ( AudibleCliException, LicenseDenied, NoDownloadUrl, - NotDownloadableAsAAX + NotDownloadableAsAAX, + ItemNotPublished ) from .utils import full_response_callback, LongestSubString @@ -114,6 +117,9 @@ class BaseItem: return images[res] def get_pdf_url(self): + if not self.is_published(): + raise ItemNotPublished(self.asin, self.publication_datetime) + if self.pdf_url is not None: domain = self._client.auth.locale.domain return f"https://www.audible.{domain}/companion-file/{self.asin}" @@ -124,6 +130,14 @@ class BaseItem: or self.content_type == "Podcast") and self.has_children: return True + def is_published(self): + if self.publication_datetime is not None: + pub_date = datetime.strptime( + self.publication_datetime, "%Y-%m-%dT%H:%M:%SZ" + ) + now = datetime.utcnow() + return now > pub_date + class LibraryItem(BaseItem): def _prepare_data(self, data: dict) -> dict: @@ -221,6 +235,9 @@ class LibraryItem(BaseItem): return False async def get_aax_url_old(self, quality: str = "high"): + if not self.is_published(): + raise ItemNotPublished(self.asin, self.publication_datetime) + if not self.is_downloadable(): raise AudibleCliException( f"{self.full_title} is not downloadable." @@ -257,6 +274,8 @@ class LibraryItem(BaseItem): return httpx.URL(link), codec_name async def get_aax_url(self, quality: str = "high"): + if not self.is_published(): + raise ItemNotPublished(self.asin, self.publication_datetime) if not self.is_downloadable(): raise AudibleCliException( @@ -282,6 +301,9 @@ class LibraryItem(BaseItem): quality: str = "high", license_response_groups: Optional[str] = None ): + if not self.is_published(): + raise ItemNotPublished(self.asin, self.publication_datetime) + if not self.is_downloadable(): raise AudibleCliException( f"{self.full_title} is not downloadable." @@ -312,14 +334,24 @@ class LibraryItem(BaseItem): "response_groups": response_groups } + headers = { + "X-Amzn-RequestId": secrets.token_hex(20).upper(), + "X-ADP-SW": "37801821", + "X-ADP-Transport": "WIFI", + "X-ADP-LTO": "120", + "X-Device-Type-Id": "A2CZJZGLK2JJVM", + "device_idiom": "phone" + } lr = await self._client.post( f"content/{self.asin}/licenserequest", - body=body + body=body, + headers=headers ) + content_license = lr["content_license"] - if lr["content_license"]["status_code"] == "Denied": - if "license_denial_reasons" in lr["content_license"]: - for reason in lr["content_license"]["license_denial_reasons"]: + if content_license["status_code"] == "Denied": + if "license_denial_reasons" in content_license: + for reason in content_license["license_denial_reasons"]: message = reason.get("message", "UNKNOWN") rejection_reason = reason.get("rejectionReason", "UNKNOWN") validation_type = reason.get("validationType", "UNKNOWN") @@ -329,15 +361,15 @@ class LibraryItem(BaseItem): f"Type: {validation_type}" ) - msg = lr["content_license"]["message"] + msg = content_license["message"] raise LicenseDenied(msg) - content_url = lr["content_license"]["content_metadata"]\ + content_url = content_license["content_metadata"]\ .get("content_url", {}).get("offline_url") if content_url is None: raise NoDownloadUrl(self.asin) - if "license_response" in lr["content_license"]: + if "license_response" in content_license: try: voucher = decrypt_voucher_from_licenserequest( self._client.auth, lr @@ -345,7 +377,7 @@ class LibraryItem(BaseItem): except Exception: logger.error(f"Decrypting voucher for {self.asin} failed") else: - lr["content_license"]["license_response"] = voucher + content_license["license_response"] = voucher else: logger.error(f"No voucher for {self.asin} found") From 33533583a2763fc88dae39e34653aa576616f8e0 Mon Sep 17 00:00:00 2001 From: mkb79 Date: Tue, 6 Sep 2022 21:24:24 +0200 Subject: [PATCH 33/79] count downloaded AYCL items --- src/audible_cli/cmds/cmd_download.py | 36 +++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/src/audible_cli/cmds/cmd_download.py b/src/audible_cli/cmds/cmd_download.py index fc3ed34..851801f 100644 --- a/src/audible_cli/cmds/cmd_download.py +++ b/src/audible_cli/cmds/cmd_download.py @@ -54,6 +54,8 @@ class DownloadCounter: self._pdf: int = 0 self._voucher: int = 0 self._voucher_saved: int = 0 + self._aycl = 0 + self._aycl_voucher = 0 @property def aax(self): @@ -71,6 +73,24 @@ class DownloadCounter: self._aaxc += 1 logger.debug(f"Currently downloaded aaxc files: {self.aaxc}") + @property + def aycl(self): + return self._aycl + + def count_aycl(self): + self._aycl += 1 + # log as error to display this message in any cases + logger.debug(f"Currently downloaded aycl files: {self.aycl}") + + @property + def aycl_voucher(self): + return self._aycl_voucher + + def count_aycl_voucher(self): + self._aycl_voucher += 1 + # log as error to display this message in any cases + logger.debug(f"Currently downloaded aycl voucher files: {self.aycl_voucher}") + @property def annotation(self): return self._annotation @@ -128,7 +148,9 @@ class DownloadCounter: "cover": self.cover, "pdf": self.pdf, "voucher": self.voucher, - "voucher_saved": self.voucher_saved + "voucher_saved": self.voucher_saved, + "aycl": self.aycl, + "aycl_voucher": self.aycl_voucher } def has_downloads(self): @@ -300,7 +322,7 @@ async def _reuse_voucher(lr_file, item): logger.debug(f"{lr_file} does not contain allowed users key.") # Verification of voucher validity - if "refresh_date" in content_license: + if "refresh_date" in content_license: refresh_date = content_license["refresh_date"] refresh_date = datetime_type.convert(refresh_date, None, None) if refresh_date < datetime.utcnow(): @@ -353,9 +375,13 @@ async def download_aaxc( logger.debug(f"Refresh date for voucher {lr_file} reached. Refreshing license.") overwrite_existing = True + is_aycl = item.benefit_id == "AYCL" + if lr is None or url is None or codec is None: url, codec, lr = await item.get_aaxc_url(quality) counter.count_voucher() + if is_aycl: + counter.count_aycl_voucher() if codec.lower() == "mpeg": ext = "mp3" @@ -391,6 +417,8 @@ async def download_aaxc( if downloaded: counter.count_aaxc() + if is_aycl: + counter.count_aycl() async def consume(queue, ignore_errors): @@ -504,6 +532,8 @@ def display_counter(): if k == "voucher_saved": k = "voucher" + elif k == "aycl_voucher": + k = "aycl voucher" elif k == "voucher": diff = v - counter.voucher_saved if diff > 0: @@ -823,4 +853,4 @@ async def cli(session, api_client, **params): consumer.cancel() await asyncio.gather(*consumers, return_exceptions=True) - display_counter() + display_counter() \ No newline at end of file From cf17c05c7ef8cca4757b2a0cd1d055b7948b4ca3 Mon Sep 17 00:00:00 2001 From: mkb79 Date: Tue, 6 Sep 2022 21:31:43 +0200 Subject: [PATCH 34/79] Update CHANGELOG.md --- CHANGELOG.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c407e2..7a4509d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,13 +6,18 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## Unreleased +- + +## [0.2.3] - 2022-09-06 + ### Added -- `start-date` and `end-date` option to `download` command -- `start-date` and `end-date` option to `library export` and `library list` command +- `--start-date` and `--end-date` option to `download` command +- `--start-date` and `--end-date` option to `library export` and `library list` command - better error handling for license requests - verify that a download link is valid -- extend `ignore-errors` flag of the download command +- make sure an item is published before downloading the aax, aaxc or pdf file +- `--ignore-errors` flag of the download command now continue, if an item failed to download ## [0.2.2] - 2022-08-09 From fb365311aea5d2d5676c5ee074ce774ceaf63009 Mon Sep 17 00:00:00 2001 From: mkb79 Date: Tue, 6 Sep 2022 21:32:03 +0200 Subject: [PATCH 35/79] release v0.2.3 --- src/audible_cli/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/audible_cli/_version.py b/src/audible_cli/_version.py index fbfd06d..316c1e2 100644 --- a/src/audible_cli/_version.py +++ b/src/audible_cli/_version.py @@ -1,7 +1,7 @@ __title__ = "audible-cli" __description__ = "Command line interface (cli) for the audible package." __url__ = "https://github.com/mkb79/audible-cli" -__version__ = "0.2.2" +__version__ = "0.2.3" __author__ = "mkb79" __author_email__ = "mkb79@hackitall.de" __license__ = "AGPL" From 1318c6d7d1ba6b888d573b24c05b70ab92fd56b8 Mon Sep 17 00:00:00 2001 From: mkb79 Date: Wed, 7 Sep 2022 12:42:16 +0200 Subject: [PATCH 36/79] Add support to download multiple cover sizes --- CHANGELOG.md | 4 +++- src/audible_cli/cmds/cmd_download.py | 36 +++++++++++++++------------- 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a4509d..4290099 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## Unreleased -- +### Added + +- Allow download multiple cover sizes at once. Each cover size must be provided with the `--cover-size` option ## [0.2.3] - 2022-09-06 diff --git a/src/audible_cli/cmds/cmd_download.py b/src/audible_cli/cmds/cmd_download.py index 851801f..168edbf 100644 --- a/src/audible_cli/cmds/cmd_download.py +++ b/src/audible_cli/cmds/cmd_download.py @@ -173,7 +173,7 @@ async def download_cover( url = item.get_cover_url(res) if url is None: logger.error( - f"No COVER found for {item.full_title} with given resolution" + f"No COVER with size {res} found for {item.full_title}" ) return @@ -446,7 +446,7 @@ def queue_job( output_dir, filename_mode, item, - cover_size, + cover_sizes, quality, overwrite_existing, aax_fallback @@ -454,16 +454,17 @@ def queue_job( base_filename = item.create_base_filename(filename_mode) if get_cover: - queue.put_nowait( - download_cover( - client=client, - output_dir=output_dir, - base_filename=base_filename, - item=item, - res=cover_size, - overwrite_existing=overwrite_existing + for cover_size in cover_sizes: + queue.put_nowait( + download_cover( + client=client, + output_dir=output_dir, + base_filename=base_filename, + item=item, + res=cover_size, + overwrite_existing=overwrite_existing + ) ) - ) if get_pdf: queue.put_nowait( @@ -602,8 +603,9 @@ def display_counter(): "--cover-size", type=click.Choice(["252", "315", "360", "408", "500", "558", "570", "882", "900", "1215"]), - default="500", - help="the cover pixel size" + default=["500"], + multiple=True, + help="The cover pixel size. This option can be provided multiple times." ) @click.option( "--chapter", @@ -709,7 +711,7 @@ async def cli(session, api_client, **params): # additional options sim_jobs = params.get("jobs") quality = params.get("quality") - cover_size = params.get("cover_size") + cover_sizes = list(set(params.get("cover_size"))) overwrite_existing = params.get("overwrite") ignore_errors = params.get("ignore_errors") no_confirm = params.get("no_confirm") @@ -736,7 +738,7 @@ async def cli(session, api_client, **params): # fetch the user library library = await Library.from_api_full_sync( api_client, - image_sizes="1215, 408, 360, 882, 315, 570, 252, 558, 900, 500", + image_sizes=", ".join(cover_sizes), bunch_size=bunch_size, response_groups=( "product_desc, media, product_attrs, relationships, " @@ -833,7 +835,7 @@ async def cli(session, api_client, **params): output_dir=odir, filename_mode=filename_mode, item=item, - cover_size=cover_size, + cover_sizes=cover_sizes, quality=quality, overwrite_existing=overwrite_existing, aax_fallback=aax_fallback @@ -853,4 +855,4 @@ async def cli(session, api_client, **params): consumer.cancel() await asyncio.gather(*consumers, return_exceptions=True) - display_counter() \ No newline at end of file + display_counter() From c1b2d1db525dca2c1d3da8b7d994268f19a0c1ef Mon Sep 17 00:00:00 2001 From: mkb79 Date: Mon, 19 Sep 2022 14:14:57 +0200 Subject: [PATCH 37/79] rework start_date and end_date option (#124) --- CHANGELOG.md | 9 +++ src/audible_cli/cmds/cmd_download.py | 88 ++++++++++++---------------- src/audible_cli/cmds/cmd_library.py | 82 +++++++------------------- src/audible_cli/decorators.py | 35 +++++++++++ src/audible_cli/models.py | 53 ++++++++++++++++- src/audible_cli/utils.py | 9 +++ 6 files changed, 163 insertions(+), 113 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4290099..f7c2240 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,15 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Allow download multiple cover sizes at once. Each cover size must be provided with the `--cover-size` option + +### Changed + +- Rework start_date and end_date option + +### Bugfix + +- In some cases, the purchase date is None. This results in an exception. Now check for purchase date or date added and skip, if date is missing + ## [0.2.3] - 2022-09-06 ### Added diff --git a/src/audible_cli/cmds/cmd_download.py b/src/audible_cli/cmds/cmd_download.py index 168edbf..10fe248 100644 --- a/src/audible_cli/cmds/cmd_download.py +++ b/src/audible_cli/cmds/cmd_download.py @@ -15,6 +15,8 @@ from click import echo from ..decorators import ( bunch_size_option, + end_date_option, + start_date_option, timeout_option, pass_client, pass_session @@ -27,7 +29,7 @@ from ..exceptions import ( VoucherNeedRefresh ) from ..models import Library -from ..utils import Downloader +from ..utils import datetime_type, Downloader logger = logging.getLogger("audible_cli.cmds.cmd_download") @@ -35,13 +37,6 @@ logger = logging.getLogger("audible_cli.cmds.cmd_download") CLIENT_HEADERS = { "User-Agent": "Audible/671 CFNetwork/1240.0.4 Darwin/20.6.0" } -datetime_type = click.DateTime([ - "%Y-%m-%d", - "%Y-%m-%dT%H:%M:%S", - "%Y-%m-%d %H:%M:%S", - "%Y-%m-%dT%H:%M:%S.%fZ", - "%Y-%m-%dT%H:%M:%SZ" -]) class DownloadCounter: @@ -617,18 +612,8 @@ def display_counter(): is_flag=True, help="saves the annotations (e.g. bookmarks, notes) as JSON file" ) -@click.option( - "--start-date", - type=datetime_type, - default="1970-1-1", - help="Only considers books added to library on or after this UTC date." -) -@click.option( - "--end-date", - type=datetime_type, - default=datetime.utcnow(), - help="Only considers books added to library on or before this UTC date." -) +@start_date_option +@end_date_option @click.option( "--no-confirm", "-y", is_flag=True, @@ -719,16 +704,20 @@ async def cli(session, api_client, **params): ignore_podcasts = params.get("ignore_podcasts") bunch_size = session.params.get("bunch_size") - start_date = params.get("start_date") - purchased_after = start_date.strftime('%Y-%m-%dT%H:%M:%S.%fZ') - end_date = params.get("end_date") - if start_date > end_date: + start_date = session.params.get("start_date") + end_date = session.params.get("end_date") + if all([start_date, end_date]) and start_date > end_date: logger.error("start date must be before or equal the end date") raise click.Abort() - logger.info(f"Selected start date: {purchased_after}") - logger.info( - f"Selected end date: {end_date.strftime('%Y-%m-%dT%H:%M:%S.%fZ')}") + if start_date is not None: + logger.info( + f"Selected start date: {start_date.strftime('%Y-%m-%dT%H:%M:%S.%fZ')}" + ) + if end_date is not None: + logger.info( + f"Selected end date: {end_date.strftime('%Y-%m-%dT%H:%M:%S.%fZ')}" + ) filename_mode = params.get("filename_mode") if filename_mode == "config": @@ -744,12 +733,13 @@ async def cli(session, api_client, **params): "product_desc, media, product_attrs, relationships, " "series, customer_rights, pdf_url" ), - purchased_after=purchased_after, + start_date=start_date, + end_date=end_date, status="Active", ) if resolve_podcats: - await library.resolve_podcats() + await library.resolve_podcats(start_date=start_date, end_date=end_date) # collect jobs jobs = [] @@ -807,7 +797,9 @@ async def cli(session, api_client, **params): if not ignore_podcasts and item.is_parent_podcast(): items.remove(item) if item._children is None: - await item.get_child_items() + await item.get_child_items( + start_date=start_date, end_date=end_date + ) for i in item._children: if i.asin not in jobs: @@ -819,27 +811,23 @@ async def cli(session, api_client, **params): odir.mkdir(parents=True) for item in items: - purchase_date = datetime_type.convert( - item.purchase_date, None, None + queue_job( + queue=queue, + get_cover=get_cover, + get_pdf=get_pdf, + get_annotation=get_annotation, + get_chapters=get_chapters, + get_aax=get_aax, + get_aaxc=get_aaxc, + client=client, + output_dir=odir, + filename_mode=filename_mode, + item=item, + cover_sizes=cover_sizes, + quality=quality, + overwrite_existing=overwrite_existing, + aax_fallback=aax_fallback ) - if start_date <= purchase_date <= end_date: - queue_job( - queue=queue, - get_cover=get_cover, - get_pdf=get_pdf, - get_annotation=get_annotation, - get_chapters=get_chapters, - get_aax=get_aax, - get_aaxc=get_aaxc, - client=client, - output_dir=odir, - filename_mode=filename_mode, - item=item, - cover_sizes=cover_sizes, - quality=quality, - overwrite_existing=overwrite_existing, - aax_fallback=aax_fallback - ) try: # schedule the consumer diff --git a/src/audible_cli/cmds/cmd_library.py b/src/audible_cli/cmds/cmd_library.py index d76d1bd..b38699f 100644 --- a/src/audible_cli/cmds/cmd_library.py +++ b/src/audible_cli/cmds/cmd_library.py @@ -1,13 +1,14 @@ import asyncio import json import pathlib -from datetime import datetime import click from click import echo from ..decorators import ( bunch_size_option, + end_date_option, + start_date_option, timeout_option, pass_client, pass_session, @@ -17,23 +18,17 @@ from ..models import Library from ..utils import export_to_csv -datetime_type = click.DateTime([ - "%Y-%m-%d", - "%Y-%m-%dT%H:%M:%S", - "%Y-%m-%d %H:%M:%S", - "%Y-%m-%dT%H:%M:%S.%fZ" -]) - - @click.group("library") def cli(): """interact with library""" -async def _get_library(session, client, purchased_after: str): +async def _get_library(session, client, resolve_podcasts): bunch_size = session.params.get("bunch_size") + start_date = session.params.get("start_date") + end_date = session.params.get("end_date") - return await Library.from_api_full_sync( + library = await Library.from_api_full_sync( client, response_groups=( "contributors, media, price, product_attrs, product_desc, " @@ -45,9 +40,15 @@ async def _get_library(session, client, purchased_after: str): "percent_complete, provided_review" ), bunch_size=bunch_size, - purchased_after=purchased_after + start_date=start_date, + end_date=end_date ) + if resolve_podcasts: + await library.resolve_podcats(start_date=start_date, end_date=end_date) + + return library + @cli.command("export") @click.option( @@ -71,18 +72,8 @@ async def _get_library(session, client, purchased_after: str): is_flag=True, help="Resolve podcasts to show all episodes" ) -@click.option( - "--start-date", - type=datetime_type, - default="1970-1-1", - help="Only considers books added to library on or after this UTC date." -) -@click.option( - "--end-date", - type=datetime_type, - default=datetime.utcnow(), - help="Only considers books added to library on or before this UTC date." -) +@start_date_option +@end_date_option @pass_session @pass_client async def export_library(session, client, **params): @@ -90,12 +81,6 @@ async def export_library(session, client, **params): @wrap_async def _prepare_item(item): - purchase_date = datetime_type.convert( - item.purchase_date, None, None - ) - if not start_date <= purchase_date <= end_date: - return None - data_row = {} for key in item: v = getattr(item, key) @@ -133,13 +118,8 @@ async def export_library(session, client, **params): suffix = "." + output_format output_filename = output_filename.with_suffix(suffix) - start_date = params.get("start_date") - purchased_after = start_date.strftime('%Y-%m-%dT%H:%M:%S.%fZ') - end_date = params.get("end_date") - - library = await _get_library(session, client, purchased_after) - if params.get("resolve_podcasts"): - await library.resolve_podcats() + resolve_podcasts = params.get("resolve_podcasts") + library = await _get_library(session, client, resolve_podcasts) keys_with_raw_values = ( "asin", "title", "subtitle", "extended_product_description", "runtime_length_min", "is_finished", @@ -180,31 +160,15 @@ async def export_library(session, client, **params): is_flag=True, help="Resolve podcasts to show all episodes" ) -@click.option( - "--start-date", - type=datetime_type, - default="1970-1-1", - help="Only considers books added to library on or after this UTC date." -) -@click.option( - "--end-date", - type=datetime_type, - default=datetime.utcnow(), - help="Only considers books added to library on or before this UTC date." -) +@start_date_option +@end_date_option @pass_session @pass_client -async def list_library(session, client, resolve_podcasts, start_date, end_date): +async def list_library(session, client, resolve_podcasts): """list titles in library""" @wrap_async def _prepare_item(item): - purchase_date = datetime_type.convert( - item.purchase_date, None, None - ) - if not start_date <= purchase_date <= end_date: - return "" - fields = [item.asin] authors = ", ".join( @@ -222,11 +186,7 @@ async def list_library(session, client, resolve_podcasts, start_date, end_date): fields.append(item.title) return ": ".join(fields) - purchased_after = start_date.strftime('%Y-%m-%dT%H:%M:%S.%fZ') - library = await _get_library(session, client, purchased_after) - - if resolve_podcasts: - await library.resolve_podcats() + library = await _get_library(session, client, resolve_podcasts) books = await asyncio.gather( *[_prepare_item(i) for i in library] diff --git a/src/audible_cli/decorators.py b/src/audible_cli/decorators.py index 9e39c3a..df43f54 100644 --- a/src/audible_cli/decorators.py +++ b/src/audible_cli/decorators.py @@ -7,6 +7,7 @@ import httpx from packaging.version import parse from .config import Session +from .utils import datetime_type from ._logging import _normalize_logger from . import __version__ @@ -236,3 +237,37 @@ def bunch_size_option(func=None, **kwargs): return option(func) return option + + +def start_date_option(func=None, **kwargs): + kwargs.setdefault("type", datetime_type) + kwargs.setdefault( + "help", + "Only considers books added to library on or after this UTC date." + ) + kwargs.setdefault("callback", add_param_to_session) + kwargs.setdefault("expose_value", False) + + option = click.option("--start-date", **kwargs) + + if callable(func): + return option(func) + + return option + + +def end_date_option(func=None, **kwargs): + kwargs.setdefault("type", datetime_type) + kwargs.setdefault( + "help", + "Only considers books added to library on or before this UTC date." + ) + kwargs.setdefault("callback", add_param_to_session) + kwargs.setdefault("expose_value", False) + + option = click.option("--end-date", **kwargs) + + if callable(func): + return option(func) + + return option diff --git a/src/audible_cli/models.py b/src/audible_cli/models.py index 99f2a3b..ddbdc32 100644 --- a/src/audible_cli/models.py +++ b/src/audible_cli/models.py @@ -476,8 +476,41 @@ class Library(BaseList): cls, api_client: audible.AsyncClient, include_total_count_header: bool = False, + start_date: Optional[datetime] = None, + end_date: Optional[datetime] = None, **request_params ): + def filter_by_date(item): + if item.purchase_date is not None: + date_added = datetime.strptime( + item.purchase_date, + "%Y-%m-%dT%H:%M:%S.%fZ" + ) + elif item.library_status.get("date_added") is not None: + date_added = datetime.strptime( + item.library_status.get("date_added"), + "%Y-%m-%dT%H:%M:%S.%fZ" + ) + else: + logger.info( + f"{item.asin}: {item.full_title} can not determine date added." + ) + return True + + if start_date is not None and start_date > date_added: + return False + # If a new episode is added to a parent podcast, the purchase_date + # and date_added is set to this date. This can makes things + # difficult to get older podcast episodes + # the end date will be filtered by the resolve_podcasts function later + if item.is_parent_podcast(): + return True + + if end_date is not None and end_date < date_added: + return False + + return True + if "response_groups" not in request_params: request_params["response_groups"] = ( "contributors, customer_rights, media, price, product_attrs, " @@ -491,6 +524,14 @@ class Library(BaseList): "periodicals, provided_review, product_details" ) + if start_date is not None: + if "purchase_date" in request_params: + raise AudibleCliException( + "Do not use purchase_date and start_date together" + ) + request_params["purchased_after"] = start_date.strftime( + "%Y-%m-%dT%H:%M:%S.%fZ") + resp: httpx.Response = await api_client.get( "library", response_callback=full_response_callback, @@ -500,6 +541,9 @@ class Library(BaseList): total_count_header = resp.headers.get("total-count") cls_instance = cls(resp_content, api_client=api_client) + if start_date is not None or end_date is not None: + cls_instance._data = list(filter(filter_by_date, cls_instance.data)) + if include_total_count_header: return cls_instance, total_count_header return cls_instance @@ -541,9 +585,14 @@ class Library(BaseList): return library - async def resolve_podcats(self): + async def resolve_podcats( + self, + start_date: Optional[datetime] = None, + end_date: Optional[datetime] = None + ): podcast_items = await asyncio.gather( - *[i.get_child_items() for i in self if i.is_parent_podcast()] + *[i.get_child_items(start_date=start_date, end_date=end_date) + for i in self if i.is_parent_podcast()] ) for i in podcast_items: self.data.extend(i.data) diff --git a/src/audible_cli/utils.py b/src/audible_cli/utils.py index 3a5425d..bc7f6e8 100644 --- a/src/audible_cli/utils.py +++ b/src/audible_cli/utils.py @@ -21,6 +21,15 @@ from .constants import DEFAULT_AUTH_FILE_ENCRYPTION logger = logging.getLogger("audible_cli.utils") +datetime_type = click.DateTime([ + "%Y-%m-%d", + "%Y-%m-%dT%H:%M:%S", + "%Y-%m-%d %H:%M:%S", + "%Y-%m-%dT%H:%M:%S.%fZ", + "%Y-%m-%dT%H:%M:%SZ" +]) + + def prompt_captcha_callback(captcha_url: str) -> str: """Helper function for handling captcha.""" From 6af331f43a657d822cd3f7a10a67f77462f61bd3 Mon Sep 17 00:00:00 2001 From: mkb79 Date: Wed, 21 Sep 2022 13:33:46 +0200 Subject: [PATCH 38/79] Release v0.2.4 --- CHANGELOG.md | 4 ++++ src/audible_cli/_version.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f7c2240..dff2013 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## Unreleased +- + +## [0.2.4] - 2022-09-21 + ### Added - Allow download multiple cover sizes at once. Each cover size must be provided with the `--cover-size` option diff --git a/src/audible_cli/_version.py b/src/audible_cli/_version.py index 316c1e2..66c9501 100644 --- a/src/audible_cli/_version.py +++ b/src/audible_cli/_version.py @@ -1,7 +1,7 @@ __title__ = "audible-cli" __description__ = "Command line interface (cli) for the audible package." __url__ = "https://github.com/mkb79/audible-cli" -__version__ = "0.2.3" +__version__ = "0.2.4" __author__ = "mkb79" __author_email__ = "mkb79@hackitall.de" __license__ = "AGPL" From 9bbfa5c1a415ee918fb879e93a4c6d504cbfa030 Mon Sep 17 00:00:00 2001 From: mkb79 Date: Thu, 23 Feb 2023 15:32:17 +0100 Subject: [PATCH 39/79] Create cmd_decrypt.py --- plugin_cmds/cmd_decrypt.py | 571 +++++++++++++++++++++++++++++++++++++ 1 file changed, 571 insertions(+) create mode 100644 plugin_cmds/cmd_decrypt.py diff --git a/plugin_cmds/cmd_decrypt.py b/plugin_cmds/cmd_decrypt.py new file mode 100644 index 0000000..f8f366b --- /dev/null +++ b/plugin_cmds/cmd_decrypt.py @@ -0,0 +1,571 @@ +"""Removes encryption of aax and aaxc files. + +This is a proof-of-concept and for testing purposes only. + +No error handling. +Need further work. Some options do not work or options are missing. + +Needs at least ffmpeg 4.4 +""" + + +import json +import operator +import pathlib +import re +import subprocess # noqa: S404 +import tempfile +import typing as t +from enum import Enum +from functools import reduce +from glob import glob +from shlex import quote +from shutil import which + +import click +from click import echo, secho + +from audible_cli.decorators import pass_session +from audible_cli.exceptions import AudibleCliException + + +class ChapterError(AudibleCliException): + """Base class for all chapter errors.""" + + +class SupportedFiles(Enum): + AAX = ".aax" + AAXC = ".aaxc" + + @classmethod + def get_supported_list(cls): + return list(set(item.value for item in cls)) + + @classmethod + def is_supported_suffix(cls, value): + return value in cls.get_supported_list() + + @classmethod + def is_supported_file(cls, value): + return pathlib.PurePath(value).suffix in cls.get_supported_list() + + +def _get_input_files( + files: t.Union[t.Tuple[str], t.List[str]], + recursive: bool = True +) -> t.List[pathlib.Path]: + filenames = [] + for filename in files: + # if the shell does not do filename globbing + expanded = list(glob(filename, recursive=recursive)) + + if ( + len(expanded) == 0 + and '*' not in filename + and not SupportedFiles.is_supported_file(filename) + ): + raise(click.BadParameter("{filename}: file not found or supported.")) + + expanded_filter = filter( + lambda x: SupportedFiles.is_supported_file(x), expanded + ) + expanded = list(map(lambda x: pathlib.Path(x).resolve(), expanded_filter)) + filenames.extend(expanded) + + return filenames + + +def recursive_lookup_dict(key: str, dictionary: t.Dict[str, t.Any]) -> t.Any: + if key in dictionary: + return dictionary[key] + for value in dictionary.values(): + if isinstance(value, dict): + try: + item = recursive_lookup_dict(key, value) + except KeyError: + continue + else: + return item + + raise KeyError + + +def get_aaxc_credentials(voucher_file: pathlib.Path): + if not voucher_file.exists() or not voucher_file.is_file(): + raise AudibleCliException(f"Voucher file {voucher_file} not found.") + + voucher_dict = json.loads(voucher_file.read_text()) + try: + key = recursive_lookup_dict("key", voucher_dict) + iv = recursive_lookup_dict("iv", voucher_dict) + except KeyError: + raise AudibleCliException(f"No key/iv found in file {voucher_file}.") from None + + return key, iv + + +class ApiChapterInfo: + def __init__(self, content_metadata: t.Dict[str, t.Any]) -> None: + chapter_info = self._parse(content_metadata) + self._chapter_info = chapter_info + + @classmethod + def from_file(cls, file: t.Union[pathlib.Path, str]) -> "ApiChapterInfo": + file = pathlib.Path(file) + if not file.exists() or not file.is_file(): + raise ChapterError(f"Chapter file {file} not found.") + content_string = pathlib.Path(file).read_text("utf-8") + content_json = json.loads(content_string) + return cls(content_json) + + @staticmethod + def _parse(content_metadata: t.Dict[str, t.Any]) -> t.Dict[str, t.Any]: + if "chapters" in content_metadata: + return content_metadata + + try: + return recursive_lookup_dict("chapter_info", content_metadata) + except KeyError: + raise ChapterError("No chapter info found.") from None + + def count_chapters(self): + return len(self.get_chapters()) + + def get_chapters(self, separate_intro_outro=False): + def extract_chapters(initial, current): + if "chapters" in current: + return initial + [current] + current["chapters"] + else: + return initial + [current] + + chapters = list( + reduce( + extract_chapters, + self._chapter_info["chapters"], + [], + ) + ) + + if separate_intro_outro: + return self._separate_intro_outro(chapters) + + return chapters + + def get_intro_duration_ms(self): + return self._chapter_info["brandIntroDurationMs"] + + def get_outro_duration_ms(self): + return self._chapter_info["brandOutroDurationMs"] + + def get_runtime_length_ms(self): + return self._chapter_info["runtime_length_ms"] + + def is_accurate(self): + return self._chapter_info["is_accurate"] + + def _separate_intro_outro(self, chapters): + echo("Separate Audible Brand Intro and Outro to own Chapter.") + chapters.sort(key=operator.itemgetter("start_offset_ms")) + + first = chapters[0] + intro_dur_ms = self.get_intro_duration_ms() + first["start_offset_ms"] = intro_dur_ms + first["start_offset_sec"] = round(first["start_offset_ms"] / 1000) + first["length_ms"] -= intro_dur_ms + + last = chapters[-1] + outro_dur_ms = self.get_outro_duration_ms() + last["length_ms"] -= outro_dur_ms + + chapters.append( + { + "length_ms": intro_dur_ms, + "start_offset_ms": 0, + "start_offset_sec": 0, + "title": "Intro", + } + ) + chapters.append( + { + "length_ms": outro_dur_ms, + "start_offset_ms": self.get_runtime_length_ms() - outro_dur_ms, + "start_offset_sec": round( + (self.get_runtime_length_ms() - outro_dur_ms) / 1000 + ), + "title": "Outro", + } + ) + chapters.sort(key=operator.itemgetter("start_offset_ms")) + + return chapters + + +class FFMeta: + SECTION = re.compile(r"\[(?P
[^]]+)\]") + OPTION = re.compile(r"(?P