From 6da1eda757a5db0f718dc1404e10049031594446 Mon Sep 17 00:00:00 2001 From: mkb79 Date: Wed, 27 Sep 2023 10:36:14 +0200 Subject: [PATCH 01/35] ci: Update build action (#158) * ci: update build action * docs: Update README.md --- .github/workflows/build.yml | 40 +++++++++++-------------------------- README.md | 3 +-- 2 files changed, 13 insertions(+), 30 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1c2055f..f35811f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -9,10 +9,12 @@ jobs: createrelease: name: Create Release - runs-on: [ubuntu-latest] + runs-on: ubuntu-latest + outputs: + release_url: ${{ steps.create-release.outputs.upload_url }} steps: - name: Create Release - id: create_release + id: create-release uses: actions/create-release@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -21,13 +23,6 @@ jobs: release_name: Release ${{ github.ref }} draft: false prerelease: false - - name: Output Release URL File - run: echo "${{ steps.create_release.outputs.upload_url }}" > release_url.txt - - name: Save Release URL File for publish - uses: actions/upload-artifact@v2 - with: - name: release_url - path: release_url.txt build: name: Build packages @@ -44,13 +39,13 @@ jobs: zip -r9 audible_linux_ubuntu_latest audible OUT_FILE_NAME: audible_linux_ubuntu_latest.zip ASSET_MIME: application/zip # application/octet-stream - - os: ubuntu-18.04 + - os: ubuntu-20.04 TARGET: linux CMD_BUILD: > pyinstaller --clean -F --hidden-import audible_cli -n audible -c pyi_entrypoint.py && cd dist/ && - zip -r9 audible_linux_ubuntu_18_04 audible - OUT_FILE_NAME: audible_linux_ubuntu_18_04.zip + zip -r9 audible_linux_ubuntu_20_04 audible + OUT_FILE_NAME: audible_linux_ubuntu_20_04.zip ASSET_MIME: application/zip # application/octet-stream - os: macos-latest TARGET: macos @@ -85,34 +80,23 @@ jobs: OUT_FILE_NAME: audible_win.zip ASSET_MIME: application/zip steps: - - uses: actions/checkout@v2 - - name: Set up Python 3.8 - uses: actions/setup-python@v2 + - uses: actions/checkout@v4 + - name: Set up Python 3.11 + uses: actions/setup-python@v4 with: - python-version: '3.8' + python-version: 3.11 - name: Install dependencies run: | python -m pip install --upgrade pip .[pyi] && pip list - name: Build with pyinstaller for ${{matrix.TARGET}} run: ${{matrix.CMD_BUILD}} - - name: Load Release URL File from release job - uses: actions/download-artifact@v2 - with: - name: release_url - path: release_url - - name: Get Release File Name & Upload URL - id: get_release_info - shell: bash - run: | - value=`cat release_url/release_url.txt` - echo ::set-output name=upload_url::$value - name: Upload Release Asset id: upload-release-asset uses: actions/upload-release-asset@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - upload_url: ${{ steps.get_release_info.outputs.upload_url }} + upload_url: ${{ needs.createrelease.outputs.release_url }} asset_path: ./dist/${{ matrix.OUT_FILE_NAME}} asset_name: ${{ matrix.OUT_FILE_NAME}} asset_content_type: ${{ matrix.ASSET_MIME}} diff --git a/README.md b/README.md index d99fd13..0600273 100644 --- a/README.md +++ b/README.md @@ -49,9 +49,8 @@ page (including beta releases). At this moment Windows, Linux and macOS are supp ### Links 1. Linux - - [debian 11 onefile](https://github.com/mkb79/audible-cli/releases/latest/download/audible_linux_debian_11.zip) - [ubuntu latest onefile](https://github.com/mkb79/audible-cli/releases/latest/download/audible_linux_ubuntu_latest.zip) - - [ubuntu 18.04 onefile](https://github.com/mkb79/audible-cli/releases/latest/download/audible_linux_ubuntu_18_04.zip) + - [ubuntu 20.04 onefile](https://github.com/mkb79/audible-cli/releases/latest/download/audible_linux_ubuntu_20_04.zip) 2. macOS - [macOS latest onefile](https://github.com/mkb79/audible-cli/releases/latest/download/audible_mac.zip) From 0f55179bff73b3c6454992cfcc32baa1842bd807 Mon Sep 17 00:00:00 2001 From: mkb79 Date: Fri, 29 Sep 2023 06:41:07 +0200 Subject: [PATCH 02/35] Create FUNDING.yml --- .github/FUNDING.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .github/FUNDING.yml diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..95697c9 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,13 @@ +# These are supported funding model platforms + +github: [mkb79] # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] +patreon: # Replace with a single Patreon username +open_collective: # Replace with a single Open Collective username +ko_fi: # Replace with a single Ko-fi username +tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel +community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry +liberapay: # Replace with a single Liberapay username +issuehunt: # Replace with a single IssueHunt username +otechie: # Replace with a single Otechie username +lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry +custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] From 275a2b2b77c4538d62dcb0a15534cda92cc6fbf2 Mon Sep 17 00:00:00 2001 From: mkb79 Date: Wed, 8 Nov 2023 22:05:22 +0100 Subject: [PATCH 03/35] Update marketplace choices in manage command (#161) * Update marketplace choices in `manage auth-file add` command * docs: update CHANGELOG.md --- CHANGELOG.md | 4 +++- src/audible_cli/cmds/cmd_manage.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 10e4556..ce68818 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## Unreleased -- +### Added + +- Update marketplace choices in `manage auth-file add` command. Now all available marketplaces are listed. ## [0.2.5] - 2022-09-26 diff --git a/src/audible_cli/cmds/cmd_manage.py b/src/audible_cli/cmds/cmd_manage.py index b4399b3..a72221f 100644 --- a/src/audible_cli/cmds/cmd_manage.py +++ b/src/audible_cli/cmds/cmd_manage.py @@ -160,7 +160,7 @@ def check_if_auth_file_not_exists(session, ctx, param, value): ) @click.option( "--country-code", "-cc", - type=click.Choice(["us", "ca", "uk", "au", "fr", "de", "jp", "it", "in"]), + type=click.Choice(AVAILABLE_MARKETPLACES), prompt="Please enter the country code", help="The country code for the marketplace you want to authenticate." ) From 72b4ff916f6156fd143c74673d06cb517a587f11 Mon Sep 17 00:00:00 2001 From: mkb79 Date: Wed, 8 Nov 2023 22:08:25 +0100 Subject: [PATCH 04/35] add freeze_support to pyinstaller entry script (#162) * fix: add freeze_support to pyinstaller entryscript * docs: update CHANGELOG.md --- CHANGELOG.md | 4 ++++ pyi_entrypoint.py | 9 +++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ce68818..3f69336 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Update marketplace choices in `manage auth-file add` command. Now all available marketplaces are listed. +### Misc + +- add `freeze_support` to pyinstaller entry script (#78) + ## [0.2.5] - 2022-09-26 ### Added diff --git a/pyi_entrypoint.py b/pyi_entrypoint.py index 633774f..c260363 100644 --- a/pyi_entrypoint.py +++ b/pyi_entrypoint.py @@ -1,4 +1,9 @@ -from audible_cli import cli +import multiprocessing -cli.main() +multiprocessing.freeze_support() + + +if __name__ == '__main__': + from audible_cli import cli + cli.main() From cda40c62d7ffbd2037c8973a197c3f006da99df1 Mon Sep 17 00:00:00 2001 From: mkb79 Date: Wed, 15 Nov 2023 07:52:49 +0100 Subject: [PATCH 05/35] fix: Fix progressbar issue (#165) * fix: Avoid tqdm progress bar interruption Avoid tqdm progress bar interruption by logger's output to console * doc: Update CHANGELOG.md --- CHANGELOG.md | 4 ++++ src/audible_cli/_logging.py | 12 ++++++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f69336..f719673 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Update marketplace choices in `manage auth-file add` command. Now all available marketplaces are listed. +### Bugfix + +- Avoid tqdm progress bar interruption by logger’s output to console. + ### Misc - add `freeze_support` to pyinstaller entry script (#78) diff --git a/src/audible_cli/_logging.py b/src/audible_cli/_logging.py index 82cbf18..47aea17 100644 --- a/src/audible_cli/_logging.py +++ b/src/audible_cli/_logging.py @@ -4,6 +4,7 @@ from typing import Optional, Union from warnings import warn import click +from tqdm import tqdm audible_cli_logger = logging.getLogger("audible_cli") @@ -100,10 +101,13 @@ class ClickHandler(logging.Handler): try: msg = self.format(record) level = record.levelname.lower() - if self.echo_kwargs.get(level): - click.echo(msg, **self.echo_kwargs[level]) - else: - click.echo(msg) + + # Avoid tqdm progress bar interruption by logger's output to console + with tqdm.external_write_mode(): + if self.echo_kwargs.get(level): + click.echo(msg, **self.echo_kwargs[level]) + else: + click.echo(msg) except Exception: self.handleError(record) From 8f8eacf3242ac5a989b26419a01a596b2df3a706 Mon Sep 17 00:00:00 2001 From: mkb79 Date: Wed, 15 Nov 2023 10:58:37 +0100 Subject: [PATCH 06/35] fix: Fixing an issue with unawaited coroutines (#166) Fixing an issue with unawaited coroutines when the download command exited abnormal. * Refactor consume function and queue_job function * doc: Update CHANGELOG.md --- CHANGELOG.md | 1 + src/audible_cli/cmds/cmd_download.py | 118 +++++++++++++-------------- 2 files changed, 60 insertions(+), 59 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f719673..f51801c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Bugfix - Avoid tqdm progress bar interruption by logger’s output to console. +- Fixing an issue with unawaited coroutines when the download command exited abnormal. ### Misc diff --git a/src/audible_cli/cmds/cmd_download.py b/src/audible_cli/cmds/cmd_download.py index 10fe248..f8d7232 100644 --- a/src/audible_cli/cmds/cmd_download.py +++ b/src/audible_cli/cmds/cmd_download.py @@ -418,9 +418,9 @@ async def download_aaxc( async def consume(queue, ignore_errors): while True: - item = await queue.get() + cmd, kwargs = await queue.get() try: - await item + await cmd(**kwargs) except Exception as e: logger.error(e) if not ignore_errors: @@ -450,73 +450,73 @@ def queue_job( if get_cover: for cover_size in cover_sizes: - queue.put_nowait( - download_cover( - client=client, - output_dir=output_dir, - base_filename=base_filename, - item=item, - res=cover_size, - overwrite_existing=overwrite_existing - ) - ) + cmd = download_cover + kwargs = { + "client": client, + "output_dir": output_dir, + "base_filename": base_filename, + "item": item, + "res": cover_size, + "overwrite_existing": overwrite_existing + } + queue.put_nowait((cmd, kwargs)) if get_pdf: - queue.put_nowait( - download_pdf( - client=client, - output_dir=output_dir, - base_filename=base_filename, - item=item, - overwrite_existing=overwrite_existing - ) - ) + cmd = download_pdf + kwargs = { + "client": client, + "output_dir": output_dir, + "base_filename": base_filename, + "item": item, + "overwrite_existing": overwrite_existing + } + queue.put_nowait((cmd, kwargs)) if get_chapters: - queue.put_nowait( - download_chapters( - output_dir=output_dir, - base_filename=base_filename, - item=item, - quality=quality, - overwrite_existing=overwrite_existing - ) - ) + cmd = download_chapters + kwargs = { + "output_dir": output_dir, + "base_filename": base_filename, + "item": item, + "quality": quality, + "overwrite_existing": overwrite_existing + } + queue.put_nowait((cmd, kwargs)) if get_annotation: - queue.put_nowait( - download_annotations( - output_dir=output_dir, - base_filename=base_filename, - item=item, - overwrite_existing=overwrite_existing - ) - ) + cmd = download_annotations + kwargs = { + "output_dir": output_dir, + "base_filename": base_filename, + "item": item, + "overwrite_existing": overwrite_existing + } + queue.put_nowait((cmd, kwargs)) if get_aax: - queue.put_nowait( - download_aax( - client=client, - output_dir=output_dir, - base_filename=base_filename, - item=item, - quality=quality, - overwrite_existing=overwrite_existing, - aax_fallback=aax_fallback - ) - ) + cmd = download_aax + kwargs = { + "client": client, + "output_dir": output_dir, + "base_filename": base_filename, + "item": item, + "quality": quality, + "overwrite_existing": overwrite_existing, + "aax_fallback": aax_fallback + } + queue.put_nowait((cmd, kwargs)) if get_aaxc: - queue.put_nowait( - download_aaxc( - client=client, - output_dir=output_dir, - base_filename=base_filename, - item=item, - quality=quality, - overwrite_existing=overwrite_existing - ) - ) + cmd = download_aaxc + kwargs = { + "client": client, + "output_dir": output_dir, + "base_filename": base_filename, + "item": item, + "quality": quality, + "overwrite_existing": overwrite_existing + } + queue.put_nowait((cmd, kwargs)) def display_counter(): From e996fb619dc194ed15beafc22ba257c6cf4903ad Mon Sep 17 00:00:00 2001 From: mkb79 Date: Thu, 16 Nov 2023 13:38:54 +0100 Subject: [PATCH 07/35] Update httpx version range to >=0.23.3 and <0.26.0 --- CHANGELOG.md | 6 +++++- setup.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f51801c..fc985ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Bugfix - Avoid tqdm progress bar interruption by logger’s output to console. -- Fixing an issue with unawaited coroutines when the download command exited abnormal. +- Fixing an issue with unawaited coroutines when the download command exited abnormal. + +### Changed + +- Update httpx version range to >=0.23.3 and <0.26.0. ### Misc diff --git a/setup.py b/setup.py index f7848f4..594ae5c 100644 --- a/setup.py +++ b/setup.py @@ -49,7 +49,7 @@ setup( "audible>=0.8.2", "click>=8", "colorama; platform_system=='Windows'", - "httpx>=0.20.0,<0.24.0", + "httpx>=0.23.3,<0.26.0", "packaging", "Pillow", "tabulate", From d463dbae945a739fab0231c8b6ff65f0b39226dd Mon Sep 17 00:00:00 2001 From: mkb79 Date: Thu, 16 Nov 2023 13:55:54 +0100 Subject: [PATCH 08/35] Release v0.2.6 --- CHANGELOG.md | 4 ++++ src/audible_cli/_version.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fc985ef..0df7f85 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## Unreleased +- + +## [0.2.6] - 2023-11-16 + ### Added - Update marketplace choices in `manage auth-file add` command. Now all available marketplaces are listed. diff --git a/src/audible_cli/_version.py b/src/audible_cli/_version.py index 222ae3a..d25f15f 100644 --- a/src/audible_cli/_version.py +++ b/src/audible_cli/_version.py @@ -1,7 +1,7 @@ __title__ = "audible-cli" __description__ = "Command line interface (cli) for the audible package." __url__ = "https://github.com/mkb79/audible-cli" -__version__ = "0.2.5" +__version__ = "0.2.6" __author__ = "mkb79" __author_email__ = "mkb79@hackitall.de" __license__ = "AGPL" From 1bfbf72f66c24776a2bdb997efacf394e7a186df Mon Sep 17 00:00:00 2001 From: mkb79 Date: Thu, 16 Nov 2023 14:02:16 +0100 Subject: [PATCH 09/35] doc: fix CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0df7f85..349f8e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,7 +27,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - add `freeze_support` to pyinstaller entry script (#78) -## [0.2.5] - 2022-09-26 +## [0.2.5] - 2023-09-26 ### Added From 8bb611e5db3ba04591fe366227b039387f6fb132 Mon Sep 17 00:00:00 2001 From: vwkd <33468089+vwkd@users.noreply.github.com> Date: Tue, 9 Jan 2024 18:42:46 +0400 Subject: [PATCH 10/35] Add `-f, --force-rebuild-chapters` option to `decrypt` (#175) --- plugin_cmds/cmd_decrypt.py | 65 ++++++++++++++++++++++++++------------ 1 file changed, 45 insertions(+), 20 deletions(-) diff --git a/plugin_cmds/cmd_decrypt.py b/plugin_cmds/cmd_decrypt.py index f8f366b..75ecda2 100644 --- a/plugin_cmds/cmd_decrypt.py +++ b/plugin_cmds/cmd_decrypt.py @@ -271,6 +271,7 @@ class FFMeta: def update_chapters_from_chapter_info( self, chapter_info: ApiChapterInfo, + force_rebuild_chapters: bool = False, separate_intro_outro: bool = False ) -> None: if not chapter_info.is_accurate(): @@ -278,9 +279,12 @@ class FFMeta: return if chapter_info.count_chapters() != self.count_chapters(): - raise ChapterError("Chapter mismatch") + if force_rebuild_chapters: + echo("Force rebuild chapters due to chapter mismatch.") + else: + raise ChapterError("Chapter mismatch") - echo(f"Found {self.count_chapters()} chapters to prepare.") + echo(f"Found {chapter_info.count_chapters()} chapters to prepare.") api_chapters = chapter_info.get_chapters(separate_intro_outro) @@ -322,7 +326,8 @@ class FfmpegFileDecrypter: tempdir: pathlib.Path, activation_bytes: t.Optional[str], rebuild_chapters: bool, - ignore_missing_chapters: bool, + force_rebuild_chapters: bool, + skip_rebuild_chapters: bool, separate_intro_outro: bool ) -> None: file_type = SupportedFiles(file.suffix) @@ -344,7 +349,8 @@ class FfmpegFileDecrypter: self._target_dir = target_dir self._tempdir = tempdir self._rebuild_chapters = rebuild_chapters - self._ignore_missing_chapters = ignore_missing_chapters + self._force_rebuild_chapters = force_rebuild_chapters + self._skip_rebuild_chapters = skip_rebuild_chapters self._separate_intro_outro = separate_intro_outro self._api_chapter: t.Optional[ApiChapterInfo] = None self._ffmeta: t.Optional[FFMeta] = None @@ -405,7 +411,7 @@ class FfmpegFileDecrypter: def rebuild_chapters(self) -> None: if not self._is_rebuilded: self.ffmeta.update_chapters_from_chapter_info( - self.api_chapter, self._separate_intro_outro + self.api_chapter, self._force_rebuild_chapters, self._separate_intro_outro ) self._is_rebuilded = True @@ -450,7 +456,9 @@ class FfmpegFileDecrypter: self.rebuild_chapters() self.ffmeta.write(metafile) except ChapterError: - if not self._ignore_missing_chapters: + if self._skip_rebuild_chapters: + echo("Skip rebuild chapters due to chapter mismatch.") + else: raise else: base_cmd.extend( @@ -501,6 +509,25 @@ class FfmpegFileDecrypter: is_flag=True, help="Rebuild chapters with chapters from voucher or chapter file." ) +@click.option( + "--force-rebuild-chapters", + "-f", + is_flag=True, + help=( + "Force rebuild chapters with chapters from voucher or chapter file " + "if the built-in chapters in the audio file mismatch. " + "Only use with `--rebuild-chapters`." + ), +) +@click.option( + "--skip-rebuild-chapters", + "-t", + is_flag=True, + help=( + "Decrypt without rebuilding chapters when chapters mismatch. " + "Only use with `--rebuild-chapters`." + ), +) @click.option( "--separate-intro-outro", "-s", @@ -510,16 +537,6 @@ class FfmpegFileDecrypter: "Only use with `--rebuild-chapters`." ), ) -@click.option( - "--ignore-missing-chapters", - "-t", - is_flag=True, - help=( - "Decrypt without rebuilding chapters when chapters are not present. " - "Otherwise an item is skipped when this option is not provided. " - "Only use with `--rebuild-chapters`." - ), -) @pass_session def cli( session, @@ -528,8 +545,9 @@ def cli( all_: bool, overwrite: bool, rebuild_chapters: bool, + force_rebuild_chapters: bool, + skip_rebuild_chapters: bool, separate_intro_outro: bool, - ignore_missing_chapters: bool ): """Decrypt audiobooks downloaded with audible-cli. @@ -543,12 +561,18 @@ def cli( ctx = click.get_current_context() ctx.fail("ffmpeg not found") - if (separate_intro_outro or ignore_missing_chapters) and not rebuild_chapters: + if (force_rebuild_chapters or skip_rebuild_chapters or separate_intro_outro) and not rebuild_chapters: raise click.BadOptionUsage( - "`--separate-intro-outro` and `--ignore-missing-chapters` can " + "`--force-rebuild-chapters`, `--skip-rebuild-chapters` and `--separate-intro-outro` can " "only be used together with `--rebuild-chapters`" ) + if force_rebuild_chapters and skip_rebuild_chapters: + raise click.BadOptionUsage( + "`--force-rebuild-chapters` and `--skip-rebuild-chapters` can " + "not be used together" + ) + if all_: if files: raise click.BadOptionUsage( @@ -565,7 +589,8 @@ def cli( tempdir=pathlib.Path(tempdir).resolve(), activation_bytes=session.auth.activation_bytes, rebuild_chapters=rebuild_chapters, - ignore_missing_chapters=ignore_missing_chapters, + force_rebuild_chapters=force_rebuild_chapters, + skip_rebuild_chapters=skip_rebuild_chapters, separate_intro_outro=separate_intro_outro ) decrypter.run() From e82aa52c02da094b463ab9743e36dd10790a7794 Mon Sep 17 00:00:00 2001 From: vwkd <33468089+vwkd@users.noreply.github.com> Date: Tue, 9 Jan 2024 19:39:55 +0400 Subject: [PATCH 11/35] add overwrite option to decrypt command (#176) --- plugin_cmds/cmd_decrypt.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/plugin_cmds/cmd_decrypt.py b/plugin_cmds/cmd_decrypt.py index 75ecda2..411a398 100644 --- a/plugin_cmds/cmd_decrypt.py +++ b/plugin_cmds/cmd_decrypt.py @@ -325,6 +325,7 @@ class FfmpegFileDecrypter: target_dir: pathlib.Path, tempdir: pathlib.Path, activation_bytes: t.Optional[str], + overwrite: bool, rebuild_chapters: bool, force_rebuild_chapters: bool, skip_rebuild_chapters: bool, @@ -348,6 +349,7 @@ class FfmpegFileDecrypter: self._credentials: t.Optional[t.Union[str, t.Tuple[str]]] = credentials self._target_dir = target_dir self._tempdir = tempdir + self._overwrite = overwrite self._rebuild_chapters = rebuild_chapters self._force_rebuild_chapters = force_rebuild_chapters self._skip_rebuild_chapters = skip_rebuild_chapters @@ -420,8 +422,11 @@ class FfmpegFileDecrypter: outfile = self._target_dir / oname if outfile.exists(): - secho(f"Skip {outfile}: already exists", fg="blue") - return + if self._overwrite: + secho(f"Overwrite {outfile}: already exists", fg="blue") + else: + secho(f"Skip {outfile}: already exists", fg="blue") + return base_cmd = [ "ffmpeg", @@ -429,6 +434,8 @@ class FfmpegFileDecrypter: "quiet", "-stats", ] + if self._overwrite: + base_cmd.append("-y") if isinstance(self._credentials, tuple): key, iv = self._credentials credentials_cmd = [ @@ -588,6 +595,7 @@ def cli( target_dir=pathlib.Path(directory).resolve(), tempdir=pathlib.Path(tempdir).resolve(), activation_bytes=session.auth.activation_bytes, + overwrite=overwrite, rebuild_chapters=rebuild_chapters, force_rebuild_chapters=force_rebuild_chapters, skip_rebuild_chapters=skip_rebuild_chapters, From 385f353403e254030646bd356a227130e98b797d Mon Sep 17 00:00:00 2001 From: mkb79 Date: Fri, 19 Jan 2024 08:23:45 +0100 Subject: [PATCH 12/35] feat: Add a resume feature for aaxc file (#164) * Add a resume feature for aaxc file * rework downloader class * add additional message * forgot to reduce MAX_FILE_READ_SIZE to 3 MB * Rework downloader module * Update CHANGELOG.md --- CHANGELOG.md | 5 +- src/audible_cli/cmds/cmd_download.py | 45 ++- src/audible_cli/downloader.py | 551 +++++++++++++++++++++++++++ 3 files changed, 585 insertions(+), 16 deletions(-) create mode 100644 src/audible_cli/downloader.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 349f8e3..25c7cde 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## Unreleased -- +### Added + +- Added a resume feature when downloading aaxc files. +- New `downlaoder` module which contains a rework of the Downloader class. ## [0.2.6] - 2023-11-16 diff --git a/src/audible_cli/cmds/cmd_download.py b/src/audible_cli/cmds/cmd_download.py index f8d7232..6b8ac23 100644 --- a/src/audible_cli/cmds/cmd_download.py +++ b/src/audible_cli/cmds/cmd_download.py @@ -21,6 +21,7 @@ from ..decorators import ( pass_client, pass_session ) +from ..downloader import Downloader as NewDownloader, Status from ..exceptions import ( AudibleCliException, DirectoryDoesNotExists, @@ -277,14 +278,24 @@ async def download_aax( filename = base_filename + f"-{codec}.aax" filepath = output_dir / filename - dl = Downloader( - url, filepath, client, overwrite_existing, - ["audio/aax", "audio/vnd.audible.aax", "audio/audible"] - ) - downloaded = await dl.run(pb=True) - if downloaded: + dl = NewDownloader( + source=url, + client=client, + expected_types=[ + "audio/aax", "audio/vnd.audible.aax", "audio/audible" + ] + ) + downloaded = await dl.run(target=filepath, force_reload=overwrite_existing) + + if downloaded.status == Status.Success: counter.count_aax() + elif downloaded.status == Status.DownloadIndividualParts: + # TODO: Add parts to download queue + logger.error( + f"Item {filepath} must be downloaded in parts. This feature will be added " + f"in the future." + ) async def _reuse_voucher(lr_file, item): @@ -398,22 +409,26 @@ async def download_aaxc( logger.info(f"Voucher file saved to {lr_file}.") counter.count_voucher_saved() - dl = Downloader( - url, - filepath, - client, - overwrite_existing, - [ + dl = NewDownloader( + source=url, + client=client, + expected_types=[ "audio/aax", "audio/vnd.audible.aax", "audio/mpeg", "audio/x-m4a", "audio/audible" - ] + ], ) - downloaded = await dl.run(pb=True) + downloaded = await dl.run(target=filepath, force_reload=overwrite_existing) - if downloaded: + if downloaded.status == Status.Success: counter.count_aaxc() if is_aycl: counter.count_aycl() + elif downloaded.status == Status.DownloadIndividualParts: + # TODO: Add parts to download queue + logger.error( + f"Item {filepath} must be downloaded in parts. This feature will be added " + f"in the future." + ) async def consume(queue, ignore_errors): diff --git a/src/audible_cli/downloader.py b/src/audible_cli/downloader.py new file mode 100644 index 0000000..2ef9391 --- /dev/null +++ b/src/audible_cli/downloader.py @@ -0,0 +1,551 @@ +import logging +import pathlib +import re +from enum import Enum, auto +from typing import Any, Callable, Dict, List, NamedTuple, Optional, Union + +import aiofiles +import click +import httpx +import tqdm +from aiofiles.os import path, unlink + +try: + from typing import Literal +except ImportError: + from typing_extensions import Literal + + +FileMode = Literal["ab", "wb"] + +logger = logging.getLogger("audible_cli.downloader") + +ACCEPT_RANGES_HEADER = "Accept-Ranges" +ACCEPT_RANGES_NONE_VALUE = "none" +CONTENT_LENGTH_HEADER = "Content-Length" +CONTENT_TYPE_HEADER = "Content-Type" +MAX_FILE_READ_SIZE = 3 * 1024 * 1024 +ETAG_HEADER = "ETag" + + +class ETag: + def __init__(self, etag: str) -> None: + self._etag = etag + + @property + def value(self) -> str: + return self._etag + + @property + def parsed_etag(self) -> str: + return re.search('"([^"]*)"', self.value).group(1) + + @property + def is_weak(self) -> bool: + return bool(re.search("^W/", self.value)) + + +class File: + def __init__(self, file: Union[pathlib.Path, str]) -> None: + if not isinstance(file, pathlib.Path): + file = pathlib.Path(file) + self._file = file + + @property + def path(self) -> pathlib.Path: + return self._file + + async def get_size(self) -> int: + if await path.isfile(self.path): + return await path.getsize(self.path) + return 0 + + async def remove(self) -> None: + if await path.isfile(self.path): + await unlink(self.path) + + async def directory_exists(self) -> bool: + return await path.isdir(self.path.parent) + + async def is_file(self) -> bool: + return await path.isfile(self.path) and not await self.is_link() + + async def is_link(self) -> bool: + return await path.islink(self.path) + + async def exists(self) -> bool: + return await path.exists(self.path) + + async def read_text_content( + self, max_bytes: int = MAX_FILE_READ_SIZE, encoding: str = "utf-8", errors=None + ) -> str: + file_size = await self.get_size() + read_size = min(max_bytes, file_size) + try: + async with aiofiles.open( + file=self.path, mode="r", encoding=encoding, errors=errors + ) as file: + return await file.read(read_size) + except Exception: # noqa + return "Unknown" + + +class ResponseInfo: + def __init__(self, response: httpx.Response) -> None: + self._response = response + self.headers: httpx.Headers = response.headers + self.status_code: int = response.status_code + self.content_length: Optional[int] = self._get_content_length(self.headers) + self.content_type: Optional[str] = self._get_content_type(self.headers) + self.accept_ranges: bool = self._does_accept_ranges(self.headers) + self.etag: Optional[ETag] = self._get_etag(self.headers) + + @property + def response(self) -> httpx.Response: + return self._response + + def supports_resume(self) -> bool: + return bool(self.accept_ranges) + + @staticmethod + def _does_accept_ranges(headers: httpx.Headers) -> bool: + # 'Accept-Ranges' indicates if the source accepts range requests, + # that let you retrieve a part of the response + accept_ranges_value = headers.get( + ACCEPT_RANGES_HEADER, ACCEPT_RANGES_NONE_VALUE + ) + does_accept_ranges = accept_ranges_value != ACCEPT_RANGES_NONE_VALUE + + return does_accept_ranges + + @staticmethod + def _get_content_length(headers: httpx.Headers) -> Optional[int]: + content_length = headers.get(CONTENT_LENGTH_HEADER) + + if content_length is not None: + return int(content_length) + + return content_length + + @staticmethod + def _get_content_type(headers: httpx.Headers) -> Optional[str]: + return headers.get(CONTENT_TYPE_HEADER) + + @staticmethod + def _get_etag(headers: httpx.Headers) -> Optional[ETag]: + etag_header = headers.get(ETAG_HEADER) + if etag_header is None: + return etag_header + return ETag(etag_header) + + +class Status(Enum): + Success = auto() + DestinationAlreadyExists = auto() + DestinationFolderNotExists = auto() + DestinationNotAFile = auto() + DownloadError = auto() + DownloadErrorStatusCode = auto() + DownloadSizeMismatch = auto() + DownloadContentTypeMismatch = auto() + DownloadIndividualParts = auto() + SourceDoesNotSupportResume = auto() + StatusCode = auto() + + +async def check_target_file_status( + target_file: File, force_reload: bool, **kwargs: Any +) -> Status: + if not await target_file.directory_exists(): + logger.error( + f"Folder {target_file.path} does not exists! Skip download." + ) + return Status.DestinationFolderNotExists + + if await target_file.exists() and not await target_file.is_file(): + logger.error( + f"Object {target_file.path} exists but is not a file. Skip download." + ) + return Status.DestinationNotAFile + + if await target_file.is_file() and not force_reload: + logger.info( + f"File {target_file.path} already exists. Skip download." + ) + return Status.DestinationAlreadyExists + + return Status.Success + + +async def check_download_size( + tmp_file: File, target_file: File, head_response: ResponseInfo, **kwargs: Any +) -> Status: + tmp_file_size = await tmp_file.get_size() + content_length = head_response.content_length + + if tmp_file_size is not None and content_length is not None: + if tmp_file_size != content_length: + logger.error( + f"Error downloading {target_file.path}. File size missmatch. " + f"Expected size: {content_length}; Downloaded: {tmp_file_size}" + ) + return Status.DownloadSizeMismatch + + return Status.Success + + +async def check_status_code( + response: ResponseInfo, tmp_file: File, target_file: File, **kwargs: Any +) -> Status: + if not 200 <= response.status_code < 400: + content = await tmp_file.read_text_content() + logger.error( + f"Error downloading {target_file.path}. Message: {content}" + ) + return Status.StatusCode + + return Status.Success + + +async def check_content_type( + response: ResponseInfo, target_file: File, tmp_file: File, + expected_types: List[str], **kwargs: Any +) -> Status: + if not expected_types: + return Status.Success + + if response.content_type not in expected_types: + content = await tmp_file.read_text_content() + logger.error( + f"Error downloading {target_file.path}. Wrong content type. " + f"Expected type(s): {expected_types}; " + f"Got: {response.content_type}; Message: {content}" + ) + return Status.DownloadContentTypeMismatch + + return Status.Success + + +def _status_for_message(message: str) -> Status: + if "please download individual parts" in message: + return Status.DownloadIndividualParts + return Status.Success + + +async def check_status_for_message( + response: ResponseInfo, tmp_file: File, **kwargs: Any +) -> Status: + if ( + response.content_length and response.content_type + and response.content_length <= MAX_FILE_READ_SIZE + and "text" in response.content_type + ): + message = await tmp_file.read_text_content() + return _status_for_message(message) + return Status.Success + + +class DownloadResult(NamedTuple): + status: Status + destination: File + head_response: Optional[ResponseInfo] + response: Optional[ResponseInfo] + message: Optional[str] + + +class DummyProgressBar: + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + pass + + def update(self, *args, **kwargs): + pass + + +def get_progressbar( + destination: pathlib.Path, total: Optional[int], start: int = 0 +) -> Union[tqdm.tqdm, DummyProgressBar]: + if total is None: + return DummyProgressBar() + + description = click.format_filename(destination, shorten=True) + progressbar = tqdm.tqdm( + desc=description, + total=total, + unit="B", + unit_scale=True, + unit_divisor=1024 + ) + if start > 0: + progressbar.update(start) + + return progressbar + + +class Downloader: + + MIN_STREAM_LENGTH = 10*1024*1024 # using stream mode if source is greater than + MIN_RESUME_FILE_LENGTH = 10*1024*1024 # keep resume file if file is greater than + RESUME_SUFFIX = ".resume" + TMP_SUFFIX = ".tmp" + + def __init__( + self, + source: httpx.URL, + client: httpx.AsyncClient, + expected_types: Optional[Union[List[str], str]] = None, + additional_headers: Optional[Dict[str, str]] = None + ) -> None: + self._source = source + self._client = client + self._expected_types = self._normalize_expected_types(expected_types) + self._additional_headers = self._normalize_headers(additional_headers) + self._head_request: Optional[ResponseInfo] = None + + @staticmethod + def _normalize_expected_types( + expected_types: Optional[Union[List[str], str]] + ) -> List[str]: + if not isinstance(expected_types, list): + if expected_types is None: + expected_types = [] + else: + expected_types = [expected_types] + return expected_types + + @staticmethod + def _normalize_headers(headers: Optional[Dict[str, str]]) -> Dict[str, str]: + if headers is None: + return {} + return headers + + async def get_head_response(self, force_recreate: bool = False) -> ResponseInfo: + if self._head_request is None or force_recreate: + head_response = await self._client.head( + self._source, headers=self._additional_headers, follow_redirects=True, + ) + self._head_request = ResponseInfo(head_response) + + return self._head_request + + async def _determine_resume_file(self, target_file: File) -> File: + head_response = await self.get_head_response() + etag = head_response.etag + resume_name = target_file.path if etag is None else etag.parsed_etag + resume_file = pathlib.Path(resume_name).with_suffix(self.RESUME_SUFFIX) + return File(resume_file) + + def _determine_tmp_file(self, target_file: File) -> File: + tmp_file = pathlib.Path(target_file.path).with_suffix(self.TMP_SUFFIX) + return File(tmp_file) + + async def _handle_tmp_file( + self, tmp_file: File, supports_resume: bool, response: ResponseInfo + ) -> None: + tmp_file_size = await tmp_file.get_size() + expected_size = response.content_length + + if ( + supports_resume and expected_size is not None + and self.MIN_RESUME_FILE_LENGTH < tmp_file_size < expected_size + ): + logger.debug(f"Keep resume file {tmp_file.path}") + else: + await tmp_file.remove() + + @staticmethod + async def _rename_file( + tmp_file: File, target_file: File, force_reload: bool, response: ResponseInfo + ) -> Status: + target_path = target_file.path + + if await target_file.exists() and force_reload: + i = 0 + while target_path.with_suffix(f"{target_path.suffix}.old.{i}").exists(): + i += 1 + target_path.rename(target_path.with_suffix(f"{target_path.suffix}.old.{i}")) + + tmp_file.path.rename(target_path) + logger.info( + f"File {target_path} downloaded in {response.response.elapsed}." + ) + return Status.Success + + @staticmethod + async def _check_and_return_download_result( + status_check_func: Callable, + tmp_file: File, + target_file: File, + response: ResponseInfo, + head_response: ResponseInfo, + expected_types: List[str] + ) -> Optional[DownloadResult]: + status = await status_check_func( + response=response, + tmp_file=tmp_file, + target_file=target_file, + expected_types=expected_types + ) + if status != Status.Success: + message = await tmp_file.read_text_content() + return DownloadResult( + status=status, + destination=target_file, + head_response=head_response, + response=response, + message=message + ) + return None + + async def _postprocessing( + self, tmp_file: File, target_file: File, response: ResponseInfo, + force_reload: bool + ) -> DownloadResult: + head_response = await self.get_head_response() + + status_checks = [ + check_status_for_message, + check_status_code, + check_status_code, + check_content_type + ] + for check in status_checks: + result = await self._check_and_return_download_result( + check, tmp_file, target_file, response, + head_response, self._expected_types + ) + if result: + return result + + await self._rename_file( + tmp_file=tmp_file, + target_file=target_file, + force_reload=force_reload, + response=response, + ) + + return DownloadResult( + status=Status.Success, + destination=target_file, + head_response=head_response, + response=response, + message=None + ) + + async def _stream_download( + self, + tmp_file: File, + target_file: File, + start: int, + progressbar: Union[tqdm.tqdm, DummyProgressBar], + force_reload: bool = True + ) -> DownloadResult: + headers = self._additional_headers.copy() + if start > 0: + headers.update(Range=f"bytes={start}-") + file_mode: FileMode = "ab" + else: + file_mode: FileMode = "wb" + + async with self._client.stream( + method="GET", url=self._source, follow_redirects=True, headers=headers + ) as response: + with progressbar: + async with aiofiles.open(tmp_file.path, mode=file_mode) as file: + async for chunk in response.aiter_bytes(): + await file.write(chunk) + progressbar.update(len(chunk)) + + return await self._postprocessing( + tmp_file=tmp_file, + target_file=target_file, + response=ResponseInfo(response=response), + force_reload=force_reload + ) + + async def _download( + self, tmp_file: File, target_file: File, start: int, force_reload: bool + ) -> DownloadResult: + headers = self._additional_headers.copy() + if start > 0: + headers.update(Range=f"bytes={start}-") + file_mode: FileMode = "ab" + else: + file_mode: FileMode = "wb" + + response = await self._client.get( + self._source, follow_redirects=True, headers=headers + ) + async with aiofiles.open(tmp_file.path, mode=file_mode) as file: + await file.write(response.content) + + return await self._postprocessing( + tmp_file=tmp_file, + target_file=target_file, + response=ResponseInfo(response=response), + force_reload=force_reload + ) + + async def run( + self, + target: pathlib.Path, + force_reload: bool = False + ) -> DownloadResult: + target_file = File(target) + destination_status = await check_target_file_status( + target_file, force_reload + ) + if destination_status != Status.Success: + return DownloadResult( + status=destination_status, + destination=target_file, + head_response=None, + response=None, + message=None + ) + + head_response = await self.get_head_response() + supports_resume = head_response.supports_resume() + if supports_resume: + tmp_file = await self._determine_resume_file(target_file=target_file) + start = await tmp_file.get_size() + else: + tmp_file = self._determine_tmp_file(target_file=target_file) + await tmp_file.remove() + start = 0 + + should_stream = False + progressbar = None + if ( + head_response.content_length is not None and + head_response.content_length >= self.MIN_STREAM_LENGTH + ): + should_stream = True + progressbar = get_progressbar( + target_file.path, head_response.content_length, start + ) + + try: + if should_stream: + return await self._stream_download( + tmp_file=tmp_file, + target_file=target_file, + start=start, + progressbar=progressbar, + force_reload=force_reload + ) + else: + return await self._download( + tmp_file=tmp_file, + target_file=target_file, + start=start, + force_reload=force_reload + ) + finally: + await self._handle_tmp_file( + tmp_file=tmp_file, + supports_resume=supports_resume, + response=head_response + ) From 56333360da339f55f05276dc02e448e7e6ae78d7 Mon Sep 17 00:00:00 2001 From: mkb79 Date: Tue, 23 Jan 2024 22:12:01 +0100 Subject: [PATCH 13/35] docs: Add install guide using pipx to README.md --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 0600273..4664543 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,14 @@ pip install . ``` +or as the best solution using [pipx](https://pipx.pypa.io/stable/) + +```shell + +pipx install audible-cli + +``` + ## Standalone executables If you don't want to install `Python` and `audible-cli` on your machine, you can From ce8f59b732dc168f0ad90b4f021172bc1c2ffb7a Mon Sep 17 00:00:00 2001 From: mkb79 Date: Mon, 29 Jan 2024 11:44:01 +0100 Subject: [PATCH 14/35] feat: downloading large audiobooks (#178) * catch asyncio.CancelledError * add download parts feature * fix downloading book in parts * fix aax_fallback * Update default timeout value to 30 * Refactor check_status_for_message function * rework is_published method to take care about AudioPartd * update cmd_download.py * Update CHANGELOG.md --- CHANGELOG.md | 1 + src/audible_cli/cli.py | 4 + src/audible_cli/cmds/cmd_download.py | 129 ++++++++++++++++++++------- src/audible_cli/decorators.py | 2 +- src/audible_cli/downloader.py | 13 ++- src/audible_cli/models.py | 12 ++- 6 files changed, 118 insertions(+), 43 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 25c7cde..d9a5635 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Added a resume feature when downloading aaxc files. - New `downlaoder` module which contains a rework of the Downloader class. +- If necessary, large audiobooks are now downloaded in parts. ## [0.2.6] - 2023-11-16 diff --git a/src/audible_cli/cli.py b/src/audible_cli/cli.py index 4bb75aa..28591a7 100644 --- a/src/audible_cli/cli.py +++ b/src/audible_cli/cli.py @@ -1,3 +1,4 @@ +import asyncio import logging import sys from pkg_resources import iter_entry_points @@ -61,6 +62,9 @@ def main(*args, **kwargs): except click.Abort: logger.error("Aborted") sys.exit(1) + except asyncio.CancelledError: + logger.error("Aborted with Asyncio CancelledError") + sys.exit(2) except AudibleCliException as e: logger.error(e) sys.exit(2) diff --git a/src/audible_cli/cmds/cmd_download.py b/src/audible_cli/cmds/cmd_download.py index 6b8ac23..3cfb20a 100644 --- a/src/audible_cli/cmds/cmd_download.py +++ b/src/audible_cli/cmds/cmd_download.py @@ -39,6 +39,8 @@ CLIENT_HEADERS = { "User-Agent": "Audible/671 CFNetwork/1240.0.4 Darwin/20.6.0" } +QUEUE = asyncio.Queue() + class DownloadCounter: def __init__(self): @@ -256,9 +258,55 @@ async def download_annotations( counter.count_annotation() +async def _get_audioparts(item): + parts = [] + child_library: Library = await item.get_child_items() + if child_library is not None: + for child in child_library: + if ( + child.content_delivery_type is not None + and child.content_delivery_type == "AudioPart" + ): + parts.append(child) + + return parts + + +async def _add_audioparts_to_queue( + client, output_dir, filename_mode, item, quality, overwrite_existing, + aax_fallback, download_mode +): + parts = await _get_audioparts(item) + + if download_mode == "aax": + get_aax = True + get_aaxc = False + else: + get_aax = False + get_aaxc = True + + for part in parts: + queue_job( + get_cover=None, + get_pdf=None, + get_annotation=None, + get_chapters=None, + get_aax=get_aax, + get_aaxc=get_aaxc, + client=client, + output_dir=output_dir, + filename_mode=filename_mode, + item=part, + cover_sizes=None, + quality=quality, + overwrite_existing=overwrite_existing, + aax_fallback=aax_fallback + ) + + async def download_aax( client, output_dir, base_filename, item, quality, overwrite_existing, - aax_fallback + aax_fallback, filename_mode ): # url, codec = await item.get_aax_url(quality) try: @@ -272,7 +320,8 @@ async def download_aax( base_filename=base_filename, item=item, quality=quality, - overwrite_existing=overwrite_existing + overwrite_existing=overwrite_existing, + filename_mode=filename_mode ) raise @@ -291,10 +340,18 @@ async def download_aax( if downloaded.status == Status.Success: counter.count_aax() elif downloaded.status == Status.DownloadIndividualParts: - # TODO: Add parts to download queue - logger.error( - f"Item {filepath} must be downloaded in parts. This feature will be added " - f"in the future." + logger.info( + f"Item {filepath} must be downloaded in parts. Adding parts to queue" + ) + await _add_audioparts_to_queue( + client=client, + output_dir=output_dir, + filename_mode=filename_mode, + item=item, + quality=quality, + overwrite_existing=overwrite_existing, + download_mode="aax", + aax_fallback=aax_fallback, ) @@ -349,8 +406,8 @@ async def _reuse_voucher(lr_file, item): async def download_aaxc( - client, output_dir, base_filename, item, - quality, overwrite_existing + client, output_dir, base_filename, item, quality, overwrite_existing, + filename_mode ): lr, url, codec = None, None, None @@ -424,16 +481,24 @@ async def download_aaxc( if is_aycl: counter.count_aycl() elif downloaded.status == Status.DownloadIndividualParts: - # TODO: Add parts to download queue - logger.error( - f"Item {filepath} must be downloaded in parts. This feature will be added " - f"in the future." + logger.info( + f"Item {filepath} must be downloaded in parts. Adding parts to queue" + ) + await _add_audioparts_to_queue( + client=client, + output_dir=output_dir, + filename_mode=filename_mode, + item=item, + quality=quality, + overwrite_existing=overwrite_existing, + aax_fallback=False, + download_mode="aaxc" ) -async def consume(queue, ignore_errors): +async def consume(ignore_errors): while True: - cmd, kwargs = await queue.get() + cmd, kwargs = await QUEUE.get() try: await cmd(**kwargs) except Exception as e: @@ -441,11 +506,10 @@ async def consume(queue, ignore_errors): if not ignore_errors: raise finally: - queue.task_done() + QUEUE.task_done() def queue_job( - queue, get_cover, get_pdf, get_annotation, @@ -474,7 +538,7 @@ def queue_job( "res": cover_size, "overwrite_existing": overwrite_existing } - queue.put_nowait((cmd, kwargs)) + QUEUE.put_nowait((cmd, kwargs)) if get_pdf: cmd = download_pdf @@ -485,7 +549,7 @@ def queue_job( "item": item, "overwrite_existing": overwrite_existing } - queue.put_nowait((cmd, kwargs)) + QUEUE.put_nowait((cmd, kwargs)) if get_chapters: cmd = download_chapters @@ -496,7 +560,7 @@ def queue_job( "quality": quality, "overwrite_existing": overwrite_existing } - queue.put_nowait((cmd, kwargs)) + QUEUE.put_nowait((cmd, kwargs)) if get_annotation: cmd = download_annotations @@ -506,7 +570,7 @@ def queue_job( "item": item, "overwrite_existing": overwrite_existing } - queue.put_nowait((cmd, kwargs)) + QUEUE.put_nowait((cmd, kwargs)) if get_aax: cmd = download_aax @@ -517,9 +581,10 @@ def queue_job( "item": item, "quality": quality, "overwrite_existing": overwrite_existing, - "aax_fallback": aax_fallback + "aax_fallback": aax_fallback, + "filename_mode": filename_mode } - queue.put_nowait((cmd, kwargs)) + QUEUE.put_nowait((cmd, kwargs)) if get_aaxc: cmd = download_aaxc @@ -529,9 +594,10 @@ def queue_job( "base_filename": base_filename, "item": item, "quality": quality, - "overwrite_existing": overwrite_existing + "overwrite_existing": overwrite_existing, + "filename_mode": filename_mode } - queue.put_nowait((cmd, kwargs)) + QUEUE.put_nowait((cmd, kwargs)) def display_counter(): @@ -683,7 +749,7 @@ async def cli(session, api_client, **params): asins = params.get("asin") titles = params.get("title") if get_all and (asins or titles): - logger.error(f"Do not mix *asin* or *title* option with *all* option.") + logger.error("Do not mix *asin* or *title* option with *all* option.") click.Abort() # what to download @@ -803,7 +869,6 @@ async def cli(session, api_client, **params): f"Skip title {title}: Not found in library" ) - queue = asyncio.Queue() for job in jobs: item = library.get_item_by_asin(job) items = [item] @@ -827,7 +892,6 @@ async def cli(session, api_client, **params): for item in items: queue_job( - queue=queue, get_cover=get_cover, get_pdf=get_pdf, get_annotation=get_annotation, @@ -844,14 +908,13 @@ async def cli(session, api_client, **params): aax_fallback=aax_fallback ) + # schedule the consumer + consumers = [ + asyncio.ensure_future(consume(ignore_errors)) for _ in range(sim_jobs) + ] try: - # schedule the consumer - consumers = [ - asyncio.ensure_future(consume(queue, ignore_errors)) for _ in range(sim_jobs) - ] # wait until the consumer has processed all items - await queue.join() - + await QUEUE.join() finally: # the consumer is still awaiting an item, cancel it for consumer in consumers: diff --git a/src/audible_cli/decorators.py b/src/audible_cli/decorators.py index df43f54..3f8397b 100644 --- a/src/audible_cli/decorators.py +++ b/src/audible_cli/decorators.py @@ -201,7 +201,7 @@ def timeout_option(func=None, **kwargs): return value kwargs.setdefault("type", click.INT) - kwargs.setdefault("default", 10) + kwargs.setdefault("default", 30) kwargs.setdefault("show_default", True) kwargs.setdefault( "help", ("Increase the timeout time if you got any TimeoutErrors. " diff --git a/src/audible_cli/downloader.py b/src/audible_cli/downloader.py index 2ef9391..61b4dbe 100644 --- a/src/audible_cli/downloader.py +++ b/src/audible_cli/downloader.py @@ -235,13 +235,12 @@ def _status_for_message(message: str) -> Status: async def check_status_for_message( response: ResponseInfo, tmp_file: File, **kwargs: Any ) -> Status: - if ( - response.content_length and response.content_type - and response.content_length <= MAX_FILE_READ_SIZE - and "text" in response.content_type - ): - message = await tmp_file.read_text_content() - return _status_for_message(message) + if response.content_type and "text" in response.content_type: + length = response.content_length or await tmp_file.get_size() + if length <= MAX_FILE_READ_SIZE: + message = await tmp_file.read_text_content() + return _status_for_message(message) + return Status.Success diff --git a/src/audible_cli/models.py b/src/audible_cli/models.py index ddbdc32..f9d2c44 100644 --- a/src/audible_cli/models.py +++ b/src/audible_cli/models.py @@ -131,9 +131,17 @@ class BaseItem: return True def is_published(self): - if self.publication_datetime is not None: + if ( + self.content_delivery_type and self.content_delivery_type == "AudioPart" + and self._parent + ): + publication_datetime = self._parent.publication_datetime + else: + publication_datetime = self.publication_datetime + + if publication_datetime is not None: pub_date = datetime.strptime( - self.publication_datetime, "%Y-%m-%dT%H:%M:%SZ" + publication_datetime, "%Y-%m-%dT%H:%M:%SZ" ) now = datetime.utcnow() return now > pub_date From 098714e410e81725374db799366ec4434cbff90e Mon Sep 17 00:00:00 2001 From: mkb79 Date: Tue, 30 Jan 2024 15:16:20 +0100 Subject: [PATCH 15/35] fix: Uncaught Exception (#181) --- src/audible_cli/cmds/cmd_download.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/audible_cli/cmds/cmd_download.py b/src/audible_cli/cmds/cmd_download.py index 3cfb20a..f1274f6 100644 --- a/src/audible_cli/cmds/cmd_download.py +++ b/src/audible_cli/cmds/cmd_download.py @@ -39,7 +39,7 @@ CLIENT_HEADERS = { "User-Agent": "Audible/671 CFNetwork/1240.0.4 Darwin/20.6.0" } -QUEUE = asyncio.Queue() +QUEUE = None class DownloadCounter: @@ -908,6 +908,10 @@ async def cli(session, api_client, **params): aax_fallback=aax_fallback ) + # set queue + global QUEUE + QUEUE = asyncio.Queue() + # schedule the consumer consumers = [ asyncio.ensure_future(consume(ignore_errors)) for _ in range(sim_jobs) From 91b3f63bd35cba638f9a0179c6893a59ecff1d1a Mon Sep 17 00:00:00 2001 From: mkb79 Date: Tue, 30 Jan 2024 15:21:22 +0100 Subject: [PATCH 16/35] fix: move QUEUE init position --- src/audible_cli/cmds/cmd_download.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/audible_cli/cmds/cmd_download.py b/src/audible_cli/cmds/cmd_download.py index f1274f6..20282c4 100644 --- a/src/audible_cli/cmds/cmd_download.py +++ b/src/audible_cli/cmds/cmd_download.py @@ -869,6 +869,10 @@ async def cli(session, api_client, **params): f"Skip title {title}: Not found in library" ) + # set queue + global QUEUE + QUEUE = asyncio.Queue() + for job in jobs: item = library.get_item_by_asin(job) items = [item] @@ -908,10 +912,6 @@ async def cli(session, api_client, **params): aax_fallback=aax_fallback ) - # set queue - global QUEUE - QUEUE = asyncio.Queue() - # schedule the consumer consumers = [ asyncio.ensure_future(consume(ignore_errors)) for _ in range(sim_jobs) From 87bedc108024c883e59875e02ee2e7e2d9b067e0 Mon Sep 17 00:00:00 2001 From: mkb79 Date: Tue, 19 Mar 2024 20:32:19 +0100 Subject: [PATCH 17/35] refactor: rework plugin module (#189) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * refactor: rework plugin module - using importlib.metadata over setuptools (pkg_resources) to get entrypoints - help text now starts with ´(P)` for plugin commands - plugin command help page now contains additional information about the source of the plugin --- CHANGELOG.md | 8 ++++++ setup.py | 3 +- src/audible_cli/cli.py | 8 ++++-- src/audible_cli/plugins.py | 56 ++++++++++++++++++++++++++------------ 4 files changed, 54 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d9a5635..4e8d0a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Added a resume feature when downloading aaxc files. - New `downlaoder` module which contains a rework of the Downloader class. - If necessary, large audiobooks are now downloaded in parts. +- Plugin command help page now contains additional information about the source of + the plugin. +- Command help text now starts with ´(P)` for plugin commands. + +### Changed + +- Rework plugin module +- using importlib.metadata over setuptools (pkg_resources) to get entrypoints ## [0.2.6] - 2023-11-16 diff --git a/setup.py b/setup.py index 594ae5c..7e0ca65 100644 --- a/setup.py +++ b/setup.py @@ -55,7 +55,8 @@ setup( "tabulate", "toml", "tqdm", - "questionary" + "questionary", + "importlib-metadata; python_version<'3.10'", ], extras_require={ 'pyi': [ diff --git a/src/audible_cli/cli.py b/src/audible_cli/cli.py index 28591a7..d896443 100644 --- a/src/audible_cli/cli.py +++ b/src/audible_cli/cli.py @@ -1,7 +1,6 @@ import asyncio import logging import sys -from pkg_resources import iter_entry_points import click @@ -18,6 +17,11 @@ from .exceptions import AudibleCliException from ._logging import click_basic_config from . import plugins +try: + from importlib.metadata import entry_points +except ImportError: # Python < 3.10 (backport) + from importlib_metadata import entry_points + logger = logging.getLogger("audible_cli") click_basic_config(logger) @@ -26,7 +30,7 @@ CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"]) @plugins.from_folder(get_plugin_dir()) -@plugins.from_entry_point(iter_entry_points(PLUGIN_ENTRY_POINT)) +@plugins.from_entry_point(entry_points(group=PLUGIN_ENTRY_POINT)) @build_in_cmds @click.group(context_settings=CONTEXT_SETTINGS) @profile_option diff --git a/src/audible_cli/plugins.py b/src/audible_cli/plugins.py index 3d370e3..b4d00f3 100644 --- a/src/audible_cli/plugins.py +++ b/src/audible_cli/plugins.py @@ -28,39 +28,49 @@ def from_folder(plugin_dir: Union[str, pathlib.Path]): """ def decorator(group): if not isinstance(group, click.Group): - raise TypeError("Plugins can only be attached to an instance of " - "click.Group()") + raise TypeError( + "Plugins can only be attached to an instance of click.Group()" + ) - pdir = pathlib.Path(plugin_dir) - cmds = [x for x in pdir.glob("cmd_*.py")] - sys.path.insert(0, str(pdir.resolve())) + plugin_path = pathlib.Path(plugin_dir).resolve() + sys.path.insert(0, str(plugin_path)) - for cmd in cmds: - mod_name = cmd.stem + for cmd_path in plugin_path.glob("cmd_*.py"): + cmd_path_stem = cmd_path.stem try: - mod = import_module(mod_name) - name = mod_name[4:] if mod.cli.name == "cli" else mod.cli.name - group.add_command(mod.cli, name=name) + mod = import_module(cmd_path_stem) + cmd = mod.cli + if cmd.name == "cli": + # if no name given to the command, use the filename + # excl. starting cmd_ as name + cmd.name = cmd_path_stem[4:] + group.add_command(cmd) + + orig_help = cmd.help or "" + new_help = ( + f"(P) {orig_help}\n\nPlugin loaded from file: {str(cmd_path)}" + ) + cmd.help = new_help except Exception: # noqa # Catch this so a busted plugin doesn't take down the CLI. # Handled by registering a dummy command that does nothing # other than explain the error. - group.add_command(BrokenCommand(mod_name[4:])) + group.add_command(BrokenCommand(cmd_path_stem[4:])) return group return decorator -def from_entry_point(entry_point_group: str): +def from_entry_point(entry_point_group): """ A decorator to register external CLI commands to an instance of `click.Group()`. Parameters ---------- - entry_point_group : iter - An iterable producing one `pkg_resources.EntryPoint()` per iteration. + entry_point_group : list + A list producing one `pkg_resources.EntryPoint()` per iteration. Returns ------- @@ -68,13 +78,23 @@ def from_entry_point(entry_point_group: str): """ def decorator(group): if not isinstance(group, click.Group): - print(type(group)) - raise TypeError("Plugins can only be attached to an instance of " - "click.Group()") + raise TypeError( + "Plugins can only be attached to an instance of click.Group()" + ) for entry_point in entry_point_group or (): try: - group.add_command(entry_point.load()) + cmd = entry_point.load() + dist_name = entry_point.dist.name + if cmd.name == "cli": + # if no name given to the command, use the filename + # excl. starting cmd_ as name + cmd.name = dist_name + group.add_command(cmd) + + orig_help = cmd.help or "" + new_help = f"(P) {orig_help}\n\nPlugin loaded from package: {dist_name}" + cmd.help = new_help except Exception: # noqa # Catch this so a busted plugin doesn't take down the CLI. # Handled by registering a dummy command that does nothing From 0fa4dfc931490b12edfe1b5b8361d9ef562ce9ec Mon Sep 17 00:00:00 2001 From: mkb79 Date: Tue, 19 Mar 2024 22:26:51 +0100 Subject: [PATCH 18/35] Release v0.3.0 --- CHANGELOG.md | 4 ++++ src/audible_cli/_version.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e8d0a7..8a012e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## Unreleased +- + +## [0.3.0] - 2024-03-19 + ### Added - Added a resume feature when downloading aaxc files. diff --git a/src/audible_cli/_version.py b/src/audible_cli/_version.py index d25f15f..416f55b 100644 --- a/src/audible_cli/_version.py +++ b/src/audible_cli/_version.py @@ -1,7 +1,7 @@ __title__ = "audible-cli" __description__ = "Command line interface (cli) for the audible package." __url__ = "https://github.com/mkb79/audible-cli" -__version__ = "0.2.6" +__version__ = "0.3.0" __author__ = "mkb79" __author_email__ = "mkb79@hackitall.de" __license__ = "AGPL" From d6ce4041d8503d9607b186f98feba2f3d20ffbdd Mon Sep 17 00:00:00 2001 From: mkb79 Date: Tue, 19 Mar 2024 23:21:24 +0100 Subject: [PATCH 19/35] fix: Fix TypeError on some Python versions (#191) --- CHANGELOG.md | 4 +++- src/audible_cli/cli.py | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a012e4..ba08a2f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## Unreleased -- +### Bugfix + +- fix a `TypeError` on some Python versions when calling `importlib.metadata.entry_points` with group argument ## [0.3.0] - 2024-03-19 diff --git a/src/audible_cli/cli.py b/src/audible_cli/cli.py index d896443..4f70d64 100644 --- a/src/audible_cli/cli.py +++ b/src/audible_cli/cli.py @@ -17,9 +17,9 @@ from .exceptions import AudibleCliException from ._logging import click_basic_config from . import plugins -try: +if sys.version_info >= (3, 10): from importlib.metadata import entry_points -except ImportError: # Python < 3.10 (backport) +else: # Python < 3.10 (backport) from importlib_metadata import entry_points From 707a4b9192a8f877646aeab7f8214d7c9f4c7150 Mon Sep 17 00:00:00 2001 From: mkb79 Date: Tue, 19 Mar 2024 23:25:15 +0100 Subject: [PATCH 20/35] Release v0.3.1 --- CHANGELOG.md | 4 ++++ src/audible_cli/_version.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ba08a2f..3acb91d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## Unreleased +- + +## [0.3.1] - 2024-03-19 + ### Bugfix - fix a `TypeError` on some Python versions when calling `importlib.metadata.entry_points` with group argument diff --git a/src/audible_cli/_version.py b/src/audible_cli/_version.py index 416f55b..50dc2fd 100644 --- a/src/audible_cli/_version.py +++ b/src/audible_cli/_version.py @@ -1,7 +1,7 @@ __title__ = "audible-cli" __description__ = "Command line interface (cli) for the audible package." __url__ = "https://github.com/mkb79/audible-cli" -__version__ = "0.3.0" +__version__ = "0.3.1" __author__ = "mkb79" __author_email__ = "mkb79@hackitall.de" __license__ = "AGPL" From 45bd5820addc44b11521f32386a63d198e49f865 Mon Sep 17 00:00:00 2001 From: mkb79 Date: Sun, 24 Mar 2024 22:53:41 +0100 Subject: [PATCH 21/35] refactor: rework skip podcast options of download command (#195) * Improve podcast ignore feature in downloader Added conditional code in the cmd_download.py file to ignore any items that are parent podcasts if the ignore_podcasts flag is set. This allows users to choose whether or not they want to download podcasts. * Prevent mixing resolve and ignore podcasts options Added a check in cmd_download.py to prevent combining both "resolve_podcasts" and "ignore_podcasts" options. A flag was added that aborts the process if both options are activated at once. Additionally, enhanced the feature to ignore parent podcasts during download, if "ignore_podcasts" flag is set. * Update CHANGELOG.md --- CHANGELOG.md | 6 +++++- src/audible_cli/cmds/cmd_download.py | 9 ++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3acb91d..c92635a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## Unreleased -- +### Changed + +- Improved podcast ignore feature in download command +- make `--ignore-podcasts` and `--resolve-podcasts` options of download command mutual + exclusive ## [0.3.1] - 2024-03-19 diff --git a/src/audible_cli/cmds/cmd_download.py b/src/audible_cli/cmds/cmd_download.py index 20282c4..8197e85 100644 --- a/src/audible_cli/cmds/cmd_download.py +++ b/src/audible_cli/cmds/cmd_download.py @@ -783,6 +783,9 @@ async def cli(session, api_client, **params): no_confirm = params.get("no_confirm") resolve_podcats = params.get("resolve_podcasts") ignore_podcasts = params.get("ignore_podcasts") + if all([resolve_podcats, ignore_podcasts]): + logger.error("Do not mix *ignore-podcasts* with *resolve-podcasts* option.") + raise click.Abort() bunch_size = session.params.get("bunch_size") start_date = session.params.get("start_date") @@ -821,6 +824,7 @@ async def cli(session, api_client, **params): if resolve_podcats: await library.resolve_podcats(start_date=start_date, end_date=end_date) + [library.data.remove(i) for i in library if i.is_parent_podcast()] # collect jobs jobs = [] @@ -878,7 +882,10 @@ async def cli(session, api_client, **params): items = [item] odir = pathlib.Path(output_dir) - if not ignore_podcasts and item.is_parent_podcast(): + if item.is_parent_podcast(): + if ignore_podcasts: + continue + items.remove(item) if item._children is None: await item.get_child_items( From 513b97a3cc58ea8ca6cfe25c2226e1fa29ccd7db Mon Sep 17 00:00:00 2001 From: mkb79 Date: Sun, 24 Mar 2024 22:58:59 +0100 Subject: [PATCH 22/35] refactor: downloader head request (#196) * Replace HEAD request with faster GET request in downloader Switched from a HEAD to a GET request without loading the body in the get_head_response method. This change improves the program's speed, as the HEAD request was taking considerably longer than a GET request to the same URI. * Update CHANGELOG.md --- CHANGELOG.md | 3 +++ src/audible_cli/downloader.py | 14 ++++++++++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c92635a..425391f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Improved podcast ignore feature in download command - make `--ignore-podcasts` and `--resolve-podcasts` options of download command mutual exclusive +- Switched from a HEAD to a GET request without loading the body in the downloader + class. This change improves the program's speed, as the HEAD request was taking + considerably longer than a GET request on some Audible pages. ## [0.3.1] - 2024-03-19 diff --git a/src/audible_cli/downloader.py b/src/audible_cli/downloader.py index 61b4dbe..145c912 100644 --- a/src/audible_cli/downloader.py +++ b/src/audible_cli/downloader.py @@ -322,10 +322,16 @@ class Downloader: async def get_head_response(self, force_recreate: bool = False) -> ResponseInfo: if self._head_request is None or force_recreate: - head_response = await self._client.head( - self._source, headers=self._additional_headers, follow_redirects=True, - ) - self._head_request = ResponseInfo(head_response) + # switched from HEAD to GET request without loading the body + # HEAD request to cds.audible.de will responded in 1 - 2 minutes + # a GET request to the same URI will take ~4-6 seconds + async with self._client.stream( + "GET", self._source, headers=self._additional_headers, + follow_redirects=True, + ) as head_response: + if head_response.request.url != self._source: + self._source = head_response.request.url + self._head_request = ResponseInfo(head_response) return self._head_request From 24c57ec73e507d6f514b7011c47b72cd33aa31aa Mon Sep 17 00:00:00 2001 From: mkb79 Date: Sun, 24 Mar 2024 23:01:53 +0100 Subject: [PATCH 23/35] Update audible-cli version The version of audible-cli has been updated from 0.3.1 to 0.3.2b1. This update may involve bugs fixes, improvements, or new features in the audible-cli package. --- src/audible_cli/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/audible_cli/_version.py b/src/audible_cli/_version.py index 50dc2fd..b85fcc2 100644 --- a/src/audible_cli/_version.py +++ b/src/audible_cli/_version.py @@ -1,7 +1,7 @@ __title__ = "audible-cli" __description__ = "Command line interface (cli) for the audible package." __url__ = "https://github.com/mkb79/audible-cli" -__version__ = "0.3.1" +__version__ = "0.3.2b1" __author__ = "mkb79" __author_email__ = "mkb79@hackitall.de" __license__ = "AGPL" From 48946ab8b87837f6263e4df890231f65316f1a95 Mon Sep 17 00:00:00 2001 From: mkb79 Date: Sun, 31 Mar 2024 11:55:23 +0200 Subject: [PATCH 24/35] feat: Add option to download flat chapters (#200) * Adjust get_content_metadata method in audible_cli model Updated the get_content_metadata method to accept a new parameter 'chapter_type'. This adjustment also enables passing additional request parameters via **request_kwargs, allowing for more flexible HTTP requests. * Add chapter type parameter to download chapters function The download_chapters function has been updated to include 'chapter_type' as a parameter to support different types of chapters. It also modifies the get_content_metadata method to handle the new argument. This makes API interaction more adaptive by facilitating more varied request parameters. * Update CHANGELOG.md --- CHANGELOG.md | 8 ++++++++ src/audible_cli/cmds/cmd_download.py | 16 +++++++++++++--- src/audible_cli/models.py | 9 +++++++-- 3 files changed, 28 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 425391f..d5ee29b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## Unreleased +### Added + +- The `--chapter-type` option is added to the download command. Chapter can now be + downloaded as `flat` or `tree` type. `tree` is the default. + ### Changed - Improved podcast ignore feature in download command @@ -14,6 +19,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Switched from a HEAD to a GET request without loading the body in the downloader class. This change improves the program's speed, as the HEAD request was taking considerably longer than a GET request on some Audible pages. +- `models.LibraryItem.get_content_metadatata` now accept a `chapter_type` argument. + Additional keyword arguments to this method are now passed through the metadata + request. ## [0.3.1] - 2024-03-19 diff --git a/src/audible_cli/cmds/cmd_download.py b/src/audible_cli/cmds/cmd_download.py index 8197e85..39a6545 100644 --- a/src/audible_cli/cmds/cmd_download.py +++ b/src/audible_cli/cmds/cmd_download.py @@ -203,7 +203,7 @@ async def download_pdf( async def download_chapters( - output_dir, base_filename, item, quality, overwrite_existing + output_dir, base_filename, item, quality, overwrite_existing, chapter_type ): if not output_dir.is_dir(): raise DirectoryDoesNotExists(output_dir) @@ -217,7 +217,7 @@ async def download_chapters( return True try: - metadata = await item.get_content_metadata(quality) + metadata = await item.get_content_metadata(quality, chapter_type=chapter_type) except NotFoundError: logger.info( f"No chapters found for {item.full_title}." @@ -521,6 +521,7 @@ def queue_job( filename_mode, item, cover_sizes, + chapter_type, quality, overwrite_existing, aax_fallback @@ -558,7 +559,8 @@ def queue_job( "base_filename": base_filename, "item": item, "quality": quality, - "overwrite_existing": overwrite_existing + "overwrite_existing": overwrite_existing, + "chapter_type": chapter_type } QUEUE.put_nowait((cmd, kwargs)) @@ -688,6 +690,12 @@ def display_counter(): is_flag=True, help="saves chapter metadata as JSON file" ) +@click.option( + "--chapter-type", + default="Tree", + type=click.Choice(["Flat", "Tree"], case_sensitive=False), + help="The chapter type." +) @click.option( "--annotation", is_flag=True, @@ -780,6 +788,7 @@ async def cli(session, api_client, **params): cover_sizes = list(set(params.get("cover_size"))) overwrite_existing = params.get("overwrite") ignore_errors = params.get("ignore_errors") + chapter_type = params.get("chapter_type") no_confirm = params.get("no_confirm") resolve_podcats = params.get("resolve_podcasts") ignore_podcasts = params.get("ignore_podcasts") @@ -914,6 +923,7 @@ async def cli(session, api_client, **params): filename_mode=filename_mode, item=item, cover_sizes=cover_sizes, + chapter_type=chapter_type, quality=quality, overwrite_existing=overwrite_existing, aax_fallback=aax_fallback diff --git a/src/audible_cli/models.py b/src/audible_cli/models.py index f9d2c44..475c8cc 100644 --- a/src/audible_cli/models.py +++ b/src/audible_cli/models.py @@ -391,15 +391,20 @@ class LibraryItem(BaseItem): return lr - async def get_content_metadata(self, quality: str = "high"): + async def get_content_metadata( + self, quality: str = "high", chapter_type: str = "Tree", **request_kwargs + ): assert quality in ("best", "high", "normal",) + assert chapter_type in ("Flat", "Tree") url = f"content/{self.asin}/metadata" params = { "response_groups": "last_position_heard, content_reference, " "chapter_info", "quality": "High" if quality in ("best", "high") else "Normal", - "drm_type": "Adrm" + "drm_type": "Adrm", + "chapter_titles_type": chapter_type, + **request_kwargs } metadata = await self._client.get(url, params=params) From 7f019494138779b2856c886754c7bf4cbbde000b Mon Sep 17 00:00:00 2001 From: mkb79 Date: Sun, 31 Mar 2024 11:57:09 +0200 Subject: [PATCH 25/35] Update audible-cli version The version of the audible-cli package has been updated from "0.3.2b1" to "0.3.2b2". This version upgrade reflects changes made to the functionality or features of the package. --- src/audible_cli/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/audible_cli/_version.py b/src/audible_cli/_version.py index b85fcc2..b46f246 100644 --- a/src/audible_cli/_version.py +++ b/src/audible_cli/_version.py @@ -1,7 +1,7 @@ __title__ = "audible-cli" __description__ = "Command line interface (cli) for the audible package." __url__ = "https://github.com/mkb79/audible-cli" -__version__ = "0.3.2b1" +__version__ = "0.3.2b2" __author__ = "mkb79" __author_email__ = "mkb79@hackitall.de" __license__ = "AGPL" From 8dc8739f66baa6677a6aa70e22ff3102c4780840 Mon Sep 17 00:00:00 2001 From: mkb79 Date: Mon, 1 Apr 2024 13:20:06 +0200 Subject: [PATCH 26/35] Rework flat chapter option (#201) * Capitalize chapter type in get_content_metadata function The get_content_metadata function in audible_cli's models.py file has been updated to capitalize the chapter type parameter. This ensures any case variations in the parameter will pass the assert condition, improving the handling of chapter types and method consistency. * Update chapter type handling in cmd_download.py Updated the cmd_download.py file to add a "config" choice to the chapter type and move the declaration of chapter_type to a more appropriate location. Also, the logging message has been updated to include the selected chapter type. This enhances traceability and ensures chapter type is handled consistently. Flat chapters can be enabled by default in the config file. In the APP or profile section must be a setting `chapter_type = "flat"`. * Update README with new `chapter_type` option Updated README.md to include a new `chapter_type` field in both the APP and profile sections. This new field allows users to specify a chapter type for the `download` command with the `--chapter-type` option. If not provided, it defaults to "tree". This change aims to increase customizability for users. * Update CHANGELOG to include new config file option The CHANGELOG has been updated to reflect the addition of the ability to set a default chapter type in the config file. This allows the user to specify whether chapters should be downloaded as `flat` or `tree` type without having to state it each time a download command is given. * Update audible-cli version Version number has been updated from "0.3.2b2" to "0.3.2b3" in the _version.py file. This indicates a new build of the code that may include minor enhancements or bug fixes. --- CHANGELOG.md | 3 ++- README.md | 7 ++++++- src/audible_cli/_version.py | 2 +- src/audible_cli/cmds/cmd_download.py | 15 ++++++++++----- src/audible_cli/models.py | 1 + 5 files changed, 20 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d5ee29b..0391f37 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Added - The `--chapter-type` option is added to the download command. Chapter can now be - downloaded as `flat` or `tree` type. `tree` is the default. + downloaded as `flat` or `tree` type. `tree` is the default. A default chapter type + can be set in the config file. ### Changed diff --git a/README.md b/README.md index 4664543..5b6a0f9 100644 --- a/README.md +++ b/README.md @@ -154,7 +154,11 @@ The APP section supports the following options: - primary_profile: The profile to use, if no other is specified - filename_mode: When using the `download` command, a filename mode can be specified here. If not present, "ascii" will be used as default. To override - these option, you can provide a mode with the `filename-mode` option of the + these option, you can provide a mode with the `--filename-mode` option of the + download command. +- chapter_type: When using the `download` command, a chapter type can be specified + here. If not present, "tree" will be used as default. To override + these option, you can provide a type with the `--chapter-type` option of the download command. #### Profile section @@ -162,6 +166,7 @@ The APP section supports the following options: - auth_file: The auth file for this profile - country_code: The marketplace for this profile - filename_mode: See APP section above. Will override the option in APP section. +- chapter_type: See APP section above. Will override the option in APP section. ## Getting started diff --git a/src/audible_cli/_version.py b/src/audible_cli/_version.py index b46f246..644bcfe 100644 --- a/src/audible_cli/_version.py +++ b/src/audible_cli/_version.py @@ -1,7 +1,7 @@ __title__ = "audible-cli" __description__ = "Command line interface (cli) for the audible package." __url__ = "https://github.com/mkb79/audible-cli" -__version__ = "0.3.2b2" +__version__ = "0.3.2b3" __author__ = "mkb79" __author_email__ = "mkb79@hackitall.de" __license__ = "AGPL" diff --git a/src/audible_cli/cmds/cmd_download.py b/src/audible_cli/cmds/cmd_download.py index 39a6545..bef408a 100644 --- a/src/audible_cli/cmds/cmd_download.py +++ b/src/audible_cli/cmds/cmd_download.py @@ -226,7 +226,7 @@ async def download_chapters( metadata = json.dumps(metadata, indent=4) async with aiofiles.open(file, "w") as f: await f.write(metadata) - logger.info(f"Chapter file saved to {file}.") + logger.info(f"Chapter file saved in style '{chapter_type.upper()}' to {file}.") counter.count_chapter() @@ -291,6 +291,7 @@ async def _add_audioparts_to_queue( get_pdf=None, get_annotation=None, get_chapters=None, + chapter_type=None, get_aax=get_aax, get_aaxc=get_aaxc, client=client, @@ -688,12 +689,12 @@ def display_counter(): @click.option( "--chapter", is_flag=True, - help="saves chapter metadata as JSON file" + help="Saves chapter metadata as JSON file." ) @click.option( "--chapter-type", - default="Tree", - type=click.Choice(["Flat", "Tree"], case_sensitive=False), + default="config", + type=click.Choice(["Flat", "Tree", "config"], case_sensitive=False), help="The chapter type." ) @click.option( @@ -788,7 +789,6 @@ async def cli(session, api_client, **params): cover_sizes = list(set(params.get("cover_size"))) overwrite_existing = params.get("overwrite") ignore_errors = params.get("ignore_errors") - chapter_type = params.get("chapter_type") no_confirm = params.get("no_confirm") resolve_podcats = params.get("resolve_podcasts") ignore_podcasts = params.get("ignore_podcasts") @@ -812,6 +812,11 @@ async def cli(session, api_client, **params): f"Selected end date: {end_date.strftime('%Y-%m-%dT%H:%M:%S.%fZ')}" ) + chapter_type = params.get("chapter_type") + if chapter_type == "config": + chapter_type = session.config.get_profile_option( + session.selected_profile, "chapter_type") or "Tree" + filename_mode = params.get("filename_mode") if filename_mode == "config": filename_mode = session.config.get_profile_option( diff --git a/src/audible_cli/models.py b/src/audible_cli/models.py index 475c8cc..1174c66 100644 --- a/src/audible_cli/models.py +++ b/src/audible_cli/models.py @@ -394,6 +394,7 @@ class LibraryItem(BaseItem): async def get_content_metadata( self, quality: str = "high", chapter_type: str = "Tree", **request_kwargs ): + chapter_type = chapter_type.capitalize() assert quality in ("best", "high", "normal",) assert chapter_type in ("Flat", "Tree") From 0a9b2f9c7dd00d5eb05015f037e0c120c76c35f5 Mon Sep 17 00:00:00 2001 From: Paul Date: Mon, 8 Apr 2024 22:58:07 +0200 Subject: [PATCH 27/35] fix: decrypt command filename escape (#202) --- plugin_cmds/cmd_decrypt.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/plugin_cmds/cmd_decrypt.py b/plugin_cmds/cmd_decrypt.py index 411a398..003e95a 100644 --- a/plugin_cmds/cmd_decrypt.py +++ b/plugin_cmds/cmd_decrypt.py @@ -19,7 +19,6 @@ import typing as t from enum import Enum from functools import reduce from glob import glob -from shlex import quote from shutil import which import click @@ -385,20 +384,20 @@ class FfmpegFileDecrypter: key, iv = self._credentials credentials_cmd = [ "-audible_key", - quote(key), + key, "-audible_iv", - quote(iv), + iv, ] else: credentials_cmd = [ "-activation_bytes", - quote(self._credentials), + self._credentials, ] base_cmd.extend(credentials_cmd) extract_cmd = [ "-i", - quote(str(self._source)), + str(self._source), "-f", "ffmetadata", str(metafile), @@ -440,20 +439,20 @@ class FfmpegFileDecrypter: key, iv = self._credentials credentials_cmd = [ "-audible_key", - quote(key), + key, "-audible_iv", - quote(iv), + iv, ] else: credentials_cmd = [ "-activation_bytes", - quote(self._credentials), + self._credentials, ] base_cmd.extend(credentials_cmd) base_cmd.extend( [ "-i", - quote(str(self._source)), + str(self._source), ] ) @@ -471,7 +470,7 @@ class FfmpegFileDecrypter: base_cmd.extend( [ "-i", - quote(str(metafile)), + str(metafile), "-map_metadata", "0", "-map_chapters", @@ -483,7 +482,7 @@ class FfmpegFileDecrypter: [ "-c", "copy", - quote(str(outfile)), + str(outfile), ] ) From 22e388dfaa40a65d811556c35fbea606f9841c03 Mon Sep 17 00:00:00 2001 From: vwkd <33468089+vwkd@users.noreply.github.com> Date: Mon, 8 Apr 2024 23:31:34 +0200 Subject: [PATCH 28/35] Add `-c, --remove-intro-outro` option to `decrypt` (#171) --- plugin_cmds/cmd_decrypt.py | 122 +++++++++++++++++++++++++++++-------- 1 file changed, 98 insertions(+), 24 deletions(-) diff --git a/plugin_cmds/cmd_decrypt.py b/plugin_cmds/cmd_decrypt.py index 003e95a..3f72d28 100644 --- a/plugin_cmds/cmd_decrypt.py +++ b/plugin_cmds/cmd_decrypt.py @@ -130,7 +130,7 @@ class ApiChapterInfo: def count_chapters(self): return len(self.get_chapters()) - def get_chapters(self, separate_intro_outro=False): + def get_chapters(self, separate_intro_outro=False, remove_intro_outro=False): def extract_chapters(initial, current): if "chapters" in current: return initial + [current] + current["chapters"] @@ -147,6 +147,8 @@ class ApiChapterInfo: if separate_intro_outro: return self._separate_intro_outro(chapters) + elif remove_intro_outro: + return self._remove_intro_outro(chapters) return chapters @@ -198,6 +200,24 @@ class ApiChapterInfo: return chapters + def _remove_intro_outro(self, chapters): + echo("Delete Audible Brand Intro and Outro.") + chapters.sort(key=operator.itemgetter("start_offset_ms")) + + intro_dur_ms = self.get_intro_duration_ms() + outro_dur_ms = self.get_outro_duration_ms() + + first = chapters[0] + first["length_ms"] -= intro_dur_ms + + for chapter in chapters[1:]: + chapter["start_offset_ms"] -= intro_dur_ms + chapter["start_offset_sec"] -= round(chapter["start_offset_ms"] / 1000) + + last = chapters[-1] + last["length_ms"] -= outro_dur_ms + + return chapters class FFMeta: SECTION = re.compile(r"\[(?P
[^]]+)\]") @@ -271,7 +291,8 @@ class FFMeta: self, chapter_info: ApiChapterInfo, force_rebuild_chapters: bool = False, - separate_intro_outro: bool = False + separate_intro_outro: bool = False, + remove_intro_outro: bool = False ) -> None: if not chapter_info.is_accurate(): echo("Metadata from API is not accurate. Skip.") @@ -285,7 +306,7 @@ class FFMeta: echo(f"Found {chapter_info.count_chapters()} chapters to prepare.") - api_chapters = chapter_info.get_chapters(separate_intro_outro) + api_chapters = chapter_info.get_chapters(separate_intro_outro, remove_intro_outro) num_chap = 0 new_chapters = {} @@ -300,6 +321,20 @@ class FFMeta: "title": chapter["title"], } self._ffmeta_parsed["CHAPTER"] = new_chapters + + def get_start_end_without_intro_outro( + self, + chapter_info: ApiChapterInfo, + ): + intro_dur_ms = chapter_info.get_intro_duration_ms() + outro_dur_ms = chapter_info.get_outro_duration_ms() + total_runtime_ms = chapter_info.get_runtime_length_ms() + + start_new = intro_dur_ms + duration_new = total_runtime_ms - intro_dur_ms - outro_dur_ms + + return start_new, duration_new + def _get_voucher_filename(file: pathlib.Path) -> pathlib.Path: @@ -328,7 +363,8 @@ class FfmpegFileDecrypter: rebuild_chapters: bool, force_rebuild_chapters: bool, skip_rebuild_chapters: bool, - separate_intro_outro: bool + separate_intro_outro: bool, + remove_intro_outro: bool ) -> None: file_type = SupportedFiles(file.suffix) @@ -353,6 +389,7 @@ class FfmpegFileDecrypter: self._force_rebuild_chapters = force_rebuild_chapters self._skip_rebuild_chapters = skip_rebuild_chapters self._separate_intro_outro = separate_intro_outro + self._remove_intro_outro = remove_intro_outro self._api_chapter: t.Optional[ApiChapterInfo] = None self._ffmeta: t.Optional[FFMeta] = None self._is_rebuilded: bool = False @@ -412,7 +449,7 @@ class FfmpegFileDecrypter: def rebuild_chapters(self) -> None: if not self._is_rebuilded: self.ffmeta.update_chapters_from_chapter_info( - self.api_chapter, self._force_rebuild_chapters, self._separate_intro_outro + self.api_chapter, self._force_rebuild_chapters, self._separate_intro_outro, self._remove_intro_outro ) self._is_rebuilded = True @@ -449,12 +486,6 @@ class FfmpegFileDecrypter: self._credentials, ] base_cmd.extend(credentials_cmd) - base_cmd.extend( - [ - "-i", - str(self._source), - ] - ) if self._rebuild_chapters: metafile = _get_ffmeta_file(self._source, self._tempdir) @@ -467,16 +498,43 @@ class FfmpegFileDecrypter: else: raise else: - base_cmd.extend( - [ - "-i", - str(metafile), - "-map_metadata", - "0", - "-map_chapters", - "1", - ] - ) + if self._remove_intro_outro: + start_new, duration_new = self.ffmeta.get_start_end_without_intro_outro(self.api_chapter) + + base_cmd.extend( + [ + "-ss", + f"{start_new}ms", + "-t", + f"{duration_new}ms", + "-i", + str(self._source), + "-i", + str(metafile), + "-map_metadata", + "0", + "-map_chapters", + "1", + ] + ) + else: + base_cmd.extend( + [ + "-i", + str(metafile), + "-map_metadata", + "0", + "-map_chapters", + "1", + ] + ) + else: + base_cmd.extend( + [ + "-i", + str(self._source), + ] + ) base_cmd.extend( [ @@ -543,6 +601,15 @@ class FfmpegFileDecrypter: "Only use with `--rebuild-chapters`." ), ) +@click.option( + "--remove-intro-outro", + "-c", + is_flag=True, + help=( + "Remove Audible Brand Intro and Outro. " + "Only use with `--rebuild-chapters`." + ), +) @pass_session def cli( session, @@ -554,6 +621,7 @@ def cli( force_rebuild_chapters: bool, skip_rebuild_chapters: bool, separate_intro_outro: bool, + remove_intro_outro: bool, ): """Decrypt audiobooks downloaded with audible-cli. @@ -567,9 +635,9 @@ def cli( ctx = click.get_current_context() ctx.fail("ffmpeg not found") - if (force_rebuild_chapters or skip_rebuild_chapters or separate_intro_outro) and not rebuild_chapters: + if (force_rebuild_chapters or skip_rebuild_chapters or separate_intro_outro or remove_intro_outro) and not rebuild_chapters: raise click.BadOptionUsage( - "`--force-rebuild-chapters`, `--skip-rebuild-chapters` and `--separate-intro-outro` can " + "`--force-rebuild-chapters`, `--skip-rebuild-chapters`, `--separate-intro-outro` and `--remove-intro-outro` can " "only be used together with `--rebuild-chapters`" ) @@ -579,6 +647,11 @@ def cli( "not be used together" ) + if separate_intro_outro and remove_intro_outro: + raise click.BadOptionUsage( + "`--separate-intro-outro` and `--remove-intro-outro` can not be used together" + ) + if all_: if files: raise click.BadOptionUsage( @@ -598,6 +671,7 @@ def cli( rebuild_chapters=rebuild_chapters, force_rebuild_chapters=force_rebuild_chapters, skip_rebuild_chapters=skip_rebuild_chapters, - separate_intro_outro=separate_intro_outro + separate_intro_outro=separate_intro_outro, + remove_intro_outro=remove_intro_outro ) decrypter.run() From 3a6db75e0d1aecf15a68dc648819272a0ea1dc28 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 16 Apr 2024 18:45:34 +0200 Subject: [PATCH 29/35] build(deps): Update httpx requirement from <0.26.0,>=0.23.3 to >=0.23.3,<0.28.0 (#185) * Update httpx requirement from <0.26.0,>=0.23.3 to >=0.23.3,<0.28.0 Updates the requirements on [httpx](https://github.com/encode/httpx) to permit the latest version. - [Release notes](https://github.com/encode/httpx/releases) - [Changelog](https://github.com/encode/httpx/blob/master/CHANGELOG.md) - [Commits](https://github.com/encode/httpx/compare/0.23.3...0.27.0) --- updated-dependencies: - dependency-name: httpx dependency-type: direct:production ... Signed-off-by: dependabot[bot] * Update httpx version range in CHANGELOG The CHANGELOG.md has been updated to reflect the changes in the httpx version range. This update includes a specification that the version should be >=0.23.3 and <0.28.0. --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: mkb79 --- CHANGELOG.md | 4 ++++ setup.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0391f37..9d1bc0d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## Unreleased +### Changed + +- Update httpx version range to >=0.23.3 and <0.28.0. + ### Added - The `--chapter-type` option is added to the download command. Chapter can now be diff --git a/setup.py b/setup.py index 7e0ca65..cb7563b 100644 --- a/setup.py +++ b/setup.py @@ -49,7 +49,7 @@ setup( "audible>=0.8.2", "click>=8", "colorama; platform_system=='Windows'", - "httpx>=0.23.3,<0.26.0", + "httpx>=0.23.3,<0.28.0", "packaging", "Pillow", "tabulate", From 705d6f09595e3979616d9b55e8e26bfc41ff5b1f Mon Sep 17 00:00:00 2001 From: mkb79 Date: Tue, 16 Apr 2024 19:39:53 +0200 Subject: [PATCH 30/35] fix: error Invalid cross-device (#204) * refactor: resume file location match target file * update CHANGELOG.md --- CHANGELOG.md | 5 +++-- src/audible_cli/downloader.py | 11 +++++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d1bc0d..776b033 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,9 +6,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## Unreleased -### Changed +### Bugfix -- Update httpx version range to >=0.23.3 and <0.28.0. +- Fixing `[Errno 18] Invalid cross-device link` when downloading files using the `--output-dir` option. This error is fixed by creating the resume file on the same location as the target file. ### Added @@ -27,6 +27,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - `models.LibraryItem.get_content_metadatata` now accept a `chapter_type` argument. Additional keyword arguments to this method are now passed through the metadata request. +- Update httpx version range to >=0.23.3 and <0.28.0. ## [0.3.1] - 2024-03-19 diff --git a/src/audible_cli/downloader.py b/src/audible_cli/downloader.py index 145c912..2e04d58 100644 --- a/src/audible_cli/downloader.py +++ b/src/audible_cli/downloader.py @@ -338,8 +338,15 @@ class Downloader: async def _determine_resume_file(self, target_file: File) -> File: head_response = await self.get_head_response() etag = head_response.etag - resume_name = target_file.path if etag is None else etag.parsed_etag - resume_file = pathlib.Path(resume_name).with_suffix(self.RESUME_SUFFIX) + + if etag is None: + resume_name = target_file.path + else: + parsed_etag = etag.parsed_etag + resume_name = target_file.path.with_name(parsed_etag) + + resume_file = resume_name.with_suffix(self.RESUME_SUFFIX) + return File(resume_file) def _determine_tmp_file(self, target_file: File) -> File: From efcad39b8edf1cdd184a0f861325a34983df9514 Mon Sep 17 00:00:00 2001 From: Isaac Lyons <51010664+snowskeleton@users.noreply.github.com> Date: Tue, 16 Apr 2024 14:10:17 -0400 Subject: [PATCH 31/35] refactor: fix typo from 'podcats' to 'podcasts' (#141) * fix typo from 'podcats' to 'podcasts' * missed on first check * Deprecate resolve_podcats method The resolve_podcats method is marked as deprecated, and a warning message is added to inform users. It is recommended to use the resolve_podcasts method. This commit helps to phase out the resolve_podcats method, aiming to eliminate any spelling errors in method naming. * Fix typo in variable name A typo in the variable name 'resolve_podcats' was corrected to 'resolve_podcasts'. This ensures that the conditional check operates as designed, without causing errors due to referencing a non-existent variable. * Fix typos and improve readability in CHANGELOG.md The commit fixes several typos including changing `resolve_podcats` to `resolve_podcasts` and fixing spelling of 'titles'. In addition, it formalizes the formatting of terms and phrases by using backticks around them for more readability. Furthermore, a missing comma has also been added to enhance sentence clarity. --------- Co-authored-by: mkb79 --- CHANGELOG.md | 8 +++++--- src/audible_cli/cmds/cmd_download.py | 8 ++++---- src/audible_cli/cmds/cmd_library.py | 2 +- src/audible_cli/models.py | 15 ++++++++++++++- 4 files changed, 24 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 776b033..1f51b3e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). Additional keyword arguments to this method are now passed through the metadata request. - Update httpx version range to >=0.23.3 and <0.28.0. +- fix typo from `resolve_podcats` to `resolve_podcasts` +- `models.Library.resolve_podcats` is now deprecated and will be removed in a future version ## [0.3.1] - 2024-03-19 @@ -118,7 +120,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - by default a licenserequest (voucher) will not include chapter information by default - moved licenserequest part from `models.LibraryItem.get_aaxc_url` to its own `models.LibraryItem.get_license` function -- allow book tiltes with hyphens (#96) +- allow book titles with hyphens (#96) - if there is no title fallback to an empty string (#98) - reduce `response_groups` for the download command to speed up fetching the library (#109) @@ -126,7 +128,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - `Extreme` quality is not supported by the Audible API anymore (#107) - download command continued execution after error (#104) -- Currently paths with dots will break the decryption (#97) +- Currently, paths with dots will break the decryption (#97) - `models.Library.from_api_full_sync` called `models.Library.from_api` with incorrect keyword arguments ### Misc @@ -189,7 +191,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Added - the `--version` option now checks if an update for `audible-cli` is available -- build macOS releases in onedir mode +- build macOS releases in `onedir` mode ### Bugfix diff --git a/src/audible_cli/cmds/cmd_download.py b/src/audible_cli/cmds/cmd_download.py index bef408a..de8c8f5 100644 --- a/src/audible_cli/cmds/cmd_download.py +++ b/src/audible_cli/cmds/cmd_download.py @@ -790,9 +790,9 @@ async def cli(session, api_client, **params): overwrite_existing = params.get("overwrite") ignore_errors = params.get("ignore_errors") no_confirm = params.get("no_confirm") - resolve_podcats = params.get("resolve_podcasts") + resolve_podcasts = params.get("resolve_podcasts") ignore_podcasts = params.get("ignore_podcasts") - if all([resolve_podcats, ignore_podcasts]): + if all([resolve_podcasts, ignore_podcasts]): logger.error("Do not mix *ignore-podcasts* with *resolve-podcasts* option.") raise click.Abort() bunch_size = session.params.get("bunch_size") @@ -836,8 +836,8 @@ async def cli(session, api_client, **params): status="Active", ) - if resolve_podcats: - await library.resolve_podcats(start_date=start_date, end_date=end_date) + if resolve_podcasts: + await library.resolve_podcasts(start_date=start_date, end_date=end_date) [library.data.remove(i) for i in library if i.is_parent_podcast()] # collect jobs diff --git a/src/audible_cli/cmds/cmd_library.py b/src/audible_cli/cmds/cmd_library.py index b38699f..956a529 100644 --- a/src/audible_cli/cmds/cmd_library.py +++ b/src/audible_cli/cmds/cmd_library.py @@ -45,7 +45,7 @@ async def _get_library(session, client, resolve_podcasts): ) if resolve_podcasts: - await library.resolve_podcats(start_date=start_date, end_date=end_date) + await library.resolve_podcasts(start_date=start_date, end_date=end_date) return library diff --git a/src/audible_cli/models.py b/src/audible_cli/models.py index 1174c66..b8dc839 100644 --- a/src/audible_cli/models.py +++ b/src/audible_cli/models.py @@ -6,6 +6,7 @@ import unicodedata from datetime import datetime from math import ceil from typing import List, Optional, Union +from warnings import warn import audible import httpx @@ -603,6 +604,18 @@ class Library(BaseList): self, start_date: Optional[datetime] = None, end_date: Optional[datetime] = None + ): + warn( + "resolve_podcats is deprecated, use resolve_podcasts instead", + DeprecationWarning, + stacklevel=2 + ) + return self.resolve_podcasts(start_date, end_date) + + async def resolve_podcasts( + self, + start_date: Optional[datetime] = None, + end_date: Optional[datetime] = None ): podcast_items = await asyncio.gather( *[i.get_child_items(start_date=start_date, end_date=end_date) @@ -668,7 +681,7 @@ class Catalog(BaseList): return cls(resp, api_client=api_client) - async def resolve_podcats(self): + async def resolve_podcasts(self): podcast_items = await asyncio.gather( *[i.get_child_items() for i in self if i.is_parent_podcast()] ) From 009a7e69ec744c524cde2c760bea024a9271798f Mon Sep 17 00:00:00 2001 From: mkb79 Date: Tue, 16 Apr 2024 20:33:44 +0200 Subject: [PATCH 32/35] fix: Raise click.Abort() in audible_cli.decorators In the audible_cli decorators file, the click.Abort() function previously was not raising an exception in case an error occurred. This has been modified to correctly raise the exception using the "raise" keyword, thereby handling exceptions accurately. --- src/audible_cli/decorators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/audible_cli/decorators.py b/src/audible_cli/decorators.py index 3f8397b..b2f1c5f 100644 --- a/src/audible_cli/decorators.py +++ b/src/audible_cli/decorators.py @@ -95,7 +95,7 @@ def version_option(func=None, **kwargs): response.raise_for_status() except Exception as e: logger.error(e) - click.Abort() + raise click.Abort() content = response.json() From 6165e95f4087e5f3308438983cac67430461cc92 Mon Sep 17 00:00:00 2001 From: mkb79 Date: Tue, 16 Apr 2024 20:34:17 +0200 Subject: [PATCH 33/35] Optimize error handling in cmd_download.py The current change replaces generic `click.Abort()` statements with `click.BadOptionUsage()` to provide clearer error messages. These updates occur when there are conflicts in options chosen (e.g. choosing both `--all` and `--asin` or `--title`), no download option is chosen, or if both *ignore-podcasts* and *resolve-podcasts* options are selected together. It is now highlighted when the start date is after the end date in the parameters as well. --- src/audible_cli/cmds/cmd_download.py | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/src/audible_cli/cmds/cmd_download.py b/src/audible_cli/cmds/cmd_download.py index de8c8f5..3b7a2b1 100644 --- a/src/audible_cli/cmds/cmd_download.py +++ b/src/audible_cli/cmds/cmd_download.py @@ -758,8 +758,10 @@ async def cli(session, api_client, **params): asins = params.get("asin") titles = params.get("title") if get_all and (asins or titles): - logger.error("Do not mix *asin* or *title* option with *all* option.") - click.Abort() + raise click.BadOptionUsage( + "--all", + "`--all` can not be used together with `--asin` or `--title`" + ) # what to download get_aax = params.get("aax") @@ -780,8 +782,10 @@ async def cli(session, api_client, **params): if not any( [get_aax, get_aaxc, get_annotation, get_chapters, get_cover, get_pdf] ): - logger.error("Please select an option what you want download.") - raise click.Abort() + raise click.BadOptionUsage( + "", + "Please select an option what you want download." + ) # additional options sim_jobs = params.get("jobs") @@ -793,15 +797,19 @@ async def cli(session, api_client, **params): resolve_podcasts = params.get("resolve_podcasts") ignore_podcasts = params.get("ignore_podcasts") if all([resolve_podcasts, ignore_podcasts]): - logger.error("Do not mix *ignore-podcasts* with *resolve-podcasts* option.") - raise click.Abort() + raise click.BadOptionUsage( + "", + "Do not mix *ignore-podcasts* with *resolve-podcasts* option." + ) bunch_size = session.params.get("bunch_size") start_date = session.params.get("start_date") end_date = session.params.get("end_date") if all([start_date, end_date]) and start_date > end_date: - logger.error("start date must be before or equal the end date") - raise click.Abort() + raise click.BadOptionUsage( + "", + "start date must be before or equal the end date" + ) if start_date is not None: logger.info( @@ -855,7 +863,7 @@ async def cli(session, api_client, **params): else: if not ignore_errors: logger.error(f"Asin {asin} not found in library.") - click.Abort() + raise click.Abort() logger.error( f"Skip asin {asin}: Not found in library" ) From 1a1f25bc2dab7ceec4426745b368fe25ac633273 Mon Sep 17 00:00:00 2001 From: mkb79 Date: Tue, 16 Apr 2024 20:41:09 +0200 Subject: [PATCH 34/35] Refactor error messages in cmd_decrypt.py The changes aim to enhance the syntax and readability of the error messages in cmd_decrypt.py. The `raise` method syntax was corrected for better readability and adjustments were made to option usage error messages to improve clarity for the user. --- plugin_cmds/cmd_decrypt.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/plugin_cmds/cmd_decrypt.py b/plugin_cmds/cmd_decrypt.py index 3f72d28..4d60906 100644 --- a/plugin_cmds/cmd_decrypt.py +++ b/plugin_cmds/cmd_decrypt.py @@ -63,7 +63,7 @@ def _get_input_files( and '*' not in filename and not SupportedFiles.is_supported_file(filename) ): - raise(click.BadParameter("{filename}: file not found or supported.")) + raise click.BadParameter("{filename}: file not found or supported.") expanded_filter = filter( lambda x: SupportedFiles.is_supported_file(x), expanded @@ -637,24 +637,28 @@ def cli( if (force_rebuild_chapters or skip_rebuild_chapters or separate_intro_outro or remove_intro_outro) and not rebuild_chapters: raise click.BadOptionUsage( - "`--force-rebuild-chapters`, `--skip-rebuild-chapters`, `--separate-intro-outro` and `--remove-intro-outro` can " - "only be used together with `--rebuild-chapters`" + "", + "`--force-rebuild-chapters`, `--skip-rebuild-chapters`, `--separate-intro-outro` " + "and `--remove-intro-outro` can only be used together with `--rebuild-chapters`" ) if force_rebuild_chapters and skip_rebuild_chapters: raise click.BadOptionUsage( + "", "`--force-rebuild-chapters` and `--skip-rebuild-chapters` can " "not be used together" ) if separate_intro_outro and remove_intro_outro: raise click.BadOptionUsage( + "", "`--separate-intro-outro` and `--remove-intro-outro` can not be used together" ) if all_: if files: raise click.BadOptionUsage( + "", "If using `--all`, no FILES arguments can be used." ) files = [f"*{suffix}" for suffix in SupportedFiles.get_supported_list()] From b3adb9a33157322cd6d79ff59f5dacf06dc3e034 Mon Sep 17 00:00:00 2001 From: Kai Norman Clasen <46302524+kai-tub@users.noreply.github.com> Date: Thu, 2 May 2024 21:49:03 +0200 Subject: [PATCH 35/35] fix: plugin decrypt rebuild-chapters without -c (#205) --- plugin_cmds/cmd_decrypt.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugin_cmds/cmd_decrypt.py b/plugin_cmds/cmd_decrypt.py index 4d60906..8a09304 100644 --- a/plugin_cmds/cmd_decrypt.py +++ b/plugin_cmds/cmd_decrypt.py @@ -520,6 +520,8 @@ class FfmpegFileDecrypter: else: base_cmd.extend( [ + "-i", + str(self._source), "-i", str(metafile), "-map_metadata",