diff --git a/src/download/classes/gallery.py b/src/download/classes/gallery.py index 422ea0a..1d062e6 100644 --- a/src/download/classes/gallery.py +++ b/src/download/classes/gallery.py @@ -1,8 +1,10 @@ #!/usr/bin/env python3 +import os +import shlex +from typing import Sequence from classes.user import User from functions import LOG from functions import load_config_variables -from functions import quote from functions import run @@ -13,43 +15,54 @@ class Gallery: self.link: str = "" self.dest: str = "" self.list: str = "" - self.opt_args: str = "" - self.command: str = "" + self.opt_args: str | Sequence[str] = "" + self.command: list[str] = [] def generate_command(self, user: User = User(1), is_comic: bool = False) -> None: """Generates a command string.""" if is_comic: configs = load_config_variables() - directory = quote(configs["comic"]["download-dir"]) - database = quote(configs["comic"]["database"]) - queue = quote(configs["comic"][f"{self.list}-list"]) if self.list else "" + directory = str(configs["comic"]["download-dir"]) + database = str(configs["comic"]["database"]) + queue = str(configs["comic"][f"{self.list}-list"]) if self.list else "" else: - directory = quote(str(user.directories[self.dest])) - database = quote(str(user.dbs["gallery"])) - queue = quote(str(user.lists[self.list])) if self.list else "" + directory = str(user.directories[self.dest]) + database = str(user.dbs["gallery"]) + queue = str(user.lists[self.list]) if self.list else "" + + command = ["gallery-dl", "--sleep", str(user.sleep)] + if self.skip_arg: + command += shlex.split(self.skip_arg) + if self.dest or is_comic: + command += ["--dest", directory] + if self.archive: + command += ["--download-archive", database] + if self.opt_args: + if isinstance(self.opt_args, str): + command += shlex.split(self.opt_args) + else: + command += list(self.opt_args) - command = f"gallery-dl --sleep {str(user.sleep)}" - command += self.skip_arg if self.skip_arg else "" - command += f" --dest {directory}" if self.dest or is_comic else "" - command += f" --download-archive {database}" if self.archive else "" - command += self.opt_args if self.opt_args else "" - # Add authentication options from environment variables - command += ' -o "extractor.pixiv.refresh-token=${GALLERY_DL_PIXIV_REFRESH_TOKEN}"' - command += ' -o "extractor.bluesky.password=${GALLERY_DL_BLUESKY_PASSWORD}"' - command += ' -o "extractor.deviantart.refresh-token=${GALLERY_DL_DEVIANTART_REFRESH_TOKEN}"' - command += ' -o "extractor.flickr.access-token=${GALLERY_DL_FLICKR_ACCESS_TOKEN}"' - command += ' -o "extractor.flickr.access-token-secret=${GALLERY_DL_FLICKR_ACCESS_TOKEN_SECRET}"' - command += ' -o "extractor.reddit.refresh-token=${GALLERY_DL_REDDIT_REFRESH_TOKEN}"' - command += ' -o "extractor.tumblr.access-token=${GALLERY_DL_TUMBLR_ACCESS_TOKEN}"' - command += ' -o "extractor.tumblr.access-token-secret=${GALLERY_DL_TUMBLR_ACCESS_TOKEN_SECRET}"' - command += ' -o "extractor.tumblr.api-key=${GALLERY_DL_TUMBLR_API_KEY}"' - command += ' -o "extractor.tumblr.api-secret=${GALLERY_DL_TUMBLR_API_SECRET}"' + auth_env = { + "extractor.pixiv.refresh-token": "GALLERY_DL_PIXIV_REFRESH_TOKEN", + "extractor.bluesky.password": "GALLERY_DL_BLUESKY_PASSWORD", + "extractor.deviantart.refresh-token": "GALLERY_DL_DEVIANTART_REFRESH_TOKEN", + "extractor.flickr.access-token": "GALLERY_DL_FLICKR_ACCESS_TOKEN", + "extractor.flickr.access-token-secret": "GALLERY_DL_FLICKR_ACCESS_TOKEN_SECRET", + "extractor.reddit.refresh-token": "GALLERY_DL_REDDIT_REFRESH_TOKEN", + "extractor.tumblr.access-token": "GALLERY_DL_TUMBLR_ACCESS_TOKEN", + "extractor.tumblr.access-token-secret": "GALLERY_DL_TUMBLR_ACCESS_TOKEN_SECRET", + "extractor.tumblr.api-key": "GALLERY_DL_TUMBLR_API_KEY", + "extractor.tumblr.api-secret": "GALLERY_DL_TUMBLR_API_SECRET", + } + for key, env_var in auth_env.items(): + command += ["-o", f"{key}={os.environ.get(env_var, '')}"] if self.link and not self.list: - command += f" {quote(self.link)}" + command.append(self.link) if self.list and not self.link: - command += f" -i {queue}" + command += ["-i", queue] LOG.debug(command) self.command = command diff --git a/src/download/download.py b/src/download/download.py index bad944d..5eea13a 100644 --- a/src/download/download.py +++ b/src/download/download.py @@ -10,11 +10,11 @@ Also following in line more posix and python rules. """ import re +from pathlib import Path import yaml from typing import Dict from functions import LOG from functions import run -from functions import quote from functions import list_lines from functions import load_config_variables from functions import parse_link @@ -57,40 +57,55 @@ def parse_gallery(gdl_list: str, user: User) -> None: gallery.run_command(ARGS.flag_verbose) -def parse_instagram(link: str) -> str: +def parse_instagram(link: str) -> list[str]: """Fix instagram links""" if "instagram" not in link: - return "" + return [] if isinstance(ARGS.post_type, list): - return f" -o include={quote(','.join(ARGS.post_type))}" - return f" -o include={quote(ARGS.post_type)}" + return ["-o", f"include={','.join(ARGS.post_type)}"] + return ["-o", f"include={ARGS.post_type}"] -def video_command(video: Video) -> str: +def video_command(video: Video): """Filters and processes the required command to download videos""" - command = "yt-dlp" + command = ["yt-dlp"] rgx_yt = re.compile(r"(https:\/\/youtube|https:\/\/www.youtube|https:\/\/youtu.be)") rgx_music = re.compile(r"(https:\/\/music.youtube.*)") if re.search(r"chaturbate", video.link): - return f"stream-dl {video.link.rstrip("/").split("/")[-1]}" + return ["stream-dl", video.link.rstrip("/").split("/")[-1]] if rgx_yt.search(video.link): - command += " --embed-subs --embed-thumbnail" - command += " --embed-metadata --embed-chapters" - command += f" -o {quote(video.dest + '/%(title)s.%(ext)s')}" + command += [ + "--embed-subs", + "--embed-thumbnail", + "--embed-metadata", + "--embed-chapters", + "-o", + f"{video.dest}/%(title)s.%(ext)s", + ] elif rgx_music.search(video.link): - command += f" --download-archive {video.database}" if video.use_archive else "" - command += " --no-playlist --newline -x" - command += " --audio-format best --add-metadata --audio-quality 0 -o" - command += f" {quote(video.dest + '/%(title)s.%(ext)s')}" + if video.use_archive: + command += ["--download-archive", video.database] + command += [ + "--no-playlist", + "--newline", + "-x", + "--audio-format", + "best", + "--add-metadata", + "--audio-quality", + "0", + "-o", + f"{video.dest}/%(title)s.%(ext)s", + ] else: # Any other video link, just do it generic - command += f" -f mp4 -o {quote(video.dest + '/%(title)s.%(ext)s')}" + command += ["-f", "mp4", "-o", f"{video.dest}/%(title)s.%(ext)s"] - LOG.info("%s %s", command, video.link) - return f"{command} {quote(video.link)}" + LOG.info("%s %s", " ".join(command), video.link) + return command + [video.link] def comic_manager(skip_arg: str, category: str) -> None: @@ -133,15 +148,19 @@ def webcomic_manager(): LOG.info("The webcomic is %s", dest) - command = f"cd {quote(dest)} && webcomix custom" - command += f" {quote(name)}" - command += " --start-url" - command += f" {quote(link)}" - command += f" --next-page-xpath={quote(nxt_code)}" - command += f" --image-xpath={quote(img_code)}" - command += " -y --cbz" + command = [ + "webcomix", + "custom", + name, + "--start-url", + link, + f"--next-page-xpath={nxt_code}", + f"--image-xpath={img_code}", + "-y", + "--cbz", + ] - run(command, ARGS.flag_verbose) + run(command, ARGS.flag_verbose, cwd=Path(dest)) def save_comic(link: str) -> None: @@ -231,8 +250,8 @@ def push_manager(user: User): video = Video() video.use_archive = ARGS.flag_archive video.link = link - video.dest = f"{user.directories['media']}" - video.database = quote(f"{user.dbs['media']}") + video.dest = str(user.directories["media"]) + video.database = str(user.dbs["media"]) run(video_command(video), ARGS.flag_verbose) for link in links_other: diff --git a/src/download/functions.py b/src/download/functions.py index 47d476a..5e54f45 100644 --- a/src/download/functions.py +++ b/src/download/functions.py @@ -6,6 +6,9 @@ import fileinput import re import os import logging +import shlex +import subprocess +from typing import Sequence from pathlib import Path import yaml @@ -60,14 +63,24 @@ def clean_cache(directory: Path): directory.rmdir() -def run(command: str, verbose: bool): +def run(command: str | Sequence[str], verbose: bool, cwd: Path | None = None) -> None: """Run command in a subprocess""" # pylint: disable=subprocess-run-check # This toggle allows for a really wasy debug when using -v if verbose: - print(command) + if isinstance(command, (list, tuple)): + print(shlex.join(command)) + else: + print(command) + return + + if isinstance(command, str): + expanded = os.path.expandvars(command) + args = shlex.split(expanded) else: - os.system(command) + args = list(command) + + subprocess.run(args, check=False, cwd=cwd) def list_lines(i: int, line: str) -> str: @@ -84,19 +97,19 @@ def sort_txt_file(file_path: Path): """Sort every line alphabetically remove duplicated and empty lines""" file = str(file_path.resolve()) - run(f"sort -u {quote(file)} -o {quote(file)}", VERBOSE_G) - run(f"sed -i '/^$/d' {quote(file)}", VERBOSE_G) - run(f'sed -i -e "s,http:,https:," {quote(file)}', VERBOSE_G) + run(["sort", "-u", file, "-o", file], VERBOSE_G) + run(["sed", "-i", "/^$/d", file], VERBOSE_G) + run(["sed", "-i", "-e", "s,http:,https:,", file], VERBOSE_G) # fix this using strip on python # line.strip("/") - run(f'sed -i -e "s,/$,," {quote(file)}', VERBOSE_G) # trailing / + run(["sed", "-i", "-e", "s,/$,,", file], VERBOSE_G) # trailing / def randomize_txt_file(file_path: Path): """Randomize the order of the lines of the txt file""" file = str(file_path.resolve()) - run(f"sort -R {quote(file)} -o {quote(file)}", VERBOSE_G) + run(["sort", "-R", file, "-o", file], VERBOSE_G) def parse_list(file):