jawz/scripts
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

migrate os.run to subprocess

Browse Source
This commit is contained in:
Danilo Reyes
2026-02-28 19:21:36 -06:00
parent 7b6e411ebc
commit 9149a03898
3 changed files with 108 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
src/download/classes/gallery.py
View File

@@ -1,8 +1,10 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
import os
import shlex
from typing import Sequence
from classes.user import User from classes.user import User
from functions import LOG from functions import LOG
from functions import load_config_variables from functions import load_config_variables
from functions import quote
from functions import run from functions import run
@@ -13,43 +15,54 @@ class Gallery:
self.link: str = "" self.link: str = ""
self.dest: str = "" self.dest: str = ""
self.list: str = "" self.list: str = ""
self.opt_args: str = "" self.opt_args: str | Sequence[str] = ""
self.command: str = "" self.command: list[str] = []
def generate_command(self, user: User = User(1), is_comic: bool = False) -> None: def generate_command(self, user: User = User(1), is_comic: bool = False) -> None:
"""Generates a command string.""" """Generates a command string."""
if is_comic: if is_comic:
configs = load_config_variables() configs = load_config_variables()
directory = quote(configs["comic"]["download-dir"]) directory = str(configs["comic"]["download-dir"])
database = quote(configs["comic"]["database"]) database = str(configs["comic"]["database"])
queue = quote(configs["comic"][f"{self.list}-list"]) if self.list else "" queue = str(configs["comic"][f"{self.list}-list"]) if self.list else ""
else: else:
directory = quote(str(user.directories[self.dest])) directory = str(user.directories[self.dest])
database = quote(str(user.dbs["gallery"])) database = str(user.dbs["gallery"])
queue = quote(str(user.lists[self.list])) if self.list else "" queue = str(user.lists[self.list]) if self.list else ""
command = f"gallery-dl --sleep {str(user.sleep)}" command = ["gallery-dl", "--sleep", str(user.sleep)]
command += self.skip_arg if self.skip_arg else "" if self.skip_arg:
command += f" --dest {directory}" if self.dest or is_comic else "" command += shlex.split(self.skip_arg)
command += f" --download-archive {database}" if self.archive else "" if self.dest or is_comic:
command += self.opt_args if self.opt_args else "" command += ["--dest", directory]
if self.archive:
command += ["--download-archive", database]
if self.opt_args:
if isinstance(self.opt_args, str):
command += shlex.split(self.opt_args)
else:
command += list(self.opt_args)
# Add authentication options from environment variables # Add authentication options from environment variables
command += ' -o "extractor.pixiv.refresh-token=${GALLERY_DL_PIXIV_REFRESH_TOKEN}"' auth_env = {
command += ' -o "extractor.bluesky.password=${GALLERY_DL_BLUESKY_PASSWORD}"' "extractor.pixiv.refresh-token": "GALLERY_DL_PIXIV_REFRESH_TOKEN",
command += ' -o "extractor.deviantart.refresh-token=${GALLERY_DL_DEVIANTART_REFRESH_TOKEN}"' "extractor.bluesky.password": "GALLERY_DL_BLUESKY_PASSWORD",
command += ' -o "extractor.flickr.access-token=${GALLERY_DL_FLICKR_ACCESS_TOKEN}"' "extractor.deviantart.refresh-token": "GALLERY_DL_DEVIANTART_REFRESH_TOKEN",
command += ' -o "extractor.flickr.access-token-secret=${GALLERY_DL_FLICKR_ACCESS_TOKEN_SECRET}"' "extractor.flickr.access-token": "GALLERY_DL_FLICKR_ACCESS_TOKEN",
command += ' -o "extractor.reddit.refresh-token=${GALLERY_DL_REDDIT_REFRESH_TOKEN}"' "extractor.flickr.access-token-secret": "GALLERY_DL_FLICKR_ACCESS_TOKEN_SECRET",
command += ' -o "extractor.tumblr.access-token=${GALLERY_DL_TUMBLR_ACCESS_TOKEN}"' "extractor.reddit.refresh-token": "GALLERY_DL_REDDIT_REFRESH_TOKEN",
command += ' -o "extractor.tumblr.access-token-secret=${GALLERY_DL_TUMBLR_ACCESS_TOKEN_SECRET}"' "extractor.tumblr.access-token": "GALLERY_DL_TUMBLR_ACCESS_TOKEN",
command += ' -o "extractor.tumblr.api-key=${GALLERY_DL_TUMBLR_API_KEY}"' "extractor.tumblr.access-token-secret": "GALLERY_DL_TUMBLR_ACCESS_TOKEN_SECRET",
command += ' -o "extractor.tumblr.api-secret=${GALLERY_DL_TUMBLR_API_SECRET}"' "extractor.tumblr.api-key": "GALLERY_DL_TUMBLR_API_KEY",
"extractor.tumblr.api-secret": "GALLERY_DL_TUMBLR_API_SECRET",
}
for key, env_var in auth_env.items():
command += ["-o", f"{key}={os.environ.get(env_var, '')}"]
if self.link and not self.list: if self.link and not self.list:
command += f" {quote(self.link)}" command.append(self.link)
if self.list and not self.link: if self.list and not self.link:
command += f" -i {queue}" command += ["-i", queue]
LOG.debug(command) LOG.debug(command)
self.command = command self.command = command

75
src/download/download.py
View File

@@ -10,11 +10,11 @@ Also following in line more posix and python rules.
""" """
import re import re
from pathlib import Path
import yaml import yaml
from typing import Dict from typing import Dict
from functions import LOG from functions import LOG
from functions import run from functions import run
from functions import quote
from functions import list_lines from functions import list_lines
from functions import load_config_variables from functions import load_config_variables
from functions import parse_link from functions import parse_link
@@ -57,40 +57,55 @@ def parse_gallery(gdl_list: str, user: User) -> None:
gallery.run_command(ARGS.flag_verbose) gallery.run_command(ARGS.flag_verbose)
def parse_instagram(link: str) -> str: def parse_instagram(link: str) -> list[str]:
"""Fix instagram links""" """Fix instagram links"""
if "instagram" not in link: if "instagram" not in link:
return "" return []
if isinstance(ARGS.post_type, list): if isinstance(ARGS.post_type, list):
return f" -o include={quote(','.join(ARGS.post_type))}" return ["-o", f"include={','.join(ARGS.post_type)}"]
return f" -o include={quote(ARGS.post_type)}" return ["-o", f"include={ARGS.post_type}"]
def video_command(video: Video) -> str: def video_command(video: Video):
"""Filters and processes the required command to download videos""" """Filters and processes the required command to download videos"""
command = "yt-dlp" command = ["yt-dlp"]
rgx_yt = re.compile(r"(https:\/\/youtube|https:\/\/www.youtube|https:\/\/youtu.be)") rgx_yt = re.compile(r"(https:\/\/youtube|https:\/\/www.youtube|https:\/\/youtu.be)")
rgx_music = re.compile(r"(https:\/\/music.youtube.*)") rgx_music = re.compile(r"(https:\/\/music.youtube.*)")
if re.search(r"chaturbate", video.link): if re.search(r"chaturbate", video.link):
return f"stream-dl {video.link.rstrip("/").split("/")[-1]}" return ["stream-dl", video.link.rstrip("/").split("/")[-1]]
if rgx_yt.search(video.link): if rgx_yt.search(video.link):
command += " --embed-subs --embed-thumbnail" command += [
command += " --embed-metadata --embed-chapters" "--embed-subs",
command += f" -o {quote(video.dest + '/%(title)s.%(ext)s')}" "--embed-thumbnail",
"--embed-metadata",
"--embed-chapters",
"-o",
f"{video.dest}/%(title)s.%(ext)s",
]
elif rgx_music.search(video.link): elif rgx_music.search(video.link):
command += f" --download-archive {video.database}" if video.use_archive else "" if video.use_archive:
command += " --no-playlist --newline -x" command += ["--download-archive", video.database]
command += " --audio-format best --add-metadata --audio-quality 0 -o" command += [
command += f" {quote(video.dest + '/%(title)s.%(ext)s')}" "--no-playlist",
"--newline",
"-x",
"--audio-format",
"best",
"--add-metadata",
"--audio-quality",
"0",
"-o",
f"{video.dest}/%(title)s.%(ext)s",
]
else: # Any other video link, just do it generic else: # Any other video link, just do it generic
command += f" -f mp4 -o {quote(video.dest + '/%(title)s.%(ext)s')}" command += ["-f", "mp4", "-o", f"{video.dest}/%(title)s.%(ext)s"]
LOG.info("%s %s", command, video.link) LOG.info("%s %s", " ".join(command), video.link)
return f"{command} {quote(video.link)}" return command + [video.link]
def comic_manager(skip_arg: str, category: str) -> None: def comic_manager(skip_arg: str, category: str) -> None:
@@ -133,15 +148,19 @@ def webcomic_manager():
LOG.info("The webcomic is %s", dest) LOG.info("The webcomic is %s", dest)
command = f"cd {quote(dest)} && webcomix custom" command = [
command += f" {quote(name)}" "webcomix",
command += " --start-url" "custom",
command += f" {quote(link)}" name,
command += f" --next-page-xpath={quote(nxt_code)}" "--start-url",
command += f" --image-xpath={quote(img_code)}" link,
command += " -y --cbz" f"--next-page-xpath={nxt_code}",
f"--image-xpath={img_code}",
"-y",
"--cbz",
]
run(command, ARGS.flag_verbose) run(command, ARGS.flag_verbose, cwd=Path(dest))
def save_comic(link: str) -> None: def save_comic(link: str) -> None:
@@ -231,8 +250,8 @@ def push_manager(user: User):
video = Video() video = Video()
video.use_archive = ARGS.flag_archive video.use_archive = ARGS.flag_archive
video.link = link video.link = link
video.dest = f"{user.directories['media']}" video.dest = str(user.directories["media"])
video.database = quote(f"{user.dbs['media']}") video.database = str(user.dbs["media"])
run(video_command(video), ARGS.flag_verbose) run(video_command(video), ARGS.flag_verbose)
for link in links_other: for link in links_other:

29
src/download/functions.py
View File

@@ -6,6 +6,9 @@ import fileinput
import re import re
import os import os
import logging import logging
import shlex
import subprocess
from typing import Sequence
from pathlib import Path from pathlib import Path
import yaml import yaml
@@ -60,14 +63,24 @@ def clean_cache(directory: Path):
directory.rmdir() directory.rmdir()
def run(command: str, verbose: bool): def run(command: str | Sequence[str], verbose: bool, cwd: Path | None = None) -> None:
"""Run command in a subprocess""" """Run command in a subprocess"""
# pylint: disable=subprocess-run-check # pylint: disable=subprocess-run-check
# This toggle allows for a really wasy debug when using -v # This toggle allows for a really wasy debug when using -v
if verbose: if verbose:
print(command) if isinstance(command, (list, tuple)):
print(shlex.join(command))
else: else:
os.system(command) print(command)
return
if isinstance(command, str):
expanded = os.path.expandvars(command)
args = shlex.split(expanded)
else:
args = list(command)
subprocess.run(args, check=False, cwd=cwd)
def list_lines(i: int, line: str) -> str: def list_lines(i: int, line: str) -> str:
@@ -84,19 +97,19 @@ def sort_txt_file(file_path: Path):
"""Sort every line alphabetically """Sort every line alphabetically
remove duplicated and empty lines""" remove duplicated and empty lines"""
file = str(file_path.resolve()) file = str(file_path.resolve())
run(f"sort -u {quote(file)} -o {quote(file)}", VERBOSE_G) run(["sort", "-u", file, "-o", file], VERBOSE_G)
run(f"sed -i '/^$/d' {quote(file)}", VERBOSE_G) run(["sed", "-i", "/^$/d", file], VERBOSE_G)
run(f'sed -i -e "s,http:,https:," {quote(file)}', VERBOSE_G) run(["sed", "-i", "-e", "s,http:,https:,", file], VERBOSE_G)
# fix this using strip on python # fix this using strip on python
# line.strip("/") # line.strip("/")
run(f'sed -i -e "s,/$,," {quote(file)}', VERBOSE_G) # trailing / run(["sed", "-i", "-e", "s,/$,,", file], VERBOSE_G) # trailing /
def randomize_txt_file(file_path: Path): def randomize_txt_file(file_path: Path):
"""Randomize the order of the """Randomize the order of the
lines of the txt file""" lines of the txt file"""
file = str(file_path.resolve()) file = str(file_path.resolve())
run(f"sort -R {quote(file)} -o {quote(file)}", VERBOSE_G) run(["sort", "-R", file, "-o", file], VERBOSE_G)
def parse_list(file): def parse_list(file):