From 171821bc9b5e370888e893ce5057af7d0cf2fabf Mon Sep 17 00:00:00 2001 From: Danilo Reyes Date: Sat, 14 Dec 2024 16:14:53 -0600 Subject: [PATCH] migrated all scripts + function to generate output --- flake.nix | 40 +++++- pkgs/tuh-activity-logger.nix | 2 +- src/scripts/ffmpeg4discord.py | 138 +++++++++++++++++++ src/scripts/ffmpreg.sh | 122 +++++++++++++++++ src/scripts/find-dup-episodes.sh | 27 ++++ src/scripts/library-report.sh | 61 +++++++++ src/scripts/manage-library.sh | 224 +++++++++++++++++++++++++++++++ src/scripts/nextcloud-cronjob.sh | 59 ++++++++ src/scripts/pika-list.sh | 51 +++++++ src/scripts/run.sh | 40 ++++++ src/scripts/split-dir.sh | 33 +++++ src/scripts/stream-dl.sh | 29 ++++ src/scripts/sub-sync.sh | 186 +++++++++++++++++++++++++ src/{ => scripts}/tasks.sh | 0 src/scripts/update-dns.sh | 33 +++++ 15 files changed, 1037 insertions(+), 8 deletions(-) create mode 100644 src/scripts/ffmpeg4discord.py create mode 100644 src/scripts/ffmpreg.sh create mode 100644 src/scripts/find-dup-episodes.sh create mode 100755 src/scripts/library-report.sh create mode 100644 src/scripts/manage-library.sh create mode 100644 src/scripts/nextcloud-cronjob.sh create mode 100644 src/scripts/pika-list.sh create mode 100644 src/scripts/run.sh create mode 100644 src/scripts/split-dir.sh create mode 100644 src/scripts/stream-dl.sh create mode 100755 src/scripts/sub-sync.sh rename src/{ => scripts}/tasks.sh (100%) create mode 100755 src/scripts/update-dns.sh diff --git a/flake.nix b/flake.nix index 4df723f..a2311fc 100644 --- a/flake.nix +++ b/flake.nix @@ -2,17 +2,43 @@ description = "Nix flake for the activity logging script"; outputs = { nixpkgs, ... }: + let + system = "x86_64-linux"; + pkgs = import nixpkgs { inherit system; }; + generatePackages = + args: + let + inherit (args) dir ext handler; + in + ./${dir} + |> builtins.readDir + |> builtins.attrNames + |> builtins.filter (file: builtins.match ".*\\.${ext}$" file != null) + |> builtins.map (file: rec { + name = builtins.replaceStrings [ ".${ext}" ] [ "" ] file; + value = handler ./${dir}/${file} name; + }) + |> builtins.listToAttrs; + in { packages.x86_64-linux = let - system = "x86_64-linux"; - pkgs = import nixpkgs { inherit system; }; + scriptBin = path: name: pkgs.writeScriptBin name (builtins.readFile path); in - { - tasks = pkgs.writeScriptBin "tasks" (builtins.readFile ./src/tasks.sh); - download = pkgs.callPackage ./pkgs/download.nix { }; - webcomix = pkgs.callPackage ./pkgs/webcomix.nix { }; - tuh-activity-logger = pkgs.callPackage ./pkgs/tuh-activity-logger.nix { }; + generatePackages { + dir = "pkgs"; + ext = "nix"; + handler = (path: name: pkgs.callPackage path { }); + } + // generatePackages { + dir = "src/scripts"; + ext = "sh"; + handler = scriptBin; + } + // generatePackages { + dir = "src/scripts"; + ext = "py"; + handler = scriptBin; }; }; } diff --git a/pkgs/tuh-activity-logger.nix b/pkgs/tuh-activity-logger.nix index b59b89f..7b7e827 100644 --- a/pkgs/tuh-activity-logger.nix +++ b/pkgs/tuh-activity-logger.nix @@ -6,7 +6,7 @@ in python3Packages.buildPythonApplication { inherit pname version; src = builtins.path { - path = ../../src/tuhmayto/.; + path = ../src/tuhmayto/.; name = "${pname}-${version}"; }; build-system = [ python3Packages.setuptools ]; diff --git a/src/scripts/ffmpeg4discord.py b/src/scripts/ffmpeg4discord.py new file mode 100644 index 0000000..c1e6f08 --- /dev/null +++ b/src/scripts/ffmpeg4discord.py @@ -0,0 +1,138 @@ +#! /usr/bin/env nix-shell +#! nix-shell -i python3 -p python3 + +# Imports +import os +import math + + +# Function for calculating the appropriate bitrate to use during conversion +def get_bitrate(duration, filesize, audio_br): + br = math.floor(filesize / duration - audio_br) + return br, br * 0.50, br * 1.45 + + +def encode(ffmpeg_string, output_name, fs): + os.system(ffmpeg_string) + end_size = ( + os.path.getsize( + "/dev/shm/ffmpeg/out/{output_name}".format(output_name=output_name) + ) + * 0.00000095367432 + ) + if end_size < fs: + print( + ffmpeg_string.replace("\t", "") + + "\nThe FFMPEG string above has yielded a file whose size is " + + str(end_size) + + "MB.\n{output_name} is ready for Discord.\n".format( + output_name=output_name + ) + ) + return False + else: + print( + ffmpeg_string.replace("\t", "") + + "\nThe FFMPEG string above has yielded a file whose size is " + + str(end_size) + + "MB.\n{output_name} is NOT ready for Discord, and will be re-run.\nMy bad.".format( + output_name=output_name + ) + ) + return True + + +def time_calculations(fname, length): + startstring = fname[0:2] + ":" + fname[2:4] + ":" + fname[4:6] + endstring = fname[7:9] + ":" + fname[9:11] + ":" + fname[11:13] + + try: + int(fname[0:6]) + startseconds = ( + int(fname[0:2]) * 60 * 60 + int(fname[2:4]) * 60 + int(fname[4:6]) + ) + try: + int(fname[11:13]) + endseconds = ( + int(fname[7:9]) * 60 * 60 + int(fname[9:11]) * 60 + int(fname[11:13]) + ) + duration = endseconds - startseconds + timestamped_section = f"-ss {startstring} -to {endstring}" + except: + duration = length - startseconds + timestamped_section = f"-ss {startstring}" + except: + duration = length + timestamped_section = "" + + return duration, timestamped_section + + +fname = os.listdir("/dev/shm/ffmpeg/in/")[0] +os.rename("/dev/shm/ffmpeg/in/" + fname, "/dev/shm/ffmpeg/in/" + fname.replace(" ", "")) +fname = fname.replace(" ", "") + +# ffprobe to calculate the total duration of the clip. +length = math.floor( + float( + os.popen( + "ffprobe -v error -show_entries format=duration -of default=noprint_wrappers=1:nokey=1 /dev/shm/ffmpeg/in/{fname}".format( + fname=fname + ) + ).read() + ) +) + +duration, timestamped_section = time_calculations(fname, length) + +run = True + +reso = os.getenv("reso") +codec = os.getenv("codec") +audio_br = os.getenv("audio_br") +audio_br = int(str(os.getenv("audio_br"))) +fs = float(str(os.getenv("fs"))) +target_fs = fs + +codecs = { + "vp9": { + "pass1": f"-vf scale={reso} -g 240 -threads 8 -speed 4 -row-mt 1 -tile-columns 2 -vsync cfr -c:v libvpx-vp9 -pass 1 -an", + "pass2": f"-vf scale={reso} -g 240 -threads 8 -speed 2 -row-mt 1 -tile-columns 2 -c:v libvpx-vp9 -c:a libopus -pass 2", + "output_name": "small_" + fname.replace(".mp4", ".webm"), + }, + "x264": { + "pass1": f"-vf scale={reso} -vsync cfr -c:v libx264 -pass 1 -an", + "pass2": f"-vf scale={reso} -c:v libx264 -c:a aac -pass 2 ", + "output_name": "small_" + fname, + }, + "x265": { + "pass1": f"-vf scale={reso} -c:v libx265 -vsync cfr -x265-params pass=1 -an", + "pass2": f"-vf scale={reso} -c:v libx265 -x265-params pass=2 -c:a aac", + "output_name": "small_" + fname, + }, +} + + +while run: + # Conversion to KiB + end_fs = fs * 8192 + br, minbr, maxbr = get_bitrate( + duration=duration, filesize=end_fs, audio_br=audio_br + ) + ffmpeg_string = f""" + ffpb {timestamped_section} -hwaccel cuda -i /dev/shm/ffmpeg/in/{fname} -y \ + {codecs[str(codec)]['pass1']} \ + -b:v {br}k -minrate {minbr}k -maxrate {maxbr}k \ + -f null /dev/null && \ + ffpb {timestamped_section} -hwaccel cuda -i /dev/shm/ffmpeg/in/{fname} \ + {codecs[str(codec)]['pass2']} \ + -b:a {audio_br}k -b:v {br}k -minrate {minbr}k -maxrate {maxbr}k \ + /dev/shm/ffmpeg/out/{codecs[str(codec)]['output_name']} -y + """ + + run = encode( + ffmpeg_string, output_name=codecs[str(codec)]["output_name"], fs=target_fs + ) + + if run: + fs = fs - 0.2 diff --git a/src/scripts/ffmpreg.sh b/src/scripts/ffmpreg.sh new file mode 100644 index 0000000..a1bcd72 --- /dev/null +++ b/src/scripts/ffmpreg.sh @@ -0,0 +1,122 @@ +#! /usr/bin/env nix-shell +#! nix-shell -i bash -p bash gum trashy fd ripgrep mediainfo + +replace_extension() { + local file_basename + file_basename=$(basename "$1") + echo "${file_basename%.*}.$2" +} + +convert_gif() { + file_newname=$(replace_extension "$1" gif) + ffpb -i "$(realpath "$1")" -vf fps=12,scale=480:-1,smartblur=ls=-0.5 "$file_newname" +} + +convert_av1() { + local file_newname + file_newname=$(replace_extension "$1" mp4) + local file_tempdest=/dev/shm/$file_newname + local file_destination + file_destination=$(dirname "$(realpath "$1")")/$file_newname + ffpb -i "$1" \ + -c:v libaom-av1 \ + -threads 12 -cpu-used 7 \ + "$file_tempdest" + trash "$1" + mv -i "$file_tempdest" "$file_destination" +} + +convert_mp4() { + local file_newname + file_newname=$(replace_extension "$1" mp4) + local file_tempdest=/dev/shm/$file_newname + local file_destination + file_destination=$(dirname "$(realpath "$1")")/$file_newname + ffpb -i "$1" \ + -c:v libx265 \ + -preset veryslow \ + "$file_tempdest" + trash "$1" + mv -i "$file_tempdest" "$file_destination" +} + +convert_discord() { + local file_newname + file_newname=$2_$(replace_extension "$1" mp4) + local dir_ram=/dev/shm/ffmpeg + mkdir -p $dir_ram/{in,out} + ffpb -hwaccel cuda -i "$(realpath "$1")" \ + -c:v h264_nvenc \ + "$dir_ram"/in/discord.mp4 + cd "$dir_ram" || exit + codec=x264 audio_br=$3 fs=$4 reso=$5 ffmpeg4discord + mv "$dir_ram"/out/small_discord.mp4 ~/"$file_newname" + command rm -rf "$dir_ram" +} + +operation=$(gum choose mp4 av1 discord nitro gif enc265) + +case $operation in + 1 | mp4) + to_convert=() + while IFS= read -r file; do + to_convert+=("$file") + done < <(fd . "$(pwd)" -tf -aL | fzf --multi -i) + for file in "${to_convert[@]}"; do + convert_mp4 "$file" + done + ;; + 2 | av1) + to_convert=() + while IFS= read -r file; do + to_convert+=("$file") + done < <(fd . "$(pwd)" -tf -aL | fzf --multi -i) + for file in "${to_convert[@]}"; do + convert_av1 "$file" + done + ;; + 3 | discord) + to_convert=() + while IFS= read -r file; do + to_convert+=("$file") + done < <(fd . "$(pwd)" -tf -aL | fzf --multi -i) + for file in "${to_convert[@]}"; do + convert_discord "$file" discord 96 8.0 "1280x720" + done + ;; + 4 | nitro) + to_convert=() + while IFS= read -r file; do + to_convert+=("$file") + done < <(fd . "$(pwd)" -tf -aL | fzf --multi -i) + for file in "${to_convert[@]}"; do + convert_discord "$file" nitro 128 50.0 "1920x1080" + done + ;; + 5 | gif) + to_convert=() + while IFS= read -r file; do + to_convert+=("$file") + done < <(fd . "$(pwd)" -tf -aL | fzf --multi -i) + for file in "${to_convert[@]}"; do + convert_gif "$file" + done + ;; + 6 | enc265) + to_convert=() + extensions=(flv m4v mpg avi mov ts mkv mp4 webm) + for ext in "${extensions[@]}"; do + while IFS= read -r file; do + if ! (mediainfo "$file" | grep Writing\ library | grep -q x265); then + to_convert+=("$file") + fi + done < <(fd . -e "$ext" -tf -aL) + done + for file in "${to_convert[@]}"; do + convert_mp4 "$file" + done + ;; + *) + echo -n "Please select a valid input" + ;; +esac diff --git a/src/scripts/find-dup-episodes.sh b/src/scripts/find-dup-episodes.sh new file mode 100644 index 0000000..0167eab --- /dev/null +++ b/src/scripts/find-dup-episodes.sh @@ -0,0 +1,27 @@ +#! /usr/bin/env nix-shell +#! nix-shell -i bash -p bash fd ripgrep + +root=/srv/pool/multimedia/media/Series +while IFS= read -r directory; do + while IFS= read -r season; do + season_episodes=() + while IFS= read -r episode; do + number="$(basename "$episode" | + rg --pcre2 -o "S\d+E\d+" | + rg --pcre2 -o "\d+$" | + awk '$0*=1')" + season_episodes+=($((number))) + done < <(fd . "$season" -tf -d1 \ + -E '*.srt' \ + -E '*.jpg' \ + -E '*.nfo' \ + -E '*.json') + dupe=$(printf '%s\n' "${season_episodes[@]}" | awk '!($0 in seen){seen[$0];next} 1') + if [[ -z $dupe ]]; then + continue + fi + echo "The episode $dupe is duplicated on $(basename "$season") of $(basename "$directory")" + echo "$season" + echo "_______________" + done < <(fd . "$directory" -td -d1) +done < <(fd . "$root" -td -d 1) diff --git a/src/scripts/library-report.sh b/src/scripts/library-report.sh new file mode 100755 index 0000000..f933429 --- /dev/null +++ b/src/scripts/library-report.sh @@ -0,0 +1,61 @@ +#! /etc/profiles/per-user/jawz/bin/bash +#! nix-shell -i bash -p bash + +manga_dir="/srv/pool/multimedia/media/Library/manga/manga" + +airing=() +completed=() +update_volumes=() +other=() + +categorize_series() { + local series_dir="$1" + local has_ch=false + local has_vol=false + + for file in "$series_dir"/*; do + if [[ $(basename "$file") == *"Ch."* ]]; then + has_ch=true + elif [[ $(basename "$file") == *"Vol."* ]]; then + has_vol=true + fi + done + + if $has_ch && $has_vol; then + update_volumes+=("$(basename "$series_dir")") + elif $has_ch; then + airing+=("$(basename "$series_dir")") + elif $has_vol; then + completed+=("$(basename "$series_dir")") + else + other+=("$(basename "$series_dir")") + fi +} + +for series in "$manga_dir"/*; do + if [ ! -d "$series" ]; then + continue + fi + categorize_series "$series" +done + +echo "=== Categorized Manga Series ===" +echo "Airing:" +for series in "${airing[@]}"; do + echo " - $series" +done + +echo "Completed:" +for series in "${completed[@]}"; do + echo " - $series" +done + +echo "Update Volumes:" +for series in "${update_volumes[@]}"; do + echo " - $series" +done + +echo "Other:" +for series in "${other[@]}"; do + echo " - $series" +done diff --git a/src/scripts/manage-library.sh b/src/scripts/manage-library.sh new file mode 100644 index 0000000..aafc42b --- /dev/null +++ b/src/scripts/manage-library.sh @@ -0,0 +1,224 @@ +#! /usr/bin/env nix-shell +#! nix-shell -i bash -p bash gum fd ripgrep eza trash-cli zip unzip + +root_directories=( + /srv/pool/multimedia/media/Library/comics + /srv/pool/multimedia/media/Library/manga/manga + /srv/pool/multimedia/media/Library/manga/webtoons + /srv/pool/multimedia/media/Library/manga/español + /srv/pool/multimedia/media/Library/webtoons +) + +newname() { + echo "$1" | sed -E "s/$2/$3/g" +} + +separator() { + gum style --foreground 7 _________________________ +} +announce_changes() { + echo "Renaming:" + gum style --foreground 1 "$1" + echo "Into:" + gum style --foreground 2 "$2" + separator +} + +rename_file() { + while IFS= read -r file; do + local original_name + original_name=$(basename "$file") + local new_name + new_name=$(newname "$(basename "$file")" "$2" "$3") + + announce_changes "$original_name" "$new_name" + command mv -n "$(dirname "$file")"/{"$original_name","$new_name"} + done < <(fd "$1" --absolute-path -tf -s "${root_directories[@]}") +} + +rename_directory() { + while IFS= read -r dir; do + local new_name + new_name=$(newname "$(basename "$dir")" "$2" "$3") + local new_dir + new_dir=$(dirname "$dir")/$new_name + + announce_changes "$dir" "$new_dir" + echo "Processing..." + if [ ! -d "$new_dir" ]; then + echo "$(basename "$new_dir") doesn't exist. Creating it." + command mkdir -p "$new_dir" + fi + if [ -d "$new_dir" ]; then + echo "$(basename "$new_dir") has been created!, moving the following files:" + eza "$dir" + fd . "$dir" -x mv -n {} "$(realpath "$new_dir")" + fi + separator + done < <(fd "$1" --absolute-path -td -s "${root_directories[@]}") +} + +rename_numbered_files() { + while IFS= read -r file; do + local original_name + original_name=$(basename "$file") + local new_name + + new_name=$(echo "$original_name" | sed -E 's/(.*) - 0*([[:digit:]]+)(\.cbz)/\1 #\2\3/') + + if [[ "$new_name" != "$original_name" ]]; then + announce_changes "$original_name" "$new_name" + command mv -n "$(dirname "$file")"/{"$original_name","$new_name"} + fi + done < <(fd --absolute-path -tf -e cbz '\-[[:space:]]0*[[:digit:]]+' "${root_directories[@]}") +} + +# Check directory existence +for dir in "${root_directories[@]}"; do + if [ -d "$dir" ]; then + continue + fi + echo "directory doesn't exist... creating $dir" + mkdir -vp "$dir" +done + +# Fix numbered issues (For migrating from old directory) +rename_numbered_files + +# Capitalize Special words +words=(special tpb full annual) +Words=(Special TPB Full Annual) +counter=0 +for word in "${words[@]}"; do + while IFS= read -r file; do + new_name=$(newname "$(basename "$file")" "$word" "${Words[$counter]}") + echo "Inproper capitalization of the word" + gum style --foreground 1 "$word" + echo "adjusting it into" + gum style --foreground 2 "${Words[$counter]}" + announce_changes "$(basename "$file")" "$new_name" + command mv -n "$(dirname "$file")"/{"$(basename "$file")","$new_name"} + done < <(fd "$word" --absolute-path -tf -s "${root_directories[@]}") + counter=$((counter + 1)) +done + +# Rename Year files +# set regex_year_grep "\([[:digit:]]{4}\)" +# set regex_year_string "(\()(\d{4})(\))" +# rename_directory $regex_year_grep $regex_year_string \$2 +# rename_file $regex_year_grep $regex_year_string \$2 + +# Rename #_ downloads +regex_hashtag="#_" +rename_directory $regex_hashtag $regex_hashtag "#" +rename_file $regex_hashtag $regex_hashtag "#" + +# Rename ~ to : +# rename_directory " ~ " " ~ " ": " +# rename_file " ~ " " ~ " ": " + +rename_keywords() { + # Followed by digit + local regex_digit_fd="$1 \d+" + local regex_digit="($1 )([[:digit:]]+)" + rename_directory "$regex_digit_fd" "$regex_digit" "\1#\2" + rename_file "$regex_digit_fd" "$regex_digit" "\1#\2" + # Without digit + regex="#$1" + rename_directory "$regex" "$regex" "$1" + rename_file "$regex" "$regex" "$1" +} + +rename_keywords TPB +rename_keywords Special +rename_keywords Annual + +# Rename #Full +rename_directory " #Full" " #Full" "" +rename_file " #Full" " #Full" "" + +# Rename double space +rename_directory " " " " " " +rename_file " " " " " " + +# Fix names +wrongnames=( + "Transformers: Spotlight" + "Dr. Stone" + i-dont-want-this-kind-of-hero + pure-of-heart + scoob-and-shag + stick-n-poke + "Houseki no Kuni" + "Gantz E" + "Gantz G" +) +rightname=( + "Transformers Spotlight:" + "Dr. STONE" + "I DON'T WANT THIS KIND OF HERO" + "Pure of Heart" + "Scoob and Shag" + "Stick n' Poke" + "Land of the Lustrous" + "Gatz:E" + "Gantz:G" +) +counter=0 +for wrongname in "${wrongnames[@]}"; do + rename_directory "$wrongname" "$wrongname" "${rightname[$counter]}" + rename_file "$wrongname" "$wrongname" "${rightname[$counter]}" + counter=$((counter + 1)) +done + +# Merge TPB (Part X) files +while IFS= read -r file; do + new_name=$(newname "$(basename "$file" .cbz)" "TPB \(Part [[:digit:]]+\)" TPB) + extract_dir=$(realpath "$(dirname "$file")"/"$new_name") + if [ ! -d "$extract_dir" ]; then + mkdir -p "$extract_dir" + fi + unzip "$file" -d "$extract_dir"/"$(basename "$file" .cbz)" + cd "$extract_dir" || exit + zip -r "$(realpath "$(dirname "$file")")"/"$new_name"\.cbz ./ + trash "$file" + trash "$extract_dir"/"$(basename "$file" .cbz)" +done < <(fd "Part \d+" --absolute-path -tf -s "${root_directories[@]}") + +webtoon_mangas=( + "Solo Leveling: Ragnarok" + "Solo Leveling" + "All Saints Street" + "God of Blackfield" + "The Non-Ability Fighter" + "Red Storm" + "Hardcore Leveling Warrior" +) + +manga_dir=/srv/pool/multimedia/media/Library/manga/manga +webtoons_dir=/srv/pool/multimedia/media/Library/manga/webtoons +errors_dir=/srv/pool/multimedia/media/Library/manga/errors +for webtoon_manga in "${webtoon_mangas[@]}"; do + if [ ! -d "$webtoons_dir/$webtoon_manga" ]; then + fd "$webtoon_manga" -td --max-depth 1 -s \ + "$manga_dir" -x mv -n {} "$webtoons_dir"/ + else + fd "$webtoon_manga" -tf --min-depth 2 -s \ + "$manga_dir" -e cbz -x mv -n {} "$webtoons_dir/$webtoon_manga" + fi +done + +if [ ! -d "$errors_dir" ]; then + mkdir -p "$errors_dir" +fi + +for directory in "${root_directories[@]}"; do + fd . -td --min-depth 2 -s \ + "$directory" -x mv -n {} "$errors_dir/{/}" +done + +fd . --absolute-path -tf -td "${root_directories[@]}" -x chown jawz:kavita {} +fd . --absolute-path -tf "${root_directories[@]}" -x chmod 664 {} +fd . --absolute-path -td "${root_directories[@]}" -x chmod 775 {} + +fd . --absolute-path -td -te "${root_directories[@]}" -x rmdir {} diff --git a/src/scripts/nextcloud-cronjob.sh b/src/scripts/nextcloud-cronjob.sh new file mode 100644 index 0000000..cbef1b0 --- /dev/null +++ b/src/scripts/nextcloud-cronjob.sh @@ -0,0 +1,59 @@ +#!/run/current-system/sw/bin/bash + +# Cron tasks +if type /run/current-system/sw/bin/nextcloud-occ 2>/dev/null; then + /run/current-system/sw/bin/nextcloud-occ preview:pre-generate + /run/current-system/sw/bin/nextcloud-occ face:background_job -t 900 +fi + +# Sync GDL stuff +root=/srv/pool/scrapping + +cd $root || exit +set -- Aqp Ghekre +for user in "$@"; do + originDir=$root/$user + destDir=/srv/pool/nextcloud/$user/files/Requested + destDirDup=/srv/pool/nextcloud/$user/files/RequestedDupePlzCheckNDel + if [ ! -d "$destDir" ]; then + echo "$destDir does not exist, creating..." + mkdir -p "$destDir" + fi + cd "$originDir" || exit + find . -type f -not -name '*.part' | while read -r file; do + destination=$destDir/"$(echo "$file" | sed "s/^\.\///")" + destinationDup=$destDirDup/"$(echo "$file" | sed "s/^\.\///")" + + if [ ! -f "$destination" ]; then + echo "Safe to move $(basename "$file")" + if [ ! -d "$(dirname "$destination")" ]; then + echo "Creating parent directory..." + mkdir -p "$(dirname "$destination")" + fi + mv -n "$file" "$destination" + else + echo "Duplicated encountered $(basename "$file")" + if [ ! -d "$(dirname "$destinationDup")" ]; then + echo "Creating parent directory..." + mkdir -p "$(dirname "$destinationDup")" + fi + mv -n "$file" "$destinationDup" + fi + + done + + find ./ -mindepth 1 -type d -empty -delete + + chown 987:988 -R "$destDir" + find "$destDir" -type d -exec chmod -R 755 {} \; + find "$destDir" -type f -exec chmod -R 644 {} \; + + if [ -d "$destDirDup" ]; then + chown 987:988 -R "$destDirDup" + find "$destDirDup" -type d -exec chmod -R 755 {} \; + find "$destDirDup" -type f -exec chmod -R 644 {} \; + fi + if type /run/current-system/sw/bin/nextcloud-occ 2>/dev/null; then + /run/current-system/sw/bin/nextcloud-occ files:scan --all + fi +done diff --git a/src/scripts/pika-list.sh b/src/scripts/pika-list.sh new file mode 100644 index 0000000..ef3b2d6 --- /dev/null +++ b/src/scripts/pika-list.sh @@ -0,0 +1,51 @@ +#! /usr/bin/env nix-shell +#! nix-shell -i bash -p bash fd borgbackup gum ripgrep + +BORG_PASSPHRASE=$(gum input --password --placeholder "Type borg password") +export BORG_PASSPHRASE + +d_root=$HOME/pika +f_string=home/jawz/.config/jawz/lists/jawz/watch.txt +d_borg=/srv/pool/backups/pika/lists + +while IFS= read -r repo; do + IFS=" " read -r -a array <<<"$repo" + repo_id="${array[0]}" + mkdir -vp "$d_root/$repo_id" && cd "$d_root/$repo_id" || exit + borg extract $d_borg::"$repo_id" $f_string + cat "$d_root/$repo_id/$f_string" >>"$d_root/master" +done < <(borg list "$d_borg") + +cd "$HOME" || exit + +sort -u "$d_root/master" -o "$d_root/sorted" +sort -u "$LW" -o "$LW" + +echo "Current $(wc -l <"$LW") archived $(wc -l <"$d_root/sorted")" + +echo "Missing lines:" +diff "$d_root/sorted" "$LW" + +# look for duped lines with different casing +echo "Duplicated lines:" +while IFS= read -r line; do + if ! [ "$line" == "${line,,}" ]; then + if rg "${line,,}" <"$LW"; then + echo "$line" + fi + fi +done <"$LW" + +# delete pika backups +if gum confirm "Limpiar pika?"; then + command rm -rf "$d_root" + while IFS= read -r repo; do + IFS=" " read -r -a array <<<"$repo" + repo_id="${array[0]}" + gum spin --spinner dot --title "Cleaning $repo_id..." -- borg delete $d_borg::"$repo_id" + done < <(borg list "$d_borg") +else + echo "Canceled, no files deleted" +fi +gum spin --spinner dot --title "Cleaning $repo_id..." -- borg compact "$d_borg" +gum spin --spinner dot --title "Cleaning $repo_id..." -- borg compact /srv/pool/backups/pika/home diff --git a/src/scripts/run.sh b/src/scripts/run.sh new file mode 100644 index 0000000..8765af7 --- /dev/null +++ b/src/scripts/run.sh @@ -0,0 +1,40 @@ +#! /usr/bin/env nix-shell +#! nix-shell -i bash -p bash zenity rmlint git gum xclip + +if [ -n "$1" ]; then + operation=$1 +else + operation=$(gum choose rmlint download git) +fi + +case $operation in +rmlint) + rmlint -g --types="duplicates" \ + --config=sh:handler=clone \ + /srv/pool/ + ;; +download) + ENTRY=$(zenity --entry --width=250 --title "Push Manager" \ + --text="Verify the following entry is correct" \ + --add-entry="Clipboard:" --entry-text "$(xclip -o -sel clip)") + if [ -n "$ENTRY" ]; then + # kgx -e "download -u jawz -i '$ENTRY'" + kgx -e "ssh miniserver ""download -u jawz -i ""$ENTRY"" "" " + else + zenity --error --width=250 \ + --text "Please verify and try again" + fi + ;; +git) + git_dir=$HOME/Development/Git + while IFS= read -r repo; do + if ! [ -d "$repo/.git" ]; then + continue + fi + cd "$repo" || exit + gum style --foreground 2 "Updating $(basename "$repo")" + git fsck --full + git pull + done < <(fd . "$git_dir" -td --absolute-path -d 1) + ;; +esac diff --git a/src/scripts/split-dir.sh b/src/scripts/split-dir.sh new file mode 100644 index 0000000..834f313 --- /dev/null +++ b/src/scripts/split-dir.sh @@ -0,0 +1,33 @@ +#! /usr/bin/env nix-shell +#! nix-shell -i bash -p bash fd + +before_count=$(fd -tf | wc -l) +i=0 + +for file in $(fd -d1 -tf -E '*.mp4' -E '*.webm'); do + dir_name=$(basename "$(pwd)")_$(printf %03d $((i / $1 + 1))) + mkdir -p "$dir_name" + mv -i "$file" "$(realpath "$dir_name")"/ + i=$((i + 1)) +done + +for file in $(fd -d1 -tf -e webm); do + mkdir -p webm + mv -i "$file" "$(realpath webm)"/ +done + +for file in $(fd -d1 -tf -e mp4); do + mkdir -p videos + mv -i "$file" "$(realpath videos)"/ +done + +after_count=$(fd -tf | wc -l) + +if [[ "$before_count" == "$after_count" ]]; then + echo "No file count differences" +else + echo "Before count: $before_count" + echo "After count: $after_count" +fi +sleep 10 +exit diff --git a/src/scripts/stream-dl.sh b/src/scripts/stream-dl.sh new file mode 100644 index 0000000..8c2b857 --- /dev/null +++ b/src/scripts/stream-dl.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env nix-shell +#! nix-shell -i bash -p bash yt-dlp curl-impersonate + +minutes=5 +time_alive=60 +sleep_time=$((minutes * 60)) +loops=$((time_alive / (sleep_time / time_alive))) + +re="[[:space:]]+" +echo "$1" +if [[ $1 =~ $re ]]; then + read -ra arr <<<"$1" + url="https://picarto.tv/${arr[0]}" +else + url="https://chaturbate.com/$1" +fi + +save_dir=/srv/pool/glue/stream-dl +if [ ! -d "$save_dir" ]; then + mkdir -p "$save_dir" +fi +cd $save_dir || exit + +for i in $(seq 1 1 "$loops"); do + waiting_time=$(((i * sleep_time) / time_alive)) + yt-dlp --hls-use-mpegts --prefer-ffmpeg -o '%(title)s.%(ext)s' "$url" + echo "sleeping for $sleep_time seconds… been waiting for $waiting_time minutes" + sleep $sleep_time +done diff --git a/src/scripts/sub-sync.sh b/src/scripts/sub-sync.sh new file mode 100755 index 0000000..34e9af5 --- /dev/null +++ b/src/scripts/sub-sync.sh @@ -0,0 +1,186 @@ +#!/run/current-system/sw/bin/bash + + +# nix-shell -i bash -p bash fd ripgrep file alass ffmpeg gum + +MEDIA_ROOT=("/srv/pool/multimedia/media/Series" "/srv/pool/multimedia/media/Movies") +REPLACE_DIR="/srv/pool/multimedia/media" +SUBTITLE_MIRROR="/srv/pool/multimedia/backups/subtitles" +RAM_SUB="/dev/shm/sub.srt" + +# BACKUPS SUBTITLES +backup_subtitles() { + while IFS= read -r subtitle; do + echo "backing up $subtitle" + dest_dir="$(dirname "$subtitle")" + dest_dir="${dest_dir/$REPLACE_DIR/$SUBTITLE_MIRROR}" + mkdir -p "$dest_dir" + cp "$subtitle" "${subtitle/$REPLACE_DIR/$SUBTITLE_MIRROR}" + done < <(fd . -tf -e srt --absolute-path "${MEDIA_ROOT[@]}") +} + +clean_up() { + while IFS= read -r directory; do + echo "cleaning up $directory" + subtitles=() + mapfile -d $'\0' subtitles < <(fd . "$directory" -e srt -tf -d 1 -0) + + if [ "${#subtitles[@]}" -lt 2 ]; then + continue + fi + + unset base_subtitle + unset subtitles_group + for subtitle in "${subtitles[@]}"; do + group=() + mapfile -d $'\0' group < <(fd --fixed-strings \ + "$(basename "$subtitle" .srt)" "$directory" \ + -d 1 -tf -0 -e srt) + for key in "${!group[@]}"; do + if ! echo "${group[$key]}" | rg -P '\.\d{1,2}(\.\w+(-\w+)?)?\.srt' -q; then + unset "group[$key]" + continue + fi + if [ -z "${group[$key]}" ]; then + continue + fi + echo "removing $(basename "$subtitle")" + rm "$subtitle" + done + done + done < <(fd . -td --absolute-path "${MEDIA_ROOT[@]}") +} + +rename_languages() { + while IFS= read -r file; do + base=$(basename "$file" .eng.srt) + dir=$(dirname "$file") + echo "renaming sub $base" + mv "$file" "$dir/$base.$2.srt" + done < <(fd . -tf --absolute-path "${MEDIA_ROOT[@]}" -e "$1.srt") +} + +sync_subtitles() { + while IFS= read -r directory; do + echo "scanning for sync $directory" + while IFS= read -r subtitle; do + echo "processing $subtitle" + video=() + extension=$(echo "$subtitle" | rg -oP "(\.\w+(-\w+)?)?\.srt") + basename="$(basename "$subtitle" "$extension")" + mapfile -d $'\0' video < <(fd "$basename" \ + "$directory" --fixed-strings \ + -e mkv -e mp4 -e avi -e webm -tf -d 1 -0) + + # skips directory if it contains more than 1 video file + # should never get triggered + if [ "${#video[@]}" -gt 1 ]; then + basename "$(dirname "$directory")" + echo "$(basename "$directory") has many video files: ${#video[@]}" + continue + fi + + # update subtitle in ram + if [ -e "$RAM_SUB" ]; then + rm "$RAM_SUB" + fi + cp "$subtitle" "$RAM_SUB" + + if [ ! -e $RAM_SUB ] && [ ! -e "${video[0]}" ]; then + continue + fi + echo "processing...$subtitle" + alass-cli "${video[0]}" "$RAM_SUB" "$subtitle" + + done < <(fd . "$directory" -tf -e srt -d 1 --newer "$1") + done < <(fd . -td --absolute-path "${MEDIA_ROOT[@]}") +} + +find_dupes() { + while IFS= read -r directory; do + videos=() + mapfile -d $'\0' videos < <(fd . \ + "$directory" -tf -d 1 -0 \ + -e mkv -e mp4 -e avi -e webm) + + if [ "${#videos[@]}" == 0 ]; then + if [[ "$directory" != *"Season"* ]]; then + continue + fi + echo "NO FILES ERROR: $directory" + fi + + if [ "${#videos[@]}" == 1 ]; then + continue + fi + + if [ "${#videos[@]}" -gt 1 ]; then + if [[ "$directory" == *"media/Movies"* ]]; then + echo "Movie directory has more than a movie" + continue + fi + for episode in "${videos[@]}"; do + episode_number="$(echo "$episode" | + rg -oP "S\d+E\d+(-E\d+)? ")" + episode_files="$( + fd "$episode_number" "$directory" --fixed-strings \ + -tf -d 1 \ + -e mkv -e mp4 -e avi -e webm | wc -l + )" + if [ "$episode_files" == 1 ]; then + continue + fi + echo ____________________________ + echo "The episode $episode_number is repeated on" + echo "$directory" + fd "$episode_number" "$directory" --fixed-strings \ + -tf -d 1 \ + -e mkv -e mp4 -e avi -e webm + done + fi + + done < <(fd . -td --absolute-path "${MEDIA_ROOT[@]}") +} + +fd . /srv/pool/multimedia/media/Series/ --owner jawz -x chown sonarr:piracy {} +fd . /srv/pool/multimedia/media/Movies/ --owner jawz -x chown radarr:piracy {} +fd . "${MEDIA_ROOT[@]}" -td -x chmod 775 {} +fd . "${MEDIA_ROOT[@]}" -tf -x chmod 664 {} + +rename_languages eng en +rename_languages spa es +rename_languages mx es_MX + +if [ -n "$1" ]; then + operation=$1 +else + operation=$(gum choose backup clean sync all) +fi +if [ -n "$2" ]; then + start_time=$2 +else + start_time="$(date '+%Y-%m-%d') 00:00:00" +fi + +case $operation in + backup) + backup_subtitles + ;; + clean) + clean_up + ;; + sync) + sync_subtitles "$start_time" + ;; + dupe) + find_dupes + ;; + all) + echo "backing up" + backup_subtitles + echo "cleaning up" + clean_up + echo "syncing" + sync_subtitles "$start_time" + ;; +esac diff --git a/src/tasks.sh b/src/scripts/tasks.sh similarity index 100% rename from src/tasks.sh rename to src/scripts/tasks.sh diff --git a/src/scripts/update-dns.sh b/src/scripts/update-dns.sh new file mode 100755 index 0000000..50c8ac3 --- /dev/null +++ b/src/scripts/update-dns.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env nix-shell +#! nix-shell -i bash -p bash curl + +function header { + echo ============================================= + echo "$1" + echo ============================================= +} + +function update { + hostname=@ + domain=$1 + password=$2 + ip=$3 + base="https://dynamicdns.park-your-domain.com/update" + + if host "$domain" 1.1.1.1 | grep "has address" | grep "$ip"; then + echo "$domain is currently set to $ip; no changes needed" + exit + fi + echo updating "$domain" + host="host=$hostname" + domain="domain=$domain" + password="password=$password" + curl "$base?$host&$domain&$password&ip=$ip" + echo +} + +ip=$(curl -s -X GET https://checkip.amazonaws.com) +header "$ip" +header NAMECHEAP +update rotehaare.art "$PASSROTE" "$ip" +update danilo-reyes.com "$PASSDANILO" "$ip"