From 19e87b429e090a328c110a94e4d19719390b8de0 Mon Sep 17 00:00:00 2001 From: Danilo Reyes Date: Wed, 6 Sep 2023 23:46:45 -0600 Subject: [PATCH] split configs into two systems... --- .gitignore => jawz/.gitignore | 0 jawz/configuration.org | 929 ++++++++++++++++++ .../dotfiles}/gallery-dl/config.json | 0 {dotfiles => jawz/dotfiles}/gopass/config.yml | 0 {dotfiles => jawz/dotfiles}/htop/htoprc | 0 {dotfiles => jawz/dotfiles}/npm/npmrc | 0 .../npm/update-notifier-npm-check.json | 0 {dotfiles => jawz/dotfiles}/wget/wgetrc | 0 jawz/hardware-configuration.nix | 139 +++ jawz/scripts/download/.env | 1 + jawz/scripts/download/.envrc | 1 + jawz/scripts/download/argparser.py | 96 ++ jawz/scripts/download/download.py | 417 ++++++++ jawz/scripts/download/functions.py | 70 ++ jawz/scripts/download/gdl_classes.py | 103 ++ jawz/scripts/download/setup.cfg | 17 + jawz/scripts/download/setup.py | 24 + jawz/scripts/download/shell.nix | 28 + jawz/scripts/ffmpeg4discord.py | 136 +++ jawz/scripts/ffmpreg.sh | 98 ++ jawz/scripts/pika-list.sh | 51 + jawz/scripts/run.sh | 48 + jawz/scripts/split-dir.sh | 28 + jawz/scripts/tasks.sh | 140 +++ workstation/.gitignore | 4 + .../configuration.org | 8 +- workstation/dotfiles/gallery-dl/config.json | 334 +++++++ workstation/dotfiles/gopass/config.yml | 10 + workstation/dotfiles/htop/htoprc | 61 ++ workstation/dotfiles/npm/npmrc | 7 + .../npm/update-notifier-npm-check.json | 4 + workstation/dotfiles/wget/wgetrc | 1 + .../hardware-configuration.nix | 0 nginx.nix => workstation/nginx.nix | 0 openldap.nix => workstation/openldap.nix | 0 workstation/scripts/chat-dl.sh | 21 + workstation/scripts/download/.env | 1 + workstation/scripts/download/.envrc | 1 + workstation/scripts/download/argparser.py | 96 ++ workstation/scripts/download/download.py | 417 ++++++++ workstation/scripts/download/functions.py | 70 ++ workstation/scripts/download/gdl_classes.py | 103 ++ workstation/scripts/download/setup.cfg | 17 + workstation/scripts/download/setup.py | 24 + workstation/scripts/download/shell.nix | 28 + workstation/scripts/ffmpeg4discord.py | 136 +++ workstation/scripts/ffmpreg.sh | 98 ++ workstation/scripts/manage-library.sh | 153 +++ workstation/scripts/nextcloud-cronjob.sh | 59 ++ workstation/scripts/pika-list.sh | 51 + workstation/scripts/run.sh | 48 + workstation/scripts/split-dir.sh | 28 + workstation/scripts/tasks.sh | 140 +++ workstation/scripts/update-dns.sh | 38 + .../secrets.nix_wip | 0 servers.nix => workstation/servers.nix | 0 56 files changed, 4280 insertions(+), 4 deletions(-) rename .gitignore => jawz/.gitignore (100%) create mode 100644 jawz/configuration.org rename {dotfiles => jawz/dotfiles}/gallery-dl/config.json (100%) rename {dotfiles => jawz/dotfiles}/gopass/config.yml (100%) rename {dotfiles => jawz/dotfiles}/htop/htoprc (100%) rename {dotfiles => jawz/dotfiles}/npm/npmrc (100%) rename {dotfiles => jawz/dotfiles}/npm/update-notifier-npm-check.json (100%) rename {dotfiles => jawz/dotfiles}/wget/wgetrc (100%) create mode 100644 jawz/hardware-configuration.nix create mode 100644 jawz/scripts/download/.env create mode 100644 jawz/scripts/download/.envrc create mode 100644 jawz/scripts/download/argparser.py create mode 100755 jawz/scripts/download/download.py create mode 100644 jawz/scripts/download/functions.py create mode 100644 jawz/scripts/download/gdl_classes.py create mode 100644 jawz/scripts/download/setup.cfg create mode 100644 jawz/scripts/download/setup.py create mode 100644 jawz/scripts/download/shell.nix create mode 100755 jawz/scripts/ffmpeg4discord.py create mode 100755 jawz/scripts/ffmpreg.sh create mode 100755 jawz/scripts/pika-list.sh create mode 100644 jawz/scripts/run.sh create mode 100755 jawz/scripts/split-dir.sh create mode 100755 jawz/scripts/tasks.sh create mode 100644 workstation/.gitignore rename configuration.org => workstation/configuration.org (99%) create mode 100755 workstation/dotfiles/gallery-dl/config.json create mode 100755 workstation/dotfiles/gopass/config.yml create mode 100755 workstation/dotfiles/htop/htoprc create mode 100755 workstation/dotfiles/npm/npmrc create mode 100755 workstation/dotfiles/npm/update-notifier-npm-check.json create mode 100755 workstation/dotfiles/wget/wgetrc rename hardware-configuration.nix => workstation/hardware-configuration.nix (100%) rename nginx.nix => workstation/nginx.nix (100%) rename openldap.nix => workstation/openldap.nix (100%) create mode 100755 workstation/scripts/chat-dl.sh create mode 100644 workstation/scripts/download/.env create mode 100644 workstation/scripts/download/.envrc create mode 100644 workstation/scripts/download/argparser.py create mode 100755 workstation/scripts/download/download.py create mode 100644 workstation/scripts/download/functions.py create mode 100644 workstation/scripts/download/gdl_classes.py create mode 100644 workstation/scripts/download/setup.cfg create mode 100644 workstation/scripts/download/setup.py create mode 100644 workstation/scripts/download/shell.nix create mode 100755 workstation/scripts/ffmpeg4discord.py create mode 100755 workstation/scripts/ffmpreg.sh create mode 100755 workstation/scripts/manage-library.sh create mode 100755 workstation/scripts/nextcloud-cronjob.sh create mode 100755 workstation/scripts/pika-list.sh create mode 100644 workstation/scripts/run.sh create mode 100755 workstation/scripts/split-dir.sh create mode 100755 workstation/scripts/tasks.sh create mode 100755 workstation/scripts/update-dns.sh rename secrets.nix_wip => workstation/secrets.nix_wip (100%) rename servers.nix => workstation/servers.nix (100%) diff --git a/.gitignore b/jawz/.gitignore similarity index 100% rename from .gitignore rename to jawz/.gitignore diff --git a/jawz/configuration.org b/jawz/configuration.org new file mode 100644 index 0000000..e666444 --- /dev/null +++ b/jawz/configuration.org @@ -0,0 +1,929 @@ +#+TITLE: JawZ NixOS main Configuration +#+AUTHOR: Danilo Reyes +#+PROPERTY: header-args :tangle configuration.nix +#+auto_tangle: t + +* TODO [0/6] +- [ ] System configurations [0/8] + - [ ] fail2ban + - [ ] Bluetooth multiple devices + pass-through + - [ ] Topgrade (perhaps unnecessary) +- [ ] dotfiles [0/4] + - [ ] migrate config to home-manager + - [ ] migrate share to home-manager + - [ ] migrate dconf to home-manager +- [-] Migrate apps [3/6] + - [-] paru + - [ ] appimages +- [-] Compile missing apps [1/8] + - [-] zap init + - [-] font-downloader + - [ ] SaveDesktop (flathub) + - [ ] gelata + - [ ] menulibre +- [ ] Misc [0/3] + - [ ] Figure out how to get rid of xterm + + +* ABOUT +Setting up the document. Also this should allow me to set up variables, and +other functions. +- Global version number so NixOS and Home-Manager are in sync +- The unstable part allows me to build packages from the unstable channel by + prepending "unstable" to a package name. +- The next part creates a simple build of some of my simple scripts, turning + them into binaries which then I can integrate into the nix-store as well as + declared systemd units. + +* DECLARATION +Here I will declare the dependencies and variables that will be used multiple +times through the config file, such as the current version of NixOS, +repositories and even some scripts that will be reused on systemd +configurations. + +** VARIABLES + +#+begin_src nix +{ config, pkgs, ... }: +let + version = "23.05"; + myEmail = "CaptainJawZ@outlook.com"; + myName = "Danilo Reyes"; + home-manager = builtins.fetchTarball "https://github.com/nix-community/home-manager/archive/release-${version}.tar.gz"; + unstable = import + (builtins.fetchTarball "https://github.com/nixos/nixpkgs/tarball/master") { + config = config.nixpkgs.config; + }; + nixGaming = import + (builtins.fetchTarball "https://github.com/fufexan/nix-gaming/archive/master.tar.gz"); + jawzTasks = pkgs.writeScriptBin + "tasks" (builtins.readFile ./scripts/tasks.sh); +in +{ # Remember to close this bracket at the end of the document +#+end_src + +** IMPORTS +These are files and modules which get loaded onto the configuration file, in the +future I may segment this file into different modules once it becomes too +cluttered, for example, I may create a module for systemd units. + +#+begin_src nix +imports = [ + ./hardware-configuration.nix + # + (import "${home-manager}/nixos") + nixGaming.nixosModules.pipewireLowLatency +]; +#+end_src + +* SYSTEM CONFIGURATION +** NETWORKING +At the moment, I don't have a wireless card on this computer, however as I build +a new system, such setting may come in handy. + +Pick *ONLY ONE* of the below networking options. +- *wireless.enable* enables wireless support via wpa_supplicant. +- *NetworkManager* it's the default of GNOME, and easiest to use and integrate. + +#+begin_src nix +networking = { + hostName = "gamingtation"; + networkmanager.enable = true; +}; +#+end_src + +** TIMEZONE & LOCALE +For some reason, useXkbConfig throws an error when building the system, either +way it is an unnecessary setting as my keyboards are the default en_US, only +locale set to Canadian out because I prefer how it displays the date. + +#+begin_src nix +time.timeZone = "America/Mexico_City"; + +i18n = { + defaultLocale = "en_CA.UTF-8"; + extraLocaleSettings = { + LC_MONETARY = "es_MX.UTF-8"; + }; +}; +console = { + font = "Lat2-Terminus16"; + keyMap = "us"; + # useXkbConfig = true; # use xkbOptions in tty. +}; +#+end_src + +* GNOME +At the time of writing this file, I require of X11, as the NVIDIA support for +Wayland is not perfect yet. At the time being, the ability to switch through +GDM from Wayland to XORG, it's pretty handy, but in the future these settings +will require an update. + +Sets up GNOME as the default desktop environment, while excluding some +undesirable packages from installing. + +#+begin_src nix +services = { + xserver = { + enable = true; + displayManager.gdm.enable = true; + desktopManager.gnome.enable = true; + layout = "us"; + libinput.enable = true; # Wacom required? + }; +}; + +environment.gnome.excludePackages = (with pkgs; [ + gnome-photos + gnome-tour + gnome-text-editor + gnome-connections + # gnome-shell-extensions + baobab +]) +++ (with pkgs.gnome; [ + # totem + gedit + gnome-music + epiphany + gnome-characters + yelp + gnome-font-viewer + cheese +]); + +# Sets up QT to use adwaita themes. +# qt = { +# enable = true; +# platformTheme = "gnome"; +# style = "adwaita"; +# }; +#+end_src + +* SOUND +In order to avoid issues with PipeWire, the wiki recommends to disable /sound.enable/ +This is a basic PipeWire configuration, in the future stuff like Bluetooth or +latency will require expanding these settings. + +#+begin_src nix +hardware.pulseaudio.enable = false; +sound.enable = false; +services.pipewire = { + enable = true; + alsa.enable = true; + alsa.support32Bit = true; + pulse.enable = true; + lowLatency = { + enable = true; + quantum = 64; + rate = 48000; + }; +}; +#+end_src + +* SECURITY +Disabled password in sudo for commodity, but this is obviously not recommended, +regarding rkit, that setting enables pipewire to run with real-time +capabilities. And lastly, the acme settings are for signing certificates. +#+begin_src nix +security = { + rtkit.enable = true; + sudo = { + enable = true; + wheelNeedsPassword = false; + }; +}; +#+end_src + +* NIXPKGS +Allow non-free, sadly is a requirement for some of my drivers, besides that, +here is a good place to declare some package overrides as well as permit unsafe +packages. + +#+begin_src nix +nixpkgs.config = { + allowUnfree = true; +}; +#+end_src + +* NORMAL USERS +Being part of the "wheel" group, means that the user has root privileges. + +#+begin_src nix +users.users.jawz = { + isNormalUser = true; + extraGroups = [ "wheel" "networkmanager" "scanner" + "lp" "piracy" "kavita" "video" + ]; + initialPassword = "password"; + openssh = { + authorizedKeys.keys = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIB5GaQM4N+yGAByibOFQOBVMV/6TjOfaGIP+NunMiK76 gpodeacero\cdreyes@100CDREYES" ]; + }; +#+end_src + +This section of the document categorizes and organizes all he packages that I +want installed, attempting to group them as dependencies of others when +necessary. + +* USER PACKAGES +Begin the block to install user packages. +#+begin_src nix +packages = (with pkgs; [ +#+end_src + +** GUI PACKAGES +All of my GUI applications categorized to make it easier to identify what each +application does, and the justification for is existence on my system. + +*** ART AND DEVELOPMENT +Art and development applications are together, as a game-developer one of my +goals is to create a workflow between this ecosystem of applications. + +#+begin_src nix +blender # cgi animation and sculpting +godot # game development +gdtoolkit # gdscript language server +krita # art to your heart desire! +# drawpile # arty party with friends!! +mypaint # not the best art program +mypaint-brushes # but it's got some +mypaint-brushes1 # nice damn brushes +pureref # create inspiration/reference boards +gimp # the coolest bestest art program to never exist +#+end_src + +*** GAMING +So far gaming has been a lot less painful than I could have originally +anticipated, most everything seems to run seamlessly. +=note= Roblox uninstalled as there is ongoing drama regarding linux users. + +#+begin_src nix +(lutris.override { + extraPkgs = pkgs: [ + winetricks + wine64Packages.stable + wineWowPackages.stable + nixGaming.packages.${pkgs.hostPlatform.system}.wine-tkg + nixGaming.packages.${pkgs.hostPlatform.system}.wine-discord-ipc-bridge + ]; +}) +heroic +vulkan-tools +# grapejuice # roblox manager +minecraft # minecraft official launcher +parsec-bin # remote gaming with friends +protonup-qt # update proton-ge +#+end_src + +*** PRODUCTIVITY +This is the section where the apps that help me be productive come, a lot of +this are not used as often as I wish… + +#+begin_src nix +libreoffice-fresh # office, but based +calibre # ugly af eBook library manager +foliate # gtk eBook reader +newsflash # feed reader, syncs with nextcloud +wike # gtk wikipedia wow! +unstable.furtherance # I made this one tehee track time utility +gnome.simple-scan # scanner +#+end_src + +*** MISC +Most of these apps, are part of the gnome circle, and I decide to install them +if just for a try and play a little. + +#+begin_src nix +# sequeler # friendly SQL client +blanket # background noise +# czkawka # duplicate finder +pika-backup # backups +gnome-obfuscate # censor private information +metadata-cleaner # remove any metadata and geolocation from files +# gnome-recipes # migrate these to mealie and delete +# denaro # manage your finances +# celeste # sync tool for any cloud provider +libgda # for pano shell extension +#+end_src + +*** MULTIMEDIA +Overwhelmingly player applications, used for videos and music, while most of my +consumption has moved towards jellyfin, it's still worth the install of most +of these, for now. + +#+begin_src nix +celluloid # video player +cozy # audiobooks player +gnome-podcasts # podcast player +handbrake # video converter, may be unnecessary +curtail # image compressor +pitivi # video editor +identity # compare images or videos +mousai # poor man shazam +tagger # tag music files +bottles # wine prefix manager +obs-studio # screen recorder & streamer +shortwave # listen to world radio +nextcloud-client # self-hosted google-drive alternative +#+end_src + +*** WEB +Stuff that I use to interact with the web, web browsers, chats, download +managers, etc. + +#+begin_src nix +firefox # web browser that allows to disable spyware +tor-browser-bundle-bin # dark web, so dark! +ungoogled-chromium # web browser with spyware included +discord # chat +telegram-desktop # furry chat +# hugo # website engine +nicotine-plus # remember Ares? +warp # never used, but supposedly cool for sharing files +#+end_src + +** COMMAND-LINE PACKAGES + +#+begin_src nix +ffmpeg # not ffmpreg, the coolest video conversion tool! +unstable.yt-dlp # downloads videos from most video websites +unstable.gallery-dl # similar to yt-dlp but for most image gallery websites +gdu # disk-space utility, somewhat useful +du-dust # rusty du +gocryptfs # encrypted filesystem! shhh!!! +exa # like ls but with colors +trashy # oop! didn't meant to delete that +rmlint # probably my favourite app, amazing dupe finder that integrates well with BTRFS +tldr # man for retards +tree-sitter # code parsing, required by Doom emacs +# torrenttools # create torrent files from the terminal! +# vcsi # video thumbnails for torrents, can I replace it with ^? +lm_sensors # for extension, displays cpu temp +#+end_src + +** MY SCRIPTS +Here I compile my own scripts into binaries + +#+begin_src nix +jawzTasks +(writeScriptBin "ffmpeg4discord" (builtins.readFile ./scripts/ffmpeg4discord.py)) +(writeScriptBin "ffmpreg" (builtins.readFile ./scripts/ffmpreg.sh)) +(writeScriptBin "split-dir" (builtins.readFile ./scripts/split-dir.sh)) +(writeScriptBin "run" (builtins.readFile ./scripts/run.sh)) +#+end_src + +** DEVELOPMENT PACKAGES + +#+begin_src nix +# required by doom emacs, but still are rather useful. +fd # modern find, faster searches +fzf # fuzzy finder! super cool and useful +ripgrep # modern grep +# languagetool # proofreader for English. check if works without the service +graphviz # graphs +tetex +# these two are for doom everywhere +xorg.xwininfo +xdotool + +# development environment +exercism # learn to code + +# SH +bats # testing system, required by Exercism +bashdb # autocomplete +shellcheck # linting +shfmt # a shell parser and formatter + +# NIX +nixfmt # linting +cachix # why spend time compiling? + +# PYTHON. +python3 # base language +# pipenv # python development workflow for humans +# poetry # dependency management made easy + +# C# & Rust +# omnisharp-roslyn # c# linter and code formatter + +# HASKELL +# cabal-install # haskell interface + +# JS +nodejs # not as bad as I thought +# jq # linting +#+end_src + +** HUNSPELL +These dictionaries work with Firefox, Doom Emacs and LibreOffice. + +#+begin_src nix +hunspell +hunspellDicts.it_IT +hunspellDicts.es_MX +hunspellDicts.en_CA +#+end_src + +** CUSTOMIZATION PACKAGES +Themes and other customization, making my DE look the way I want is one of the +main draws of Linux for me. + +#+begin_src nix +# Themes +adw-gtk3 +# gradience # theme customizer, allows you to modify adw-gtk3 themes +gnome.gnome-tweaks # tweaks for the gnome desktop environment +qgnomeplatform + +# Fonts +(nerdfonts.override { + fonts = [ "Agave" "CascadiaCode" "SourceCodePro" + "Ubuntu" "FiraCode" "Iosevka" ]; +}) +symbola +(papirus-icon-theme.override { + color = "adwaita"; +}) +#+end_src + +** PYTHON + +#+begin_src nix +]) ++ (with pkgs.python3Packages; [ + flake8 # wraper for pyflakes, pycodestyle and mccabe + isort # sort Python imports + nose # testing and running python scripts + pyflakes # checks source code for errors + pytest # framework for writing tests + speedtest-cli # check internet speed from the comand line + editorconfig # follow rules of contributin + black # Python code formatter + pylint # bug and style checker for python + (buildPythonApplication rec { + pname = "download"; + version = "1.5"; + src = ./scripts/download/.; + doCheck = false; + buildInputs = [ setuptools ]; + propagatedBuildInputs = + [ pyyaml types-pyyaml ]; + }) + (buildPythonApplication rec { + pname = "ffpb"; + version = "0.4.1"; + src = fetchPypi { + inherit pname version; + sha256 = "sha256-7eVqbLpMHS1sBw2vYS4cTtyVdnnknGtEI8190VlXflk="; + }; + doCheck = false; + buildInputs = [ setuptools ]; + propagatedBuildInputs = + [ tqdm ]; + }) +#+end_src + +** BAT-EXTRAS + +#+begin_src nix +]) ++ (with pkgs.bat-extras; [ + batman # man pages + batpipe # piping + batgrep # ripgrep + batdiff # this is getting crazy! + batwatch # probably my next best friend + prettybat # trans your sourcecode! +#+end_src + +** GNOME EXTENSIONS + +#+begin_src nix +]) ++ (with pkgs.gnomeExtensions; [ + appindicator # applets for open applications + gsconnect # sync data and notifications from your phone + freon # hardware temperature monitor + panel-scroll # scroll well to change workspaces + reading-strip # like putting a finger on every line I read + tactile # window manager + pano # clipboard manager + blur-my-shell # make the overview more visually appealing + # burn-my-windows + # forge # window manager +# ]) ++ (with unstable.pkgs.gnomeExtensions; [ +#+end_src + +** NODEJS PACKAGES + +#+begin_src nix +]) ++ (with pkgs.nodePackages; [ + dockerfile-language-server-nodejs # LSP + bash-language-server # LSP + pyright # LSP + markdownlint-cli # Linter + prettier # Linter + pnpm # Package manager +]); }; # <--- end of package list +#+end_src + +* HOME-MANAGER +** HOME-MANAGER SETTINGS +These make it so packages install to '/etc' rather than the user home directory, +also allow for upgrades when rebuilding the system. + +#+begin_src nix +home-manager.useUserPackages = true; +home-manager.useGlobalPkgs = true; +home-manager.users.jawz = { config, pkgs, ... }:{ + home.stateVersion = "${version}"; +#+end_src + +** DOTFILES +*** BASH + +#+begin_src nix +programs.bash = { + enable = true; + historyFile = "\${XDG_STATE_HOME}/bash/history"; + historyControl = [ "erasedups" ]; + shellAliases = { + ls = "exa --icons --group-directories-first"; + edit = "emacsclient -t"; + # comic = "download -u jawz -i \"$(cat $LC | fzf --multi --exact -i)\""; + # gallery = "download -u jawz -i \"$(cat $LW | fzf --multi --exact -i)\""; + # open_gallery = "cd /mnt/disk2/scrapping/JawZ/gallery-dl && xdg-open $(fd . ./ Husbands -tdirectory -d 1 | fzf -i)\""; + unique_extensions = "fd -tf | rev | cut -d. -f1 | rev | tr '[:upper:]' '[:lower:]' | sort | uniq --count | sort -rn"; + cp = "cp -i"; + mv = "mv -i"; + mkcd = "mkdir -pv \"$1\" && cd \"$1\" || exit"; + mkdir = "mkdir -p"; + rm = "trash"; + ".." = "cd .."; + "..." = "cd ../.."; + ".3" = "cd ../../.."; + ".4" = "cd ../../../.."; + ".5" = "cd ../../../../.."; + dl = "download -u jawz -i"; + e = "edit"; + c = "cat"; + f = "fzf --multi --exact -i"; + sc = "systemctl --user"; + jc = "journalctl --user -xefu"; + }; + enableVteIntegration = true; + initExtra = '' +#+end_src + +#+begin_src bash +$HOME/.local/bin/pokemon-colorscripts -r --no-title +# Lists +list_root="${config.home.homeDirectory}"/.config/jawz/lists/jawz +export LW=$list_root/watch.txt +export LI=$list_root/instant.txt +export LC=$list_root/comic.txt +export command_timeout=30 + +# GPG_TTY=$(tty) +# export GPG_TTY + +if command -v fzf-share >/dev/null; then + source "$(fzf-share)/key-bindings.bash" + source "$(fzf-share)/completion.bash" +fi + +nixos-reload () { + nix-store --add-fixed sha256 /home/jawz/Development/NixOS/scripts/PureRef-1.11.1_x64.Appimage + nixfmt /home/jawz/Development/NixOS/*.nix + sudo nixos-rebuild switch -I nixos-config=/home/jawz/Development/NixOS/jawz/configuration.nix +} +#+end_src +#+begin_src nix + ''; +}; +#+end_src + +*** OTHER + +#+begin_src nix +programs = { + emacs = { + enable = true; + }; + direnv = { + enable = true; + enableBashIntegration = true; + nix-direnv.enable = true; + }; + bat = { + enable = true; + config = { + pager = "less -FR"; + theme = "base16"; + }; + }; + git = { + enable = true; + userName = "${myName}"; + userEmail = "${myEmail}"; + }; + htop = { + enable = true; + package = pkgs.htop-vim; + }; +}; +#+end_src + +*** XDG + +#+begin_src nix +xdg = { + enable = true; + userDirs = { + enable = true; + createDirectories = false; + desktop = "${config.home.homeDirectory}"; + documents = "${config.home.homeDirectory}/Documents"; + download = "${config.home.homeDirectory}/Downloads"; + music = "${config.home.homeDirectory}/Music"; + pictures = "${config.home.homeDirectory}/Pictures"; + templates = "${config.home.homeDirectory}/.local/share/Templates"; + videos = "${config.home.homeDirectory}/Videos"; + }; + configFile = { + "wgetrc".source = ./dotfiles/wget/wgetrc; + "configstore/update-notifier-npm-check.json".source = ./dotfiles/npm/update-notifier-npm-check.json; + "npm/npmrc".source = ./dotfiles/npm/npmrc; + "gallery-dl/config.json".source = ./dotfiles/gallery-dl/config.json; + "htop/htoprc".source = ./dotfiles/htop/htoprc; + }; +}; +#+end_src + +** USER-SERVICES + +#+begin_src nix +services = { + lorri.enable = true; + emacs = { + enable = true; + defaultEditor = true; + package = pkgs.emacs; + }; +}; +#+end_src + +** CLOSING HOME-MANAGER + +#+begin_src nix +}; +#+end_src + +* ENVIRONMENT PACKAGES +These are a MUST to ensure the optimal function of nix, without these, recovery +may be challenging. + +#+begin_src nix +environment.systemPackages = with pkgs; [ + wget +]; +#+end_src + +* ENVIRONMENT VARIABLES + +#+begin_src nix +environment.variables = rec { + # PATH + XDG_CACHE_HOME = "\${HOME}/.cache"; + XDG_CONFIG_HOME = "\${HOME}/.config"; + XDG_BIN_HOME = "\${HOME}/.local/bin"; + XDG_DATA_HOME = "\${HOME}/.local/share"; + XDG_STATE_HOME = "\${HOME}/.local/state"; + + # DEV PATH + CABAL_DIR = "\${XDG_CACHE_HOME}/cabal"; + CARGO_HOME = "\${XDG_DATA_HOME}/cargo"; + GEM_HOME = "\${XDG_DATA_HOME}/ruby/gems"; + GEM_PATH = "\${XDG_DATA_HOME}/ruby/gems"; + GEM_SPEC_CACHE = "\${XDG_DATA_HOME}/ruby/specs"; + GOPATH = "\${XDG_DATA_HOME}/go"; + NPM_CONFIG_USERCONFIG = "\${XDG_CONFIG_HOME}/npm/npmrc"; + PNPM_HOME = "\${XDG_DATA_HOME}/pnpm"; + PSQL_HISTORY="\${XDG_DATA_HOME}/psql_history"; + REDISCLI_HISTFILE="\${XDG_DATA_HOME}/redis/rediscli_history"; + WINEPREFIX="\${XDG_DATA_HOME}/wine"; + + # OPTIONS + HISTFILE = "\${XDG_STATE_HOME}/bash/history"; + LESSHISTFILE = "-"; + GHCUP_USE_XDG_DIRS = "true"; + RIPGREP_CONFIG_PATH = "\${XDG_CONFIG_HOME}/ripgrep/ripgreprc"; + ELECTRUMDIR = "\${XDG_DATA_HOME}/electrum"; + VISUAL = "emacsclient -ca emacs"; + WGETRC = "\${XDG_CONFIG_HOME}/wgetrc"; + XCOMPOSECACHE = "${XDG_CACHE_HOME}/X11/xcompose"; + "_JAVA_OPTIONS" = "-Djava.util.prefs.userRoot=\${XDG_CONFIG_HOME}/java"; + DOCKER_CONFIG="\${XDG_CONFIG_HOME}/docker"; + + # NVIDIA + CUDA_CACHE_PATH = "\${XDG_CACHE_HOME}/nv"; + # WEBKIT_DISABLE_COMPOSITING_MODE = "1"; + # GBM_BACKEND = "nvidia-drm"; + # "__GLX_VENDOR_LIBRARY_NAME" = "nvidia"; + + # Themes + # GTK_THEME = "Adwaita:light"; + # QT_QPA_PLATFORMTHEME = "adwaita"; + # QT_STYLE_OVERRIDE = "adwaita"; + CALIBRE_USE_SYSTEM_THEME = "1"; + + PATH = [ + "\${HOME}/.local/bin" + "\${XDG_CONFIG_HOME}/emacs/bin" + "\${XDG_DATA_HOME}/npm/bin" + "\${XDG_DATA_HOME}/pnpm" + ]; +}; +#+end_src + +* PROGRAMS +Some programs get enabled and installed through here, as well as the activation +of some services. + +#+begin_src nix +programs = { + starship.enable = true; + fzf.fuzzyCompletion = true; + neovim = { + enable = true; + vimAlias = true; + }; + gnupg.agent = { + enable = true; + enableSSHSupport = true; + }; + geary = { + enable = true; + }; + steam = { + enable = true; + remotePlay.openFirewall = true; + dedicatedServer.openFirewall = true; + }; +}; +#+end_src + +* SERVICES +Miscellaneous services, most of which are managed by systemd. + +#+begin_src nix +services = { + printing = { + enable = true; + drivers = [ pkgs.hplip pkgs.hplipWithPlugin ]; + }; + avahi.enable = true; + avahi.nssmdns = true; + fstrim.enable = true; + btrfs.autoScrub = { + enable = true; + fileSystems = [ + "/" + ]; + }; + openssh = { + enable = true; + ports = [ 25552 ]; + settings = { + PasswordAuthentication = true; + KbdInteractiveAuthentication = true; + }; + startWhenNeeded = true; + listenAddresses = [ + { + addr = "0.0.0.0"; + port = 25552; + } + ]; + }; +}; +#+end_src + +* SYSTEMD +Home-manager, is not as flushed out when it comes to creating systemd units, so +the best way to define them for now, is using nix. + +#+begin_src nix +systemd = { + services = { }; + timers = { }; + user = { + services = { + tasks = { + restartIfChanged = true; + description = "Run a tasks script which keeps a lot of things organized"; + wantedBy = [ "default.target" ]; + path = [ + pkgs.bash + pkgs.nix + jawzTasks + ]; + serviceConfig = { + Restart = "on-failure"; + RestartSec = 30; + ExecStart = "${jawzTasks}/bin/tasks"; + }; + }; + }; + timers = { + tasks = { + enable = true; + description = "Run a tasks script which keeps a lot of things organized"; + wantedBy = [ "timers.target" ]; + timerConfig = { + OnCalendar = "*:0/10"; + }; + }; + }; + }; +}; +#+end_src + +* FIREWALL +Open ports in the firewall. +=TIP= list what app a port belongs to in a table. + +#+begin_src nix +networking = { + firewall = let + open_firewall_ports = [ + 25552 # ssh + ]; + open_firewall_port_ranges = [ + { from = 1714; to = 1764; } # kdeconnect + ]; + in + { + enable = true; + allowedTCPPorts = open_firewall_ports; + allowedUDPPorts = open_firewall_ports; + allowedTCPPortRanges = open_firewall_port_ranges; + allowedUDPPortRanges = open_firewall_port_ranges; + }; +}; +#+end_src + +* MISC SETTINGS +** ENABLE FONTCONFIG +If enabled, a Fontconfig configuration file will point to a set of default +fonts. If you don't care about running X11 applications or any other program +that uses Fontconfig, you can turn this option off and prevent a dependency on +all those fonts. +=tip= once that Wayland is ready for deployment, I probably can remove this +setting. + +#+begin_src nix +fonts.fontconfig.enable = true; +#+end_src + +* FINAL SYSTEM CONFIGURATIONS +The first setting creates a copy the NixOS configuration file and link it from +the resulting system (/run/current-system/configuration.nix). This is useful in +case you accidentally delete configuration.nix. + +The version value determines the NixOS release from which the default settings for +stateful data, like file locations and database versions on your system. +It‘s perfectly fine and recommended to leave this value at the release version +of the first install of this system. + +Lastly I configure in here Cachix repositories, which is a website that keeps a +cache of nixbuilds for easy quick deployments without having to compile +everything from scratch. + +#+begin_src nix +system = { + copySystemConfiguration = true; + stateVersion = "${version}"; +}; +nix = { + settings = { + substituters = [ + "https://nix-gaming.cachix.org" + "https://nixpkgs-python.cachix.org" + "https://devenv.cachix.org" + "https://cuda-maintainers.cachix.org" + ]; + trusted-public-keys = [ + "nix-gaming.cachix.org-1:nbjlureqMbRAxR1gJ/f3hxemL9svXaZF/Ees8vCUUs4=" + "nixpkgs-python.cachix.org-1:hxjI7pFxTyuTHn2NkvWCrAUcNZLNS3ZAvfYNuYifcEU=" + "devenv.cachix.org-1:w1cLUi8dv3hnoSPGAuibQv+f9TZLr6cv/Hm9XgU50cw=" + "cuda-maintainers.cachix.org-1:0dq3bujKpuEPMCX6U4WylrUDZ9JyUG0VpVZa7CNfq5E=" + ]; + }; + gc = { + automatic = true; + dates = "weekly"; + }; +}; +} +#+end_src + +# LocalWords: useXkbConfig Wayland XORG NIXPKGS diff --git a/dotfiles/gallery-dl/config.json b/jawz/dotfiles/gallery-dl/config.json similarity index 100% rename from dotfiles/gallery-dl/config.json rename to jawz/dotfiles/gallery-dl/config.json diff --git a/dotfiles/gopass/config.yml b/jawz/dotfiles/gopass/config.yml similarity index 100% rename from dotfiles/gopass/config.yml rename to jawz/dotfiles/gopass/config.yml diff --git a/dotfiles/htop/htoprc b/jawz/dotfiles/htop/htoprc similarity index 100% rename from dotfiles/htop/htoprc rename to jawz/dotfiles/htop/htoprc diff --git a/dotfiles/npm/npmrc b/jawz/dotfiles/npm/npmrc similarity index 100% rename from dotfiles/npm/npmrc rename to jawz/dotfiles/npm/npmrc diff --git a/dotfiles/npm/update-notifier-npm-check.json b/jawz/dotfiles/npm/update-notifier-npm-check.json similarity index 100% rename from dotfiles/npm/update-notifier-npm-check.json rename to jawz/dotfiles/npm/update-notifier-npm-check.json diff --git a/dotfiles/wget/wgetrc b/jawz/dotfiles/wget/wgetrc similarity index 100% rename from dotfiles/wget/wgetrc rename to jawz/dotfiles/wget/wgetrc diff --git a/jawz/hardware-configuration.nix b/jawz/hardware-configuration.nix new file mode 100644 index 0000000..32705f3 --- /dev/null +++ b/jawz/hardware-configuration.nix @@ -0,0 +1,139 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +let + unstable = import + (builtins.fetchTarball "https://github.com/nixos/nixpkgs/tarball/master") { + config = config.nixpkgs.config; + }; +in { + imports = [ (modulesPath + "/installer/scan/not-detected.nix") ]; + boot = { + #plymouth = { enable = true; }; + loader = { + efi = { + canTouchEfiVariables = true; + efiSysMountPoint = "/boot/efi"; + }; + grub = { + enable = true; + device = "nodev"; + efiSupport = true; + enableCryptodisk = true; + }; + }; + initrd.luks.devices = { + nvme = { + + device = "/dev/disk/by-uuid/e9618e85-a631-4374-b2a4-22c376d6e41b"; + preLVM = true; + }; + }; + kernelModules = [ "kvm-intel" ]; + kernel.sysctl = { "vm.swappiness" = 80; }; + extraModulePackages = [ ]; + initrd = { + availableKernelModules = + [ "xhci_pci" "ahci" "usbhid" "nvme" "usb_storage" "sd_mod" ]; + kernelModules = [ ]; + }; + }; + + fileSystems."/" = { + device = "/dev/mapper/nvme"; + fsType = "btrfs"; + options = [ + "subvol=nixos" + "ssd" + "compress=zstd:3" + "x-systemd.device-timeout=0" + "space_cache=v2" + "commit=120" + "datacow" + "noatime" + ]; + }; + + fileSystems."/home" = { + device = "/dev/mapper/nvme"; + fsType = "btrfs"; + options = [ + "subvol=home" + "ssd" + "compress=zstd:3" + "x-systemd.device-timeout=0" + "space_cache=v2" + "commit=120" + "datacow" + ]; + }; + + fileSystems."/boot" = { + device = "/dev/disk/by-uuid/ac6d349a-96b9-499e-9009-229efd7743a5"; + fsType = "ext4"; + }; + + fileSystems."/boot/efi" = { + device = "/dev/disk/by-uuid/B05D-B5FB"; + fsType = "vfat"; + }; + + swapDevices = [{ + device = "/dev/disk/by-uuid/5772497d-5e0b-4cec-b8d8-828601589ab4"; + randomEncryption = { + enable = true; + cipher = "aes-xts-plain64"; + keySize = 512; + sectorSize = 4096; + }; + }]; + + # Enables DHCP on each ethernet and wireless interface. In case of scripted networking + # (the default) this is the recommended approach. When using systemd-networkd it's + # still possible to use this option, but it's recommended to use it in conjunction + # with explicit per-interface declarations with `networking.interfaces..useDHCP`. + networking.useDHCP = lib.mkDefault true; + # networking.interfaces.enp0s31f6.useDHCP = lib.mkDefault true; + + nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; + powerManagement.cpuFreqGovernor = lib.mkDefault "performance"; + + # nixpkgs.config.packageOverrides = pkgs: { + # vaapiIntel = pkgs.vaapiIntel.override { enableHybridCodec = true; }; + # }; + + nixpkgs.config = { allowUnfree = true; }; + services.xserver.videoDrivers = [ "nvidia" ]; + hardware = { + nvidia = { + modesetting.enable = true; + powerManagement.enable = true; + }; + sane = { + enable = true; + extraBackends = [ pkgs.hplip pkgs.hplipWithPlugin ]; + }; + + cpu.amd.updateMicrocode = + lib.mkDefault config.hardware.enableRedistributableFirmware; + bluetooth.enable = true; + # opentabletdriver = { + # enable = true; + # package = unstable.opentabletdriver; + # daemon.enable = false; + # }; + opengl = { + enable = true; + driSupport = true; + driSupport32Bit = true; + # extraPackages = with pkgs; [ + # intel-media-driver # LIBVA_DRIVER_NAME=iHD + # vaapiIntel # LIBVA_DRIVER_NAME=i965 (older but works better for Firefox/Chromium) + # vaapiVdpau + # libvdpau-va-gl + # ]; + }; + }; +} diff --git a/jawz/scripts/download/.env b/jawz/scripts/download/.env new file mode 100644 index 0000000..6b75134 --- /dev/null +++ b/jawz/scripts/download/.env @@ -0,0 +1 @@ +CONFIG_FILE = "/home/jawz/.config/jawz/config.yaml" diff --git a/jawz/scripts/download/.envrc b/jawz/scripts/download/.envrc new file mode 100644 index 0000000..1d953f4 --- /dev/null +++ b/jawz/scripts/download/.envrc @@ -0,0 +1 @@ +use nix diff --git a/jawz/scripts/download/argparser.py b/jawz/scripts/download/argparser.py new file mode 100644 index 0000000..a4cd6b9 --- /dev/null +++ b/jawz/scripts/download/argparser.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +"""Setup the argparser""" +import argparse + +scrapper_types = ( + "push", + "gallery", + "instagram", + "kemono", + "comic", + "manga", + "webcomic", +) +# Define types of instagram stories +instagram_types = ["posts", "reels", "channel", "stories", "highlights"] + + +def argparser(users: list) -> argparse.Namespace: + """Returns an argparser to evaluate user input""" + # ARG PARSER + parser = argparse.ArgumentParser( + prog="Downloader", + description="Download images and galleries from a wide array of websites" + " either by using links or chosing from user define lists." + " This program also takes care of archiving tasks," + " that keep the run time fast and prevents downloading duplicates.", + ) + # Chose the type of scrapper + parser.add_argument( + choices=scrapper_types, + nargs="?", + dest="scrapper", + help="Select a scrapper.", + ) + # Parse user list + parser.add_argument( + "-u", + "--user", + choices=users, + dest="user", + help="Selects the personal user list to process. Defaults to everyone", + default="everyone", + type=str, + ) + # Parse individual links + parser.add_argument( + "-i", + "--input", + nargs="*", + dest="link", + action="append", + help="Download the provided links", + type=str, + ) + # Set the print list flag + parser.add_argument( + "-l", + "--list", + dest="flag_list", + action="store_true", + help="Prints a list of all the added links and prompts for a choice", + ) + # Set the use archiver flag + parser.add_argument( + "-a", + "--no-archive", + dest="flag_archive", + action="store_false", + help="Disables the archiver flag", + ) + # Set the skip flag + parser.add_argument( + "-s", + "--no_skip", + dest="flag_skip", + action="store_false", + help="Disables the skip function, downloads the entire gallery", + ) + parser.add_argument( + "-v", + "--verbose", + dest="flag_verbose", + action="store_true", + help="Prints the generated commands instead of running them", + ) + parser.add_argument( + "-t", + "--type-post", + choices=instagram_types, + nargs="*", + dest="post_type", + help="Filters posts on instagram by type", + default=instagram_types, + type=str, + ) + return parser.parse_args() diff --git a/jawz/scripts/download/download.py b/jawz/scripts/download/download.py new file mode 100755 index 0000000..66b5d55 --- /dev/null +++ b/jawz/scripts/download/download.py @@ -0,0 +1,417 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +Rewriting of the download manager script +with the intention to make it +more modular with the use of flags +in order to avoid unnecesary modifications +to the cofig files. +Also following in line more posix and python rules. +""" + +import re +import time +import logging +import yaml +from functions import run +from functions import quote +from functions import list_lines +from functions import load_config_variables +from argparser import argparser +from gdl_classes import User + +# GLOBAL VARIABLE SECTION +# Store the name of the main binaries early in the code +BIN_GALLERY = "gallery-dl" +BIN_YOUTUBE = "yt-dlp" +# SKIP = "3" +CONFIGS = load_config_variables() + +LOGGER = logging.getLogger() +HANDLER = logging.StreamHandler() +FORMATTER = logging.Formatter( + "[%(filename)s][%(levelname)s] %(funcName)s '%(message)s'" +) +HANDLER.setFormatter(FORMATTER) +LOGGER.addHandler(HANDLER) +LOGGER.setLevel(logging.INFO) + +# Enable a default "everyone" flag for when running stuff like download gallery +USERS = ["everyone"] +for dictionary in CONFIGS["users"]: + USERS.append(dictionary["name"]) + +ARGS = argparser(USERS) + + +def get_index(value: str) -> int: + """Find the index in the config file""" + for i, dic in enumerate(CONFIGS["users"]): + if dic["name"] == value: + LOGGER.debug("%s is %s", dic["name"], i) + return i + return -1 + + +def parse_gallery(gdl_list: str, user: User): + """Processes the gallery-dl command based on the selected gallery""" + # skip_arg = f" -A {SKIP}" if ARGS.flag_skip else "" + skip_arg = " -o skip=true" if not ARGS.flag_skip else "" + LOGGER.debug(skip_arg) + + # Send the list to gallery-dl + download_gallery( + ARGS.flag_archive, + skip_arg, + "", + str(user.sleep), + quote(f"{user.dir_download}"), + quote(f"{user.archive_gallery}"), + quote(gdl_list), + parse_instagram(gdl_list), + ) + + +def parse_instagram(link: str) -> str: + """Fix instagram links""" + if "instagram" not in link: + return "" + if isinstance(ARGS.post_type, list): + string = f" -o include={quote(','.join(ARGS.post_type))}" + LOGGER.debug(string) + return string + string = f" -o include={quote(ARGS.post_type)}" + LOGGER.debug(string) + return string + + +def parse_link(link: str) -> str: + """Fixes links""" + if not re.search(r"(twitter\.com\/\w+(\/)?(?!.*status))", link): + LOGGER.debug("No modifications needed for the link %s", link) + return link + # if url contains /media at the end just write the line + fixed_link = re.sub(r"\/$|\/media(\/?)$", "", link) + "/media" + LOGGER.debug("Processed link %s", fixed_link) + return fixed_link + + +def download_gallery( + use_archive: bool, + skip_arg: str = "", + link: str = "", + sleep: str = "0", + destination: str = "", + database: str = "", + queue: str = "", + opt_args: str = "", +): + """Processes the command string to run the gallery archiver""" + command = f"{BIN_GALLERY} --sleep {sleep}" + if skip_arg != "": + command += skip_arg + if destination != "": + command += f" --dest {destination}" + if use_archive: + command += f" --download-archive {database}" + if opt_args != "": + command += opt_args + if link != "" and queue == "": + LOGGER.info("link: %s", quote(link)) + command += f" {link}" + if queue != "" and link == "": + LOGGER.info("queue: %s", queue) + command += f" -i {queue}" + LOGGER.debug(command) + run(command, ARGS.flag_verbose) + + +def download_youtube( + use_archive: bool, + link: str = "", + destination: str = "", + database: str = "", +): + """Filters and processes the required command to download videos""" + command = BIN_YOUTUBE + + if re.search(r"(https:\/\/youtube|https:\/\/www.youtube|https:\/\/youtu.be)", link): + command += f' -o {quote(destination + "/%(title)s.%(ext)s")}' + + elif re.search(r"(https:\/\/music.youtube.*)", link): + if use_archive: + command += f" --download-archive {database}" + command += f""" \ + --no-playlist --newline -x \ + --audio-format best --add-metadata --audio-quality 0 -o \ + {quote(destination + '/%(title)s.%(ext)s')} \ + """ + + elif re.search(r"chaturbate", link): + # Re-runs the program every 30 seconds in case the stream goes private or dc + for i in range(1, 41): # For a 20 minute total + run( + f""" + {BIN_YOUTUBE} \ + --hls-use-mpegts --prefer-ffmpeg \ + -o {quote(destination + '/%(title)s.%(ext)s')} \ + {link} + """, + ARGS.flag_verbose, + ) + time.sleep(30) + LOGGER.info("waited for %s minutes", i * 30 / 60) + + else: # Any other video link, just do it generic + command += f" -f mp4 -o {quote(destination + '/%(title)s.%(ext)s')}" + LOGGER.info("%s %s", command, link) + run(f"{command} {link}", ARGS.flag_verbose) + + +def comic_manager(skip_arg: str, category: str): + """Process the information to download manga""" + re_cat = "" + if category == "manga": + re_cat = "manga|webtoon" + elif category == "comic": + re_cat = "readcomiconline" + + with open(CONFIGS["comic"]["list"], encoding="utf-8") as list_comic: + for graphic_novel in [line.rstrip() for line in list_comic]: + # Search for mangas but exclude comics + if not re.search(re_cat, graphic_novel): + LOGGER.debug("%s does not match regex espression", graphic_novel) + continue + download_gallery( + ARGS.flag_archive, + skip_arg, + quote(graphic_novel), + "0", + CONFIGS["comic"]["download-directory"], + CONFIGS["comic"]["archive"], + "", + "", + ) + + +def webcomic_manager(): + """Process the information to download webcomics""" + webcomic_list = CONFIGS["comic"]["webcomic-list"] + with open(webcomic_list, encoding="utf-8") as open_list: + webcomic_file = yaml.safe_load(open_list) + + # Create a list of all the available webcomics for the user to chose from + for index, entry in enumerate(webcomic_file["Webcomics"]): + print(list_lines(index, entry["name"])) + + # Prompt for a choice + usr_input = int(input("Select your comic: ")) + # Determines where the webcomic will be downloaded + rating = webcomic_file["Webcomics"][usr_input]["type"] + webcomic_category = webcomic_file["Global"][f"{rating}_directory"] + LOGGER.debug("The webcomic is %s", webcomic_category) + command = f"""cd {quote(webcomic_category)} && webcomix custom \ + {quote(webcomic_file["Webcomics"][usr_input]["name"])} \ + --start-url \ + {quote(webcomic_file["Webcomics"][usr_input]["url"])} \ + --next-page-xpath={quote(webcomic_file["Webcomics"][usr_input]["next_code"])} \ + --image-xpath={quote(webcomic_file["Webcomics"][usr_input]["image_code"])} \ + -y --cbz""" + LOGGER.debug(command) + run(command, ARGS.flag_verbose) + + +def push_manager(user: User): + """Filters out the URL to use the appropiate downloader""" + # Creates an array which will store any links that should use youtube-dl + link_video_cache = [] + re_links = re.compile( + r"(twitter\.com\/\w+((?=.*media)|(?!.*status)))" + r"|(men\.wikifeet)" + r"|(furaffinity\.net\/user\/)" + r"|((deviantart\.com\/\w+(?!.*\/art\/)))" + r"|(furaffinity\.net\/gallery\/)" + r"|(furaffinity\.net\/scraps\/)" + r"|(furaffinity\.net\/favorites\/)" + r"|(instagram.com(?!\/p\/)\/\w+)" + r"|(e621\.net((?=\/post\/)|(?!\/posts\/)))" + r"|(flickr\.com\/photos\/\w+\/(?!\d+))" + r"|(tumblr\.com(?!\/post\/))" + r"|(kemono\.party\/(fanbox|gumroad|patreon)(?!\/user\/\d+\/post))" + r"|(blogspot\.com(?!\/))" + r"|(rule34\.paheal\.net\/post\/(?!view))" + r"|(rule34\.xxx\/index\.php\?page\=post&s=(?!view))" + r"|(pixiv\.net\/(en\/)?((?=users)|(?!artwork)))" + r"|(reddit\.com\/(user|u))" + r"|(baraag\.net\/((@\w+)|(?!\/\d+)))" + r"|(pinterest\.com\/(?!pin\/\d+))" + r"|(redgifs\.com\/(users|u|(?!watch)))", + ) + with open(user.list_push, encoding="utf-8") as list_push: + for link in [line.rstrip() for line in list_push]: + LOGGER.debug("Processing %s", link) + # Flush the push list, cleans all the contents + with open(user.list_push, "w", encoding="utf-8") as list_push: + list_push.close() + # VIDEOS + if re.search(r"youtu.be|youtube|pornhub|xtube|xvideos|chaturbate", link): + LOGGER.debug("Matched type yt-dlp") + link_video_cache.append(link) + # Search for gallery links, these will be added to a list after downloading + elif re.search(re_links, link): + LOGGER.debug("Matched type gallery-dl") + # skip_arg = f" -A {SKIP}" if ARGS.flag_skip else "" + skip_arg = " -o skip=true" if not ARGS.flag_skip else "" + LOGGER.debug("Skip: %s, link: %s", skip_arg, parse_instagram(link)) + download_gallery( + ARGS.flag_archive, + skip_arg, + quote(f"{parse_link(link)}"), + f"{user.sleep}", + quote(f"{user.dir_download}"), + quote(f"{user.archive_gallery}"), + "", + f"{parse_instagram(link)}", + ) + # Record the gallery link, so it remains on the watch list + with open(user.list_master, "a", encoding="utf-8") as w_file, open( + user.list_master, "r", encoding="utf-8" + ) as r_file: + content = r_file.read().lower() + if parse_link(link).lower() in content: + LOGGER.info("Gallery repeated, not saving") + continue + LOGGER.info("New gallery, saving") + w_file.write(parse_link(str(link)) + "\n") + + # Searches for comic/manga links + elif re.search(r"readcomiconline|mangahere|mangadex|webtoons", link): + # Toggle for comic/manga skip flag + if ARGS.flag_skip and re.search(r"readcomiconline", link): + skip_arg = " --chapter-range 1" + elif ARGS.flag_skip and re.search(r"mangahere|webtoons", link): + skip_arg = " --chapter-range 1-5" + else: + skip_arg = "" + LOGGER.debug(skip_arg) + + download_gallery( + ARGS.flag_archive, + skip_arg, + quote(link), + "0", + CONFIGS["comic"]["download-directory"], + CONFIGS["comic"]["archive"], + "", + "", + ) + # Add comic/manga link to the list + list_gn = CONFIGS["comic"]["list"] + with open(list_gn, "a", encoding="utf-8") as w_file, open( + list_gn, "r", encoding="utf-8" + ) as r_file: + content = r_file.read().lower() + if parse_link(link).lower() in content: + LOGGER.info("Graphic novel repeated, not saving") + continue + LOGGER.info("New graphic novel, saving") + w_file.write(link + "\n") + # Download generic links, the -o flag overwrites config file and + # downloads the files into the root destination + else: + LOGGER.info("Other type of download %s", link) + download_gallery( + False, + " -o directory='[]'", + quote(link), + "0", + quote(str(user.dir_push)), + "", + "", + "", + ) + # Send the video links to youtube-dl + for link in link_video_cache: + download_youtube( + ARGS.flag_archive, + quote(link), + f"{user.dir_media_download}", + quote(f"{user.archive_media}"), + ) + + +def scrapper_manager(user: User): + # pylint: disable=too-many-branches + """Analyze the user arguments and call in functions""" + if not ARGS.scrapper: # Check if a scrapper was selected + return + + if re.search(r"gallery|instagram|kemono", ARGS.scrapper): + # skip_arg = f" -A {SKIP}" if ARGS.flag_skip else "" + skip_arg = " -o skip=true" if not ARGS.flag_skip else "" + LOGGER.debug(skip_arg) + if ARGS.scrapper == "gallery": + parse_gallery(f"{user.list_main}", user) + elif ARGS.scrapper == "instagram": + parse_gallery(f"{user.list_instagram}", user) + elif ARGS.scrapper == "kemono": + parse_gallery(f"{user.list_kemono}", user) + elif ARGS.scrapper in "push": + push_manager(user) + elif ARGS.scrapper in "comic": + skip_arg = " --chapter-range 1" if ARGS.flag_skip else "" + LOGGER.debug(skip_arg) + comic_manager(skip_arg, "comic") + elif ARGS.scrapper in "manga": + skip_arg = " --chapter-range 1-5" if ARGS.flag_skip else "" + LOGGER.debug(skip_arg) + comic_manager(skip_arg, "manga") + elif ARGS.scrapper in "webcomic": + webcomic_manager() + + +def main(): + """Main module to decide what to do based on the parsed arguments""" + if ARGS.scrapper: + if (ARGS.user in "everyone") and ( + re.search(r"push|gallery|instagram|kemono", ARGS.scrapper) + ): + for current_user in CONFIGS["users"]: + user = User(get_index(current_user["name"])) + user.list_manager() + LOGGER.info("Scrapping %s for %s", ARGS.scrapper, current_user["name"]) + scrapper_manager(user) + elif re.search(r"comic|manga|webcomic", ARGS.scrapper): + user = User(get_index("jawz")) + user.list_manager() + LOGGER.info("Scrapping %s", ARGS.scrapper) + scrapper_manager(user) + else: + # Create the lists to scrap + user = User(get_index(ARGS.user)) + user.list_manager() + scrapper_manager(user) + elif ARGS.link: + LOGGER.debug(ARGS.link) + if re.search(r"everyone|jawz", ARGS.user): + # Create the lists to scrap + user = User(get_index("jawz")) + user.list_manager() + else: + # Create the lists to scrap + user = User(get_index(ARGS.user)) + user.list_manager() + for arg_link in ARGS.link[0]: + LOGGER.debug(arg_link) + if ARGS.flag_verbose: + LOGGER.debug( + "%s >> %s", quote(parse_link(arg_link)), quote(user.list_push) + ) + else: + with open(user.list_push, "a", encoding="utf-8") as open_file: + open_file.write(parse_link(arg_link) + "\n") + push_manager(user) + + +if __name__ == "__main__": + main() diff --git a/jawz/scripts/download/functions.py b/jawz/scripts/download/functions.py new file mode 100644 index 0000000..902703f --- /dev/null +++ b/jawz/scripts/download/functions.py @@ -0,0 +1,70 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +"""Personal functions to aid on multiple scripts""" +import sys +import fileinput +import re +import os +from pathlib import Path +import yaml + +VERBOSE_G = False + + +def load_config_variables(): + """Loads all the variables from the config file""" + config_file = Path("~/.config/jawz/config.yaml") + with open(config_file.expanduser(), encoding="utf-8") as open_file: + return yaml.safe_load(open_file) + + +def run(command: str, verbose: bool): + """Run command in a subprocess""" + # pylint: disable=subprocess-run-check + # This toggle allows for a really wasy debug when using -v + if verbose: + print(command) + else: + os.system(command) + + +def list_lines(i: int, line: str) -> str: + """Create a numbered list""" + return f"{i}) {line}" + + +def quote(line: str) -> str: + """Quote the line""" + return f'"{line}"' + + +def sort_txt_file(file_path: Path): + """Sort every line alphabetically + remove duplicated and empty lines""" + file = str(file_path.resolve()) + run(f"sort -u {quote(file)} -o {quote(file)}", VERBOSE_G) + run(f"sed -i '/^$/d' {quote(file)}", VERBOSE_G) + run(f'sed -i -e "s,http:,https:," {quote(file)}', VERBOSE_G) + # fix this using strip on python + # line.strip("/") + run(f'sed -i -e "s,/$,," {quote(file)}', VERBOSE_G) # trailing / + + +def randomize_txt_file(file_path: Path): + """Randomize the order of the + lines of the txt file""" + file = str(file_path.resolve()) + run(f"sort -R {quote(file)} -o {quote(file)}", VERBOSE_G) + + +def parse_list(file): + """Replace http with https and remove trailing /""" + for line in fileinput.input(file, inplace=True): + sys.stdout.write(str(line).replace("http://", "https://")) + with open(file, "r+", encoding="utf-8") as open_file: + f_content = open_file.read() + f_content = re.compile(r"\/$", 0).sub(r"\/$", "") + open_file.seek(0) + open_file.truncate() + print(f_content) + sort_txt_file(file) diff --git a/jawz/scripts/download/gdl_classes.py b/jawz/scripts/download/gdl_classes.py new file mode 100644 index 0000000..7186d3f --- /dev/null +++ b/jawz/scripts/download/gdl_classes.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +"""Define the user class to populate and setup the download environment""" +import re +from pathlib import Path +from functions import sort_txt_file, randomize_txt_file, load_config_variables + +config_variables = load_config_variables() + + +class User: + """Populate the directory for each user""" + + # pylint: disable=too-many-instance-attributes + def __init__(self, index): + self.user = config_variables["users"][index] + self.config = config_variables["global"] + self.name = self.user["name"] + self.sleep = self.config["sleep"] + # Directories + self.dir_cache = Path(self.config["cache-directory"]) / self.name + self.dir_log = Path(self.config["log-directory"]) + self.dir_archive = Path(self.config["archive-directory"]) + self.dir_download = Path(self.user["download-directory"]) + self.dir_media_download = Path(self.user["media-directory"]) + self.dir_push = Path(self.user["push-directory"]) + self.dir_master_list = Path(self.config["list-dir"]) / self.name + # Files + self.archive_gallery = self.dir_archive / f"{self.name}.sqlite3" + self.archive_media = self.dir_archive / f"{self.name}_ytdl.txt" + # Lists + self.list_master = self.dir_master_list / "watch.txt" + self.list_push = self.dir_master_list / "instant.txt" + self.list_instagram = self.dir_cache / "instagram.txt" + self.list_kemono = self.dir_cache / "kemono.txt" + self.list_main = self.dir_cache / "main.txt" + + def create_directories(self): + """Create user directories if they don't exist""" + if self.dir_cache.is_dir(): + for file in self.dir_cache.iterdir(): + if file.is_file(): + file.unlink() + for file in self.dir_cache.iterdir(): + if file.is_dir(): + file.rmdir() + self.dir_cache.rmdir() + # Create directories + self.dir_cache.mkdir(parents=True, exist_ok=True) + self.dir_log.mkdir(parents=True, exist_ok=True) + self.dir_archive.mkdir(parents=True, exist_ok=True) + self.dir_download.mkdir(parents=True, exist_ok=True) + self.dir_media_download.mkdir(parents=True, exist_ok=True) + self.dir_push.mkdir(parents=True, exist_ok=True) + # Check for the existence of core files + if not Path(self.archive_gallery).is_file(): + self.archive_gallery.touch() + if not Path(self.archive_media).is_file(): + self.archive_media.touch() + if not self.dir_master_list.is_dir(): + print(f"ERROR: Directory for user {self.name} doesn't exist") + if not Path(self.list_master).is_file(): + self.list_master.touch() + if not Path(self.list_push).is_file(): + self.list_push.touch() + # Create temporary lists + for gdl_list in ("instagram", "kemono", "main"): + Path(self.dir_cache.resolve() / f"{gdl_list}.txt").touch() + + def list_manager(self): + """Manage all the user list and create sub-lists""" + # sort_txt_file(self.list_master) + self.create_directories() # Call the function to create necesary cache dirs + with open(self.list_master, encoding="utf-8") as list_master: + # Create temporary list files segmented per scrapper + for line in [line.rstrip() for line in list_master]: + # WIKIFEET + with open(self.list_main, "a", encoding="utf-8") as list_main, open( + self.list_kemono, "a", encoding="utf-8" + ) as list_kemono, open( + self.list_instagram, "a", encoding="utf-8" + ) as list_instagram: + if re.search(r"kemono.party", line): + list_kemono.write(line + "\n") + elif re.search(r"instagram", line): + list_instagram.write(line + "\n") + elif re.search(r"wikifeet", line): + continue + # list_main.write(line + "\n") + elif re.search(r"furaffinity", line): + list_main.write(line + "\n") + elif re.search(r"twitter", line): + # if url contains /media at the end just write the line + if re.search(r"\/media$", line): + list_main.write(line + "\n") + else: + # if does not contain /media at the end then add /media + list_main.write(line + "/media" + "\n") + else: + list_main.write(line + "\n") + sort_txt_file(self.list_kemono) + # Try to avoid getting banned by shuffling download order + randomize_txt_file(self.list_instagram) + randomize_txt_file(self.list_main) diff --git a/jawz/scripts/download/setup.cfg b/jawz/scripts/download/setup.cfg new file mode 100644 index 0000000..a992005 --- /dev/null +++ b/jawz/scripts/download/setup.cfg @@ -0,0 +1,17 @@ +[metadata] +name = download +version = 1.5 + +[options] +py_modules = + download + functions + argparser + gdl_classes + +[options.entry_points] +console_scripts = + download = download:main + +# [aliases] +# test = pytest diff --git a/jawz/scripts/download/setup.py b/jawz/scripts/download/setup.py new file mode 100644 index 0000000..5f03bb0 --- /dev/null +++ b/jawz/scripts/download/setup.py @@ -0,0 +1,24 @@ +from setuptools import setup + +setup() +# import os +# from setuptools import find_packages +# from distutils.core import setup + +# import setuptools + +# # User-friendly description from README.md +# current_directory = os.path.dirname(os.path.abspath(__file__)) +# try: +# with open(os.path.join(current_directory, "README.md"), encoding="utf-8") as f: +# long_description = f.read() +# except Exception: +# long_description = "" + +# setup( +# name="download", +# # packages=["argparser", "functions"], +# version="1.5.0", +# scripts=["download.py"], +# # entry_points={"console_scripts": ["download = download:main"]}, +# ) diff --git a/jawz/scripts/download/shell.nix b/jawz/scripts/download/shell.nix new file mode 100644 index 0000000..bcccd5e --- /dev/null +++ b/jawz/scripts/download/shell.nix @@ -0,0 +1,28 @@ +{ pkgs ? import { } }: + +with pkgs; + +mkShell { + packages = [ + (python3.withPackages (ps: + with ps; [ + setuptools + pyyaml + types-pyyaml + # (buildPythonApplication rec { + # pname = "webcomix"; + # version = "3.6.6"; + # src = fetchPypi { + # inherit pname version; + # sha256 = "sha256-hCnic8Rd81qY1R1XMrSME5ntYTSvZu4/ANp03nCmLKU="; + # }; + # doCheck = false; + # propagatedBuildInputs = + # [ click scrapy scrapy-splash scrapy-fake-useragent tqdm ]; + # }) + ])) + ]; + buildInputs = [ + + ]; +} diff --git a/jawz/scripts/ffmpeg4discord.py b/jawz/scripts/ffmpeg4discord.py new file mode 100755 index 0000000..9a28c76 --- /dev/null +++ b/jawz/scripts/ffmpeg4discord.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 + +# Imports +import os +import math + +# Function for calculating the appropriate bitrate to use during conversion +def get_bitrate(duration, filesize, audio_br): + br = math.floor(filesize / duration - audio_br) + return br, br * 0.50, br * 1.45 + + +def encode(ffmpeg_string, output_name, fs): + os.system(ffmpeg_string) + end_size = ( + os.path.getsize( + "/dev/shm/ffmpeg/out/{output_name}".format(output_name=output_name) + ) + * 0.00000095367432 + ) + if end_size < fs: + print( + ffmpeg_string.replace("\t", "") + + "\nThe FFMPEG string above has yielded a file whose size is " + + str(end_size) + + "MB.\n{output_name} is ready for Discord.\n".format( + output_name=output_name + ) + ) + return False + else: + print( + ffmpeg_string.replace("\t", "") + + "\nThe FFMPEG string above has yielded a file whose size is " + + str(end_size) + + "MB.\n{output_name} is NOT ready for Discord, and will be re-run.\nMy bad.".format( + output_name=output_name + ) + ) + return True + + +def time_calculations(fname, length): + startstring = fname[0:2] + ":" + fname[2:4] + ":" + fname[4:6] + endstring = fname[7:9] + ":" + fname[9:11] + ":" + fname[11:13] + + try: + int(fname[0:6]) + startseconds = ( + int(fname[0:2]) * 60 * 60 + int(fname[2:4]) * 60 + int(fname[4:6]) + ) + try: + int(fname[11:13]) + endseconds = ( + int(fname[7:9]) * 60 * 60 + int(fname[9:11]) * 60 + int(fname[11:13]) + ) + duration = endseconds - startseconds + timestamped_section = f"-ss {startstring} -to {endstring}" + except: + duration = length - startseconds + timestamped_section = f"-ss {startstring}" + except: + duration = length + timestamped_section = "" + + return duration, timestamped_section + + +fname = os.listdir("/dev/shm/ffmpeg/in/")[0] +os.rename("/dev/shm/ffmpeg/in/" + fname, "/dev/shm/ffmpeg/in/" + fname.replace(" ", "")) +fname = fname.replace(" ", "") + +# ffprobe to calculate the total duration of the clip. +length = math.floor( + float( + os.popen( + "ffprobe -v error -show_entries format=duration -of default=noprint_wrappers=1:nokey=1 /dev/shm/ffmpeg/in/{fname}".format( + fname=fname + ) + ).read() + ) +) + +duration, timestamped_section = time_calculations(fname, length) + +run = True + +reso = os.getenv("reso") +codec = os.getenv("codec") +audio_br = os.getenv("audio_br") +audio_br = int(str(os.getenv("audio_br"))) +fs = float(str(os.getenv("fs"))) +target_fs = fs + +codecs = { + "vp9": { + "pass1": f"-vf scale={reso} -g 240 -threads 8 -speed 4 -row-mt 1 -tile-columns 2 -vsync cfr -c:v libvpx-vp9 -pass 1 -an", + "pass2": f"-vf scale={reso} -g 240 -threads 8 -speed 2 -row-mt 1 -tile-columns 2 -c:v libvpx-vp9 -c:a libopus -pass 2", + "output_name": "small_" + fname.replace(".mp4", ".webm"), + }, + "x264": { + "pass1": f"-vf scale={reso} -vsync cfr -c:v libx264 -pass 1 -an", + "pass2": f"-vf scale={reso} -c:v libx264 -c:a aac -pass 2 ", + "output_name": "small_" + fname, + }, + "x265": { + "pass1": f"-vf scale={reso} -c:v libx265 -vsync cfr -x265-params pass=1 -an", + "pass2": f"-vf scale={reso} -c:v libx265 -x265-params pass=2 -c:a aac", + "output_name": "small_" + fname, + }, +} + + +while run: + # Conversion to KiB + end_fs = fs * 8192 + br, minbr, maxbr = get_bitrate( + duration=duration, filesize=end_fs, audio_br=audio_br + ) + ffmpeg_string = f""" + ffpb {timestamped_section} -hwaccel cuda -i /dev/shm/ffmpeg/in/{fname} -y \ + {codecs[str(codec)]['pass1']} \ + -b:v {br}k -minrate {minbr}k -maxrate {maxbr}k \ + -f null /dev/null && \ + ffpb {timestamped_section} -hwaccel cuda -i /dev/shm/ffmpeg/in/{fname} \ + {codecs[str(codec)]['pass2']} \ + -b:a {audio_br}k -b:v {br}k -minrate {minbr}k -maxrate {maxbr}k \ + /dev/shm/ffmpeg/out/{codecs[str(codec)]['output_name']} -y + """ + + run = encode( + ffmpeg_string, output_name=codecs[str(codec)]["output_name"], fs=target_fs + ) + + if run: + fs = fs - 0.2 diff --git a/jawz/scripts/ffmpreg.sh b/jawz/scripts/ffmpreg.sh new file mode 100755 index 0000000..8bbbf0d --- /dev/null +++ b/jawz/scripts/ffmpreg.sh @@ -0,0 +1,98 @@ +#! /usr/bin/env nix-shell +#! nix-shell -i bash -p bash gum trashy fd ripgrep mediainfo + +replace_extension() { + local file_basename + file_basename=$(basename "$1") + echo "${file_basename%.*}.$2" +} + +convert_gif() { + file_newname=$(replace_extension "$1" gif) + ffpb -i "$(realpath "$1")" -vf fps=12,scale=480:-1,smartblur=ls=-0.5 "$file_newname" +} + +convert_mp4() { + local file_newname + file_newname=$(replace_extension "$1" mp4) + local file_tempdest=/dev/shm/$file_newname + local file_destination + file_destination=$(dirname "$(realpath "$1")")/$file_newname + ffpb -i "$1" \ + -c:v libx265 \ + "$file_tempdest" + trash "$1" + mv -i "$file_tempdest" "$file_destination" +} + +convert_discord() { + local file_newname + file_newname=$2_$(replace_extension "$1" mp4) + local dir_ram=/dev/shm/ffmpeg + mkdir -p $dir_ram/{in,out} + ffpb -hwaccel cuda -i "$(realpath "$1")" \ + -c:v h264_nvenc \ + "$dir_ram"/in/discord.mp4 + cd "$dir_ram" || exit + codec=x264 audio_br=$3 fs=$4 reso=$5 ffmpeg4discord + mv "$dir_ram"/out/small_discord.mp4 ~/"$file_newname" + command rm -rf "$dir_ram" +} + +operation=$(gum choose mp4 discord nitro gif enc265) + +case $operation in + 1 | mp4) + to_convert=() + while IFS= read -r file; do + to_convert+=("$file") + done < <(fd . "$(pwd)" -tf -aL | fzf --multi -i) + for file in "${to_convert[@]}"; do + convert_mp4 "$file" + done + ;; + 2 | discord) + to_convert=() + while IFS= read -r file; do + to_convert+=("$file") + done < <(fd . "$(pwd)" -tf -aL | fzf --multi -i) + for file in "${to_convert[@]}"; do + convert_discord "$file" discord 96 8.0 "1280x720" + done + ;; + 3 | nitro) + to_convert=() + while IFS= read -r file; do + to_convert+=("$file") + done < <(fd . "$(pwd)" -tf -aL | fzf --multi -i) + for file in "${to_convert[@]}"; do + convert_discord "$file" nitro 128 50.0 "1920x1080" + done + ;; + 4 | gif) + to_convert=() + while IFS= read -r file; do + to_convert+=("$file") + done < <(fd . "$(pwd)" -tf -aL | fzf --multi -i) + for file in "${to_convert[@]}"; do + convert_gif "$file" + done + ;; + 5 | enc265) + to_convert=() + extensions=(flv m4v mpg avi mov ts mkv mp4 webm) + for ext in "${extensions[@]}"; do + while IFS= read -r file; do + if ! (mediainfo "$file" | grep Writing\ library | grep -q x265); then + to_convert+=("$file") + fi + done < <(fd . -e "$ext" -tf -aL) + done + for file in "${to_convert[@]}"; do + convert_mp4 "$file" + done + ;; + *) + echo -n "Please select a valid input" + ;; +esac diff --git a/jawz/scripts/pika-list.sh b/jawz/scripts/pika-list.sh new file mode 100755 index 0000000..ba8f046 --- /dev/null +++ b/jawz/scripts/pika-list.sh @@ -0,0 +1,51 @@ +#! /usr/bin/env nix-shell +#! nix-shell -i bash -p bash fd borgbackup gum ripgrep + +BORG_PASSPHRASE=$(gum input --password --placeholder "Type borg password") +export BORG_PASSPHRASE + +d_root=$HOME/pika +f_string=home/jawz/.config/jawz/lists/jawz/watch.txt +d_borg=/mnt/disk1/backups/pika/lists + +while IFS= read -r repo; do + IFS=" " read -r -a array <<<"$repo" + repo_id="${array[0]}" + mkdir -vp "$d_root/$repo_id" && cd "$d_root/$repo_id" || exit + borg extract $d_borg::"$repo_id" $f_string + cat "$d_root/$repo_id/$f_string" >>"$d_root/master" +done < <(borg list "$d_borg") + +cd "$HOME" || exit + +sort -u "$d_root/master" -o "$d_root/sorted" +sort -u "$LW" -o "$LW" + +echo "Current $(wc -l <"$LW") archived $(wc -l <"$d_root/sorted")" + +echo "Missing lines:" +diff "$d_root/sorted" "$LW" + +# look for duped lines with different casing +echo "Duplicated lines:" +while IFS= read -r line; do + if ! [ "$line" == "${line,,}" ]; then + if rg "${line,,}" <"$LW"; then + echo "$line" + fi + fi +done <"$LW" + +# delete pika backups +if gum confirm "Limpiar pika?"; then + command rm -rf "$d_root" + while IFS= read -r repo; do + IFS=" " read -r -a array <<<"$repo" + repo_id="${array[0]}" + gum spin --spinner dot --title "Cleaning $repo_id..." -- borg delete $d_borg::"$repo_id" + done < <(borg list "$d_borg") +else + echo "Canceled, no files deleted" +fi +gum spin --spinner dot --title "Cleaning $repo_id..." -- borg compact "$d_borg" +gum spin --spinner dot --title "Cleaning $repo_id..." -- borg compact /mnt/disk1/backups/pika/home diff --git a/jawz/scripts/run.sh b/jawz/scripts/run.sh new file mode 100644 index 0000000..6b112d7 --- /dev/null +++ b/jawz/scripts/run.sh @@ -0,0 +1,48 @@ +#! /usr/bin/env nix-shell +#! nix-shell -i bash -p bash gnome.zenity rmlint git gum xclip + +if [ -n "$1" ]; then + operation=$1 +else + operation=$(gum choose rmlint_1 rmlint_2 download git) +fi + +case $operation in + # onlyfans) + # source ~/Development/Python/onlyfans/bin/activate.fish + # python ~/Development/Git/OnlyFans/start_ofd.py + # deactivate + rmlint_1) + rmlint -g --types="duplicates" \ + --config=sh:handler=clone \ + /mnt/disk1/personal + ;; + rmlint_2) + rmlint -g --types="duplicates" \ + --config=sh:handler=clone \ + /mnt/disk2/{glue,home,personal,scrapping} + ;; + download) + ENTRY=$(zenity --entry --width=250 --title "Push Manager" \ + --text="Verify the following entry is correct" \ + --add-entry="Clipboard:" --entry-text "$(xclip -o -sel clip)") + if [ -n "$ENTRY" ]; then + kgx -e "download -u jawz -i '$ENTRY'" + else + zenity --error --width=250 \ + --text "Please verify and try again" + fi + ;; + git) + git_dir=$HOME/Development/Git + while IFS= read -r repo; do + if ! [ -d "$repo/.git" ]; then + continue + fi + cd "$repo" || exit + gum style --foreground 2 "Updating $(basename "$repo")" + git fsck --full + git pull + done < <(fd . "$git_dir" -td --absolute-path -d 1) + ;; +esac diff --git a/jawz/scripts/split-dir.sh b/jawz/scripts/split-dir.sh new file mode 100755 index 0000000..2c37eae --- /dev/null +++ b/jawz/scripts/split-dir.sh @@ -0,0 +1,28 @@ +#! /usr/bin/env nix-shell +#! nix-shell -i bash -p bash fd + +before_count=$(fd -tf | wc -l) +i=0 + +for file in $(fd -d1 -tf -E '*.mp4'); do + dir_name=$(basename "$(pwd)")_$(printf %03d $((i / $1 + 1))) + mkdir -p "$dir_name" + mv -i "$file" "$(realpath "$dir_name")"/ + i=$((i + 1)) +done + +for file in $(fd -d1 -tf -e mp4); do + mkdir -p videos + mv -i "$file" "$(realpath videos)"/ +done + +after_count=$(fd -tf | wc -l) + +if [[ "$before_count" == "$after_count" ]]; then + echo "No file count differences" +else + echo "Before count: $before_count" + echo "After count: $after_count" +fi +sleep 10 +exit diff --git a/jawz/scripts/tasks.sh b/jawz/scripts/tasks.sh new file mode 100755 index 0000000..8a92ddc --- /dev/null +++ b/jawz/scripts/tasks.sh @@ -0,0 +1,140 @@ +#! /usr/bin/env nix-shell +#! nix-shell -i bash -p bash trashy fd ripgrep file + +directories=("$HOME/Pictures/To Organize/" "$HOME/Downloads/") + +replace_extension() { + local file_basename + file_basename=$(basename "$1") + echo "${file_basename%.*}.$2" +} + +generate_random_number() { + local min=0 + local max=9999999999 + printf "%010d\n" $((min + RANDOM % max)) +} + +test_name() { + local random_number + random_number=$(generate_random_number) + while (($(fd "$random_number"* "$HOME/Pictures/" "$HOME/Downloads/" -tf | wc -l) > 0)); do + echo "Conflicts found, generating a new filename" + random_number=$(generate_random_number) + echo "$random_number" + done + echo "$random_number" +} + +while IFS= read -r file; do + regex_str='source|tenor|media|duckduckgo\.com|giphy|' + regex_str+='(? 0)); then + while IFS= read -r file; do + date=$(stat -c "%y" "$file" | rg -o "\d{4}-\d{2}-\d{2}") + year=$(echo "$date" | rg -o "\d{4}") + month=$(echo "$date" | rg -o "\d{4}-\d{2}" | rg -o --pcre2 "(?<=-)\d{2}") + dest_dir=$(realpath "$screenshots/$year/$month") + echo "Moving screenshot $(basename "$file") into $dest_dir" + mkdir -vp "$dest_dir" + command mv -n "$file" "$dest_dir/" + done < <(fd . "$screenshots" --absolute-path -tf -d 1) +fi + +# Where steam screenshots are stored, may need to replace with ur ID +dir_steam=$XDG_DATA_HOME/Steam/userdata/107446271/760/remote +declare -A games +# Insert here new games, put between [] the ID of the game +# You can find it by visiting the $dir_steam directory +# the ID is simply the name of the folder in there. +games+=( + [386360]=Smite + [960090]="Bloons Tower Defense 6" + [648800]=Raft + [262060]="Darkest Dungeon" + [234140]="Mad Max" + [433340]="Slime Rancher" +) + +for key in "${!games[@]}"; do + # Modify this to store your screenshots somewhere else + dir_dest=$(realpath "$HOME/Pictures/Screenshots/Games")/${games[$key]} + dir_game=$(realpath "$dir_steam")/$key/screenshots + # If there are not screenshots currently stored, why bother lol + if ! [[ -d $dir_game ]]; then # + continue + fi + # If screenshots exist however... + if (($(fd . "$dir_game" -d 1 -tf | wc -l) > 0)); then + # Create destination directory + mkdir -vp "$dir_dest" + echo "Moving ${games[$key]} screenshots..." + fd . "$dir_game" -d 1 -tf -x mv -n {} "$dir_dest"/ + # Delete thumnnails + echo "Deleting ${games[$key]} thumbnails..." + rm -rf "$dir_game"/thumbnails + fi +done +# Clearing up empty directories +fd . "$dir_steam" -td -te -x trash {} + +cyberpunk_dir=$HOME/Games/cyberpunk-2077/drive_c/users/jawz/Pictures/"Cyberpunk 2077" +if [[ -d $cyberpunk_dir ]]; then + while IFS= read -r file; do + echo "Moving cyberpunk screenshots" + command mv -n "$file" "$HOME/Pictures/Screenshots/Games/Cyberpunk 2077/" + done < <(fd . "$cyberpunk_dir" -tf) +fi + +proton_dir=$HOME/.steam/steam/compatibilitytools.d +if [[ -d "$proton_dir" ]]; then + while IFS= read -r protonver; do + lutrisdir=$XDG_DATA_HOME/lutris/runners/wine/$(basename "$protonver") + if ! [ -d "$lutrisdir" ] && ! [ -L "$lutrisdir" ]; then + echo "Symlink $lutrisdir doesn't exist, creating link..." + ln -s "$(realpath "$protonver")"/files "$lutrisdir" + fi + done < <(fd . "$proton_dir" -d 1 -td) +fi +fd . "$XDG_DATA_HOME/lutris/runners/wine" -d 1 -tl -x trash {} + +while IFS= read -r file; do + ext=$(file --mime-type "$file" | rg -o '\w+$') + correct_ext=${ext,,} + filename=$(basename -- "$file") + current_ext="${filename##*.}" + filename="${filename%.*}" + if echo "$correct_ext" | rg -q 'jpe|jpg|jpeg|png|gif'; then + if [ "$current_ext" != "$correct_ext" ]; then + echo "The file $(basename "$file")" \ + "will be renamed, the propper extension is $correct_ext" + new_name="$filename".$correct_ext + command mv -n "$(dirname "$file")"/{"$(basename "$file")","$new_name"} + fi + fi +done < <(fd . "${directories[@]}" -d 1 -tf) + +files_home_clean=(.pki HuionCore.pid DriverUI.pid huion.log) +for file in "${files_home_clean[@]}"; do + file=$HOME/$file + if [ -e "$file" ]; then + rm -rf "$file" + fi +done diff --git a/workstation/.gitignore b/workstation/.gitignore new file mode 100644 index 0000000..985d863 --- /dev/null +++ b/workstation/.gitignore @@ -0,0 +1,4 @@ +/dotfiles/*.Appimage +/scripts/download/.direnv/ +/configuration.nix +/scripts/PureRef-1.11.1_x64.Appimage diff --git a/configuration.org b/workstation/configuration.org similarity index 99% rename from configuration.org rename to workstation/configuration.org index 8fae71f..382b4d6 100755 --- a/configuration.org +++ b/workstation/configuration.org @@ -72,7 +72,7 @@ cluttered, for example, I may create a module for systemd units. #+begin_src nix imports = [ ./hardware-configuration.nix - # ./servers.nix + ./servers.nix # ./openldap.nix # (import "${home-manager}/nixos") @@ -884,8 +884,8 @@ services = { enable = true; ports = [ 25152 ]; settings = { - PasswordAuthentication = false; - KbdInteractiveAuthentication = false; + PasswordAuthentication = true; + KbdInteractiveAuthentication = true; }; startWhenNeeded = true; listenAddresses = [ @@ -916,7 +916,7 @@ systemd = { user = { services = { HentaiAtHome = { - enable = true; + enable = false; restartIfChanged = true; description = "Run hentai@home server"; wantedBy = [ "default.target" ]; diff --git a/workstation/dotfiles/gallery-dl/config.json b/workstation/dotfiles/gallery-dl/config.json new file mode 100755 index 0000000..06fbfcb --- /dev/null +++ b/workstation/dotfiles/gallery-dl/config.json @@ -0,0 +1,334 @@ +{ + "extractor": { + "skip": "abort:5", + "cookies": [ + "firefox", + "yw8fhvh4.default-release", + "gnomekeyring" + ], + "user-agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/537.36", + "retries": 10, + "sleep-request": 0, + "directlink": { + "directory": [], + "filename": "{filename}.{extension}" + }, + "twitter": { + "skip": "abort:1", + "directory": [ + "{user[name]}" + ], + "retweets": false, + "videos": true, + "logout": true + }, + "flickr": { + "directory": [ + "{category}", + "{owner[username]}" + ], + "size-max": "Original", + "access-token": "72157720849409732-e83af94a8ca145aa", + "access-token-secret": "0c7e86529694756a" + }, + "pinterest": { + "directory": [ + "{board[owner][username]}", + "{board[name]}" + ] + }, + "wikifeet": { + "page-reverse": true, + "directory": [ + "{category}", + "{celebrity}" + ] + }, + "instagram": { + "sleep-request": "15-45", + "sleep": "2-10", + "directory": [ + "{username}" + ], + "parent-directory": true, + "highlights": { + "reverse": "true", + "directory": [ + "{username}" + ] + }, + "stories": { + "reverse": "true", + "directory": [ + "{username}" + ] + }, + "tagged": { + "directory": [ + "{tagged_username}", + "tagged" + ] + } + }, + "kemonoparty": { + "directory": [ + "{category}", + "{user}" + ], + "retries": 10, + "timeout": 5, + "filename": "{id}_{filename}.{extension}" + }, + "exhentai": { + "directory": [ + "{category}", + "{title}" + ] + }, + "tumblr": { + "directory": [ + "{blog_name}" + ], + "access-token": "WTt2nJdHLJAOQMpTbnMBGYqeJwoBeY2HDRztDPjf4HnqJ65rnT", + "access-token-secret": "0mI7ZWmD9CJPrQ1jjXvMGLjvJa44kOtgcKHtwz8LsAVDcODMPi", + "external": true, + "inline": true, + "posts": "all", + "reblogs": false, + "parent-directory": true, + "api-key": "uhBUtgPaX9gl7eaD8suGWW6ZInRedQoVT6xsZzopljy0jXHqm5", + "api-secret": "D3FDj1INyPzXikVpp4jmzSqjlC9czFUQ8oj2I883PSYJdqwURv" + }, + "deviantart": { + "client-id": "20016", + "client-secret": "52e1f9b0cb26e673da36f69e2ddd0e9a", + "refresh-token": "cc862526cb515d82e750c099aa7f32a29087c961", + "directory": [ + "{username}" + ], + "include": "gallery,scraps", + "flat": true, + "original": true, + "mature": true, + "auto-watch": true, + "auto-unwatch": true + }, + "furaffinity": { + "directory": [ + "{user}", + "{subcategory}" + ], + "include": [ + "scraps", + "gallery" + ] + }, + "patreon": { + "directory": [ + "(Patreon) {creator[vanity]}", + "({date:%Y%m%d}) {title} ({id})" + ], + "filename": "{filename}.{num}.{extension}", + "browser": "firefox" + }, + "blogger": { + "directory": [ + "{blog[name]}", + "{post[author]}", + "{post[title]} - [{post[id]}]" + ], + "filename": "{filename} - {num}.{extension}" + }, + "artstation": { + "directory": [ + "{userinfo[username]}" + ], + "external": true + }, + "gfycat": { + "format": "webm" + }, + "reddit": { + "user-agent": "Python:gallery-dl:v1.0 (by /u/captainjawz)", + "client-id": "T7nZ6WZ3_onJWBhLP8r08g", + "refresh-token": "184157546842-bkMXgGYWzkwGSgXTeC8mMmaDZouhUQ", + "directory": [ + "{author}" + ], + "parent-directory": true + }, + "redgifs": { + "reverse": "true", + "directory": [ + "{userName}" + ] + }, + "imgur": { + "mp4": true + }, + "paheal": { + "directory": [ + "Husbands", + "{search_tags}" + ] + }, + "rule34": { + "directory": [ + "Husbands", + "{search_tags}" + ] + }, + "e621": { + "directory": [ + "Husbands", + "{search_tags}" + ] + }, + "baraag": { + "directory": [ + "{account[username]}" + ] + }, + "pixiv": { + "refresh-token": "O4kc9tTzGItuuacDcfmevW6NELjm5CJdWiAbZdUv3Kk", + "directory": [ + "{user[account]} - {user[id]}" + ], + "ugoira": true, + "favorite": { + "directory": [ + "{user_bookmark[account]} - {user_bookmark[id]}", + "Bookmarks" + ] + }, + "postprocessors": [ + { + "name": "ugoira", + "extension": "webm", + "keep-files": false, + "whitelist": [ + "pixiv" + ], + "ffmpeg-twopass": true, + "ffmpeg-args": [ + "-c:v", + "libvpx", + "-crf", + "4", + "-b:v", + "5000k", + "-an" + ] + } + ] + }, + "readcomiconline": { + "chapter-reverse": true, + "directory": [ + "Comics", + "{comic}", + "{comic} #{issue}" + ], + "quality": "hq", + "captcha": "wait", + "postprocessors": [ + "cbz" + ] + }, + "kissmanga": { + "chapter-reverse": true, + "directory": [ + "Manga", + "{manga}", + "{manga} Ch.{chapter}{chapter_minor}" + ], + "captcha": "wait", + "postprocessors": [ + "cbz" + ] + }, + "mangahere": { + "chapter-reverse": true, + "directory": [ + "Manga", + "{manga}", + "{manga} Ch.{chapter}{chapter_minor}" + ], + "postprocessors": [ + "cbz" + ] + }, + "mangadex": { + "chapter-reverse": true, + "chapter-filter": "lang == 'en'", + "directory": [ + "Manga", + "{manga}", + "{manga} Ch.{chapter}{chapter_minor}" + ], + "postprocessors": [ + "cbz" + ] + }, + "mangareader": { + "chapter-reverse": true, + "directory": [ + "Manga", + "{manga}", + "{manga} Ch.{chapter}{chapter_minor}" + ], + "postprocessors": [ + "cbz" + ] + }, + "mangapanda": { + "chapter-reverse": true, + "directory": [ + "Manga", + "{manga}", + "{manga} Ch.{chapter}{chapter_minor}" + ], + "postprocessors": [ + "cbz" + ] + }, + "webtoons": { + "chapter-reverse": true, + "directory": [ + "Webtoons", + "{comic}", + "{comic} #{episode}" + ], + "postprocessors": [ + "cbz" + ] + } + }, + "output": { + "mode": "auto" + }, + "downloader": { + "part": true, + "part-directory": "/home/jawz/.cache/gallery-dl", + "ytdl": { + "logging": true, + "format": "bestvideo+bestaudio/best", + "module": "yt_dlp", + "forward-cookies": true + }, + "http": { + "rate": null, + "retries": 5, + "timeout": 10.0, + "verify": true + } + }, + "postprocessor": { + "cbz": { + "name": "zip", + "compression": "store", + "mode": "safe", + "extension": "cbz" + } + } +} diff --git a/workstation/dotfiles/gopass/config.yml b/workstation/dotfiles/gopass/config.yml new file mode 100755 index 0000000..ece10ca --- /dev/null +++ b/workstation/dotfiles/gopass/config.yml @@ -0,0 +1,10 @@ +autoclip: true +autoimport: false +cliptimeout: 45 +exportkeys: false +nopager: false +notifications: false +parsing: true +path: /home/jawz/.local/share/pass +safecontent: true +mounts: {} diff --git a/workstation/dotfiles/htop/htoprc b/workstation/dotfiles/htop/htoprc new file mode 100755 index 0000000..79ff45d --- /dev/null +++ b/workstation/dotfiles/htop/htoprc @@ -0,0 +1,61 @@ +# Beware! This file is rewritten by htop when settings are changed in the interface. +# The parser is also very primitive, and not human-friendly. +htop_version=3.2.1 +config_reader_min_version=3 +fields=18 0 123 124 46 47 38 50 1 +hide_kernel_threads=0 +hide_userland_threads=0 +shadow_other_users=0 +show_thread_names=0 +show_program_path=0 +highlight_base_name=1 +highlight_deleted_exe=1 +highlight_megabytes=1 +highlight_threads=1 +highlight_changes=0 +highlight_changes_delay_secs=5 +find_comm_in_cmdline=1 +strip_exe_from_cmdline=1 +show_merged_command=1 +header_margin=1 +screen_tabs=1 +detailed_cpu_time=0 +cpu_count_from_one=1 +show_cpu_usage=1 +show_cpu_frequency=1 +show_cpu_temperature=1 +degree_fahrenheit=0 +update_process_names=0 +account_guest_in_cpu_meter=0 +color_scheme=3 +enable_mouse=1 +delay=15 +hide_function_bar=0 +header_layout=two_67_33 +column_meters_0=LeftCPUs Swap Tasks NetworkIO Memory +column_meter_modes_0=1 1 2 2 2 +column_meters_1=RightCPUs Hostname Uptime LoadAverage +column_meter_modes_1=1 2 2 2 +tree_view=1 +sort_key=38 +tree_sort_key=0 +sort_direction=-1 +tree_sort_direction=1 +tree_view_always_by_pid=1 +all_branches_collapsed=1 +screen:Main=NICE PID COMM EXE PERCENT_CPU PERCENT_MEM M_VIRT NLWP Command +.sort_key=M_VIRT +.tree_sort_key=PID +.tree_view=1 +.tree_view_always_by_pid=1 +.sort_direction=-1 +.tree_sort_direction=1 +.all_branches_collapsed=1 +screen:I/O=PID USER IO_PRIORITY IO_RATE IO_READ_RATE IO_WRITE_RATE PERCENT_SWAP_DELAY PERCENT_IO_DELAY Command +.sort_key=IO_RATE +.tree_sort_key=PID +.tree_view=0 +.tree_view_always_by_pid=0 +.sort_direction=-1 +.tree_sort_direction=1 +.all_branches_collapsed=0 diff --git a/workstation/dotfiles/npm/npmrc b/workstation/dotfiles/npm/npmrc new file mode 100755 index 0000000..8932649 --- /dev/null +++ b/workstation/dotfiles/npm/npmrc @@ -0,0 +1,7 @@ +user=0 +unsafe-perm=true +prefix=${XDG_DATA_HOME}/npm +cache=${XDG_CACHE_HOME}/npm +tmp=${XDG_RUNTIME_DIR}/npm +init-module=${XDG_CONFIG_HOME}/npm/config/npm-init.js +store-dir=${XDG_DATA_HOME}/pnpm-store diff --git a/workstation/dotfiles/npm/update-notifier-npm-check.json b/workstation/dotfiles/npm/update-notifier-npm-check.json new file mode 100755 index 0000000..d2275b3 --- /dev/null +++ b/workstation/dotfiles/npm/update-notifier-npm-check.json @@ -0,0 +1,4 @@ +{ + "optOut": false, + "lastUpdateCheck": 1646662583446 +} \ No newline at end of file diff --git a/workstation/dotfiles/wget/wgetrc b/workstation/dotfiles/wget/wgetrc new file mode 100755 index 0000000..ed7eedf --- /dev/null +++ b/workstation/dotfiles/wget/wgetrc @@ -0,0 +1 @@ +hsts-file = /home/jawz/.cache/wget-hsts diff --git a/hardware-configuration.nix b/workstation/hardware-configuration.nix similarity index 100% rename from hardware-configuration.nix rename to workstation/hardware-configuration.nix diff --git a/nginx.nix b/workstation/nginx.nix similarity index 100% rename from nginx.nix rename to workstation/nginx.nix diff --git a/openldap.nix b/workstation/openldap.nix similarity index 100% rename from openldap.nix rename to workstation/openldap.nix diff --git a/workstation/scripts/chat-dl.sh b/workstation/scripts/chat-dl.sh new file mode 100755 index 0000000..e927214 --- /dev/null +++ b/workstation/scripts/chat-dl.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env nix-shell +#! nix-shell -i bash -p bash yt-dlp + +minutes=10 +time_alive=60 +sleep_time=$((minutes * 60)) +loops=$((time_alive / (sleep_time / time_alive))) +url="https://chaturbate.com/$1" + +save_dir=/mnt/disk2/glue/Tuhmayto +if [ ! -d "$save_dir" ]; then + mkdir -p "$save_dir" +fi +cd $save_dir || exit + +for i in $(seq 1 1 "$loops"); do + waiting_time=$(((i * sleep_time) / time_alive)) + yt-dlp --hls-use-mpegts --prefer-ffmpeg -o '%(title)s.%(ext)s' "$url" + echo "sleeping for $sleep_time seconds… been waiting for $waiting_time minutes" + sleep $sleep_time +done diff --git a/workstation/scripts/download/.env b/workstation/scripts/download/.env new file mode 100644 index 0000000..6b75134 --- /dev/null +++ b/workstation/scripts/download/.env @@ -0,0 +1 @@ +CONFIG_FILE = "/home/jawz/.config/jawz/config.yaml" diff --git a/workstation/scripts/download/.envrc b/workstation/scripts/download/.envrc new file mode 100644 index 0000000..1d953f4 --- /dev/null +++ b/workstation/scripts/download/.envrc @@ -0,0 +1 @@ +use nix diff --git a/workstation/scripts/download/argparser.py b/workstation/scripts/download/argparser.py new file mode 100644 index 0000000..a4cd6b9 --- /dev/null +++ b/workstation/scripts/download/argparser.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +"""Setup the argparser""" +import argparse + +scrapper_types = ( + "push", + "gallery", + "instagram", + "kemono", + "comic", + "manga", + "webcomic", +) +# Define types of instagram stories +instagram_types = ["posts", "reels", "channel", "stories", "highlights"] + + +def argparser(users: list) -> argparse.Namespace: + """Returns an argparser to evaluate user input""" + # ARG PARSER + parser = argparse.ArgumentParser( + prog="Downloader", + description="Download images and galleries from a wide array of websites" + " either by using links or chosing from user define lists." + " This program also takes care of archiving tasks," + " that keep the run time fast and prevents downloading duplicates.", + ) + # Chose the type of scrapper + parser.add_argument( + choices=scrapper_types, + nargs="?", + dest="scrapper", + help="Select a scrapper.", + ) + # Parse user list + parser.add_argument( + "-u", + "--user", + choices=users, + dest="user", + help="Selects the personal user list to process. Defaults to everyone", + default="everyone", + type=str, + ) + # Parse individual links + parser.add_argument( + "-i", + "--input", + nargs="*", + dest="link", + action="append", + help="Download the provided links", + type=str, + ) + # Set the print list flag + parser.add_argument( + "-l", + "--list", + dest="flag_list", + action="store_true", + help="Prints a list of all the added links and prompts for a choice", + ) + # Set the use archiver flag + parser.add_argument( + "-a", + "--no-archive", + dest="flag_archive", + action="store_false", + help="Disables the archiver flag", + ) + # Set the skip flag + parser.add_argument( + "-s", + "--no_skip", + dest="flag_skip", + action="store_false", + help="Disables the skip function, downloads the entire gallery", + ) + parser.add_argument( + "-v", + "--verbose", + dest="flag_verbose", + action="store_true", + help="Prints the generated commands instead of running them", + ) + parser.add_argument( + "-t", + "--type-post", + choices=instagram_types, + nargs="*", + dest="post_type", + help="Filters posts on instagram by type", + default=instagram_types, + type=str, + ) + return parser.parse_args() diff --git a/workstation/scripts/download/download.py b/workstation/scripts/download/download.py new file mode 100755 index 0000000..66b5d55 --- /dev/null +++ b/workstation/scripts/download/download.py @@ -0,0 +1,417 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +Rewriting of the download manager script +with the intention to make it +more modular with the use of flags +in order to avoid unnecesary modifications +to the cofig files. +Also following in line more posix and python rules. +""" + +import re +import time +import logging +import yaml +from functions import run +from functions import quote +from functions import list_lines +from functions import load_config_variables +from argparser import argparser +from gdl_classes import User + +# GLOBAL VARIABLE SECTION +# Store the name of the main binaries early in the code +BIN_GALLERY = "gallery-dl" +BIN_YOUTUBE = "yt-dlp" +# SKIP = "3" +CONFIGS = load_config_variables() + +LOGGER = logging.getLogger() +HANDLER = logging.StreamHandler() +FORMATTER = logging.Formatter( + "[%(filename)s][%(levelname)s] %(funcName)s '%(message)s'" +) +HANDLER.setFormatter(FORMATTER) +LOGGER.addHandler(HANDLER) +LOGGER.setLevel(logging.INFO) + +# Enable a default "everyone" flag for when running stuff like download gallery +USERS = ["everyone"] +for dictionary in CONFIGS["users"]: + USERS.append(dictionary["name"]) + +ARGS = argparser(USERS) + + +def get_index(value: str) -> int: + """Find the index in the config file""" + for i, dic in enumerate(CONFIGS["users"]): + if dic["name"] == value: + LOGGER.debug("%s is %s", dic["name"], i) + return i + return -1 + + +def parse_gallery(gdl_list: str, user: User): + """Processes the gallery-dl command based on the selected gallery""" + # skip_arg = f" -A {SKIP}" if ARGS.flag_skip else "" + skip_arg = " -o skip=true" if not ARGS.flag_skip else "" + LOGGER.debug(skip_arg) + + # Send the list to gallery-dl + download_gallery( + ARGS.flag_archive, + skip_arg, + "", + str(user.sleep), + quote(f"{user.dir_download}"), + quote(f"{user.archive_gallery}"), + quote(gdl_list), + parse_instagram(gdl_list), + ) + + +def parse_instagram(link: str) -> str: + """Fix instagram links""" + if "instagram" not in link: + return "" + if isinstance(ARGS.post_type, list): + string = f" -o include={quote(','.join(ARGS.post_type))}" + LOGGER.debug(string) + return string + string = f" -o include={quote(ARGS.post_type)}" + LOGGER.debug(string) + return string + + +def parse_link(link: str) -> str: + """Fixes links""" + if not re.search(r"(twitter\.com\/\w+(\/)?(?!.*status))", link): + LOGGER.debug("No modifications needed for the link %s", link) + return link + # if url contains /media at the end just write the line + fixed_link = re.sub(r"\/$|\/media(\/?)$", "", link) + "/media" + LOGGER.debug("Processed link %s", fixed_link) + return fixed_link + + +def download_gallery( + use_archive: bool, + skip_arg: str = "", + link: str = "", + sleep: str = "0", + destination: str = "", + database: str = "", + queue: str = "", + opt_args: str = "", +): + """Processes the command string to run the gallery archiver""" + command = f"{BIN_GALLERY} --sleep {sleep}" + if skip_arg != "": + command += skip_arg + if destination != "": + command += f" --dest {destination}" + if use_archive: + command += f" --download-archive {database}" + if opt_args != "": + command += opt_args + if link != "" and queue == "": + LOGGER.info("link: %s", quote(link)) + command += f" {link}" + if queue != "" and link == "": + LOGGER.info("queue: %s", queue) + command += f" -i {queue}" + LOGGER.debug(command) + run(command, ARGS.flag_verbose) + + +def download_youtube( + use_archive: bool, + link: str = "", + destination: str = "", + database: str = "", +): + """Filters and processes the required command to download videos""" + command = BIN_YOUTUBE + + if re.search(r"(https:\/\/youtube|https:\/\/www.youtube|https:\/\/youtu.be)", link): + command += f' -o {quote(destination + "/%(title)s.%(ext)s")}' + + elif re.search(r"(https:\/\/music.youtube.*)", link): + if use_archive: + command += f" --download-archive {database}" + command += f""" \ + --no-playlist --newline -x \ + --audio-format best --add-metadata --audio-quality 0 -o \ + {quote(destination + '/%(title)s.%(ext)s')} \ + """ + + elif re.search(r"chaturbate", link): + # Re-runs the program every 30 seconds in case the stream goes private or dc + for i in range(1, 41): # For a 20 minute total + run( + f""" + {BIN_YOUTUBE} \ + --hls-use-mpegts --prefer-ffmpeg \ + -o {quote(destination + '/%(title)s.%(ext)s')} \ + {link} + """, + ARGS.flag_verbose, + ) + time.sleep(30) + LOGGER.info("waited for %s minutes", i * 30 / 60) + + else: # Any other video link, just do it generic + command += f" -f mp4 -o {quote(destination + '/%(title)s.%(ext)s')}" + LOGGER.info("%s %s", command, link) + run(f"{command} {link}", ARGS.flag_verbose) + + +def comic_manager(skip_arg: str, category: str): + """Process the information to download manga""" + re_cat = "" + if category == "manga": + re_cat = "manga|webtoon" + elif category == "comic": + re_cat = "readcomiconline" + + with open(CONFIGS["comic"]["list"], encoding="utf-8") as list_comic: + for graphic_novel in [line.rstrip() for line in list_comic]: + # Search for mangas but exclude comics + if not re.search(re_cat, graphic_novel): + LOGGER.debug("%s does not match regex espression", graphic_novel) + continue + download_gallery( + ARGS.flag_archive, + skip_arg, + quote(graphic_novel), + "0", + CONFIGS["comic"]["download-directory"], + CONFIGS["comic"]["archive"], + "", + "", + ) + + +def webcomic_manager(): + """Process the information to download webcomics""" + webcomic_list = CONFIGS["comic"]["webcomic-list"] + with open(webcomic_list, encoding="utf-8") as open_list: + webcomic_file = yaml.safe_load(open_list) + + # Create a list of all the available webcomics for the user to chose from + for index, entry in enumerate(webcomic_file["Webcomics"]): + print(list_lines(index, entry["name"])) + + # Prompt for a choice + usr_input = int(input("Select your comic: ")) + # Determines where the webcomic will be downloaded + rating = webcomic_file["Webcomics"][usr_input]["type"] + webcomic_category = webcomic_file["Global"][f"{rating}_directory"] + LOGGER.debug("The webcomic is %s", webcomic_category) + command = f"""cd {quote(webcomic_category)} && webcomix custom \ + {quote(webcomic_file["Webcomics"][usr_input]["name"])} \ + --start-url \ + {quote(webcomic_file["Webcomics"][usr_input]["url"])} \ + --next-page-xpath={quote(webcomic_file["Webcomics"][usr_input]["next_code"])} \ + --image-xpath={quote(webcomic_file["Webcomics"][usr_input]["image_code"])} \ + -y --cbz""" + LOGGER.debug(command) + run(command, ARGS.flag_verbose) + + +def push_manager(user: User): + """Filters out the URL to use the appropiate downloader""" + # Creates an array which will store any links that should use youtube-dl + link_video_cache = [] + re_links = re.compile( + r"(twitter\.com\/\w+((?=.*media)|(?!.*status)))" + r"|(men\.wikifeet)" + r"|(furaffinity\.net\/user\/)" + r"|((deviantart\.com\/\w+(?!.*\/art\/)))" + r"|(furaffinity\.net\/gallery\/)" + r"|(furaffinity\.net\/scraps\/)" + r"|(furaffinity\.net\/favorites\/)" + r"|(instagram.com(?!\/p\/)\/\w+)" + r"|(e621\.net((?=\/post\/)|(?!\/posts\/)))" + r"|(flickr\.com\/photos\/\w+\/(?!\d+))" + r"|(tumblr\.com(?!\/post\/))" + r"|(kemono\.party\/(fanbox|gumroad|patreon)(?!\/user\/\d+\/post))" + r"|(blogspot\.com(?!\/))" + r"|(rule34\.paheal\.net\/post\/(?!view))" + r"|(rule34\.xxx\/index\.php\?page\=post&s=(?!view))" + r"|(pixiv\.net\/(en\/)?((?=users)|(?!artwork)))" + r"|(reddit\.com\/(user|u))" + r"|(baraag\.net\/((@\w+)|(?!\/\d+)))" + r"|(pinterest\.com\/(?!pin\/\d+))" + r"|(redgifs\.com\/(users|u|(?!watch)))", + ) + with open(user.list_push, encoding="utf-8") as list_push: + for link in [line.rstrip() for line in list_push]: + LOGGER.debug("Processing %s", link) + # Flush the push list, cleans all the contents + with open(user.list_push, "w", encoding="utf-8") as list_push: + list_push.close() + # VIDEOS + if re.search(r"youtu.be|youtube|pornhub|xtube|xvideos|chaturbate", link): + LOGGER.debug("Matched type yt-dlp") + link_video_cache.append(link) + # Search for gallery links, these will be added to a list after downloading + elif re.search(re_links, link): + LOGGER.debug("Matched type gallery-dl") + # skip_arg = f" -A {SKIP}" if ARGS.flag_skip else "" + skip_arg = " -o skip=true" if not ARGS.flag_skip else "" + LOGGER.debug("Skip: %s, link: %s", skip_arg, parse_instagram(link)) + download_gallery( + ARGS.flag_archive, + skip_arg, + quote(f"{parse_link(link)}"), + f"{user.sleep}", + quote(f"{user.dir_download}"), + quote(f"{user.archive_gallery}"), + "", + f"{parse_instagram(link)}", + ) + # Record the gallery link, so it remains on the watch list + with open(user.list_master, "a", encoding="utf-8") as w_file, open( + user.list_master, "r", encoding="utf-8" + ) as r_file: + content = r_file.read().lower() + if parse_link(link).lower() in content: + LOGGER.info("Gallery repeated, not saving") + continue + LOGGER.info("New gallery, saving") + w_file.write(parse_link(str(link)) + "\n") + + # Searches for comic/manga links + elif re.search(r"readcomiconline|mangahere|mangadex|webtoons", link): + # Toggle for comic/manga skip flag + if ARGS.flag_skip and re.search(r"readcomiconline", link): + skip_arg = " --chapter-range 1" + elif ARGS.flag_skip and re.search(r"mangahere|webtoons", link): + skip_arg = " --chapter-range 1-5" + else: + skip_arg = "" + LOGGER.debug(skip_arg) + + download_gallery( + ARGS.flag_archive, + skip_arg, + quote(link), + "0", + CONFIGS["comic"]["download-directory"], + CONFIGS["comic"]["archive"], + "", + "", + ) + # Add comic/manga link to the list + list_gn = CONFIGS["comic"]["list"] + with open(list_gn, "a", encoding="utf-8") as w_file, open( + list_gn, "r", encoding="utf-8" + ) as r_file: + content = r_file.read().lower() + if parse_link(link).lower() in content: + LOGGER.info("Graphic novel repeated, not saving") + continue + LOGGER.info("New graphic novel, saving") + w_file.write(link + "\n") + # Download generic links, the -o flag overwrites config file and + # downloads the files into the root destination + else: + LOGGER.info("Other type of download %s", link) + download_gallery( + False, + " -o directory='[]'", + quote(link), + "0", + quote(str(user.dir_push)), + "", + "", + "", + ) + # Send the video links to youtube-dl + for link in link_video_cache: + download_youtube( + ARGS.flag_archive, + quote(link), + f"{user.dir_media_download}", + quote(f"{user.archive_media}"), + ) + + +def scrapper_manager(user: User): + # pylint: disable=too-many-branches + """Analyze the user arguments and call in functions""" + if not ARGS.scrapper: # Check if a scrapper was selected + return + + if re.search(r"gallery|instagram|kemono", ARGS.scrapper): + # skip_arg = f" -A {SKIP}" if ARGS.flag_skip else "" + skip_arg = " -o skip=true" if not ARGS.flag_skip else "" + LOGGER.debug(skip_arg) + if ARGS.scrapper == "gallery": + parse_gallery(f"{user.list_main}", user) + elif ARGS.scrapper == "instagram": + parse_gallery(f"{user.list_instagram}", user) + elif ARGS.scrapper == "kemono": + parse_gallery(f"{user.list_kemono}", user) + elif ARGS.scrapper in "push": + push_manager(user) + elif ARGS.scrapper in "comic": + skip_arg = " --chapter-range 1" if ARGS.flag_skip else "" + LOGGER.debug(skip_arg) + comic_manager(skip_arg, "comic") + elif ARGS.scrapper in "manga": + skip_arg = " --chapter-range 1-5" if ARGS.flag_skip else "" + LOGGER.debug(skip_arg) + comic_manager(skip_arg, "manga") + elif ARGS.scrapper in "webcomic": + webcomic_manager() + + +def main(): + """Main module to decide what to do based on the parsed arguments""" + if ARGS.scrapper: + if (ARGS.user in "everyone") and ( + re.search(r"push|gallery|instagram|kemono", ARGS.scrapper) + ): + for current_user in CONFIGS["users"]: + user = User(get_index(current_user["name"])) + user.list_manager() + LOGGER.info("Scrapping %s for %s", ARGS.scrapper, current_user["name"]) + scrapper_manager(user) + elif re.search(r"comic|manga|webcomic", ARGS.scrapper): + user = User(get_index("jawz")) + user.list_manager() + LOGGER.info("Scrapping %s", ARGS.scrapper) + scrapper_manager(user) + else: + # Create the lists to scrap + user = User(get_index(ARGS.user)) + user.list_manager() + scrapper_manager(user) + elif ARGS.link: + LOGGER.debug(ARGS.link) + if re.search(r"everyone|jawz", ARGS.user): + # Create the lists to scrap + user = User(get_index("jawz")) + user.list_manager() + else: + # Create the lists to scrap + user = User(get_index(ARGS.user)) + user.list_manager() + for arg_link in ARGS.link[0]: + LOGGER.debug(arg_link) + if ARGS.flag_verbose: + LOGGER.debug( + "%s >> %s", quote(parse_link(arg_link)), quote(user.list_push) + ) + else: + with open(user.list_push, "a", encoding="utf-8") as open_file: + open_file.write(parse_link(arg_link) + "\n") + push_manager(user) + + +if __name__ == "__main__": + main() diff --git a/workstation/scripts/download/functions.py b/workstation/scripts/download/functions.py new file mode 100644 index 0000000..902703f --- /dev/null +++ b/workstation/scripts/download/functions.py @@ -0,0 +1,70 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +"""Personal functions to aid on multiple scripts""" +import sys +import fileinput +import re +import os +from pathlib import Path +import yaml + +VERBOSE_G = False + + +def load_config_variables(): + """Loads all the variables from the config file""" + config_file = Path("~/.config/jawz/config.yaml") + with open(config_file.expanduser(), encoding="utf-8") as open_file: + return yaml.safe_load(open_file) + + +def run(command: str, verbose: bool): + """Run command in a subprocess""" + # pylint: disable=subprocess-run-check + # This toggle allows for a really wasy debug when using -v + if verbose: + print(command) + else: + os.system(command) + + +def list_lines(i: int, line: str) -> str: + """Create a numbered list""" + return f"{i}) {line}" + + +def quote(line: str) -> str: + """Quote the line""" + return f'"{line}"' + + +def sort_txt_file(file_path: Path): + """Sort every line alphabetically + remove duplicated and empty lines""" + file = str(file_path.resolve()) + run(f"sort -u {quote(file)} -o {quote(file)}", VERBOSE_G) + run(f"sed -i '/^$/d' {quote(file)}", VERBOSE_G) + run(f'sed -i -e "s,http:,https:," {quote(file)}', VERBOSE_G) + # fix this using strip on python + # line.strip("/") + run(f'sed -i -e "s,/$,," {quote(file)}', VERBOSE_G) # trailing / + + +def randomize_txt_file(file_path: Path): + """Randomize the order of the + lines of the txt file""" + file = str(file_path.resolve()) + run(f"sort -R {quote(file)} -o {quote(file)}", VERBOSE_G) + + +def parse_list(file): + """Replace http with https and remove trailing /""" + for line in fileinput.input(file, inplace=True): + sys.stdout.write(str(line).replace("http://", "https://")) + with open(file, "r+", encoding="utf-8") as open_file: + f_content = open_file.read() + f_content = re.compile(r"\/$", 0).sub(r"\/$", "") + open_file.seek(0) + open_file.truncate() + print(f_content) + sort_txt_file(file) diff --git a/workstation/scripts/download/gdl_classes.py b/workstation/scripts/download/gdl_classes.py new file mode 100644 index 0000000..7186d3f --- /dev/null +++ b/workstation/scripts/download/gdl_classes.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +"""Define the user class to populate and setup the download environment""" +import re +from pathlib import Path +from functions import sort_txt_file, randomize_txt_file, load_config_variables + +config_variables = load_config_variables() + + +class User: + """Populate the directory for each user""" + + # pylint: disable=too-many-instance-attributes + def __init__(self, index): + self.user = config_variables["users"][index] + self.config = config_variables["global"] + self.name = self.user["name"] + self.sleep = self.config["sleep"] + # Directories + self.dir_cache = Path(self.config["cache-directory"]) / self.name + self.dir_log = Path(self.config["log-directory"]) + self.dir_archive = Path(self.config["archive-directory"]) + self.dir_download = Path(self.user["download-directory"]) + self.dir_media_download = Path(self.user["media-directory"]) + self.dir_push = Path(self.user["push-directory"]) + self.dir_master_list = Path(self.config["list-dir"]) / self.name + # Files + self.archive_gallery = self.dir_archive / f"{self.name}.sqlite3" + self.archive_media = self.dir_archive / f"{self.name}_ytdl.txt" + # Lists + self.list_master = self.dir_master_list / "watch.txt" + self.list_push = self.dir_master_list / "instant.txt" + self.list_instagram = self.dir_cache / "instagram.txt" + self.list_kemono = self.dir_cache / "kemono.txt" + self.list_main = self.dir_cache / "main.txt" + + def create_directories(self): + """Create user directories if they don't exist""" + if self.dir_cache.is_dir(): + for file in self.dir_cache.iterdir(): + if file.is_file(): + file.unlink() + for file in self.dir_cache.iterdir(): + if file.is_dir(): + file.rmdir() + self.dir_cache.rmdir() + # Create directories + self.dir_cache.mkdir(parents=True, exist_ok=True) + self.dir_log.mkdir(parents=True, exist_ok=True) + self.dir_archive.mkdir(parents=True, exist_ok=True) + self.dir_download.mkdir(parents=True, exist_ok=True) + self.dir_media_download.mkdir(parents=True, exist_ok=True) + self.dir_push.mkdir(parents=True, exist_ok=True) + # Check for the existence of core files + if not Path(self.archive_gallery).is_file(): + self.archive_gallery.touch() + if not Path(self.archive_media).is_file(): + self.archive_media.touch() + if not self.dir_master_list.is_dir(): + print(f"ERROR: Directory for user {self.name} doesn't exist") + if not Path(self.list_master).is_file(): + self.list_master.touch() + if not Path(self.list_push).is_file(): + self.list_push.touch() + # Create temporary lists + for gdl_list in ("instagram", "kemono", "main"): + Path(self.dir_cache.resolve() / f"{gdl_list}.txt").touch() + + def list_manager(self): + """Manage all the user list and create sub-lists""" + # sort_txt_file(self.list_master) + self.create_directories() # Call the function to create necesary cache dirs + with open(self.list_master, encoding="utf-8") as list_master: + # Create temporary list files segmented per scrapper + for line in [line.rstrip() for line in list_master]: + # WIKIFEET + with open(self.list_main, "a", encoding="utf-8") as list_main, open( + self.list_kemono, "a", encoding="utf-8" + ) as list_kemono, open( + self.list_instagram, "a", encoding="utf-8" + ) as list_instagram: + if re.search(r"kemono.party", line): + list_kemono.write(line + "\n") + elif re.search(r"instagram", line): + list_instagram.write(line + "\n") + elif re.search(r"wikifeet", line): + continue + # list_main.write(line + "\n") + elif re.search(r"furaffinity", line): + list_main.write(line + "\n") + elif re.search(r"twitter", line): + # if url contains /media at the end just write the line + if re.search(r"\/media$", line): + list_main.write(line + "\n") + else: + # if does not contain /media at the end then add /media + list_main.write(line + "/media" + "\n") + else: + list_main.write(line + "\n") + sort_txt_file(self.list_kemono) + # Try to avoid getting banned by shuffling download order + randomize_txt_file(self.list_instagram) + randomize_txt_file(self.list_main) diff --git a/workstation/scripts/download/setup.cfg b/workstation/scripts/download/setup.cfg new file mode 100644 index 0000000..a992005 --- /dev/null +++ b/workstation/scripts/download/setup.cfg @@ -0,0 +1,17 @@ +[metadata] +name = download +version = 1.5 + +[options] +py_modules = + download + functions + argparser + gdl_classes + +[options.entry_points] +console_scripts = + download = download:main + +# [aliases] +# test = pytest diff --git a/workstation/scripts/download/setup.py b/workstation/scripts/download/setup.py new file mode 100644 index 0000000..5f03bb0 --- /dev/null +++ b/workstation/scripts/download/setup.py @@ -0,0 +1,24 @@ +from setuptools import setup + +setup() +# import os +# from setuptools import find_packages +# from distutils.core import setup + +# import setuptools + +# # User-friendly description from README.md +# current_directory = os.path.dirname(os.path.abspath(__file__)) +# try: +# with open(os.path.join(current_directory, "README.md"), encoding="utf-8") as f: +# long_description = f.read() +# except Exception: +# long_description = "" + +# setup( +# name="download", +# # packages=["argparser", "functions"], +# version="1.5.0", +# scripts=["download.py"], +# # entry_points={"console_scripts": ["download = download:main"]}, +# ) diff --git a/workstation/scripts/download/shell.nix b/workstation/scripts/download/shell.nix new file mode 100644 index 0000000..bcccd5e --- /dev/null +++ b/workstation/scripts/download/shell.nix @@ -0,0 +1,28 @@ +{ pkgs ? import { } }: + +with pkgs; + +mkShell { + packages = [ + (python3.withPackages (ps: + with ps; [ + setuptools + pyyaml + types-pyyaml + # (buildPythonApplication rec { + # pname = "webcomix"; + # version = "3.6.6"; + # src = fetchPypi { + # inherit pname version; + # sha256 = "sha256-hCnic8Rd81qY1R1XMrSME5ntYTSvZu4/ANp03nCmLKU="; + # }; + # doCheck = false; + # propagatedBuildInputs = + # [ click scrapy scrapy-splash scrapy-fake-useragent tqdm ]; + # }) + ])) + ]; + buildInputs = [ + + ]; +} diff --git a/workstation/scripts/ffmpeg4discord.py b/workstation/scripts/ffmpeg4discord.py new file mode 100755 index 0000000..9a28c76 --- /dev/null +++ b/workstation/scripts/ffmpeg4discord.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 + +# Imports +import os +import math + +# Function for calculating the appropriate bitrate to use during conversion +def get_bitrate(duration, filesize, audio_br): + br = math.floor(filesize / duration - audio_br) + return br, br * 0.50, br * 1.45 + + +def encode(ffmpeg_string, output_name, fs): + os.system(ffmpeg_string) + end_size = ( + os.path.getsize( + "/dev/shm/ffmpeg/out/{output_name}".format(output_name=output_name) + ) + * 0.00000095367432 + ) + if end_size < fs: + print( + ffmpeg_string.replace("\t", "") + + "\nThe FFMPEG string above has yielded a file whose size is " + + str(end_size) + + "MB.\n{output_name} is ready for Discord.\n".format( + output_name=output_name + ) + ) + return False + else: + print( + ffmpeg_string.replace("\t", "") + + "\nThe FFMPEG string above has yielded a file whose size is " + + str(end_size) + + "MB.\n{output_name} is NOT ready for Discord, and will be re-run.\nMy bad.".format( + output_name=output_name + ) + ) + return True + + +def time_calculations(fname, length): + startstring = fname[0:2] + ":" + fname[2:4] + ":" + fname[4:6] + endstring = fname[7:9] + ":" + fname[9:11] + ":" + fname[11:13] + + try: + int(fname[0:6]) + startseconds = ( + int(fname[0:2]) * 60 * 60 + int(fname[2:4]) * 60 + int(fname[4:6]) + ) + try: + int(fname[11:13]) + endseconds = ( + int(fname[7:9]) * 60 * 60 + int(fname[9:11]) * 60 + int(fname[11:13]) + ) + duration = endseconds - startseconds + timestamped_section = f"-ss {startstring} -to {endstring}" + except: + duration = length - startseconds + timestamped_section = f"-ss {startstring}" + except: + duration = length + timestamped_section = "" + + return duration, timestamped_section + + +fname = os.listdir("/dev/shm/ffmpeg/in/")[0] +os.rename("/dev/shm/ffmpeg/in/" + fname, "/dev/shm/ffmpeg/in/" + fname.replace(" ", "")) +fname = fname.replace(" ", "") + +# ffprobe to calculate the total duration of the clip. +length = math.floor( + float( + os.popen( + "ffprobe -v error -show_entries format=duration -of default=noprint_wrappers=1:nokey=1 /dev/shm/ffmpeg/in/{fname}".format( + fname=fname + ) + ).read() + ) +) + +duration, timestamped_section = time_calculations(fname, length) + +run = True + +reso = os.getenv("reso") +codec = os.getenv("codec") +audio_br = os.getenv("audio_br") +audio_br = int(str(os.getenv("audio_br"))) +fs = float(str(os.getenv("fs"))) +target_fs = fs + +codecs = { + "vp9": { + "pass1": f"-vf scale={reso} -g 240 -threads 8 -speed 4 -row-mt 1 -tile-columns 2 -vsync cfr -c:v libvpx-vp9 -pass 1 -an", + "pass2": f"-vf scale={reso} -g 240 -threads 8 -speed 2 -row-mt 1 -tile-columns 2 -c:v libvpx-vp9 -c:a libopus -pass 2", + "output_name": "small_" + fname.replace(".mp4", ".webm"), + }, + "x264": { + "pass1": f"-vf scale={reso} -vsync cfr -c:v libx264 -pass 1 -an", + "pass2": f"-vf scale={reso} -c:v libx264 -c:a aac -pass 2 ", + "output_name": "small_" + fname, + }, + "x265": { + "pass1": f"-vf scale={reso} -c:v libx265 -vsync cfr -x265-params pass=1 -an", + "pass2": f"-vf scale={reso} -c:v libx265 -x265-params pass=2 -c:a aac", + "output_name": "small_" + fname, + }, +} + + +while run: + # Conversion to KiB + end_fs = fs * 8192 + br, minbr, maxbr = get_bitrate( + duration=duration, filesize=end_fs, audio_br=audio_br + ) + ffmpeg_string = f""" + ffpb {timestamped_section} -hwaccel cuda -i /dev/shm/ffmpeg/in/{fname} -y \ + {codecs[str(codec)]['pass1']} \ + -b:v {br}k -minrate {minbr}k -maxrate {maxbr}k \ + -f null /dev/null && \ + ffpb {timestamped_section} -hwaccel cuda -i /dev/shm/ffmpeg/in/{fname} \ + {codecs[str(codec)]['pass2']} \ + -b:a {audio_br}k -b:v {br}k -minrate {minbr}k -maxrate {maxbr}k \ + /dev/shm/ffmpeg/out/{codecs[str(codec)]['output_name']} -y + """ + + run = encode( + ffmpeg_string, output_name=codecs[str(codec)]["output_name"], fs=target_fs + ) + + if run: + fs = fs - 0.2 diff --git a/workstation/scripts/ffmpreg.sh b/workstation/scripts/ffmpreg.sh new file mode 100755 index 0000000..8bbbf0d --- /dev/null +++ b/workstation/scripts/ffmpreg.sh @@ -0,0 +1,98 @@ +#! /usr/bin/env nix-shell +#! nix-shell -i bash -p bash gum trashy fd ripgrep mediainfo + +replace_extension() { + local file_basename + file_basename=$(basename "$1") + echo "${file_basename%.*}.$2" +} + +convert_gif() { + file_newname=$(replace_extension "$1" gif) + ffpb -i "$(realpath "$1")" -vf fps=12,scale=480:-1,smartblur=ls=-0.5 "$file_newname" +} + +convert_mp4() { + local file_newname + file_newname=$(replace_extension "$1" mp4) + local file_tempdest=/dev/shm/$file_newname + local file_destination + file_destination=$(dirname "$(realpath "$1")")/$file_newname + ffpb -i "$1" \ + -c:v libx265 \ + "$file_tempdest" + trash "$1" + mv -i "$file_tempdest" "$file_destination" +} + +convert_discord() { + local file_newname + file_newname=$2_$(replace_extension "$1" mp4) + local dir_ram=/dev/shm/ffmpeg + mkdir -p $dir_ram/{in,out} + ffpb -hwaccel cuda -i "$(realpath "$1")" \ + -c:v h264_nvenc \ + "$dir_ram"/in/discord.mp4 + cd "$dir_ram" || exit + codec=x264 audio_br=$3 fs=$4 reso=$5 ffmpeg4discord + mv "$dir_ram"/out/small_discord.mp4 ~/"$file_newname" + command rm -rf "$dir_ram" +} + +operation=$(gum choose mp4 discord nitro gif enc265) + +case $operation in + 1 | mp4) + to_convert=() + while IFS= read -r file; do + to_convert+=("$file") + done < <(fd . "$(pwd)" -tf -aL | fzf --multi -i) + for file in "${to_convert[@]}"; do + convert_mp4 "$file" + done + ;; + 2 | discord) + to_convert=() + while IFS= read -r file; do + to_convert+=("$file") + done < <(fd . "$(pwd)" -tf -aL | fzf --multi -i) + for file in "${to_convert[@]}"; do + convert_discord "$file" discord 96 8.0 "1280x720" + done + ;; + 3 | nitro) + to_convert=() + while IFS= read -r file; do + to_convert+=("$file") + done < <(fd . "$(pwd)" -tf -aL | fzf --multi -i) + for file in "${to_convert[@]}"; do + convert_discord "$file" nitro 128 50.0 "1920x1080" + done + ;; + 4 | gif) + to_convert=() + while IFS= read -r file; do + to_convert+=("$file") + done < <(fd . "$(pwd)" -tf -aL | fzf --multi -i) + for file in "${to_convert[@]}"; do + convert_gif "$file" + done + ;; + 5 | enc265) + to_convert=() + extensions=(flv m4v mpg avi mov ts mkv mp4 webm) + for ext in "${extensions[@]}"; do + while IFS= read -r file; do + if ! (mediainfo "$file" | grep Writing\ library | grep -q x265); then + to_convert+=("$file") + fi + done < <(fd . -e "$ext" -tf -aL) + done + for file in "${to_convert[@]}"; do + convert_mp4 "$file" + done + ;; + *) + echo -n "Please select a valid input" + ;; +esac diff --git a/workstation/scripts/manage-library.sh b/workstation/scripts/manage-library.sh new file mode 100755 index 0000000..f2334ee --- /dev/null +++ b/workstation/scripts/manage-library.sh @@ -0,0 +1,153 @@ +#! /usr/bin/env nix-shell +#! nix-shell -i bash -p bash gum fd ripgrep exa trashy zip unzip + +root_directories=( + ~/Multimedia/Library/Comics + ~/Multimedia/Library/Manga + ~/Multimedia/Library/Webtoons +) + +newname() { + echo "$1" | sed -E "s/$2/$3/g" +} + +separator() { + gum style --foreground 7 _________________________ +} +announce_changes() { + echo "Renaming:" + gum style --foreground 1 "$1" + echo "Into:" + gum style --foreground 2 "$2" + separator +} + +rename_file() { + while IFS= read -r file; do + local original_name + original_name=$(basename "$file") + local new_name + new_name=$(newname "$(basename "$file")" "$2" "$3") + + announce_changes "$original_name" "$new_name" + command mv -n "$(dirname "$file")"/{"$original_name","$new_name"} + done < <(fd "$1" --absolute-path -tf -s "${root_directories[@]}") +} + +rename_directory() { + while IFS= read -r dir; do + local new_name + new_name=$(newname "$(basename "$dir")" "$2" "$3") + local new_dir + new_dir=$(dirname "$dir")/$new_name + + announce_changes "$dir" "$new_dir" + echo "Processing..." + if [ ! -d "$new_dir" ]; then + echo "$(basename "$new_dir") doesn't exist. Creating it." + command mkdir -p "$new_dir" + fi + if [ -d "$new_dir" ]; then + echo "$(basename "$new_dir") has been created!, moving the following files:" + exa "$dir" + fd . "$dir" -x mv -n {} "$(realpath "$new_dir")" + fi + separator + done < <(fd "$1" --absolute-path -td -s "${root_directories[@]}") +} + +# Capitalize Special words +words=(special tpb full annual) +Words=(Special TPB Full Annual) +counter=0 +for word in "${words[@]}"; do + while IFS= read -r file; do + new_name=$(newname "$(basename "$file")" "$word" "${Words[$counter]}") + echo "Inproper capitalization of the word" + gum style --foreground 1 "$word" + echo "adjusting it into" + gum style --foreground 2 "${Words[$counter]}" + announce_changes "$(basename "$file")" "$new_name" + command mv -n "$(dirname "$file")"/{"$(basename "$file")","$new_name"} + done < <(fd "$word" --absolute-path -tf -s "${root_directories[@]}") + counter=$((counter + 1)) +done + +# Rename Year files +# set regex_year_grep "\([[:digit:]]{4}\)" +# set regex_year_string "(\()(\d{4})(\))" +# rename_directory $regex_year_grep $regex_year_string \$2 +# rename_file $regex_year_grep $regex_year_string \$2 + +# Rename #_ downloads +regex_hashtag="#_" +rename_directory $regex_hashtag $regex_hashtag "#" +rename_file $regex_hashtag $regex_hashtag "#" + +rename_keywords() { + # Followed by digit + local regex_digit_fd="$1 \d+" + local regex_digit="($1 )([[:digit:]]+)" + rename_directory "$regex_digit_fd" "$regex_digit" "\1#\2" + rename_file "$regex_digit_fd" "$regex_digit" "\1#\2" + # Without digit + regex="#$1" + rename_directory "$regex" "$regex" "$1" + rename_file "$regex" "$regex" "$1" +} + +rename_keywords TPB +rename_keywords Special +rename_keywords Annual + +# Rename #Full +rename_directory " #Full" " #Full" "" +rename_file " #Full" " #Full" "" + +# Rename double space +rename_directory " " " " " " +rename_file " " " " " " + +# Fix names +wrongnames=( + "Dr. Stone" + i-dont-want-this-kind-of-hero + pure-of-heart + scoob-and-shag + stick-n-poke + "Houseki no Kuni" + "Gantz E" + "Gantz G" +) +rightname=( + "Dr. STONE" + "I DON'T WANT THIS KIND OF HERO" + "Pure of Heart" + "Scoob and Shag" + "Stick n' Poke" + "Land of the Lustrous" + "Gatz:E" + "Gantz:G" +) +counter=0 +for wrongname in "${wrongnames[@]}"; do + rename_directory "$wrongname" "$wrongname" "${rightname[$counter]}" + rename_file "$wrongname" "$wrongname" "${rightname[$counter]}" + counter=$((counter + 1)) +done + +# Merge TPB (Part X) files +while IFS= read -r file; do + new_name=$(newname "$(basename "$file" .cbz)" "TPB \(Part [[:digit:]]+\)" TPB) + extract_dir=$(realpath "$(dirname "$file")"/"$new_name") + if [ ! -d "$extract_dir" ]; then + mkdir -p "$extract_dir" + fi + unzip "$file" -d "$extract_dir"/"$(basename "$file" .cbz)" + cd "$extract_dir" || exit + zip -r "$(realpath "$(dirname "$file")")"/"$new_name"\.cbz ./ + trash "$file" + trash "$extract_dir"/"$(basename "$file" .cbz)" +done < <(fd "Part \d+" --absolute-path -tf -s "${root_directories[@]}") + +fd . --absolute-path -td -te "${root_directories[@]}" -x trash {} diff --git a/workstation/scripts/nextcloud-cronjob.sh b/workstation/scripts/nextcloud-cronjob.sh new file mode 100755 index 0000000..5eb60de --- /dev/null +++ b/workstation/scripts/nextcloud-cronjob.sh @@ -0,0 +1,59 @@ +#!/run/current-system/sw/bin/bash + +# Cron tasks +if type /run/current-system/sw/bin/nextcloud-occ 2>/dev/null; then + /run/current-system/sw/bin/nextcloud-occ preview:pre-generate + /run/current-system/sw/bin/nextcloud-occ face:background_job -t 900 +fi + +# Sync GDL stuff +root=/mnt/disk2/scrapping + +cd $root || exit +set -- Aqp Ghekre +for user in "$@"; do + originDir=$root/$user + destDir=/mnt/disk1/nextcloud/$user/files/Requested + destDirDup=/mnt/disk1/nextcloud/$user/files/RequestedDupePlzCheckNDel + if [ ! -d "$destDir" ]; then + echo "$destDir does not exist, creating..." + mkdir -p "$destDir" + fi + cd "$originDir" || exit + find . -type f | while read -r file; do + destination=$destDir/"$(echo "$file" | sed "s/^\.\///")" + destinationDup=$destDirDup/"$(echo "$file" | sed "s/^\.\///")" + + if [ ! -f "$destination" ]; then + echo "Safe to move $(basename "$file")" + if [ ! -d "$(dirname "$destination")" ]; then + echo "Creating parent directory..." + mkdir -p "$(dirname "$destination")" + fi + mv -n "$file" "$destination" + else + echo "Duplicated encountered $(basename "$file")" + if [ ! -d "$(dirname "$destinationDup")" ]; then + echo "Creating parent directory..." + mkdir -p "$(dirname "$destinationDup")" + fi + mv -n "$file" "$destinationDup" + fi + + done + + find ./ -mindepth 1 -type d -empty -delete + + chown 990:990 -R "$destDir" + find "$destDir" -type d -exec chmod -R 755 {} \; + find "$destDir" -type f -exec chmod -R 644 {} \; + + if [ -d "$destDirDup" ]; then + chown 990:990 -R "$destDirDup" + find "$destDirDup" -type d -exec chmod -R 755 {} \; + find "$destDirDup" -type f -exec chmod -R 644 {} \; + fi + if type /run/current-system/sw/bin/nextcloud-occ 2>/dev/null; then + /run/current-system/sw/bin/nextcloud-occ files:scan --all + fi +done diff --git a/workstation/scripts/pika-list.sh b/workstation/scripts/pika-list.sh new file mode 100755 index 0000000..ba8f046 --- /dev/null +++ b/workstation/scripts/pika-list.sh @@ -0,0 +1,51 @@ +#! /usr/bin/env nix-shell +#! nix-shell -i bash -p bash fd borgbackup gum ripgrep + +BORG_PASSPHRASE=$(gum input --password --placeholder "Type borg password") +export BORG_PASSPHRASE + +d_root=$HOME/pika +f_string=home/jawz/.config/jawz/lists/jawz/watch.txt +d_borg=/mnt/disk1/backups/pika/lists + +while IFS= read -r repo; do + IFS=" " read -r -a array <<<"$repo" + repo_id="${array[0]}" + mkdir -vp "$d_root/$repo_id" && cd "$d_root/$repo_id" || exit + borg extract $d_borg::"$repo_id" $f_string + cat "$d_root/$repo_id/$f_string" >>"$d_root/master" +done < <(borg list "$d_borg") + +cd "$HOME" || exit + +sort -u "$d_root/master" -o "$d_root/sorted" +sort -u "$LW" -o "$LW" + +echo "Current $(wc -l <"$LW") archived $(wc -l <"$d_root/sorted")" + +echo "Missing lines:" +diff "$d_root/sorted" "$LW" + +# look for duped lines with different casing +echo "Duplicated lines:" +while IFS= read -r line; do + if ! [ "$line" == "${line,,}" ]; then + if rg "${line,,}" <"$LW"; then + echo "$line" + fi + fi +done <"$LW" + +# delete pika backups +if gum confirm "Limpiar pika?"; then + command rm -rf "$d_root" + while IFS= read -r repo; do + IFS=" " read -r -a array <<<"$repo" + repo_id="${array[0]}" + gum spin --spinner dot --title "Cleaning $repo_id..." -- borg delete $d_borg::"$repo_id" + done < <(borg list "$d_borg") +else + echo "Canceled, no files deleted" +fi +gum spin --spinner dot --title "Cleaning $repo_id..." -- borg compact "$d_borg" +gum spin --spinner dot --title "Cleaning $repo_id..." -- borg compact /mnt/disk1/backups/pika/home diff --git a/workstation/scripts/run.sh b/workstation/scripts/run.sh new file mode 100644 index 0000000..6b112d7 --- /dev/null +++ b/workstation/scripts/run.sh @@ -0,0 +1,48 @@ +#! /usr/bin/env nix-shell +#! nix-shell -i bash -p bash gnome.zenity rmlint git gum xclip + +if [ -n "$1" ]; then + operation=$1 +else + operation=$(gum choose rmlint_1 rmlint_2 download git) +fi + +case $operation in + # onlyfans) + # source ~/Development/Python/onlyfans/bin/activate.fish + # python ~/Development/Git/OnlyFans/start_ofd.py + # deactivate + rmlint_1) + rmlint -g --types="duplicates" \ + --config=sh:handler=clone \ + /mnt/disk1/personal + ;; + rmlint_2) + rmlint -g --types="duplicates" \ + --config=sh:handler=clone \ + /mnt/disk2/{glue,home,personal,scrapping} + ;; + download) + ENTRY=$(zenity --entry --width=250 --title "Push Manager" \ + --text="Verify the following entry is correct" \ + --add-entry="Clipboard:" --entry-text "$(xclip -o -sel clip)") + if [ -n "$ENTRY" ]; then + kgx -e "download -u jawz -i '$ENTRY'" + else + zenity --error --width=250 \ + --text "Please verify and try again" + fi + ;; + git) + git_dir=$HOME/Development/Git + while IFS= read -r repo; do + if ! [ -d "$repo/.git" ]; then + continue + fi + cd "$repo" || exit + gum style --foreground 2 "Updating $(basename "$repo")" + git fsck --full + git pull + done < <(fd . "$git_dir" -td --absolute-path -d 1) + ;; +esac diff --git a/workstation/scripts/split-dir.sh b/workstation/scripts/split-dir.sh new file mode 100755 index 0000000..2c37eae --- /dev/null +++ b/workstation/scripts/split-dir.sh @@ -0,0 +1,28 @@ +#! /usr/bin/env nix-shell +#! nix-shell -i bash -p bash fd + +before_count=$(fd -tf | wc -l) +i=0 + +for file in $(fd -d1 -tf -E '*.mp4'); do + dir_name=$(basename "$(pwd)")_$(printf %03d $((i / $1 + 1))) + mkdir -p "$dir_name" + mv -i "$file" "$(realpath "$dir_name")"/ + i=$((i + 1)) +done + +for file in $(fd -d1 -tf -e mp4); do + mkdir -p videos + mv -i "$file" "$(realpath videos)"/ +done + +after_count=$(fd -tf | wc -l) + +if [[ "$before_count" == "$after_count" ]]; then + echo "No file count differences" +else + echo "Before count: $before_count" + echo "After count: $after_count" +fi +sleep 10 +exit diff --git a/workstation/scripts/tasks.sh b/workstation/scripts/tasks.sh new file mode 100755 index 0000000..8a92ddc --- /dev/null +++ b/workstation/scripts/tasks.sh @@ -0,0 +1,140 @@ +#! /usr/bin/env nix-shell +#! nix-shell -i bash -p bash trashy fd ripgrep file + +directories=("$HOME/Pictures/To Organize/" "$HOME/Downloads/") + +replace_extension() { + local file_basename + file_basename=$(basename "$1") + echo "${file_basename%.*}.$2" +} + +generate_random_number() { + local min=0 + local max=9999999999 + printf "%010d\n" $((min + RANDOM % max)) +} + +test_name() { + local random_number + random_number=$(generate_random_number) + while (($(fd "$random_number"* "$HOME/Pictures/" "$HOME/Downloads/" -tf | wc -l) > 0)); do + echo "Conflicts found, generating a new filename" + random_number=$(generate_random_number) + echo "$random_number" + done + echo "$random_number" +} + +while IFS= read -r file; do + regex_str='source|tenor|media|duckduckgo\.com|giphy|' + regex_str+='(? 0)); then + while IFS= read -r file; do + date=$(stat -c "%y" "$file" | rg -o "\d{4}-\d{2}-\d{2}") + year=$(echo "$date" | rg -o "\d{4}") + month=$(echo "$date" | rg -o "\d{4}-\d{2}" | rg -o --pcre2 "(?<=-)\d{2}") + dest_dir=$(realpath "$screenshots/$year/$month") + echo "Moving screenshot $(basename "$file") into $dest_dir" + mkdir -vp "$dest_dir" + command mv -n "$file" "$dest_dir/" + done < <(fd . "$screenshots" --absolute-path -tf -d 1) +fi + +# Where steam screenshots are stored, may need to replace with ur ID +dir_steam=$XDG_DATA_HOME/Steam/userdata/107446271/760/remote +declare -A games +# Insert here new games, put between [] the ID of the game +# You can find it by visiting the $dir_steam directory +# the ID is simply the name of the folder in there. +games+=( + [386360]=Smite + [960090]="Bloons Tower Defense 6" + [648800]=Raft + [262060]="Darkest Dungeon" + [234140]="Mad Max" + [433340]="Slime Rancher" +) + +for key in "${!games[@]}"; do + # Modify this to store your screenshots somewhere else + dir_dest=$(realpath "$HOME/Pictures/Screenshots/Games")/${games[$key]} + dir_game=$(realpath "$dir_steam")/$key/screenshots + # If there are not screenshots currently stored, why bother lol + if ! [[ -d $dir_game ]]; then # + continue + fi + # If screenshots exist however... + if (($(fd . "$dir_game" -d 1 -tf | wc -l) > 0)); then + # Create destination directory + mkdir -vp "$dir_dest" + echo "Moving ${games[$key]} screenshots..." + fd . "$dir_game" -d 1 -tf -x mv -n {} "$dir_dest"/ + # Delete thumnnails + echo "Deleting ${games[$key]} thumbnails..." + rm -rf "$dir_game"/thumbnails + fi +done +# Clearing up empty directories +fd . "$dir_steam" -td -te -x trash {} + +cyberpunk_dir=$HOME/Games/cyberpunk-2077/drive_c/users/jawz/Pictures/"Cyberpunk 2077" +if [[ -d $cyberpunk_dir ]]; then + while IFS= read -r file; do + echo "Moving cyberpunk screenshots" + command mv -n "$file" "$HOME/Pictures/Screenshots/Games/Cyberpunk 2077/" + done < <(fd . "$cyberpunk_dir" -tf) +fi + +proton_dir=$HOME/.steam/steam/compatibilitytools.d +if [[ -d "$proton_dir" ]]; then + while IFS= read -r protonver; do + lutrisdir=$XDG_DATA_HOME/lutris/runners/wine/$(basename "$protonver") + if ! [ -d "$lutrisdir" ] && ! [ -L "$lutrisdir" ]; then + echo "Symlink $lutrisdir doesn't exist, creating link..." + ln -s "$(realpath "$protonver")"/files "$lutrisdir" + fi + done < <(fd . "$proton_dir" -d 1 -td) +fi +fd . "$XDG_DATA_HOME/lutris/runners/wine" -d 1 -tl -x trash {} + +while IFS= read -r file; do + ext=$(file --mime-type "$file" | rg -o '\w+$') + correct_ext=${ext,,} + filename=$(basename -- "$file") + current_ext="${filename##*.}" + filename="${filename%.*}" + if echo "$correct_ext" | rg -q 'jpe|jpg|jpeg|png|gif'; then + if [ "$current_ext" != "$correct_ext" ]; then + echo "The file $(basename "$file")" \ + "will be renamed, the propper extension is $correct_ext" + new_name="$filename".$correct_ext + command mv -n "$(dirname "$file")"/{"$(basename "$file")","$new_name"} + fi + fi +done < <(fd . "${directories[@]}" -d 1 -tf) + +files_home_clean=(.pki HuionCore.pid DriverUI.pid huion.log) +for file in "${files_home_clean[@]}"; do + file=$HOME/$file + if [ -e "$file" ]; then + rm -rf "$file" + fi +done diff --git a/workstation/scripts/update-dns.sh b/workstation/scripts/update-dns.sh new file mode 100755 index 0000000..5585ee2 --- /dev/null +++ b/workstation/scripts/update-dns.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env nix-shell +#! nix-shell -i bash -p bash curl jq dig + +# Shell script to update namecheap.com dynamic dns +# for a domain to your external IP address + +# namecheap +hostnames=(cloud @) +domain=rotehaare.art +password=60d672be5d9d4828a0f96264babe0ac1 + +ip=$(curl -s ipecho.net/plain) +for hostname in "${hostnames[@]}"; do + curl "https://dynamicdns.park-your-domain.com/update?host=$hostname&domain=$domain&password=$password&ip=$ip" +done + +# cloudflare +zone_id=833996ed25eb09f1a50606e0457790e4 +record=servidos.lat +record_id=6b117173e53a7511ba36ceb9637ede63 +cloudflare_token=VdKosfThQmOcuywLOUq9DY4-df9EmbHrDWyf_vUb + +# get record_id +# curl -s -X GET "https://api.cloudflare.com/client/v4/zones/${zone_id}/dns_records?type=A&name=${record}" \ +# -H "Authorization: Bearer ${cloudflare_token}" \ +# -H "Content-Type: application/json" | jq -r '{"result"}[] | .[0] | .id' + +curr_ip=$(curl -s -X GET https://checkip.amazonaws.com) +curr_reg=$(dig ${record} +short @1.1.1.1) +if echo "${curr_reg}" | grep "${curr_ip}"; then + echo "$(date --rfc-3339=seconds) - OK - Current record matches current IP (${curr_ip})" +else + curl -s -X PUT "https://api.cloudflare.com/client/v4/zones/${zone_id}/dns_records/${record_id}" \ + -H "Authorization: Bearer ${cloudflare_token}" \ + -H "Content-Type: application/json" \ + --data "{\"type\":\"A\",\"name\":\"${record}\",\"content\":\"$curr_ip\",\"ttl\":1,\"proxied\":false}" >/dev/null + echo "$(date --rfc-3339=seconds) - NOK - Record Updated to $curr_ip from ${curr_reg}" +fi diff --git a/secrets.nix_wip b/workstation/secrets.nix_wip similarity index 100% rename from secrets.nix_wip rename to workstation/secrets.nix_wip diff --git a/servers.nix b/workstation/servers.nix similarity index 100% rename from servers.nix rename to workstation/servers.nix