split configs into two systems...

workstation/scripts/download/gdl_classes.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python3
+"""Define the user class to populate and setup the download environment"""
+import re
+from pathlib import Path
+from functions import sort_txt_file, randomize_txt_file, load_config_variables
+
+config_variables = load_config_variables()
+
+
+class User:
+    """Populate the directory for each user"""
+
+    # pylint: disable=too-many-instance-attributes
+    def __init__(self, index):
+        self.user = config_variables["users"][index]
+        self.config = config_variables["global"]
+        self.name = self.user["name"]
+        self.sleep = self.config["sleep"]
+        # Directories
+        self.dir_cache = Path(self.config["cache-directory"]) / self.name
+        self.dir_log = Path(self.config["log-directory"])
+        self.dir_archive = Path(self.config["archive-directory"])
+        self.dir_download = Path(self.user["download-directory"])
+        self.dir_media_download = Path(self.user["media-directory"])
+        self.dir_push = Path(self.user["push-directory"])
+        self.dir_master_list = Path(self.config["list-dir"]) / self.name
+        # Files
+        self.archive_gallery = self.dir_archive / f"{self.name}.sqlite3"
+        self.archive_media = self.dir_archive / f"{self.name}_ytdl.txt"
+        # Lists
+        self.list_master = self.dir_master_list / "watch.txt"
+        self.list_push = self.dir_master_list / "instant.txt"
+        self.list_instagram = self.dir_cache / "instagram.txt"
+        self.list_kemono = self.dir_cache / "kemono.txt"
+        self.list_main = self.dir_cache / "main.txt"
+
+    def create_directories(self):
+        """Create user directories if they don't exist"""
+        if self.dir_cache.is_dir():
+            for file in self.dir_cache.iterdir():
+                if file.is_file():
+                    file.unlink()
+            for file in self.dir_cache.iterdir():
+                if file.is_dir():
+                    file.rmdir()
+            self.dir_cache.rmdir()
+        # Create directories
+        self.dir_cache.mkdir(parents=True, exist_ok=True)
+        self.dir_log.mkdir(parents=True, exist_ok=True)
+        self.dir_archive.mkdir(parents=True, exist_ok=True)
+        self.dir_download.mkdir(parents=True, exist_ok=True)
+        self.dir_media_download.mkdir(parents=True, exist_ok=True)
+        self.dir_push.mkdir(parents=True, exist_ok=True)
+        # Check for the existence of core files
+        if not Path(self.archive_gallery).is_file():
+            self.archive_gallery.touch()
+        if not Path(self.archive_media).is_file():
+            self.archive_media.touch()
+        if not self.dir_master_list.is_dir():
+            print(f"ERROR: Directory for user {self.name} doesn't exist")
+        if not Path(self.list_master).is_file():
+            self.list_master.touch()
+        if not Path(self.list_push).is_file():
+            self.list_push.touch()
+        # Create temporary lists
+        for gdl_list in ("instagram", "kemono", "main"):
+            Path(self.dir_cache.resolve() / f"{gdl_list}.txt").touch()
+
+    def list_manager(self):
+        """Manage all the user list and create sub-lists"""
+        # sort_txt_file(self.list_master)
+        self.create_directories()  # Call the function to create necesary cache dirs
+        with open(self.list_master, encoding="utf-8") as list_master:
+            # Create temporary list files segmented per scrapper
+            for line in [line.rstrip() for line in list_master]:
+                # WIKIFEET
+                with open(self.list_main, "a", encoding="utf-8") as list_main, open(
+                    self.list_kemono, "a", encoding="utf-8"
+                ) as list_kemono, open(
+                    self.list_instagram, "a", encoding="utf-8"
+                ) as list_instagram:
+                    if re.search(r"kemono.party", line):
+                        list_kemono.write(line + "\n")
+                    elif re.search(r"instagram", line):
+                        list_instagram.write(line + "\n")
+                    elif re.search(r"wikifeet", line):
+                        continue
+                        # list_main.write(line + "\n")
+                    elif re.search(r"furaffinity", line):
+                        list_main.write(line + "\n")
+                    elif re.search(r"twitter", line):
+                        # if url contains /media at the end just write the line
+                        if re.search(r"\/media$", line):
+                            list_main.write(line + "\n")
+                        else:
+                            # if does not contain /media at the end then add /media
+                            list_main.write(line + "/media" + "\n")
+                    else:
+                        list_main.write(line + "\n")
+        sort_txt_file(self.list_kemono)
+        # Try to avoid getting banned by shuffling download order
+        randomize_txt_file(self.list_instagram)
+        randomize_txt_file(self.list_main)