104 lines
4.8 KiB
Python
104 lines
4.8 KiB
Python
#!/usr/bin/env python3
|
|
"""Define the user class to populate and setup the download environment"""
|
|
import re
|
|
from pathlib import Path
|
|
from functions import sort_txt_file, randomize_txt_file, load_config_variables
|
|
|
|
config_variables = load_config_variables()
|
|
|
|
|
|
class User:
|
|
"""Populate the directory for each user"""
|
|
|
|
# pylint: disable=too-many-instance-attributes
|
|
def __init__(self, index):
|
|
self.user = config_variables["users"][index]
|
|
self.config = config_variables["global"]
|
|
self.name = self.user["name"]
|
|
self.sleep = self.config["sleep"]
|
|
# Directories
|
|
self.dir_cache = Path(self.config["cache-directory"]) / self.name
|
|
self.dir_log = Path(self.config["log-directory"])
|
|
self.dir_archive = Path(self.config["archive-directory"])
|
|
self.dir_download = Path(self.user["download-directory"])
|
|
self.dir_media_download = Path(self.user["media-directory"])
|
|
self.dir_push = Path(self.user["push-directory"])
|
|
self.dir_master_list = Path(self.config["list-dir"]) / self.name
|
|
# Files
|
|
self.archive_gallery = self.dir_archive / f"{self.name}.sqlite3"
|
|
self.archive_media = self.dir_archive / f"{self.name}_ytdl.txt"
|
|
# Lists
|
|
self.list_master = self.dir_master_list / "watch.txt"
|
|
self.list_push = self.dir_master_list / "instant.txt"
|
|
self.list_instagram = self.dir_cache / "instagram.txt"
|
|
self.list_kemono = self.dir_cache / "kemono.txt"
|
|
self.list_main = self.dir_cache / "main.txt"
|
|
|
|
def create_directories(self):
|
|
"""Create user directories if they don't exist"""
|
|
if self.dir_cache.is_dir():
|
|
for file in self.dir_cache.iterdir():
|
|
if file.is_file():
|
|
file.unlink()
|
|
for file in self.dir_cache.iterdir():
|
|
if file.is_dir():
|
|
file.rmdir()
|
|
self.dir_cache.rmdir()
|
|
# Create directories
|
|
self.dir_cache.mkdir(parents=True, exist_ok=True)
|
|
self.dir_log.mkdir(parents=True, exist_ok=True)
|
|
self.dir_archive.mkdir(parents=True, exist_ok=True)
|
|
self.dir_download.mkdir(parents=True, exist_ok=True)
|
|
self.dir_media_download.mkdir(parents=True, exist_ok=True)
|
|
self.dir_push.mkdir(parents=True, exist_ok=True)
|
|
# Check for the existence of core files
|
|
if not Path(self.archive_gallery).is_file():
|
|
self.archive_gallery.touch()
|
|
if not Path(self.archive_media).is_file():
|
|
self.archive_media.touch()
|
|
if not self.dir_master_list.is_dir():
|
|
print(f"ERROR: Directory for user {self.name} doesn't exist")
|
|
if not Path(self.list_master).is_file():
|
|
self.list_master.touch()
|
|
if not Path(self.list_push).is_file():
|
|
self.list_push.touch()
|
|
# Create temporary lists
|
|
for gdl_list in ("instagram", "kemono", "main"):
|
|
Path(self.dir_cache.resolve() / f"{gdl_list}.txt").touch()
|
|
|
|
def list_manager(self):
|
|
"""Manage all the user list and create sub-lists"""
|
|
# sort_txt_file(self.list_master)
|
|
self.create_directories() # Call the function to create necesary cache dirs
|
|
with open(self.list_master, encoding="utf-8") as list_master:
|
|
# Create temporary list files segmented per scrapper
|
|
for line in [line.rstrip() for line in list_master]:
|
|
# WIKIFEET
|
|
with open(self.list_main, "a", encoding="utf-8") as list_main, open(
|
|
self.list_kemono, "a", encoding="utf-8"
|
|
) as list_kemono, open(
|
|
self.list_instagram, "a", encoding="utf-8"
|
|
) as list_instagram:
|
|
if re.search(r"kemono.party", line):
|
|
list_kemono.write(line + "\n")
|
|
elif re.search(r"instagram", line):
|
|
list_instagram.write(line + "\n")
|
|
elif re.search(r"wikifeet", line):
|
|
continue
|
|
# list_main.write(line + "\n")
|
|
elif re.search(r"furaffinity", line):
|
|
list_main.write(line + "\n")
|
|
elif re.search(r"twitter", line):
|
|
# if url contains /media at the end just write the line
|
|
if re.search(r"\/media$", line):
|
|
list_main.write(line + "\n")
|
|
else:
|
|
# if does not contain /media at the end then add /media
|
|
list_main.write(line + "/media" + "\n")
|
|
else:
|
|
list_main.write(line + "\n")
|
|
sort_txt_file(self.list_kemono)
|
|
# Try to avoid getting banned by shuffling download order
|
|
randomize_txt_file(self.list_instagram)
|
|
randomize_txt_file(self.list_main)
|