diff --git a/src/download/admin.py b/src/download/admin.py index f3ca382..e4c46e1 100644 --- a/src/download/admin.py +++ b/src/download/admin.py @@ -160,15 +160,90 @@ def cmd_users(args: argparse.Namespace) -> None: def cmd_import(args: argparse.Namespace) -> None: configs = load_config_variables() with db.connect(configs) as conn: + imported_paths = [] for entry in configs["users"]: user = entry["name"] lists_dir = Path(configs["global"]["lists-dir"]) / user master = lists_dir / "watch.txt" result = db.import_master_list(conn, user, master) + if result["status"] == "ok": + imported_paths.append(str(master)) print(f"{user}: {result}") + if result.get("duplicates"): + print(f"{user} duplicates:") + for dup in result["duplicates"]: + print(f" {dup}") + if imported_paths: + print("Imported lists:") + for path in imported_paths: + print(f" {path}") conn.commit() +def parse_list_file(path: Path) -> dict: + enabled: set[str] = set() + disabled: set[str] = set() + if not path.is_file(): + return {"enabled": enabled, "disabled": disabled} + with open(path, "r", encoding="utf-8") as r_file: + for raw in r_file: + line = raw.strip() + if not line: + continue + if line.startswith("#"): + url = line.lstrip("#").strip() + if url: + disabled.add(db.normalize_url(url)) + else: + enabled.add(db.normalize_url(line)) + return {"enabled": enabled, "disabled": disabled} + + +def cmd_validate_import(args: argparse.Namespace) -> None: + configs = load_config_variables() + with db.connect(configs) as conn: + for entry in configs["users"]: + user = entry["name"] + lists_dir = Path(configs["global"]["lists-dir"]) / user + master = lists_dir / "watch.txt" + list_sets = parse_list_file(master) + + rows = db.get_links_by_user(conn, user) + db_enabled = set() + db_disabled = set() + for row in rows: + norm = db.normalize_url(row["url_original"]) + if row["enabled"] and not row["banned_at"]: + db_enabled.add(norm) + else: + db_disabled.add(norm) + + missing_enabled = list_sets["enabled"] - db_enabled + missing_disabled = list_sets["disabled"] - db_disabled + extra_enabled = db_enabled - list_sets["enabled"] + extra_disabled = db_disabled - list_sets["disabled"] + + print(f"{user}:") + if missing_enabled: + print(" Missing enabled in DB:") + for url in sorted(missing_enabled): + print(f" {url}") + if missing_disabled: + print(" Missing disabled in DB:") + for url in sorted(missing_disabled): + print(f" {url}") + if extra_enabled: + print(" Extra enabled in DB:") + for url in sorted(extra_enabled): + print(f" {url}") + if extra_disabled: + print(" Extra disabled in DB:") + for url in sorted(extra_disabled): + print(f" {url}") + if not any([missing_enabled, missing_disabled, extra_enabled, extra_disabled]): + print(" OK") + + def cmd_user_rename(args: argparse.Namespace) -> None: configs = load_config_variables() with db.connect(configs) as conn: @@ -245,6 +320,9 @@ def build_parser() -> argparse.ArgumentParser: p_user_rename.add_argument("new") p_user_rename.set_defaults(func=cmd_user_rename) + p_validate = sub.add_parser("validate-import") + p_validate.set_defaults(func=cmd_validate_import) + return parser diff --git a/src/download/db.py b/src/download/db.py index 4ea5f47..1e67f32 100644 --- a/src/download/db.py +++ b/src/download/db.py @@ -347,6 +347,13 @@ def get_links( return conn.execute(f"SELECT * FROM links {clause} ORDER BY user_name, id", params).fetchall() +def get_links_by_user(conn: sqlite3.Connection, user_name: str) -> list[sqlite3.Row]: + return conn.execute( + "SELECT * FROM links WHERE user_name = ? ORDER BY id", + (user_name,), + ).fetchall() + + def import_master_list(conn: sqlite3.Connection, user_name: str, path: Path) -> dict: if not path.is_file(): return {"status": "missing", "path": str(path)} @@ -356,15 +363,33 @@ def import_master_list(conn: sqlite3.Connection, user_name: str, path: Path) -> added = 0 exists = 0 removed = 0 + duplicates: list[str] = [] for line in lines: - result = add_link(conn, user_name, line, assume_yes=True, source="import") + disabled = False + raw = line + if raw.startswith("#"): + disabled = True + raw = raw.lstrip("#").strip() + if not raw: + continue + + result = add_link(conn, user_name, raw, assume_yes=True, source="import") if result["status"] == "added": added += 1 + if disabled: + set_enabled(conn, user_name, raw, enabled=False) elif result["status"] == "exists": exists += 1 + duplicates.append(raw) elif result["status"] == "removed": removed += 1 - return {"status": "ok", "added": added, "exists": exists, "removed": removed} + return { + "status": "ok", + "added": added, + "exists": exists, + "removed": removed, + "duplicates": duplicates, + } def bulk_rename_handle(