admin import/validate

src/download/admin.py

@@ -160,15 +160,90 @@ def cmd_users(args: argparse.Namespace) -> None:
 def cmd_import(args: argparse.Namespace) -> None:
     configs = load_config_variables()
     with db.connect(configs) as conn:
+        imported_paths = []
         for entry in configs["users"]:
             user = entry["name"]
             lists_dir = Path(configs["global"]["lists-dir"]) / user
             master = lists_dir / "watch.txt"
             result = db.import_master_list(conn, user, master)
+            if result["status"] == "ok":
+                imported_paths.append(str(master))
             print(f"{user}: {result}")
+            if result.get("duplicates"):
+                print(f"{user} duplicates:")
+                for dup in result["duplicates"]:
+                    print(f"  {dup}")
+        if imported_paths:
+            print("Imported lists:")
+            for path in imported_paths:
+                print(f"  {path}")
         conn.commit()
 
 
+def parse_list_file(path: Path) -> dict:
+    enabled: set[str] = set()
+    disabled: set[str] = set()
+    if not path.is_file():
+        return {"enabled": enabled, "disabled": disabled}
+    with open(path, "r", encoding="utf-8") as r_file:
+        for raw in r_file:
+            line = raw.strip()
+            if not line:
+                continue
+            if line.startswith("#"):
+                url = line.lstrip("#").strip()
+                if url:
+                    disabled.add(db.normalize_url(url))
+            else:
+                enabled.add(db.normalize_url(line))
+    return {"enabled": enabled, "disabled": disabled}
+
+
+def cmd_validate_import(args: argparse.Namespace) -> None:
+    configs = load_config_variables()
+    with db.connect(configs) as conn:
+        for entry in configs["users"]:
+            user = entry["name"]
+            lists_dir = Path(configs["global"]["lists-dir"]) / user
+            master = lists_dir / "watch.txt"
+            list_sets = parse_list_file(master)
+
+            rows = db.get_links_by_user(conn, user)
+            db_enabled = set()
+            db_disabled = set()
+            for row in rows:
+                norm = db.normalize_url(row["url_original"])
+                if row["enabled"] and not row["banned_at"]:
+                    db_enabled.add(norm)
+                else:
+                    db_disabled.add(norm)
+
+            missing_enabled = list_sets["enabled"] - db_enabled
+            missing_disabled = list_sets["disabled"] - db_disabled
+            extra_enabled = db_enabled - list_sets["enabled"]
+            extra_disabled = db_disabled - list_sets["disabled"]
+
+            print(f"{user}:")
+            if missing_enabled:
+                print("  Missing enabled in DB:")
+                for url in sorted(missing_enabled):
+                    print(f"    {url}")
+            if missing_disabled:
+                print("  Missing disabled in DB:")
+                for url in sorted(missing_disabled):
+                    print(f"    {url}")
+            if extra_enabled:
+                print("  Extra enabled in DB:")
+                for url in sorted(extra_enabled):
+                    print(f"    {url}")
+            if extra_disabled:
+                print("  Extra disabled in DB:")
+                for url in sorted(extra_disabled):
+                    print(f"    {url}")
+            if not any([missing_enabled, missing_disabled, extra_enabled, extra_disabled]):
+                print("  OK")
+
+
 def cmd_user_rename(args: argparse.Namespace) -> None:
     configs = load_config_variables()
     with db.connect(configs) as conn:
@@ -245,6 +320,9 @@ def build_parser() -> argparse.ArgumentParser:
     p_user_rename.add_argument("new")
     p_user_rename.set_defaults(func=cmd_user_rename)
 
+    p_validate = sub.add_parser("validate-import")
+    p_validate.set_defaults(func=cmd_validate_import)
+
     return parser
 
 

src/download/db.py

@@ -347,6 +347,13 @@ def get_links(
     return conn.execute(f"SELECT * FROM links {clause} ORDER BY user_name, id", params).fetchall()
 
 
+def get_links_by_user(conn: sqlite3.Connection, user_name: str) -> list[sqlite3.Row]:
+    return conn.execute(
+        "SELECT * FROM links WHERE user_name = ? ORDER BY id",
+        (user_name,),
+    ).fetchall()
+
+
 def import_master_list(conn: sqlite3.Connection, user_name: str, path: Path) -> dict:
     if not path.is_file():
         return {"status": "missing", "path": str(path)}
@@ -356,15 +363,33 @@ def import_master_list(conn: sqlite3.Connection, user_name: str, path: Path) ->
     added = 0
     exists = 0
     removed = 0
+    duplicates: list[str] = []
     for line in lines:
-        result = add_link(conn, user_name, line, assume_yes=True, source="import")
+        disabled = False
+        raw = line
+        if raw.startswith("#"):
+            disabled = True
+            raw = raw.lstrip("#").strip()
+        if not raw:
+            continue
+
+        result = add_link(conn, user_name, raw, assume_yes=True, source="import")
         if result["status"] == "added":
             added += 1
+            if disabled:
+                set_enabled(conn, user_name, raw, enabled=False)
         elif result["status"] == "exists":
             exists += 1
+            duplicates.append(raw)
         elif result["status"] == "removed":
             removed += 1
-    return {"status": "ok", "added": added, "exists": exists, "removed": removed}
+    return {
+        "status": "ok",
+        "added": added,
+        "exists": exists,
+        "removed": removed,
+        "duplicates": duplicates,
+    }
 
 
 def bulk_rename_handle(