freshrss favorite archiver

.env.example

@@ -14,3 +14,15 @@ FLATNOTES_SECRET_KEY=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
 METUBE_PASSWORD_HASH=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
 
 TRANSMUTE_SECRET_KEY=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+
+FRESHRSS_DB=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+ARCHIVE_INDEX=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+
+METUBE_URL=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+METUBE_USER=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+METUBE_PASS=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+
+NTFY_URL=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+NTFY_TOKEN=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+
+DOCKER_COMPOSE_DIR=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx

bootstrap.sh

@@ -166,13 +166,17 @@ mkdir -p \
     /opt/server/caddy/data \
     /opt/server/caddy/config \
     /opt/server/resspublica/feed \
-    /opt/server/resspublica/images
+    /opt/server/resspublica/images \
+    /opt/server/fav-archiver
 
 sudo chown -R 1000:1000 /opt/server/filebrowser # somehow filebrowser cant acces it
 
 sudo chown -R tomasr:tomasr /opt/server/resspublica
 sudo chmod -R u+rwX /opt/server/resspublica
 
+sudo chown -R tomasr:tomasr /opt/server/fav-archiver
+sudo chmod -R u+rwX /opt/server/fav-archiver
+
 echo
 echo "Configure rclone now."
 rclone config

freshrss-fav-archiver.py

@@ -0,0 +1,186 @@
+import os
+import time
+import sqlite3
+import requests
+import subprocess
+
+from urllib.parse import urlparse
+
+def load_env_file(path=".env"):
+    if not os.path.exists(path):
+        return
+
+    with open(path) as f:
+        for line in f:
+            line = line.strip()
+
+            # ignore empty lines and comments
+            if not line or line.startswith("#"):
+                continue
+
+            key, value = line.split("=", 1)
+            os.environ.setdefault(key, value)
+
+
+load_env_file()
+
+FRESHRSS_DB = os.environ["FRESHRSS_DB"]
+ARCHIVE_INDEX = os.environ["ARCHIVE_INDEX"]
+
+METUBE_URL = os.environ["METUBE_URL"]
+METUBE_USER = os.environ["METUBE_USER"]
+METUBE_PASS = os.environ["METUBE_PASS"]
+
+NTFY_URL = os.environ["NTFY_URL"]
+NTFY_TOKEN = os.environ["NTFY_TOKEN"]
+
+ARCHIVEBOX_COMPOSE_DIR = os.environ["ARCHIVEBOX_COMPOSE_DIR"]
+
+VIDEO_HOSTS = {
+    "youtube.com",
+    "www.youtube.com",
+    "youtu.be",
+    "vimeo.com",
+    "www.vimeo.com",
+    "dailymotion.com",
+    "www.dailymotion.com",
+}
+
+
+def get_favorites():
+    conn = sqlite3.connect(FRESHRSS_DB)
+    conn.row_factory = sqlite3.Row
+
+    cur = conn.cursor()
+    cur.execute("""
+        SELECT title, link
+        FROM entry
+        WHERE is_favorite = 1
+        ORDER BY date DESC
+    """)
+
+    result = [
+        {
+            "title": row["title"],
+            "link": row["link"]
+        }
+        for row in cur.fetchall()
+    ]
+
+    conn.close()
+    return result
+
+
+def load_index():
+    if not os.path.exists(ARCHIVE_INDEX):
+        return set()
+
+    with open(ARCHIVE_INDEX, "r") as f:
+        return set(line.strip() for line in f if line.strip())
+
+
+def save_to_index(url):
+    with open(ARCHIVE_INDEX, "a") as f:
+        f.write(url + "\n")
+
+
+def is_video_url(url):
+    host = urlparse(url).netloc.lower()
+    return any(host.endswith(site) for site in VIDEO_HOSTS)
+
+
+def send_to_metube(url):
+    """
+    MeTube API endpoint:
+    POST /add
+    """
+    try:
+        r = requests.post(
+            f"{METUBE_URL}/add",
+            json={
+                "url": url
+            },
+            auth=(METUBE_USER, METUBE_PASS),
+            timeout=30
+        )
+
+        return r.status_code in range(200, 300)
+
+    except Exception as e:
+        print("MeTube error:", e)
+        return False
+
+def send_to_archivebox(url):
+    try:
+        r = subprocess.run(
+            [
+                "docker",
+                "compose",
+                "-f",
+                f"{ARCHIVEBOX_COMPOSE_DIR}/docker-compose.yml",
+                "exec",
+                "-T",
+                "archivebox",
+                "archivebox",
+                "add",
+                url,
+            ],
+            capture_output=True,
+            text=True,
+            timeout=300
+        )
+
+        if r.returncode != 0:
+            print("ArchiveBox error:")
+            print(r.stderr)
+            return False
+
+        print(r.stdout)
+        return True
+
+    except Exception as e:
+        print("ArchiveBox error:", e)
+        return False
+
+def notify_failure(url):
+    try:
+        requests.post(
+            NTFY_URL,
+            data=f"Failed to archive {url}",
+            auth=("", NTFY_TOKEN),
+            timeout=10
+        )
+    except Exception as e:
+        print("ntfy error:", e)
+
+
+def process():
+    indexed = load_index()
+    favorites = get_favorites()
+
+    for article in favorites:
+        url = article["link"]
+
+        if url in indexed:
+            continue
+
+        print("Archiving:", url)
+
+        if is_video_url(url):
+            success = send_to_metube(url)
+        else:
+            success = send_to_archivebox(url)
+
+        if success:
+            print("Success")
+            save_to_index(url)
+        else:
+            print("Failed")
+            notify_failure(url)
+
+        # avoid hammering services
+        time.sleep(60)
+
+
+if __name__ == "__main__":
+    process()

systemd/freshrss-fav-archiver.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=FreshRSS favorite article archiver
+After=network-online.target
+
+[Service]
+Type=oneshot
+WorkingDirectory=/home/tomasr/vps-config
+ExecStart=/usr/bin/python3 /home/tomasr/vps-config/freshrss-fav-archiver.py
+
+StandardOutput=journal
+StandardError=journal