diff --git a/.env.example b/.env.example index 3330752..34538da 100644 --- a/.env.example +++ b/.env.example @@ -14,3 +14,15 @@ FLATNOTES_SECRET_KEY=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx METUBE_PASSWORD_HASH=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx TRANSMUTE_SECRET_KEY=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + +FRESHRSS_DB=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +ARCHIVE_INDEX=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + +METUBE_URL=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +METUBE_USER=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +METUBE_PASS=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + +NTFY_URL=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +NTFY_TOKEN=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + +DOCKER_COMPOSE_DIR=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx diff --git a/bootstrap.sh b/bootstrap.sh index 4318783..8b47be2 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -166,13 +166,17 @@ mkdir -p \ /opt/server/caddy/data \ /opt/server/caddy/config \ /opt/server/resspublica/feed \ - /opt/server/resspublica/images + /opt/server/resspublica/images \ + /opt/server/fav-archiver sudo chown -R 1000:1000 /opt/server/filebrowser # somehow filebrowser cant acces it sudo chown -R tomasr:tomasr /opt/server/resspublica sudo chmod -R u+rwX /opt/server/resspublica +sudo chown -R tomasr:tomasr /opt/server/fav-archiver +sudo chmod -R u+rwX /opt/server/fav-archiver + echo echo "Configure rclone now." rclone config diff --git a/freshrss-fav-archiver.py b/freshrss-fav-archiver.py new file mode 100644 index 0000000..7000060 --- /dev/null +++ b/freshrss-fav-archiver.py @@ -0,0 +1,186 @@ +import os +import time +import sqlite3 +import requests +import subprocess + +from urllib.parse import urlparse + +def load_env_file(path=".env"): + if not os.path.exists(path): + return + + with open(path) as f: + for line in f: + line = line.strip() + + # ignore empty lines and comments + if not line or line.startswith("#"): + continue + + key, value = line.split("=", 1) + os.environ.setdefault(key, value) + + +load_env_file() + +FRESHRSS_DB = os.environ["FRESHRSS_DB"] +ARCHIVE_INDEX = os.environ["ARCHIVE_INDEX"] + +METUBE_URL = os.environ["METUBE_URL"] +METUBE_USER = os.environ["METUBE_USER"] +METUBE_PASS = os.environ["METUBE_PASS"] + +NTFY_URL = os.environ["NTFY_URL"] +NTFY_TOKEN = os.environ["NTFY_TOKEN"] + +ARCHIVEBOX_COMPOSE_DIR = os.environ["ARCHIVEBOX_COMPOSE_DIR"] + +VIDEO_HOSTS = { + "youtube.com", + "www.youtube.com", + "youtu.be", + "vimeo.com", + "www.vimeo.com", + "dailymotion.com", + "www.dailymotion.com", +} + + +def get_favorites(): + conn = sqlite3.connect(FRESHRSS_DB) + conn.row_factory = sqlite3.Row + + cur = conn.cursor() + cur.execute(""" + SELECT title, link + FROM entry + WHERE is_favorite = 1 + ORDER BY date DESC + """) + + result = [ + { + "title": row["title"], + "link": row["link"] + } + for row in cur.fetchall() + ] + + conn.close() + return result + + +def load_index(): + if not os.path.exists(ARCHIVE_INDEX): + return set() + + with open(ARCHIVE_INDEX, "r") as f: + return set(line.strip() for line in f if line.strip()) + + +def save_to_index(url): + with open(ARCHIVE_INDEX, "a") as f: + f.write(url + "\n") + + +def is_video_url(url): + host = urlparse(url).netloc.lower() + return any(host.endswith(site) for site in VIDEO_HOSTS) + + +def send_to_metube(url): + """ + MeTube API endpoint: + POST /add + """ + try: + r = requests.post( + f"{METUBE_URL}/add", + json={ + "url": url + }, + auth=(METUBE_USER, METUBE_PASS), + timeout=30 + ) + + return r.status_code in range(200, 300) + + except Exception as e: + print("MeTube error:", e) + return False + +def send_to_archivebox(url): + try: + r = subprocess.run( + [ + "docker", + "compose", + "-f", + f"{ARCHIVEBOX_COMPOSE_DIR}/docker-compose.yml", + "exec", + "-T", + "archivebox", + "archivebox", + "add", + url, + ], + capture_output=True, + text=True, + timeout=300 + ) + + if r.returncode != 0: + print("ArchiveBox error:") + print(r.stderr) + return False + + print(r.stdout) + return True + + except Exception as e: + print("ArchiveBox error:", e) + return False + +def notify_failure(url): + try: + requests.post( + NTFY_URL, + data=f"Failed to archive {url}", + auth=("", NTFY_TOKEN), + timeout=10 + ) + except Exception as e: + print("ntfy error:", e) + + +def process(): + indexed = load_index() + favorites = get_favorites() + + for article in favorites: + url = article["link"] + + if url in indexed: + continue + + print("Archiving:", url) + + if is_video_url(url): + success = send_to_metube(url) + else: + success = send_to_archivebox(url) + + if success: + print("Success") + save_to_index(url) + else: + print("Failed") + notify_failure(url) + + # avoid hammering services + time.sleep(60) + + +if __name__ == "__main__": + process() diff --git a/systemd/freshrss-fav-archiver.service b/systemd/freshrss-fav-archiver.service new file mode 100644 index 0000000..fd7f5f9 --- /dev/null +++ b/systemd/freshrss-fav-archiver.service @@ -0,0 +1,11 @@ +[Unit] +Description=FreshRSS favorite article archiver +After=network-online.target + +[Service] +Type=oneshot +WorkingDirectory=/home/tomasr/vps-config +ExecStart=/usr/bin/python3 /home/tomasr/vps-config/freshrss-fav-archiver.py + +StandardOutput=journal +StandardError=journal