Написали класс для сканирования

main.py

@@ -1,19 +1,79 @@
 import requests
 from bs4 import BeautifulSoup
 import re
+import sqlite3
+from datetime import datetime
+
+class Spyder:
+    def __init__(self):
+        self.db_name = 'database.db'
+        self.conn = sqlite3.connect(self.db_name)
+        self.cursor = self.conn.cursor()
+    def init_db(self):
+        query = '''CREATE TABLE links(
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            link TEXT NOT NULL,
+            last_scan INTEGER DEFAULT 0,
+            title TEXT DEFAULT ''
+        )'''
+        self.cursor.execute(query)
+    def scan(self,url):
+        if url.endswith('.pdf'):
+            return
+        headers = {
+            'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36'
+            }
+        try:
+            response = requests.get(
+                url,
+                headers=headers,
+                timeout=5
+            )
+        except:
+            self.update_url(url,'Scan error')
+            return
+        html = response.text
+        soup = BeautifulSoup(html,'html.parser')
+        result = soup.find_all("a", href=True)
+        title = soup.title.text
+        self.update_url(url,title)
+        for el in result:
+            if el['href'].endswith('.pdf'):
+                continue
+            if not el['href'].startswith('http'):
+                match = re.findall(r'https?://[A-Za-z0-9.-]+',url)
+                link = f"{match[0]}{el['href']}"
+            else:
+                link = el['href']
+            query = f'SELECT id FROM links WHERE link = ?'
+            self.cursor.execute(query,(link,))
+            if self.cursor.fetchone():
+                continue
+            query = f'INSERT INTO links(link) VALUES(?)'
+            self.cursor.execute(query,(link,))
+            self.conn.commit()
+    def update_url(self,url,title):
+        query = "SELECT id FROM links WHERE link = ?"
+        self.cursor.execute(query,(url,))
+        link_id = self.cursor.fetchone()
+        if link_id:
+            query = "UPDATE links SET title=?, last_scan=? WHERE id=?"
+            self.cursor.execute(query,(title,datetime.now().timestamp(),link_id[0]))
+            self.conn.commit()
+    def get_url(self):
+        query = "SELECT link FROM links ORDER BY last_scan LIMIT 1"
+        self.cursor.execute(query)
+        return self.cursor.fetchone()
+    def close(self):
+        self.conn.close()
+
+if __name__ == '__main__':
+    spyder = Spyder()
+    #spyder.init_db()
+    #spyder.scan('https://habr.com/ru/news/1046177/')
+    for _ in range(5):
+        url = spyder.get_url()
+        print(url[0])
+        spyder.scan(url[0])
+    spyder.close()
 
-headers = {
-    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36'
-    }
-response = requests.get(
-    'https://online.top-academy.ru/education/ochno',
-    headers=headers
-    )
-html = response.text
-soup = BeautifulSoup(html,'html.parser')
-result = soup.find_all("a", class_='styles_container__Yf7Qk')
-for el in result:
-    name = el.find("span")
-    name = re.search(r'<span>([^<>]+)</span>',str(name))
-    if name:
-        print(name.group(1))