Написали класс для сканирования

main.py

@@ -1,127 +1,79 @@
+import requests
+from bs4 import BeautifulSoup
+import re
 import sqlite3
+from datetime import datetime
 
-db_name = 'mydb.db'
+class Spyder:
+    def __init__(self):
+        self.db_name = 'database.db'
+        self.conn = sqlite3.connect(self.db_name)
+        self.cursor = self.conn.cursor()
+    def init_db(self):
+        query = '''CREATE TABLE links(
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            link TEXT NOT NULL,
+            last_scan INTEGER DEFAULT 0,
+            title TEXT DEFAULT ''
+        )'''
+        self.cursor.execute(query)
+    def scan(self,url):
+        if url.endswith('.pdf'):
+            return
+        headers = {
+            'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36'
+            }
+        try:
+            response = requests.get(
+                url,
+                headers=headers,
+                timeout=5
+            )
+        except:
+            self.update_url(url,'Scan error')
+            return
+        html = response.text
+        soup = BeautifulSoup(html,'html.parser')
+        result = soup.find_all("a", href=True)
+        title = soup.title.text
+        self.update_url(url,title)
+        for el in result:
+            if el['href'].endswith('.pdf'):
+                continue
+            if not el['href'].startswith('http'):
+                match = re.findall(r'https?://[A-Za-z0-9.-]+',url)
+                link = f"{match[0]}{el['href']}"
+            else:
+                link = el['href']
+            query = f'SELECT id FROM links WHERE link = ?'
+            self.cursor.execute(query,(link,))
+            if self.cursor.fetchone():
+                continue
+            query = f'INSERT INTO links(link) VALUES(?)'
+            self.cursor.execute(query,(link,))
+            self.conn.commit()
+    def update_url(self,url,title):
+        query = "SELECT id FROM links WHERE link = ?"
+        self.cursor.execute(query,(url,))
+        link_id = self.cursor.fetchone()
+        if link_id:
+            query = "UPDATE links SET title=?, last_scan=? WHERE id=?"
+            self.cursor.execute(query,(title,datetime.now().timestamp(),link_id[0]))
+            self.conn.commit()
+    def get_url(self):
+        query = "SELECT link FROM links ORDER BY last_scan LIMIT 1"
+        self.cursor.execute(query)
+        return self.cursor.fetchone()
+    def close(self):
+        self.conn.close()
 
-class Field:
-    def __init__(self,name,column_type):
-        self.name = name
-        self.type = column_type
-    def __str__(self):
-        return f'{self.name}:{self.type}'
+if __name__ == '__main__':
+    spyder = Spyder()
+    #spyder.init_db()
+    #spyder.scan('https://habr.com/ru/news/1046177/')
+    for _ in range(5):
+        url = spyder.get_url()
+        print(url[0])
+        spyder.scan(url[0])
+    spyder.close()
 
-class IntegerField(Field):
-    def __init__(self,name):
-        super().__init__(name,'INTEGER')
-
-class StringField(Field):
-    def __init__(self,name):
-        super().__init__(name,'TEXT')
-    
-class ModelMeta(type):
-    def __new__(cls, name, bases, attrs):
-        if name == 'BaseModel':
-            return super().__new__(cls,name,bases,attrs)
-
-        fields = dict()
-        for k,v in attrs.items():
-            if isinstance(v,Field):
-                fields[k] = v
-
-        attrs['_fields'] = fields
-        return super().__new__(cls,name,bases,attrs)
-        
-
-class BaseModel(metaclass=ModelMeta):
-    def __init__(self,**kwargs):
-        for k,v in kwargs.items():
-            setattr(self,k,v)   #{'name': 'Alex'} self.name = 'Alex'
-
-    @classmethod
-    def get_by_id(cls,id):
-        conn = sqlite3.connect(db_name)
-        cursor = conn.cursor()
-        property_list = ['id']+[f"{k}" for k, v in cls.__dict__.items() if isinstance(v, property)]
-        property_string = ','.join(property_list)
-        query = f"SELECT {property_string} FROM {cls.__name__} WHERE id = ?"
-        cursor.execute(query,(id,))
-        result = cursor.fetchone()
-        property_dict = dict()
-        for index in range(len(property_list)):
-            property_dict[property_list[index]] = result[index]
-        conn.close()
-        return cls(**property_dict)
-
-    def save(self):
-        conn = sqlite3.connect(db_name)
-        cursor = conn.cursor()
-
-        fields = []
-        values = []
-        for name, field in self._fields.items():
-            fields.append(field.name)
-            values.append(getattr(self,name,None))
-
-        query =f'''INSERT INTO {self.__class__.__name__.lower()}({','.join(fields)})
-                    VALUES({','.join(['?']*len(values))})'''
-
-        print(query,values)
-        conn.close()
-'''
-        property_list = [f"{k}" for k, v in self.__class__.__dict__.items() if isinstance(v, property)]
-        
-        if self.__id != None:
-            update_string = ','.join([f'`{p_name}`= ? ' for p_name in property_list])
-            update_values = [getattr(self,p) for p in property_list] + [self.__id]
-            query = f"UPDATE {self.__class__.__name__} SET {update_string} WHERE id = ?"
-            print(query,update_values)
-            cursor.execute(query,update_values)
-            conn.commit()
-        else:
-            property_count = ('?,'*len(property_list))[:-1]
-            property_names = ','.join(property_list)
-            property_values = [getattr(self,p) for p in property_list]
-            query = f"INSERT INTO {self.__class__.__name__}({property_names}) VALUES({property_count})"
-            cursor.execute(query,property_values)
-            conn.commit()
-            self.__id = cursor.lastrowid
-'''
-
-def validate_email(func):
-    def wrapper(self,*args,**kwargs):
-        if hasattr(self,'email'):
-            if '@' not in self.email:
-                raise ValueError('Некоректный email')
-            
-        return func(self,*args,**kwargs)
-    return wrapper
-
-class User(BaseModel):
-    id = IntegerField('id')
-    name = StringField('name')
-    age = Field('age','REAL')
-    email = StringField('email')
-
-    @validate_email
-    def save(self):
-        super().save()
-
-'''
-class Product(BaseModel):
-    def __init__(self,name,category,price,id=None):
-        super().__init__(id)
-        self.__name = name
-        self.__category = category
-        self.__price = price
-    @property
-    def name(self):
-        return self.__name
-    @property
-    def category(self):
-        return self.__category
-    @property
-    def price(self):
-        return self.__price
-'''
-
-user1 = User(name='John',age=18, email='john@example.com')
-user1.save()