1 Commits

+58 -74
View File
@@ -1,79 +1,63 @@
import requests
from bs4 import BeautifulSoup
import re
import sqlite3 import sqlite3
from datetime import datetime
class Spyder: db_name = 'mydb.db'
def __init__(self):
self.db_name = 'database.db' class User:
self.conn = sqlite3.connect(self.db_name) def __init__(self,name,email,age):
self.cursor = self.conn.cursor() self.__id = None
def init_db(self): self.__name = name
query = '''CREATE TABLE links( self.__email = email
id INTEGER PRIMARY KEY AUTOINCREMENT, self.__age = age
link TEXT NOT NULL,
last_scan INTEGER DEFAULT 0, @property
title TEXT DEFAULT '' def name(self):
)''' return self.__name
self.cursor.execute(query) @name.setter
def scan(self,url): def name(self,value):
if url.endswith('.pdf'): self.__name = value
return
headers = { @property
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36' def email(self):
} return self.__email
try:
response = requests.get( @email.setter
url, def email(self,value):
headers=headers, if '@' in value:
timeout=5 self.__email = value
)
except:
self.update_url(url,'Scan error')
return
html = response.text
soup = BeautifulSoup(html,'html.parser')
result = soup.find_all("a", href=True)
title = soup.title.text
self.update_url(url,title)
for el in result:
if el['href'].endswith('.pdf'):
continue
if not el['href'].startswith('http'):
match = re.findall(r'https?://[A-Za-z0-9.-]+',url)
link = f"{match[0]}{el['href']}"
else: else:
link = el['href'] raise ValueError('Не корректный адрес почты')
query = f'SELECT id FROM links WHERE link = ?' @property
self.cursor.execute(query,(link,)) def age(self):
if self.cursor.fetchone(): return self.__age
continue @age.setter
query = f'INSERT INTO links(link) VALUES(?)' def age(self,value):
self.cursor.execute(query,(link,)) if 0 <= value <= 120:
self.conn.commit() self.__age = value
def update_url(self,url,title): else:
query = "SELECT id FROM links WHERE link = ?" raise ValueError('Неправильно указан возраст')
self.cursor.execute(query,(url,))
link_id = self.cursor.fetchone()
if link_id:
query = "UPDATE links SET title=?, last_scan=? WHERE id=?"
self.cursor.execute(query,(title,datetime.now().timestamp(),link_id[0]))
self.conn.commit()
def get_url(self):
query = "SELECT link FROM links ORDER BY last_scan LIMIT 1"
self.cursor.execute(query)
return self.cursor.fetchone()
def close(self):
self.conn.close()
if __name__ == '__main__': def save(self):
spyder = Spyder() conn = sqlite3.connect(db_name)
#spyder.init_db() cursor = conn.cursor()
#spyder.scan('https://habr.com/ru/news/1046177/') property_list = [f"{k}" for k, v in User.__dict__.items() if isinstance(v, property)]
for _ in range(5): if '_User__id' in self.__dict__.keys():
url = spyder.get_url() update_string = ','.join([f'`{p_name}`= ? ' for p_name in property_list])
print(url[0]) update_values = [getattr(self,p) for p in property_list] + [self.__id]
spyder.scan(url[0]) query = f"UPDATE {User.__name__} SET {update_string} WHERE id = ?"
spyder.close() print(query,update_values)
else:
property_count = ('?,'*len(property_list))[:-1] #','.join(['?' for _ in range(len(property_list))])
property_names = ','.join(property_list)
property_values = [getattr(self,p) for p in property_list]
query = f"INSERT INTO {User.__name__}({property_names}) VALUES({property_count})"
print(query,property_values)
cursor.execute(query,property_values)
conn.commit()
self.__id = cursor.lastrowid
conn.close()
#print(User.__dict__)
user1 = User('Igor','igor@example.com',44)
user1.save()
user1.age = 55
user1.save()