4 Commits

2 changed files with 123 additions and 75 deletions
+122 -74
View File
@@ -1,79 +1,127 @@
import requests
from bs4 import BeautifulSoup
import re
import sqlite3 import sqlite3
from datetime import datetime
class Spyder: db_name = 'mydb.db'
def __init__(self):
self.db_name = 'database.db' class Field:
self.conn = sqlite3.connect(self.db_name) def __init__(self,name,column_type):
self.cursor = self.conn.cursor() self.name = name
def init_db(self): self.type = column_type
query = '''CREATE TABLE links( def __str__(self):
id INTEGER PRIMARY KEY AUTOINCREMENT, return f'{self.name}:{self.type}'
link TEXT NOT NULL,
last_scan INTEGER DEFAULT 0, class IntegerField(Field):
title TEXT DEFAULT '' def __init__(self,name):
)''' super().__init__(name,'INTEGER')
self.cursor.execute(query)
def scan(self,url): class StringField(Field):
if url.endswith('.pdf'): def __init__(self,name):
return super().__init__(name,'TEXT')
headers = {
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36' class ModelMeta(type):
} def __new__(cls, name, bases, attrs):
try: if name == 'BaseModel':
response = requests.get( return super().__new__(cls,name,bases,attrs)
url,
headers=headers, fields = dict()
timeout=5 for k,v in attrs.items():
) if isinstance(v,Field):
except: fields[k] = v
self.update_url(url,'Scan error')
return attrs['_fields'] = fields
html = response.text return super().__new__(cls,name,bases,attrs)
soup = BeautifulSoup(html,'html.parser')
result = soup.find_all("a", href=True)
title = soup.title.text class BaseModel(metaclass=ModelMeta):
self.update_url(url,title) def __init__(self,**kwargs):
for el in result: for k,v in kwargs.items():
if el['href'].endswith('.pdf'): setattr(self,k,v) #{'name': 'Alex'} self.name = 'Alex'
continue
if not el['href'].startswith('http'): @classmethod
match = re.findall(r'https?://[A-Za-z0-9.-]+',url) def get_by_id(cls,id):
link = f"{match[0]}{el['href']}" conn = sqlite3.connect(db_name)
cursor = conn.cursor()
property_list = ['id']+[f"{k}" for k, v in cls.__dict__.items() if isinstance(v, property)]
property_string = ','.join(property_list)
query = f"SELECT {property_string} FROM {cls.__name__} WHERE id = ?"
cursor.execute(query,(id,))
result = cursor.fetchone()
property_dict = dict()
for index in range(len(property_list)):
property_dict[property_list[index]] = result[index]
conn.close()
return cls(**property_dict)
def save(self):
conn = sqlite3.connect(db_name)
cursor = conn.cursor()
fields = []
values = []
for name, field in self._fields.items():
fields.append(field.name)
values.append(getattr(self,name,None))
query =f'''INSERT INTO {self.__class__.__name__.lower()}({','.join(fields)})
VALUES({','.join(['?']*len(values))})'''
print(query,values)
conn.close()
'''
property_list = [f"{k}" for k, v in self.__class__.__dict__.items() if isinstance(v, property)]
if self.__id != None:
update_string = ','.join([f'`{p_name}`= ? ' for p_name in property_list])
update_values = [getattr(self,p) for p in property_list] + [self.__id]
query = f"UPDATE {self.__class__.__name__} SET {update_string} WHERE id = ?"
print(query,update_values)
cursor.execute(query,update_values)
conn.commit()
else: else:
link = el['href'] property_count = ('?,'*len(property_list))[:-1]
query = f'SELECT id FROM links WHERE link = ?' property_names = ','.join(property_list)
self.cursor.execute(query,(link,)) property_values = [getattr(self,p) for p in property_list]
if self.cursor.fetchone(): query = f"INSERT INTO {self.__class__.__name__}({property_names}) VALUES({property_count})"
continue cursor.execute(query,property_values)
query = f'INSERT INTO links(link) VALUES(?)' conn.commit()
self.cursor.execute(query,(link,)) self.__id = cursor.lastrowid
self.conn.commit() '''
def update_url(self,url,title):
query = "SELECT id FROM links WHERE link = ?"
self.cursor.execute(query,(url,))
link_id = self.cursor.fetchone()
if link_id:
query = "UPDATE links SET title=?, last_scan=? WHERE id=?"
self.cursor.execute(query,(title,datetime.now().timestamp(),link_id[0]))
self.conn.commit()
def get_url(self):
query = "SELECT link FROM links ORDER BY last_scan LIMIT 1"
self.cursor.execute(query)
return self.cursor.fetchone()
def close(self):
self.conn.close()
if __name__ == '__main__': def validate_email(func):
spyder = Spyder() def wrapper(self,*args,**kwargs):
#spyder.init_db() if hasattr(self,'email'):
#spyder.scan('https://habr.com/ru/news/1046177/') if '@' not in self.email:
for _ in range(5): raise ValueError('Некоректный email')
url = spyder.get_url()
print(url[0])
spyder.scan(url[0])
spyder.close()
return func(self,*args,**kwargs)
return wrapper
class User(BaseModel):
id = IntegerField('id')
name = StringField('name')
age = Field('age','REAL')
email = StringField('email')
@validate_email
def save(self):
super().save()
'''
class Product(BaseModel):
def __init__(self,name,category,price,id=None):
super().__init__(id)
self.__name = name
self.__category = category
self.__price = price
@property
def name(self):
return self.__name
@property
def category(self):
return self.__category
@property
def price(self):
return self.__price
'''
user1 = User(name='John',age=18, email='john@example.com')
user1.save()
BIN
View File
Binary file not shown.