import requests
from bs4 import BeautifulSoup
import re
import sqlite3
from datetime import datetime

class Spyder:
    def __init__(self):
        self.db_name = 'database.db'
        self.conn = sqlite3.connect(self.db_name)
        self.cursor = self.conn.cursor()
    def init_db(self):
        query = '''CREATE TABLE links(
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            link TEXT NOT NULL,
            last_scan INTEGER DEFAULT 0,
            title TEXT DEFAULT ''
        )'''
        self.cursor.execute(query)
    def scan(self,url):
        if url.endswith('.pdf'):
            return
        headers = {
            'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36'
            }
        try:
            response = requests.get(
                url,
                headers=headers,
                timeout=5
            )
        except:
            self.update_url(url,'Scan error')
            return
        html = response.text
        soup = BeautifulSoup(html,'html.parser')
        result = soup.find_all("a", href=True)
        title = soup.title.text
        self.update_url(url,title)
        for el in result:
            if el['href'].endswith('.pdf'):
                continue
            if not el['href'].startswith('http'):
                match = re.findall(r'https?://[A-Za-z0-9.-]+',url)
                link = f"{match[0]}{el['href']}"
            else:
                link = el['href']
            query = f'SELECT id FROM links WHERE link = ?'
            self.cursor.execute(query,(link,))
            if self.cursor.fetchone():
                continue
            query = f'INSERT INTO links(link) VALUES(?)'
            self.cursor.execute(query,(link,))
            self.conn.commit()
    def update_url(self,url,title):
        query = "SELECT id FROM links WHERE link = ?"
        self.cursor.execute(query,(url,))
        link_id = self.cursor.fetchone()
        if link_id:
            query = "UPDATE links SET title=?, last_scan=? WHERE id=?"
            self.cursor.execute(query,(title,datetime.now().timestamp(),link_id[0]))
            self.conn.commit()
    def get_url(self):
        query = "SELECT link FROM links ORDER BY last_scan LIMIT 1"
        self.cursor.execute(query)
        return self.cursor.fetchone()
    def close(self):
        self.conn.close()

if __name__ == '__main__':
    spyder = Spyder()
    #spyder.init_db()
    #spyder.scan('https://habr.com/ru/news/1046177/')
    for _ in range(5):
        url = spyder.get_url()
        print(url[0])
        spyder.scan(url[0])
    spyder.close()