import traceback import datetime import json import time import os import shutil import time import locale from PIL import Image from bs4 import BeautifulSoup from selenium import webdriver from selenium.webdriver.common.keys import Keys from selenium.webdriver.remote.webelement import WebElement from selenium.common.exceptions import NoSuchElementException def datuak_jaso(url2): try: driver2 = webdriver.Chrome('../../chromedriver/chromedriver') driver2.get(url2) pub_data = meta(driver2.page_source) img_src = None try: img1class = driver2.find_element_by_class_name( 'c-mainarticle__img') if img1class: img_src = img1class.get_attribute("src") except NoSuchElementException: img1class = None return { "argazkia": img_src, "pub_data": pub_data } except Exception as e: print("eeeeeee", e) traceback.print_exc() finally: if driver2: driver2.close() def meta(html_content) -> datetime: soup = BeautifulSoup(html_content, 'html.parser') meta_tag = soup.find('meta', {'property': 'article:published_time'}) if meta_tag: published_time = meta_tag.get('content') # datetime_object = datetime.datetime.strptime( published_time, '%Y-%m-%dT%H:%M:%S%z') return published_time else: print("No se encontró la etiqueta meta con property='article:published_time'.") return None def save_screenshot(driver: webdriver.Chrome, path: str = '/tmp/screenshot.png') -> None: tmp_path="/tmp/screen1.png" # Ref: https://stackoverflow.com/a/52572919/ original_size = driver.get_window_size() required_width = driver.execute_script( 'return document.body.parentNode.scrollWidth') required_height = driver.execute_script( 'return document.body.parentNode.scrollHeight') driver.set_window_size(required_width, required_height) # driver.save_screenshot(path) # has scrollbar driver.find_element_by_tag_name( 'body').screenshot(tmp_path) # avoids scrollbar driver.set_window_size(original_size['width'], original_size['height']) image = Image.open( tmp_path ) w2=400 h2=800 print(f"Original size : {image.size}") resize1 = image.resize(( w2, round(image.size[1]*w2/2 / image.size[0])*2 )) print(f"Original size1 : {resize1.size}") resize2 = resize1.crop((0, 0, w2, h2)) print(f"Original size1 : {resize2.size}") print("image path: ", path ) resize2.save( path ) os.unlink(tmp_path) def ireki_url(url)->webdriver: driver = webdriver.Chrome('../../chromedriver/chromedriver') driver.get(url) time.sleep(2) try: decpush = driver.find_element_by_id('onesignal-slidedown-cancel-button') decpush.click() except NoSuchElementException: pass try: decookies = driver.find_element_by_id('CybotCookiebotDialogBodyButtonDecline') decookies.click() except NoSuchElementException: pass time.sleep(2) altura_total = driver.execute_script( "return Math.max( document.body.scrollHeight, document.body.offsetHeight, document.documentElement.clientHeight, document.documentElement.scrollHeight, document.documentElement.offsetHeight);") # Establece el tamaño de la ventana del navegador para abarcar toda la altura de la página driver.set_window_size(driver.execute_script( "return window.innerWidth;"), altura_total) return driver def find( webd, type, value): el=None try: if type=="tag": el = webd.find_element_by_tag_name( value ) elif type=="class": el = webd.find_element_by_class_name(value) else: pass except NoSuchElementException: pass return el def itxi_url(driver:webdriver): if driver: driver.close() efemeride_guztiak=[] with open('../dat/efemerideak.json', 'r') as f: efemeride_guztiak = json.load(f) src_efemerideak=[ { "hedabidea":"argia.eus", "logo":"dat/Argia-logoa-2019.png", "url": "https://www.argia.eus/efemerideen-kanala?hilabetea=$hilabetea&eguna=$eguna" } ] locale.setlocale(locale.LC_TIME, 'eu_ES.UTF-8') for efemeride1 in src_efemerideak: gaur = datetime.datetime.now() eguna_key= str(gaur.day) + '-' + str(gaur.month) hilabetea = gaur.strftime('%B').capitalize() url=efemeride1['url'].replace('$hilabetea', str(gaur.month) ).replace('$eguna',str(gaur.day) ) driver=ireki_url(url) time.sleep(3) kaxak = driver.find_element_by_class_name('kanala-azken-elementuak').find_elements_by_class_name('kaxa') for kaxa1 in kaxak: url_t=None url_img=None tit=None urtea=None at=find( kaxa1, 'tag', 'a') if at: url_t = at.get_attribute("href") ka=find( kaxa1, 'tag', 'img') if ka: url_img = ka.get_attribute("src") ur=find(kaxa1,'class','urtea') if ur: urtea=ur.text ti=find(kaxa1,'class','izenburua') if ti: tit=ti.text hedabide2=next( (x for x in efemeride_guztiak if x['hedabidea'] == efemeride1['hedabidea']), None ) if not hedabide2: a1={ "hedabidea": efemeride1['hedabidea'], "data": "", "logo": efemeride1['logo'], "efemerideak": [] } efemeride_guztiak.append(a1) hedabide2=next( (x for x in efemeride_guztiak if x['hedabidea'] == efemeride1['hedabidea']), None ) #Gaurkoak ez direnak ezabatu. hedabide2['efemerideak']=[ x for x in hedabide2['efemerideak'] if 'eguna' in x and x['eguna']==eguna_key ] urlold=[ x for x in hedabide2['efemerideak'] if x['url']==url_t ] if not urlold: f={ 'id': int(time.time()*1000), 'titulua': urtea, 'deskripzioa': tit, 'argazkia': url_img, 'url': url_t, 'eguna': eguna_key } hedabide2['data']=hilabetea+"k "+str(gaur.day) hedabide2['efemerideak'].append(f) itxi_url(driver) with open('../dat/efemerideak.json', 'w', encoding='utf-8') as f: json.dump(efemeride_guztiak, f, ensure_ascii=False, indent=4)