#!/usr/local/bin/python3 # this is the telegram version ... created with the @botFather of telegram # bot name: bully3 # bot userId: bully3_bot # t.me/bully3_bot # token for HTTP API: 696861399:AAHoNorIizQ3pE4-nUE1IIPU9ywyvrn32gs # description of the Bot API, see this page: https://core.telegram.org/bots/api # telegram server: https://api.telgram.org/bot696861399:AAHoNorIizQ3pE4-nUE1IIPU9ywyvrn32gs/getUpdates # ssh -R 80:localhost:5000 serveo.net starten um die WebApp nach aussen sichtbar zu machen # not needed anymore : !!!! ngrok starten ausserhalb des virtuellen env: ngrok http 5000 # in twilio WebHooks die Https Adresse die ngrok beim startup zeigt eintragen inkl. /wa am Ende # python virtualles env aufbauen und darin Flask installieren # für windows: http://timmyreilly.azurewebsites.net/python-flask-windows-development-environment-setup/ # python searchwebsite_wa.py starten ... import sys import os.path import threading import requests import re from bs4 import BeautifulSoup import time from time import sleep from datetime import datetime from datetime import timedelta import random import telegram from telegram.error import NetworkError, Unauthorized my_song1 = "" my_interpret1 = "" my_song2 = "" my_interpret2 = "" test = True if test: # bully4 telegram_token = '669736539:AAGBydjzxqllnmKSt5Oguz8kYg05VqLG72A' telegram_root_clients = ["31165108"] telegram_admins_clients = ["31165108"] # ["31165108","625794308"] telegram_clients = ["31165108"] # ["31165108","625794308"] else: # bully3 telegram_token = '696861399:AAHoNorIizQ3pE4-nUE1IIPU9ywyvrn32gs' telegram_root_clients = ["31165108"] telegram_admins_clients = ["31165108", "625794308"] telegram_clients = ["31165108", "625794308"] ######################################################### verbose_start = "06:00" verbose_stop = "23:00" song1 = "" full_song1 = "" interpret1 = "" full_interpret1 = "" song2 = "" full_song2 = "" interpret2 = "" full_interpret2 = "" currentmsg = "" telegram_error = False telegram_error_msg = "" parse_error = False parse_error_msg = "" run_thread = 1 lastchecktime = "" nextchecktime = "" checkcounter = 0 errorcounter = 0 lastwwwparsetime = "" fullcurrenttime = "" file_name = "./bully_search.txt" foundsongslist = [] update_id = None def telegram_ahoy_reply(body): global song1 global interpret1 global song2 global interpret2 global currentmsg global lastchecktime global fullcurrenttime global lastwwwparsetime global checkcounter global errorcounter global foundsongslist global nextchecktime if (re.search("^list$", body, re.IGNORECASE) is not None) | (re.search("^l$", body, re.IGNORECASE) is not None): soup_page(15) message = "Latest list is:\n" + currentmsg lastchecktime = "" else: if body == "?" : message = "#1: '" + interpret1 + "'\n - '" + song1 + "'\n\n" + "#2: '" + interpret2 + "'\n - '" + song2 + "'\n\n" message = message + "https://www.antenne.de/programm/aktionen/bullis-geschenkt\n" else: if re.search("^s1=(.+)", body, re.IGNORECASE): song1 = re.search("^s1=(.+)",body, re.IGNORECASE).group(1) message = "Song1 changed to: '" + song1 + "'\n" + "Interpret1-Song1:\n'" + interpret1 + "' - '" + song1 + "'\n\n" + "Interpret2-Song2:\n'" + interpret2 + "' - '" + song2 + "'\n\n" else: if re.search("^i1=(.+)", body, re.IGNORECASE): interpret1 = re.search("^i1=(.+)",body, re.IGNORECASE).group(1) message = "Interpret1 changed to: '" + interpret1 + "'\n" + "#1: '" + interpret1 + "'\n - '" + song1 + "'\n\n" + "#2: '" + interpret2 + "' - '" + song2 + "'\n\n" else: if re.search("^s2=(.+)", body, re.IGNORECASE): song2 = re.search("^s2=(.+)", body, re.IGNORECASE).group(1) message = "Song2 changed to: '" + song2 + "'\n" + "#1: '" + interpret1 + "'\n - '" + song1 + "'\n\n" + "#2 :'" + interpret2 + "'\n - '" + song2 + "'\n\n" else: if re.search("^i2=(.+)", body, re.IGNORECASE): interpret2 = re.search("^i2=(.+)", body, re.IGNORECASE).group(1) message = "Interpret2 changed to: '" + interpret2 + "'\n" + "#1: '" + interpret1 + "'\n - '" + song1 + "'\n\n" + "#2: '" + interpret2 + "'\n - '" + song2 + "'\n\n" else: if re.search("^w$", body, re.IGNORECASE): write_to_file() message = "Written songs and interprets to file.\n" + "You can read them with 'r'.\n" + "#1: '" + interpret1 + "'\n - '" + song1 + "'\n\n" + "#2: '" + interpret2 + "'\n - '" + song2 + "'\n\n" else: if re.search("^r$", body, re.IGNORECASE): read_from_file() message = "Read songs and interprets from file.\n" + "You can write them with 'w'.\n" + "#1: '" + interpret1 + "'\n - '" + song1 + "'\n\n" + "#2: '" + interpret2 + "'\n - '" + song2 + "'\n\n" else: if re.search("^is1=(.+),(.+)", body, re.IGNORECASE): interpret1 = re.search("^is1=(.+),(.+)", body, re.IGNORECASE).group(1) song1 = re.search("^is1=(.+),(.+)", body, re.IGNORECASE).group(2) message = "#1 changed to: '" + interpret1 + "'-'" + song1 + "'\n" + "Interpret1-Song1:\n'" + interpret1 + "' - '" + song1 + "'\n\n" + "Interpret2-Song2:\n'" + interpret2 + "' - '" + song2 + "'\n\n" else: if re.search("^is2=(.+),(.+)", body, re.IGNORECASE): interpret2 = re.search("^is2=(.+),(.+)", body, re.IGNORECASE).group(1) song2 = re.search("^is2=(.+),(.+)", body, re.IGNORECASE).group(2) message = "#2 changed to: '" + interpret2 + "'-'" + song2 + "'\n" + "Interpret1-Song1:\n'" + interpret1 + "' - '" + song1 + "'\n\n" + "Interpret2-Song2:\n'" + interpret2 + "' - '" + song2 + "'\n\n" else: if re.search("^f", body, re.IGNORECASE): message = "List of songs and theire playtimes:\n\n" for foundsong in foundsongslist: message = message + foundsong + "\n" else: if re.search("^s", body, re.IGNORECASE): message = "Statistics:\n\n" message = message + "Last overall checktime: " + fullcurrenttime + "\n" message = message + "Last automatic checktime: " + lastwwwparsetime + "\n" message = message + "Next checktime: " + nextchecktime + "\n" message = message + "Number of checkcalls: " + str(checkcounter) + "\n" message = message + "Number of errors: " + str(errorcounter) + "\n" else: if re.search("^c", body, re.IGNORECASE): message = "Clear list of when the songs and playtimes.\n\n" foundsongslist = [] else: if re.search("^g", body, re.IGNORECASE): message = soup_2songs_page(20) else: message = "Send: \n\n'list' or 'l' for latest playlist.\n '?' for songs/title looking for.\n 's1=songname' to change song1.\n" message = message + " 'i1=artist' to change interpret1.\n 's2=songname' to change song2.\n 'i2=artist' to change interpret2.\n" message = message + " 'is1=artist,song to change #1 \n 'is2=artist,song to change #2 \n " message = message + " 'w' write i&s to file.\n 'r' read i&s from file\n" message = message + " 'f' shows found songs.\n 'c' clears list with found songs\n" message = message + " 's' shows statistics.\n 'g' gets song1,2 and interprets1,2" return message def write_to_file(): global file_name global song1, interpret1, song2, interpret2 file = open(file_name, "w") file.write(interpret1 + "\n") file.write(song1 + "\n") file.write(interpret2 + "\n") file.write(song2 + "\n") file.close() def read_from_file(): global song1, interpret1, song2, interpret2, my_song1, my_interpret1, my_song2, my_interpret2 global full_song1 global full_interpret1 global full_song2 global full_interpret2 global file_name if os.path.isfile(file_name): file = open(file_name,"r") interpret1 = file.readline().replace('\n', '') song1 = file.readline().replace('\n', '') interpret2 = file.readline().replace('\n', '') song2 = file.readline().replace('\n', '') file.close() else: song1 = my_song1 interpret1 =my_interpret1 song2 = my_song2 interpret2 = my_interpret2 full_song1 = song1 full_interpret1 = interpret1 full_song2 = song2 full_interpret2 = interpret2 def check_for_alarm_case(found_interpret_song_1, found_interpret_song_1_time,found_interpret_song_2, found_interpret_song_2_time, sendall, wait): global full_song1 global full_interpret1 global full_song2 global full_interpret2 global fullcurrenttime global lastchecktime callermsg = "" waittime = wait sendmsg = False if found_interpret_song_1 == 1 or found_interpret_song_1 == 2 or found_interpret_song_2 == 1 or found_interpret_song_2 == 2: if found_interpret_song_1 == 2: if found_interpret_song_2 == 1: #song1==2 & song2==1 callermsg = fullcurrenttime + " ++ BULLI DOPPEL ALARM! ++\n" callermsg = callermsg + "\n" callermsg = callermsg + "+ JETZT ANRUFEN!!! 01375/100 100 +\n" callermsg = callermsg + "\n" callermsg = callermsg + "Beide Songs (sind ge)laufen!" callermsg = callermsg + "\n" # send that every time ... sendmsg = True else: #song1==2 & song2==other than 1 ... callermsg = fullcurrenttime + " -- FEHLALARM! 2.Song falsch --\n" if sendall is True: # dont send that at night ... sendmsg = True else: # song1 is 1 or other, song2 is all if found_interpret_song_1 == 1: if found_interpret_song_2 == 2: callermsg = fullcurrenttime + " -- FEHLALARM! 2.Song lief zuerst. --\n" if sendall is True: # dont send that at night ... sendmsg = True else: waittime = 8 + random.randint(-2, 2) callermsg = fullcurrenttime + " ++ BULLI ERSTER ALARM! ++\n" callermsg = callermsg + found_interpret_song_1_time + ": der ERSTE Song läuft!\n" callermsg = callermsg + "Es läuft:\n" callermsg = callermsg + full_interpret1 + " '" + full_song1 + "'\n" callermsg = callermsg + "Es fehlt noch:\n" callermsg = callermsg + full_interpret2 + " '" + full_song2 + "'\n\n" callermsg = callermsg + "Jetzt sofort +491375/100 100 anrufen!\n" callermsg = callermsg + "https://www.antenne.de/musik/song-suche/ \n" callermsg = callermsg + "https://www.webradio.de/antenne-bayern/\n" callermsg = callermsg + "\n" if sendall is True: # don't send that at night ... sendmsg = True else: # song2 is 1 or 2, song1 is other then 2 or 1 if found_interpret_song_2 == 1: callermsg = fullcurrenttime + " -- ENTSPANNEN! 2.Song ohne 1. --\n" callermsg = callermsg + found_interpret_song_2_time + ": läuft der 2.Song!\n" callermsg = callermsg + "Es läuft:\n" callermsg = callermsg + full_interpret2 + " '" + full_song2 + "'\n" callermsg = callermsg + "Es sollte aber zuerst laufen:\n" callermsg = callermsg + full_interpret1 + " '" + full_song1 + "'\n" callermsg = callermsg + "\n" callermsg = callermsg + "Kein Grund anzurufen.\n" if sendall is True: # dont send that at night ... sendmsg = True return(callermsg, waittime, sendmsg) def soup_page(max_chars): global interpret1 global interpret2 global song1 global song2 global full_song1 global full_interpret1 global full_song2 global full_interpret2 global currentmsg global lastchecktime global checkcounter global fullcurrenttime global parse_error global parse_error_msg global errorcounter global foundsongslist i1 = re.compile(remove_signs_from_string(interpret1), re.IGNORECASE) t1 = re.compile(remove_signs_from_string(song1), re.IGNORECASE) i2 = re.compile(remove_signs_from_string(interpret2), re.IGNORECASE) t2 = re.compile(remove_signs_from_string(song2), re.IGNORECASE) currtime_hour = time.strftime("%H") currtime_min = time.strftime("%M") currtime_sec = time.strftime("%S") fullcurrenttime = currtime_hour + ":" + currtime_min + ":" + currtime_sec found_interpret_song_1 = 0 found_interpret_song_1_time = 0 found_interpret_song_2 = 0 found_interpret_song_2_time = 0 itemnr = 0 checkcounter = checkcounter + 1 whatsappmsg = "[" + str(checkcounter) + "]" + " Checking time: " + fullcurrenttime + " Uhr.\n\n" resultschanged = True try: page = requests.get('https://www.antenne.de/musik/song-suche/') if page.ok == True: try: soup = BeautifulSoup(page.content, 'html.parser') music_results = soup.find(class_="music_results") music_results_items = music_results.find_all(class_="music_results__content") for found_item in music_results_items: itemnr = itemnr + 1 single_musik_result = found_item.find(class_="medium-8 columns") song = single_musik_result.find(class_="song_title").get_text() song = remove_signs_from_string(song) interpret = single_musik_result.find(class_="artist").get_text() interpret = remove_signs_from_string(interpret) ps = single_musik_result.find_all('p') lief = ps[1].get_text() # finde den Zeitstempel zeit = re.search("um (.+?) Uhr", lief).group(1) if (itemnr == 1): if (lastchecktime == zeit): resultschanged = False else: resultschanged = True lastchecktime = zeit if (i1.search(interpret)): if (t1.search(song)): whatsappmsg = whatsappmsg + "1>>" full_interpret1 = interpret full_song1 = song found_interpret_song_1 = itemnr found_interpret_song_1_time = zeit newelement = found_interpret_song_1_time + " " + full_interpret1 + " '" + full_song1 + "'" if newelement not in foundsongslist: foundsongslist.append(newelement) else: whatsappmsg = whatsappmsg + "1 >" if (i2.search(interpret)): if (t2.search(song)): whatsappmsg = whatsappmsg + "2>>" full_interpret2 = interpret full_song2 = song found_interpret_song_2 = itemnr found_interpret_song_2_time = zeit newelement = found_interpret_song_2_time + " " + full_interpret2 + " '" + full_song2 + "'" if newelement not in foundsongslist: foundsongslist.append(newelement) else: whatsappmsg = whatsappmsg + "2 >" if (interpret[:max_chars] != interpret): interpret = interpret[:max_chars-3] + "..." if (song[:max_chars] != song): song = song[:max_chars-3] + "..." whatsappmsg = whatsappmsg + zeit + " " + interpret + " '" + song + "'\n" if parse_error == True: parse_error_msg = "PASS - parsing of webpage seems to work properly again ...\n" print(parse_error_msg) try: send_telegram_err_message(parse_error_msg) except: print("Message couldn't be sent to root.") errorcounter = errorcounter + 1 parse_error = False except: if parse_error == False: parse_error_msg = "ERROR - parsing of webpage failed but will try again ...\n" print(parse_error_msg) try: send_telegram_err_message(parse_error_msg) except: print("Message couldn't be sent to root.") parse_error = True errorcounter = errorcounter + 1 else: if parse_error == False: parse_error_msg = "ERROR - reading of webpage failed but will try again...\n" print(parse_error_msg) try: send_telegram_err_message(parse_error_msg) except: print("Message couldn't be sent to root.") parse_error = True errorcounter = errorcounter + 1 except: whatsappmsg = whatsappmsg + "Failed to read webpage - but will retry!!!\n\n" errorcounter = errorcounter + 1 pass currentmsg = whatsappmsg return( whatsappmsg, resultschanged, found_interpret_song_1,found_interpret_song_1_time, found_interpret_song_2,found_interpret_song_2_time ) def remove_signs_from_string(string): string = string.replace("'", "") string = string.replace("´", "") string = string.replace("’", "") string = re.sub(' +', ' ', string) string = string.strip() return string def soup_2songs_page(max_chars): global interpret1 global interpret2 global song1 global song2 global full_song1 global full_interpret1 global full_song2 global full_interpret2 global currentmsg global lastchecktime global fullcurrenttime global parse_error global parse_error_msg global errorcounter global foundsongslist iinterpret1 = interpret1 iinterpret2 = interpret2 isong1 = song1 isong2 = song2 currtime_hour = time.strftime("%H") currtime_min = time.strftime("%M") currtime_sec = time.strftime("%S") fullcurrenttime = currtime_hour + ":" + currtime_min + ":" + currtime_sec itemnr = 0 whatsappmsg = " Getting time: " + fullcurrenttime + " Uhr.\n\n" try: page = requests.get('https://www.antenne.de/programm/aktionen/bullis-geschenkt/') if page.ok is True: #print("opened page ...") try: soup = BeautifulSoup(page.content, 'html.parser') search_results = soup.find(class_="pagecontent") search_results_items = search_results.find_all(class_="text-left") is_nr = 1; for found_item in search_results_items: itemnr = itemnr + 1 item_text = found_item.get_text() if "Bulli" not in item_text: if " - " in item_text: # now split into interpret and song isstr = re.search(" *(.+?) - (.+?)$", item_text) if isstr: if is_nr == 1: iinterpret1 = isstr.group(1) isong1 = isstr.group(2) #print("Interpret1 '" + iinterpret1 + "' - Song1 '" + isong1 + "'") is_nr = is_nr + 1 elif is_nr == 2 : iinterpret2 = isstr.group(1) isong2 = isstr.group(2) #print("Interpret2 '" + iinterpret2 + "' - Song2 '" + isong2 + "'") is_nr = is_nr + 1 except: if parse_error is False: parse_error_msg = "ERROR - parsing of webpage failed but will try again ...\n" print(parse_error_msg) try: send_telegram_err_message(parse_error_msg) except: print("Message couldn't be sent to root.") parse_error = True errorcounter = errorcounter + 1 else: if parse_error is False: parse_error_msg = "ERROR - reading of webpage failed but will try again...\n" print(parse_error_msg) try: send_telegram_err_message(parse_error_msg) except: print("Message couldn't be sent to root.") parse_error = True errorcounter = errorcounter + 1 except: whatsappmsg = whatsappmsg + "Failed to read webpage - but will retry!!!\n\n" errorcounter = errorcounter + 1 pass if isong1 != song1 or isong2 != song2 or iinterpret1 != interpret1 or iinterpret2 != interpret2: whatsappmsg = whatsappmsg + "Interpret/Songs have changed:\n" whatsappmsg = whatsappmsg + "old 1: '" + interpret1 + "' - '" + song1 + "'\n" whatsappmsg = whatsappmsg + "new 1: '" + iinterpret1 + "' - '" + isong1 + "'\n" whatsappmsg = whatsappmsg + "old 2: '" + interpret2 + "' - '" + song2 + "'\n" whatsappmsg = whatsappmsg + "new 2: '" + iinterpret2 + "' - '" + isong2 + "'\n" interpret1 = iinterpret1 interpret2 = iinterpret2 song2 = isong2 song1 = isong1 write_to_file() foundsongslist = [] else: whatsappmsg = whatsappmsg + "Interpret/Songs haven't changed:\n" whatsappmsg = whatsappmsg + "1: '" + interpret1 + "' - '" + song1 + "'\n" whatsappmsg = whatsappmsg + "2: '" + interpret2 + "' - '" + song2 + "'\n" return (whatsappmsg) def parse_www(): global run_thread global currentmsg global file_name global song1 global interpret1 global song2 global interpret2 global lastwwwparsetime global nextchecktime read_from_file() # wait between two requests waittime_s = 15 nexttime_hour = time.strftime("%H") nexttime_min = time.strftime("%M") global lastchecktime startime = time.strftime("%H:%M:%S") telegrammsg = "Scan of Bully-page started " + startime + " Uhr.\n\n" telegrammsg = telegrammsg + "-> Song1:\n '" + interpret1 + "'-'" + song1 + "'\n-> Song2:\n '" + interpret2 + "'-'" + song2 + "'\n" print(telegrammsg) send_all_telegram_message(telegrammsg) while run_thread == 1: currtime_hour = time.strftime("%H") currtime_min = time.strftime("%M") currtime_sec = time.strftime("%S") lastwwwparsetime = currtime_hour + ":" + currtime_min + ":" + currtime_sec soup_2songs_page(40) (telegrammsg, resultschanged, found_interpret_song_1, found_interpret_song_1_time, found_interpret_song_2, found_interpret_song_2_time) = soup_page(40) waittime = waittime_s + random.randint(-10, 10) if ((lastchecktime > verbose_start) & (lastchecktime < verbose_stop)): sendall = True else: sendall = False (callermsg, waittime, sendmsg ) = check_for_alarm_case(found_interpret_song_1, found_interpret_song_1_time, found_interpret_song_2, found_interpret_song_2_time, sendall, waittime) # convert timestring into time() time_object = datetime.now() + timedelta(seconds=waittime) nexttime_hour = time_object.strftime("%H") nexttime_min = time_object.strftime("%M") nexttime_sec = time_object.strftime("%S") nextchecktime = nexttime_hour + ":" + nexttime_min + ":" + nexttime_sec currentmsg = telegrammsg + "\nNext checking time: " + nexttime_hour + ":" + nexttime_min + " Uhr.\n" if(resultschanged == True): telegrammsg = callermsg + telegrammsg + "\n" else: telegrammsg = "\nNo changes. Titel at " + lastchecktime + " Uhr is the last..\n" if(sendall == True): telegrammsg = telegrammsg + "Wait for " + str(waittime) + " seconds before next check. \n" telegrammsg = telegrammsg + "My next checking time: " + nexttime_hour + ":" + nexttime_min + ":" + nexttime_sec + " Uhr. " #print(telegrammsg) if(resultschanged == True): if(sendmsg == True): send_all_telegram_message(callermsg) time.sleep(waittime) stoptime = time.strftime("%H:%M:%S") telegrammsg = "Scan of Bully-page stopped " + stoptime + " Uhr. Bye.\n" print(telegrammsg) send_all_telegram_message(telegrammsg) def send_all_telegram_message(text): global telegram_clients for chat_id in telegram_clients: bully3_bot.send_message(chat_id=chat_id, text=text) def send_telegram_err_message(text): global telegram_root_clients for chat_id in telegram_root_clients: bully3_bot.send_message(chat_id=chat_id, text=text) def telegram_bot(): global run_thread global telegram_error global errorcounter global update_id try: update_id = bully3_bot.get_updates()[0].update_id except IndexError: update_id = None while 1 == run_thread: try: reply_telegram() except NetworkError: sleep(1) except Unauthorized: # The user has removed or blocked the bot. update_id += 1 def reply_telegram(): global update_id # Request updates after the last update_id for update in bully3_bot.get_updates(offset=update_id, timeout=10): update_id = update.update_id + 1 if update.message: # your bot can receive updates without messages # Reply to the message response = telegram_ahoy_reply(update.message.text) update.message.reply_text(response) class scanThread (threading.Thread): def __init__(self, threadID, name): threading.Thread.__init__(self) self.threadID = threadID self.name = name def run(self): print ("----> Starting " + self.name) parse_www() print ("----> Exiting " + self.name) class telegramThread (threading.Thread): def __init__(self, threadID, name): threading.Thread.__init__(self) self.threadID = threadID self.name = name def run(self): print ("----> Starting " + self.name) telegram_bot() print ("----> Exiting " + self.name) bully3_bot = telegram.Bot(telegram_token) if __name__ == "__main__": try: scanwwthread = scanThread(1,"parse_www") scanwwthread.start() telegramthread = telegramThread(1,"telegram_bot") telegramthread.start() except KeyboardInterrupt: print("Detected CTRL-C ... stopping the process. Please wait ...\n") run_thread = 0 scanwwthread.join() telegramthread.join() sys.exit()