bully/searchwebsite_telegram.py

657 lines
28 KiB
Python
Raw Normal View History

2018-11-25 13:34:20 +01:00
#!/usr/local/bin/python3
# this is the telegram version ... created with the @botFather of telegram
# bot name: bully3
# bot userId: bully3_bot
# t.me/bully3_bot
# token for HTTP API: 696861399:AAHoNorIizQ3pE4-nUE1IIPU9ywyvrn32gs
# description of the Bot API, see this page: https://core.telegram.org/bots/api
# telegram server: https://api.telgram.org/bot696861399:AAHoNorIizQ3pE4-nUE1IIPU9ywyvrn32gs/getUpdates
# ssh -R 80:localhost:5000 serveo.net starten um die WebApp nach aussen sichtbar zu machen
# not needed anymore : !!!! ngrok starten ausserhalb des virtuellen env: ngrok http 5000
# in twilio WebHooks die Https Adresse die ngrok beim startup zeigt eintragen inkl. /wa am Ende
# python virtualles env aufbauen und darin Flask installieren
# für windows: http://timmyreilly.azurewebsites.net/python-flask-windows-development-environment-setup/
# python searchwebsite_wa.py starten ...
my_song1 = "Zusammen"
my_interpret1 = "Fantastischen Vier"
my_song2 = "Beat it"
my_interpret2 = "Michael Jackson"
test = False
if test:
#bully4
telegram_token = '669736539:AAGBydjzxqllnmKSt5Oguz8kYg05VqLG72A'
telegram_root_clients = ["31165108"]
telegram_admins_clients = ["31165108"] # ["31165108","625794308"]
telegram_clients = ["31165108"] # ["31165108","625794308"]
else:
#bully3
telegram_token = '696861399:AAHoNorIizQ3pE4-nUE1IIPU9ywyvrn32gs'
telegram_root_clients = ["31165108"]
telegram_admins_clients = ["31165108","625794308"]
telegram_clients = ["31165108","625794308"]
#########################################################
verbose_start = "06:00"
verbose_stop = "23:00"
song1 = "Zusammen"
full_song1 = ""
interpret1 = "Fantastischen Vier"
full_interpret1 = ""
song2 = "Beat it"
full_song2 = ""
interpret2 = "Michael Jackson"
full_interpret2 = ""
currentmsg = ""
telegram_error = False
telegram_error_msg = ""
parse_error = False
parse_error_msg = ""
run_thread = 1
lastchecktime = ""
nextchecktime = ""
checkcounter = 0
errorcounter = 0
lastwwwparsetime = ""
fullcurrenttime = ""
file_name = "./bully_search.txt"
foundsongslist = []
import sys
import subprocess
from subprocess import check_output
import os.path
import threading
import requests
import re
from bs4 import BeautifulSoup
import time
from datetime import datetime
from datetime import timedelta
import random
def telegram_ahoy_reply(body):
global song1
global interpret1
global song2
global interpret2
global currentmsg
global lastchecktime
global fullcurrenttime
global lastwwwparsetime
global checkcounter
global errorcounter
global foundsongslist
global nextchecktime
message = ""
if((re.search("^list$", body, re.IGNORECASE)!=None) | (re.search("^l$", body, re.IGNORECASE)!=None)):
soup_page(15)
message = "Latest list is:\n" + currentmsg
lastchecktime = ""
else:
if(body == "?"):
message = "#1: '" + interpret1 + "'\n - '" + song1 + "'\n\n" + "#2: '" + interpret2 + "'\n - '" + song2 + "'\n\n"
message = message + "https://www.antenne.de/programm/aktionen/vw-bullis-geschenkt/los-gehts\n"
else:
if(re.search("^s1=(.+)", body, re.IGNORECASE)):
song1 = re.search("^s1=(.+)",body, re.IGNORECASE).group(1)
message = "Song1 changed to: '" + song1 + "'\n" + "Interpret1-Song1:\n'" + interpret1 + "' - '" + song1 + "'\n\n" + "Interpret2-Song2:\n'" + interpret2 + "' - '" + song2 + "'\n\n"
else:
if(re.search("^i1=(.+)", body, re.IGNORECASE)):
interpret1 = re.search("^i1=(.+)",body, re.IGNORECASE).group(1)
message = "Interpret1 changed to: '" + interpret1 + "'\n" + "#1: '" + interpret1 + "'\n - '" + song1 + "'\n\n" + "#2: '" + interpret2 + "' - '" + song2 + "'\n\n"
else:
if (re.search("^s2=(.+)", body, re.IGNORECASE)):
song2 = re.search("^s2=(.+)", body, re.IGNORECASE).group(1)
message = "Song2 changed to: '" + song2 + "'\n" + "#1: '" + interpret1 + "'\n - '" + song1 + "'\n\n" + "#2 :'" + interpret2 + "'\n - '" + song2 + "'\n\n"
else:
if (re.search("^i2=(.+)", body, re.IGNORECASE)):
interpret2 = re.search("^i2=(.+)", body, re.IGNORECASE).group(1)
message = "Interpret2 changed to: '" + interpret2 + "'\n" + "#1: '" + interpret1 + "'\n - '" + song1 + "'\n\n" + "#2: '" + interpret2 + "'\n - '" + song2 + "'\n\n"
else:
if (re.search("^w$", body, re.IGNORECASE)):
write_to_file()
message = "Written songs and interprets to file.\n" + "You can read them with 'r'.\n" + "#1: '" + interpret1 + "'\n - '" + song1 + "'\n\n" + "#2: '" + interpret2 + "'\n - '" + song2 + "'\n\n"
else:
if (re.search("^r$", body, re.IGNORECASE)):
read_from_file()
message = "Read songs and interprets from file.\n" + "You can write them with 'w'.\n" + "#1: '" + interpret1 + "'\n - '" + song1 + "'\n\n" + "#2: '" + interpret2 + "'\n - '" + song2 + "'\n\n"
else:
if (re.search("^is1=(.+),(.+)", body, re.IGNORECASE)):
interpret1 = re.search("^is1=(.+),(.+)", body, re.IGNORECASE).group(1)
song1 = re.search("^is1=(.+),(.+)", body, re.IGNORECASE).group(2)
message = "#1 changed to: '" + interpret1 + "'-'" + song1 + "'\n" + "Interpret1-Song1:\n'" + interpret1 + "' - '" + song1 + "'\n\n" + "Interpret2-Song2:\n'" + interpret2 + "' - '" + song2 + "'\n\n"
else:
if (re.search("^is2=(.+),(.+)", body, re.IGNORECASE)):
interpret2 = re.search("^is2=(.+),(.+)", body, re.IGNORECASE).group(1)
song2 = re.search("^is2=(.+),(.+)", body, re.IGNORECASE).group(2)
message = "#2 changed to: '" + interpret2 + "'-'" + song2 + "'\n" + "Interpret1-Song1:\n'" + interpret1 + "' - '" + song1 + "'\n\n" + "Interpret2-Song2:\n'" + interpret2 + "' - '" + song2 + "'\n\n"
else:
if (re.search("^f", body, re.IGNORECASE)):
message = "List of songs and theire playtimes:\n\n"
for foundsong in foundsongslist:
message = message + foundsong + "\n"
else:
if (re.search("^s", body, re.IGNORECASE)):
message = "Statistics:\n\n"
message = message + "Last overall checktime: " + fullcurrenttime + "\n"
message = message + "Last automatic checktime: " + lastwwwparsetime + "\n"
message = message + "Next checktime: " + nextchecktime + "\n"
message = message + "Number of checkcalls: " + str(checkcounter) + "\n"
message = message + "Number of errors: " + str(errorcounter) + "\n"
else:
if (re.search("^c", body, re.IGNORECASE)):
message = "Clear list of when the songs and playtimes.\n\n"
foundsongslist = []
else:
message = "Send: \n\n'list' or 'l' for latest playlist.\n '?' for songs/title looking for.\n 's1=songname' to change song1.\n"
message = message + " 'i1=artist' to change interpret1.\n 's2=songname' to change song2.\n 'i2=artist' to change interpret2.\n"
message = message + " 'is1=artist,song to change #1 \n 'is2=artist,song to change #2 \n "
message = message + " 'w' write i&s to file.\n 'r' read i&s from file\n"
message = message + " 'f' shows found songs.\n 'c' clears list with found songs\n"
message = message + " 's' shows statistics.\n"
return message
def write_to_file():
global file_name
global song1, interpret1, song2, interpret2
file = open(file_name, "w")
file.write(interpret1 + "\n")
file.write(song1 + "\n")
file.write(interpret2 + "\n")
file.write(song2 + "\n")
file.close()
def read_from_file():
global song1, interpret1, song2, interpret2, my_song1, my_interpret1, my_song2, my_interpret2
global full_song1
global full_interpret1
global full_song2
global full_interpret2
global file_name
if (os.path.isfile(file_name)):
file = open(file_name,"r")
interpret1 = file.readline().replace('\n', '')
song1 = file.readline().replace('\n', '')
interpret2 = file.readline().replace('\n', '')
song2 = file.readline().replace('\n', '')
file.close()
else:
song1 = my_song1
interpret1 =my_interpret1
song2 = my_song2
interpret2 = my_interpret2
full_song1 = song1
full_interpret1 = interpret1
full_song2 = song2
full_interpret2 = interpret2
def check_for_alarm_case(found_interpret_song_1, found_interpret_song_1_time,found_interpret_song_2, found_interpret_song_2_time, sendall, wait):
global full_song1
global full_interpret1
global full_song2
global full_interpret2
global fullcurrenttime
global lastchecktime
global foundsongslist
callermsg = ""
waittime = wait
sendmsg = False
if (((found_interpret_song_1 > 0) | (found_interpret_song_1 > 0))):
if ((found_interpret_song_1 == 2) & (found_interpret_song_2 == 1)):
callermsg = fullcurrenttime + " ++ BULLI DOPPEL ALARM! ++\n"
callermsg = callermsg + "\n"
callermsg = callermsg + "+ JETZT ANRUFEN!!! 01375/100 100 +\n"
callermsg = callermsg + "\n"
callermsg = callermsg + "Beide Songs (sind ge)laufen!"
callermsg = callermsg + "\n"
newelement = found_interpret_song_2_time + " " + full_interpret2 + " '" + full_song2 + "'"
if newelement not in foundsongslist:
foundsongslist.append(newelement)
print("List is: ", foundsongslist)
# send taht everytime ...
sendmsg = True
else:
if (found_interpret_song_1 == 1):
waittime = 8 + random.randint(-2, 2)
callermsg = fullcurrenttime + " ++ BULLI ERSTER ALARM! ++\n"
callermsg = callermsg + found_interpret_song_1_time + ": der ERSTE Song läuft!\n"
callermsg = callermsg + "Es läuft:\n"
callermsg = callermsg + full_interpret1 + " '" + full_song1 + "'\n"
callermsg = callermsg + "Es fehlt noch:\n"
callermsg = callermsg + full_interpret2 + " '" + full_song2 + "'\n\n"
callermsg = callermsg + "Jetzt sofort +491375/100 100 anrufen!\n"
callermsg = callermsg + "https://www.antenne.de/musik/song-suche/ \n"
callermsg = callermsg + "https://www.webradio.de/antenne-bayern/\n"
callermsg = callermsg + "\n"
newelement = found_interpret_song_1_time + " " + full_interpret1 + " '" + full_song1 + "'"
if newelement not in foundsongslist:
foundsongslist.append(newelement)
print("List is: ", foundsongslist)
if (sendall == True):
# dont send that at night ...
sendmsg = True
else:
if ((found_interpret_song_1 == 2) & (found_interpret_song_2 != 1)):
callermsg = fullcurrenttime + " -- FEHLALARM! 2.Song falsch --\n"
if (sendall == True):
# dont send that at night ...
sendmsg = True
else:
if (found_interpret_song_2 == 1):
callermsg = fullcurrenttime + " -- ENTSPANNEN! 2.Song ohne 1. --\n"
callermsg = callermsg + found_interpret_song_2_time + ": läuft der 2.Song!\n"
callermsg = callermsg + "Es läuft:\n"
callermsg = callermsg + full_interpret2 + " '" + full_song2 + "'\n"
callermsg = callermsg + "Es sollte aber zuerst laufen:\n"
callermsg = callermsg + full_interpret1 + " '" + full_song1 + "'\n"
callermsg = callermsg + "\n"
callermsg = callermsg + "Kein Grund anzurufen.\n"
newelement = found_interpret_song_2_time + " " + full_interpret2 + " '" + full_song2 + "'"
if newelement not in foundsongslist:
foundsongslist.append(newelement)
print("List is: ", foundsongslist)
if (sendall == True):
# dont send that at night ...
sendmsg = True
return(callermsg, waittime, sendmsg)
def soup_page(max_chars):
global interpret1
global interpret2
global song1
global song2
global full_song1
global full_interpret1
global full_song2
global full_interpret2
global currentmsg
global lastchecktime
global checkcounter
global fullcurrenttime
global parse_error
global parse_error_msg
global errorcounter
i1 = re.compile(interpret1, re.IGNORECASE)
t1 = re.compile(song1, re.IGNORECASE)
i2 = re.compile(interpret2, re.IGNORECASE)
t2 = re.compile(song2, re.IGNORECASE)
currtime_hour = time.strftime("%H")
currtime_min = time.strftime("%M")
currtime_sec = time.strftime("%S")
fullcurrenttime = currtime_hour + ":" + currtime_min + ":" + currtime_sec
found_interpret_song_1 = 0
found_interpret_song_1_time = 0
found_interpret_song_2 = 0
found_interpret_song_2_time = 0
itemnr = 0
checkcounter = checkcounter + 1
whatsappmsg = "[" + str(checkcounter) + "]" + " Checking time: " + fullcurrenttime + " Uhr.\n\n"
resultschanged = True
try:
page = requests.get('https://www.antenne.de/musik/song-suche/')
if page.ok == True:
try:
soup = BeautifulSoup(page.content, 'html.parser')
music_results = soup.find(class_="music_results")
music_results_items = music_results.find_all(class_="music_results__content")
for found_item in music_results_items:
itemnr = itemnr + 1
single_musik_result = found_item.find(class_="medium-8 columns")
song = single_musik_result.find(class_="song_title").get_text()
interpret = single_musik_result.find(class_="artist").get_text()
ps = single_musik_result.find_all('p')
lief = ps[1].get_text()
# finde den Zeitstempel
zeit = re.search("um (.+?) Uhr", lief).group(1)
if (itemnr == 1):
if (lastchecktime == zeit):
resultschanged = False
else:
resultschanged = True
lastchecktime = zeit
if (i1.search(interpret)):
if (t1.search(song)):
whatsappmsg = whatsappmsg + "1>>"
full_interpret1 = interpret
full_song1 = song
found_interpret_song_1 = itemnr
found_interpret_song_1_time = zeit
else:
whatsappmsg = whatsappmsg + "1 >"
if (i2.search(interpret)):
if (t2.search(song)):
whatsappmsg = whatsappmsg + "2>>"
full_interpret2 = interpret
full_song2 = song
found_interpret_song_2 = itemnr
found_interpret_song_2_time = zeit
else:
whatsappmsg = whatsappmsg + "2 >"
if (interpret[:max_chars] != interpret):
interpret = interpret[:max_chars-3] + "..."
if (song[:max_chars] != song):
song = song[:max_chars-3] + "..."
whatsappmsg = whatsappmsg + zeit + " " + interpret + " '" + song + "'\n"
if parse_error == True:
parse_error_msg = "PASS - parsing of webpage seems to work properly again ...\n"
print(parse_error_msg)
try:
bully3_bot.send_err_message(parse_error_msg)
except:
print("Message couldn't be sent to root.")
errorcounter = errorcounter + 1
parse_error = False
except:
if parse_error == False:
parse_error_msg = "ERROR - parsing of webpage failed but will try again ...\n"
print(parse_error_msg)
try:
bully3_bot.send_err_message(parse_error_msg)
except:
print("Message couldn't be sent to root.")
parse_error = True
errorcounter = errorcounter + 1
else:
if parse_error == False:
parse_error_msg = "ERROR - reading of webpage failed but will try again...\n"
print(parse_error_msg)
try:
bully3_bot.send_err_message(parse_error_msg)
except:
print("Message couldn't be sent to root.")
parse_error = True
errorcounter = errorcounter + 1
except:
whatsappmsg = whatsappmsg + "Failed to read webpage - but will retry!!!\n\n"
errorcounter = errorcounter + 1
pass
currentmsg = whatsappmsg
return( whatsappmsg, resultschanged, found_interpret_song_1,found_interpret_song_1_time, found_interpret_song_2,found_interpret_song_2_time )
def parse_www():
global run_thread
global currentmsg
global file_name
global song1
global interpret1
global song2
global interpret2
global lastwwwparsetime
global nextchecktime
read_from_file()
# wait between two requests
waittime_s = 70
nexttime_hour = time.strftime("%H")
nexttime_min = time.strftime("%M")
global lastchecktime
startime = time.strftime("%H:%M:%S")
telegrammsg = "Scan of Bully-page started " + startime + " Uhr.\n\n"
telegrammsg = telegrammsg + "-> Song1:\n '" + interpret1 + "'-'" + song1 + "'\n-> Song2:\n '" + interpret2 + "'-'" + song2 + "'\n"
print(telegrammsg)
bully3_bot.send_all_message(telegrammsg)
while run_thread == 1:
currtime_hour = time.strftime("%H")
currtime_min = time.strftime("%M")
currtime_sec = time.strftime("%S")
lastwwwparsetime = currtime_hour + ":" + currtime_min + ":" + currtime_sec
(telegrammsg, resultschanged, found_interpret_song_1, found_interpret_song_1_time, found_interpret_song_2,
found_interpret_song_2_time) = soup_page(40)
waittime = waittime_s + random.randint(-20, 20)
if ((lastchecktime > verbose_start) & (lastchecktime < verbose_stop)):
sendall = True
else:
sendall = False
(callermsg, waittime, sendmsg ) = check_for_alarm_case(found_interpret_song_1, found_interpret_song_1_time,
found_interpret_song_2, found_interpret_song_2_time, sendall, waittime)
# convert timestring into time()
time_object = datetime.now() + timedelta(seconds=waittime)
nexttime_hour = time_object.strftime("%H")
nexttime_min = time_object.strftime("%M")
nexttime_sec = time_object.strftime("%S")
nextchecktime = nexttime_hour + ":" + nexttime_min + ":" + nexttime_sec
currentmsg = telegrammsg + "\nNext checking time: " + nexttime_hour + ":" + nexttime_min + " Uhr.\n"
if(resultschanged == True):
telegrammsg = callermsg + telegrammsg + "\n"
else:
telegrammsg = "\nNo changes. Titel at " + lastchecktime + " Uhr is the last..\n"
if(sendall == True):
telegrammsg = telegrammsg + "Wait for " + str(waittime) + " seconds before next check. \n"
telegrammsg = telegrammsg + "My next checking time: " + nexttime_hour + ":" + nexttime_min + ":" + nexttime_sec + " Uhr. "
#print(telegrammsg)
if(resultschanged == True):
if(sendmsg == True):
bully3_bot.send_all_message(callermsg)
time.sleep(waittime)
stoptime = time.strftime("%H:%M:%S")
telegrammsg = "Scan of Bully-page stopped " + stoptime + " Uhr. Bye.\n"
print(telegrammsg)
bully3_bot.send_all_message(telegrammsg)
def telegram_bot():
global run_thread
global telegram_error
global errorcounter
new_offset = 0
bully3_bot.get_updates(new_offset)
all_messages = bully3_bot.get_updates(new_offset)
if (len(all_messages) > 0):
for current_message in all_messages:
current_message_id = current_message['update_id']
new_offset = current_message_id
while 1==run_thread:
try:
all_messages = bully3_bot.get_updates(new_offset)
if( len(all_messages)>0):
for current_message in all_messages:
reponse = ""
print("Received: ",current_message)
#/if 'edited_message' not in
# current_message_type = 'edited_message'
current_message_id = current_message['update_id']
if 'message' in current_message:
mtype = 'message'
elif 'edited_message' in current_message:
mtype = 'edited_message'
else:
mtype = 'unknown'
if mtype != 'unknown':
current_chat_id = current_message[mtype]['chat']['id']
if 'text' not in current_message[mtype]:
response = "Welcome!"
else:
current_chat_text = current_message[mtype]['text']
#print("ID ",current_chat_id , "send: ", current_chat_text)
response = telegram_ahoy_reply(current_chat_text)
bully3_bot.send_message(current_chat_id, response)
else:
telegram_error_msg = "ERROR - unknown message type of '" + mtype + "'...\n"
bully3_bot.send_err_message(telegram_error_msg)
errorcounter = errorcounter + 1
new_offset = current_message_id +1
if telegram_error == True:
telegram_error_msg = "PASS - handling of telegram messages seems to work properly again ...\n"
print(telegram_error_msg)
try:
bully3_bot.send_err_message(telegram_error_msg)
telegram_error = False
except:
print("Message couldn't be sent to root.")
errorcounter = errorcounter + 1
pass
except:
if telegram_error == False:
telegram_error_msg = "ERROR during handling of telegram messages .. but will continue here...\n"
telegram_error_msg = telegram_error_msg + "Unexpected error:{}".format(sys.exc_info()[0])
print(telegram_error_msg)
try:
bully3_bot.send_err_message(telegram_error_msg)
except:
print("Error couldn't be sent to root.")
pass
telegram_error = True
errorcounter = errorcounter + 1
pass
class scanThread (threading.Thread):
def __init__(self, threadID, name):
threading.Thread.__init__(self)
self.threadID = threadID
self.name = name
def run(self):
print ("----> Starting " + self.name)
parse_www()
print ("----> Exiting " + self.name)
class telegramThread (threading.Thread):
def __init__(self, threadID, name):
threading.Thread.__init__(self)
self.threadID = threadID
self.name = name
def run(self):
print ("----> Starting " + self.name)
telegram_bot()
print ("----> Exiting " + self.name)
class TelegramBotHandler:
def __init__(self, token):
self.token = token
self.api_url = "https://api.telegram.org/bot{}/".format(token)
#url = "https://api.telegram.org/bot<token>/"
def get_updates(self, offset=0, timeout=30):
method = 'getUpdates'
params = {'timeout': timeout, 'offset': offset}
result_json = ""
try:
resp = requests.get(self.api_url + method, params)
if 'result' in resp.json():
result_json = resp.json()['result']
except:
errorcounter = errorcounter + 1
pass
return result_json
def send_message(self, chat_id, text):
params = {'chat_id': chat_id, 'text': text, 'parse_mode': 'HTML'}
method = 'sendMessage'
resp = requests.post(self.api_url + method, params)
return resp
def send_all_message(self, text):
global telegram_clients
for chat_id in telegram_clients:
params = {'chat_id': chat_id, 'text': text, 'parse_mode': 'HTML'}
method = 'sendMessage'
resp = requests.post(self.api_url + method, params)
return resp
def send_err_message(self, text):
global telegram_root_clients
for chat_id in telegram_root_clients:
params = {'chat_id': chat_id, 'text': text, 'parse_mode': 'HTML'}
method = 'sendMessage'
resp = requests.post(self.api_url + method, params)
return resp
def get_first_update(self):
get_result = self.get_updates()
if len(get_result) > 0:
last_update = get_result[0]
else:
last_update = None
return last_update
bully3_bot = TelegramBotHandler(telegram_token)
if __name__ == "__main__":
try:
scanwwthread = scanThread(1,"parse_www")
scanwwthread.start()
telegramthread = telegramThread(1,"telegram_bot")
telegramthread.start()
except KeyboardInterrupt:
print("Detected CTRL-C ... stopping the process. Please wait ...\n")
run_thread = 0
scanwwthread.join()
telegramthread.join()
sys.exit()