fenoglio/bully
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

last changes added like new style of song page

Browse source
This commit is contained in:
jnoack@itsblue.de 2019-02-06 17:05:22 +01:00
parent cf7dd64d46
commit 6369ef3f52
1 changed files with 796 additions and 710 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

334
searchwebsite_telegram2.py Normal file → Executable file
View file

@ -9,16 +9,6 @@
# description of the Bot API, see this page: https://core.telegram.org/bots/api # description of the Bot API, see this page: https://core.telegram.org/bots/api
# telegram server: https://api.telgram.org/bot696861399:AAHoNorIizQ3pE4-nUE1IIPU9ywyvrn32gs/getUpdates # telegram server: https://api.telgram.org/bot696861399:AAHoNorIizQ3pE4-nUE1IIPU9ywyvrn32gs/getUpdates
# ssh -R 80:localhost:5000 serveo.net starten um die WebApp nach aussen sichtbar zu machen
# not needed anymore : !!!! ngrok starten ausserhalb des virtuellen env: ngrok http 5000
# in twilio WebHooks die Https Adresse die ngrok beim startup zeigt eintragen inkl. /wa am Ende
# python virtualles env aufbauen und darin Flask installieren
# für windows: http://timmyreilly.azurewebsites.net/python-flask-windows-development-environment-setup/
# python searchwebsite_wa.py starten ...
import sys import sys
import os.path import os.path
import threading import threading
@ -33,10 +23,10 @@ import random
import telegram import telegram
from telegram.error import NetworkError, Unauthorized from telegram.error import NetworkError, Unauthorized
my_song1 = "" my_song1 = "unknownsong1"
my_interpret1 = "" my_interpret1 = "unknownart1"
my_song2 = "" my_song2 = "unknownsong2"
my_interpret2 = "" my_interpret2 = "unknownart2"
test = True test = True
@ -67,6 +57,8 @@ full_song2 = ""
interpret2 = "" interpret2 = ""
full_interpret2 = "" full_interpret2 = ""
auto_check_for_is1_is2 = True
currentmsg = "" currentmsg = ""
telegram_error = False telegram_error = False
telegram_error_msg = "" telegram_error_msg = ""
@ -75,12 +67,15 @@ parse_error_msg = ""
run_thread = 1 run_thread = 1
lastchecktime = "" lastchecktime = ""
nextchecktime = "" nextchecktime = ""
starttime = ""
checkcounter = 0 checkcounter = 0
errorcounter = 0 errorcounter = 0
nroftries = 0
lastwwwparsetime = "" lastwwwparsetime = ""
fullcurrenttime = "" fullcurrenttime = ""
file_name = "./bully_search.txt" file_name = "./bully_search.txt"
send_false_alarms = False
foundsongslist = [] foundsongslist = []
@ -100,6 +95,8 @@ def telegram_ahoy_reply(body):
global errorcounter global errorcounter
global foundsongslist global foundsongslist
global nextchecktime global nextchecktime
global starttime
global auto_check_for_is1_is2
if (re.search("^list$", body, re.IGNORECASE) is not None) | (re.search("^l$", body, re.IGNORECASE) is not None): if (re.search("^list$", body, re.IGNORECASE) is not None) | (re.search("^l$", body, re.IGNORECASE) is not None):
soup_page(15) soup_page(15)
@ -151,6 +148,15 @@ def telegram_ahoy_reply(body):
else: else:
if re.search("^s", body, re.IGNORECASE): if re.search("^s", body, re.IGNORECASE):
message = "Statistics:\n\n" message = "Statistics:\n\n"
message = message + "Start time: " + starttime + "\n"
if auto_check_for_is1_is2:
message = message + "Auto getting is1,is2: ON\n"
else:
message = message + "Auto getting is1,is2: OFF\n"
if send_false_alarms:
message = message + "Sending false alarms: ON\n"
else:
message = message + "Sending false alarms: ON\n"
message = message + "Last overall checktime: " + fullcurrenttime + "\n" message = message + "Last overall checktime: " + fullcurrenttime + "\n"
message = message + "Last automatic checktime: " + lastwwwparsetime + "\n" message = message + "Last automatic checktime: " + lastwwwparsetime + "\n"
message = message + "Next checktime: " + nextchecktime + "\n" message = message + "Next checktime: " + nextchecktime + "\n"
@ -162,14 +168,34 @@ def telegram_ahoy_reply(body):
foundsongslist = [] foundsongslist = []
else: else:
if re.search("^g", body, re.IGNORECASE): if re.search("^g", body, re.IGNORECASE):
message = soup_2songs_page(20) message = soup_2songs_page()
else: else:
message = "Send: \n\n'list' or 'l' for latest playlist.\n '?' for songs/title looking for.\n 's1=songname' to change song1.\n" if re.search("^a", body, re.IGNORECASE):
message = message + " 'i1=artist' to change interpret1.\n 's2=songname' to change song2.\n 'i2=artist' to change interpret2.\n" message = "Automatic getting songs1,2 and interprets1,2:\n"
message = message + " 'is1=artist,song to change #1 \n 'is2=artist,song to change #2 \n " if auto_check_for_is1_is2:
message = message + " 'w' write i&s to file.\n 'r' read i&s from file\n" auto_check_for_is1_is2 = False
message = message + " 'f' shows found songs.\n 'c' clears list with found songs\n" message = message + "SWITCHED OFF now."
message = message + " 's' shows statistics.\n 'g' gets song1,2 and interprets1,2" else:
auto_check_for_is1_is2 = True
message = message + "SWITCHED ON now."
else:
if re.search("^m", body, re.IGNORECASE):
message = "Sending even false alarms:\n"
if send_false_alarms:
send_false_alarms = False
message = message + "SWITCHED OFF now."
else:
send_false_alarms = True
message = message + "SWITCHED ON now."
else:
message = "Send: \n\n'list' or 'l' for latest playlist.\n '?' for songs/title looking for.\n 's1=songname' to change song1.\n"
message = message + " 'i1=artist' to change interpret1.\n 's2=songname' to change song2.\n 'i2=artist' to change interpret2.\n"
message = message + " 'is1=artist,song to change #1 \n 'is2=artist,song to change #2 \n "
message = message + " 'w' write i&s to file.\n 'r' read i&s from file\n"
message = message + " 'f' shows found songs.\n 'c' clears list with found songs\n"
message = message + " 's' shows statistics.\n 'g' gets song1,2 and interprets1,2\n"
message = message + " 'a' toggle auto get is1,is2\n 'm' toggle false alarms\n"
return message return message
@ -218,6 +244,7 @@ def check_for_alarm_case(found_interpret_song_1, found_interpret_song_1_time,fou
global full_interpret2 global full_interpret2
global fullcurrenttime global fullcurrenttime
global lastchecktime global lastchecktime
global send_false_alarms
callermsg = "" callermsg = ""
waittime = wait waittime = wait
sendmsg = False sendmsg = False
@ -249,7 +276,7 @@ def check_for_alarm_case(found_interpret_song_1, found_interpret_song_1_time,fou
callermsg = fullcurrenttime + " -- FEHLALARM! 2.Song lief zuerst. --\n" callermsg = fullcurrenttime + " -- FEHLALARM! 2.Song lief zuerst. --\n"
if sendall is True: if sendall is True:
# dont send that at night ... # dont send that at night ...
sendmsg = True sendmsg = send_false_alarms
else: else:
waittime = 8 + random.randint(-2, 2) waittime = 8 + random.randint(-2, 2)
callermsg = fullcurrenttime + " ++ BULLI ERSTER ALARM! ++\n" callermsg = fullcurrenttime + " ++ BULLI ERSTER ALARM! ++\n"
@ -277,12 +304,35 @@ def check_for_alarm_case(found_interpret_song_1, found_interpret_song_1_time,fou
callermsg = callermsg + "\n" callermsg = callermsg + "\n"
callermsg = callermsg + "Kein Grund anzurufen.\n" callermsg = callermsg + "Kein Grund anzurufen.\n"
if sendall is True: if sendall is True:
# dont send that at night ... # dont send that at night ...
sendmsg = True sendmsg = send_false_alarms
return(callermsg, waittime, sendmsg) return(callermsg, waittime, sendmsg)
def current_day_string(currtime):
wdays = ["So", "Mo", "Di", "Mi" , "Do", "Fr", "Sa"]
currtime_weekday = wdays[int(time.strftime("%w"))]
currtime_day = time.strftime("%d")
currtime_month = time.strftime("%m")
fullcurrentday = currtime_weekday + "," + currtime_day + "." + currtime_month + " " +currtime
#print("Currentday: " + fullcurrentday);
return fullcurrentday
def current_time_string():
currtime_hour = time.strftime("%H")
currtime_min = time.strftime("%M")
currtime_sec = time.strftime("%S")
fullcurrenttime = currtime_hour + ":" + currtime_min + ":" + currtime_sec
#print("Currenttime: " + fullcurrenttime);
return fullcurrenttime
def soup_page(max_chars): def soup_page(max_chars):
@ -302,17 +352,14 @@ def soup_page(max_chars):
global parse_error_msg global parse_error_msg
global errorcounter global errorcounter
global foundsongslist global foundsongslist
global nroftries
i1 = re.compile(remove_signs_from_string(interpret1), re.IGNORECASE) i1 = re.compile(remove_signs_from_string(interpret1), re.IGNORECASE)
t1 = re.compile(remove_signs_from_string(song1), re.IGNORECASE) t1 = re.compile(remove_signs_from_string(song1), re.IGNORECASE)
i2 = re.compile(remove_signs_from_string(interpret2), re.IGNORECASE) i2 = re.compile(remove_signs_from_string(interpret2), re.IGNORECASE)
t2 = re.compile(remove_signs_from_string(song2), re.IGNORECASE) t2 = re.compile(remove_signs_from_string(song2), re.IGNORECASE)
currtime_hour = time.strftime("%H") fullcurrenttime = current_day_string(current_time_string())
currtime_min = time.strftime("%M")
currtime_sec = time.strftime("%S")
fullcurrenttime = currtime_hour + ":" + currtime_min + ":" + currtime_sec
found_interpret_song_1 = 0 found_interpret_song_1 = 0
found_interpret_song_1_time = 0 found_interpret_song_1_time = 0
@ -332,7 +379,7 @@ def soup_page(max_chars):
try: try:
soup = BeautifulSoup(page.content, 'html.parser') soup = BeautifulSoup(page.content, 'html.parser')
music_results = soup.find(class_="music_results") music_results = soup.find(class_="music_results")
music_results_items = music_results.find_all(class_="music_results__content") music_results_items = music_results.find_all(class_="music_results__content", limit = 5)
for found_item in music_results_items: for found_item in music_results_items:
itemnr = itemnr + 1 itemnr = itemnr + 1
@ -358,7 +405,7 @@ def soup_page(max_chars):
full_interpret1 = interpret full_interpret1 = interpret
full_song1 = song full_song1 = song
found_interpret_song_1 = itemnr found_interpret_song_1 = itemnr
found_interpret_song_1_time = zeit found_interpret_song_1_time = current_day_string(zeit)
newelement = found_interpret_song_1_time + " " + full_interpret1 + " '" + full_song1 + "'" newelement = found_interpret_song_1_time + " " + full_interpret1 + " '" + full_song1 + "'"
if newelement not in foundsongslist: if newelement not in foundsongslist:
@ -373,7 +420,7 @@ def soup_page(max_chars):
full_interpret2 = interpret full_interpret2 = interpret
full_song2 = song full_song2 = song
found_interpret_song_2 = itemnr found_interpret_song_2 = itemnr
found_interpret_song_2_time = zeit found_interpret_song_2_time = current_day_string(zeit)
newelement = found_interpret_song_2_time + " " + full_interpret2 + " '" + full_song2 + "'" newelement = found_interpret_song_2_time + " " + full_interpret2 + " '" + full_song2 + "'"
if newelement not in foundsongslist: if newelement not in foundsongslist:
@ -390,21 +437,25 @@ def soup_page(max_chars):
if parse_error == True: if parse_error == True:
parse_error_msg = "PASS - parsing of webpage seems to work properly again ...\n" parse_error_msg = "PASS - parsing of webpage seems to work properly again ...\n"
print(parse_error_msg) print(parse_error_msg)
try: if nroftries > 5:
send_telegram_err_message(parse_error_msg) try:
except: send_telegram_err_message(parse_error_msg)
print("Message couldn't be sent to root.") except:
errorcounter = errorcounter + 1 print("Message couldn't be sent to root.")
errorcounter = errorcounter + 1
nroftries = 0
parse_error = False parse_error = False
except: except:
if parse_error == False: if parse_error == False:
parse_error_msg = "ERROR - parsing of webpage failed but will try again ...\n" parse_error_msg = "ERROR - parsing of webpage failed but will try again ...\n"
print(parse_error_msg) print(parse_error_msg)
try: if nroftries > 5:
send_telegram_err_message(parse_error_msg) try:
except: send_telegram_err_message(parse_error_msg)
print("Message couldn't be sent to root.") except:
print("Message couldn't be sent to root.")
nroftries = nroftries + 1
parse_error = True parse_error = True
errorcounter = errorcounter + 1 errorcounter = errorcounter + 1
@ -412,15 +463,19 @@ def soup_page(max_chars):
if parse_error == False: if parse_error == False:
parse_error_msg = "ERROR - reading of webpage failed but will try again...\n" parse_error_msg = "ERROR - reading of webpage failed but will try again...\n"
print(parse_error_msg) print(parse_error_msg)
try: if nroftries > 5:
send_telegram_err_message(parse_error_msg) try:
except: send_telegram_err_message(parse_error_msg)
print("Message couldn't be sent to root.") except:
print("Message couldn't be sent to root.")
nroftries = nroftries + 1
parse_error = True parse_error = True
errorcounter = errorcounter + 1 errorcounter = errorcounter + 1
except: except:
whatsappmsg = whatsappmsg + "Failed to read webpage - but will retry!!!\n\n" if nroftries > 5:
whatsappmsg = whatsappmsg + "Failed to read webpage - but will retry!!!\n\n"
nroftries = nroftries + 1
errorcounter = errorcounter + 1 errorcounter = errorcounter + 1
pass pass
@ -435,7 +490,7 @@ def remove_signs_from_string(string):
string = string.strip() string = string.strip()
return string return string
def soup_2songs_page(max_chars): def soup_2songs_page():
global interpret1 global interpret1
global interpret2 global interpret2
@ -452,39 +507,39 @@ def soup_2songs_page(max_chars):
global parse_error_msg global parse_error_msg
global errorcounter global errorcounter
global foundsongslist global foundsongslist
global auto_check_for_is1_is2
iinterpret1 = interpret1
iinterpret2 = interpret2
isong1 = song1
isong2 = song2
currtime_hour = time.strftime("%H") whatsappmsg = ""
currtime_min = time.strftime("%M") if auto_check_for_is1_is2:
currtime_sec = time.strftime("%S")
fullcurrenttime = currtime_hour + ":" + currtime_min + ":" + currtime_sec iinterpret1 = interpret1
itemnr = 0 iinterpret2 = interpret2
isong1 = song1
isong2 = song2
whatsappmsg = " Getting time: " + fullcurrenttime + " Uhr.\n\n" fullcurrenttime = current_day_string(current_time_string())
itemnr = 0
try: whatsappmsg = " Getting time: " + fullcurrenttime + " Uhr.\n\n"
page = requests.get('https://www.antenne.de/programm/aktionen/bullis-geschenkt/')
if page.ok is True: try:
#print("opened page ...") page = requests.get('https://www.antenne.de/programm/aktionen/bullis-geschenkt/')
try:
soup = BeautifulSoup(page.content, 'html.parser')
search_results = soup.find(class_="pagecontent")
search_results_items = search_results.find_all(class_="text-left")
is_nr = 1; if page.ok is True:
for found_item in search_results_items: #print("opened page ...")
itemnr = itemnr + 1 try:
item_text = found_item.get_text() soup = BeautifulSoup(page.content, 'html.parser')
if "Bulli" not in item_text: search_results = soup.find(class_="pagecontent")
if " - " in item_text: search_results_items = search_results.find_all("h5")
is_nr = 1;
for found_item in search_results_items:
item_text = found_item.get_text()
#print(item_text)
if ":" in item_text:
# now split into interpret and song # now split into interpret and song
isstr = re.search(" *(.+?) - (.+?)$", item_text) isstr = re.search(" *(.+?) *: *(.+?)$", item_text)
if isstr: if isstr:
if is_nr == 1: if is_nr == 1:
iinterpret1 = isstr.group(1) iinterpret1 = isstr.group(1)
@ -496,51 +551,69 @@ def soup_2songs_page(max_chars):
isong2 = isstr.group(2) isong2 = isstr.group(2)
#print("Interpret2 '" + iinterpret2 + "' - Song2 '" + isong2 + "'") #print("Interpret2 '" + iinterpret2 + "' - Song2 '" + isong2 + "'")
is_nr = is_nr + 1 is_nr = is_nr + 1
except: if parse_error == True:
parse_error_msg = "PASS - parsing of webpage seems to work properly again ...\n"
print(parse_error_msg)
if nroftries > 5:
try:
send_telegram_err_message(parse_error_msg)
except:
print("Message couldn't be sent to root.")
errorcounter = errorcounter + 1
nroftries = 0
parse_error = False
except:
if parse_error is False:
parse_error_msg = "ERROR - parsing of webpage failed but will try again ...\n"
print(parse_error_msg)
if nroftries>5 :
try:
send_telegram_err_message(parse_error_msg)
except:
print("Message couldn't be sent to root.")
nroftries = nroftries + 1
parse_error = True
errorcounter = errorcounter + 1
else:
if parse_error is False: if parse_error is False:
parse_error_msg = "ERROR - parsing of webpage failed but will try again ...\n" parse_error_msg = "ERROR - reading of webpage failed but will try again...\n"
print(parse_error_msg) print(parse_error_msg)
try: if nroftries >5 :
send_telegram_err_message(parse_error_msg) try:
except: send_telegram_err_message(parse_error_msg)
print("Message couldn't be sent to root.") except:
print("Message couldn't be sent to root.")
nroftries = nroftries + 1
parse_error = True parse_error = True
errorcounter = errorcounter + 1 errorcounter = errorcounter + 1
except:
if nroftries > 5:
whatsappmsg = whatsappmsg + "Failed to read webpage - but will retry!!!\n\n"
nroftries = nroftries + 1
errorcounter = errorcounter + 1
pass
if isong1 != song1 or isong2 != song2 or iinterpret1 != interpret1 or iinterpret2 != interpret2:
whatsappmsg = whatsappmsg + "Interpret/Songs have changed:\n"
whatsappmsg = whatsappmsg + "old 1: '" + interpret1 + "' - '" + song1 + "'\n"
whatsappmsg = whatsappmsg + "new 1: '" + iinterpret1 + "' - '" + isong1 + "'\n"
whatsappmsg = whatsappmsg + "old 2: '" + interpret2 + "' - '" + song2 + "'\n"
whatsappmsg = whatsappmsg + "new 2: '" + iinterpret2 + "' - '" + isong2 + "'\n"
interpret1 = iinterpret1
interpret2 = iinterpret2
song2 = isong2
song1 = isong1
write_to_file()
foundsongslist = []
else: else:
if parse_error is False: whatsappmsg = whatsappmsg + "Interpret/Songs haven't changed:\n"
parse_error_msg = "ERROR - reading of webpage failed but will try again...\n" whatsappmsg = whatsappmsg + "1: '" + interpret1 + "' - '" + song1 + "'\n"
print(parse_error_msg) whatsappmsg = whatsappmsg + "2: '" + interpret2 + "' - '" + song2 + "'\n"
try:
send_telegram_err_message(parse_error_msg)
except:
print("Message couldn't be sent to root.")
parse_error = True
errorcounter = errorcounter + 1
except:
whatsappmsg = whatsappmsg + "Failed to read webpage - but will retry!!!\n\n"
errorcounter = errorcounter + 1
pass
if isong1 != song1 or isong2 != song2 or iinterpret1 != interpret1 or iinterpret2 != interpret2:
whatsappmsg = whatsappmsg + "Interpret/Songs have changed:\n"
whatsappmsg = whatsappmsg + "old 1: '" + interpret1 + "' - '" + song1 + "'\n"
whatsappmsg = whatsappmsg + "new 1: '" + iinterpret1 + "' - '" + isong1 + "'\n"
whatsappmsg = whatsappmsg + "old 2: '" + interpret2 + "' - '" + song2 + "'\n"
whatsappmsg = whatsappmsg + "new 2: '" + iinterpret2 + "' - '" + isong2 + "'\n"
interpret1 = iinterpret1
interpret2 = iinterpret2
song2 = isong2
song1 = isong1
write_to_file()
foundsongslist = []
else:
whatsappmsg = whatsappmsg + "Interpret/Songs haven't changed:\n"
whatsappmsg = whatsappmsg + "1: '" + interpret1 + "' - '" + song1 + "'\n"
whatsappmsg = whatsappmsg + "2: '" + interpret2 + "' - '" + song2 + "'\n"
return (whatsappmsg) return (whatsappmsg)
@ -554,35 +627,48 @@ def parse_www():
global interpret2 global interpret2
global lastwwwparsetime global lastwwwparsetime
global nextchecktime global nextchecktime
global lastchecktime
global starttime
read_from_file() read_from_file()
# wait between two requests # wait between two requests
waittime_s = 15 waittime_min_s = 15
waittime_max_s = 60
waittime_s = waittime_min_s
waittime_search_s = 600
nexttime_hour = time.strftime("%H") nextsearchchecktime = datetime.now()
nexttime_min = time.strftime("%M")
global lastchecktime
startime = time.strftime("%H:%M:%S") soup_2songs_page()
telegrammsg = "Scan of Bully-page started " + startime + " Uhr.\n\n"
starttime = current_day_string(current_time_string())
telegrammsg = "Scan of Bully-page started " + starttime + " Uhr.\n\n"
telegrammsg = telegrammsg + "-> Song1:\n '" + interpret1 + "'-'" + song1 + "'\n-> Song2:\n '" + interpret2 + "'-'" + song2 + "'\n" telegrammsg = telegrammsg + "-> Song1:\n '" + interpret1 + "'-'" + song1 + "'\n-> Song2:\n '" + interpret2 + "'-'" + song2 + "'\n"
print(telegrammsg) print(telegrammsg)
send_all_telegram_message(telegrammsg) send_all_telegram_message(telegrammsg)
while run_thread == 1: while run_thread == 1:
currtime_hour = time.strftime("%H") lastwwwparsetime = current_day_string(current_time_string())
currtime_min = time.strftime("%M")
currtime_sec = time.strftime("%S") if nextsearchchecktime < datetime.now():
lastwwwparsetime = currtime_hour + ":" + currtime_min + ":" + currtime_sec soup_2songs_page()
nextsearchchecktime = datetime.now() + timedelta(seconds=waittime_search_s)
soup_2songs_page(40)
(telegrammsg, resultschanged, found_interpret_song_1, found_interpret_song_1_time, found_interpret_song_2, (telegrammsg, resultschanged, found_interpret_song_1, found_interpret_song_1_time, found_interpret_song_2,
found_interpret_song_2_time) = soup_page(40) found_interpret_song_2_time) = soup_page(40)
waittime = waittime_s + random.randint(-10, 10) currtime_min = int(time.strftime("%M"))
if (currtime_min >= 30 and currtime_min <= 50) or (currtime_min >= 0 and currtime_min <= 20):
waittime_s = waittime_min_s
#print("set min waittime: " + str(waittime_s))
else:
waittime_s = waittime_max_s
#print("set max waittime: " + str(waittime_s))
waittime = waittime_s + random.randint(-10, 5)
if ((lastchecktime > verbose_start) & (lastchecktime < verbose_stop)): if ((lastchecktime > verbose_start) & (lastchecktime < verbose_stop)):
sendall = True sendall = True
@ -618,8 +704,8 @@ def parse_www():
time.sleep(waittime) time.sleep(waittime)
stoptime = time.strftime("%H:%M:%S") stoptime = current_day_string(current_time_string())
telegrammsg = "Scan of Bully-page stopped " + stoptime + " Uhr. Bye.\n" telegrammsg = "Scan of Bully-page stopped " + stoptime + " Uhr. Was started at " + starttime + ". Bye.\n"
print(telegrammsg) print(telegrammsg)
send_all_telegram_message(telegrammsg) send_all_telegram_message(telegrammsg)