last changes added like new style of song page

searchwebsite_telegram2.py

@@ -9,16 +9,6 @@
 # description of the Bot API, see this page: https://core.telegram.org/bots/api
 # telegram server: https://api.telgram.org/bot696861399:AAHoNorIizQ3pE4-nUE1IIPU9ywyvrn32gs/getUpdates
 
-
-
-
-# ssh -R 80:localhost:5000 serveo.net starten um die WebApp nach aussen sichtbar zu machen
-# not needed anymore : !!!! ngrok starten ausserhalb des  virtuellen env: ngrok http 5000
-# in twilio WebHooks die Https Adresse die ngrok beim startup zeigt eintragen inkl. /wa am Ende
-# python virtualles env aufbauen und darin Flask installieren
-# für windows: http://timmyreilly.azurewebsites.net/python-flask-windows-development-environment-setup/
-# python searchwebsite_wa.py starten ...
-
 import sys
 import os.path
 import threading
@@ -33,10 +23,10 @@ import random
 import telegram
 from telegram.error import NetworkError, Unauthorized
 
-my_song1 = ""
+my_song1 = "unknownsong1"
-my_interpret1 = ""
+my_interpret1 = "unknownart1"
-my_song2 = ""
+my_song2 = "unknownsong2"
-my_interpret2 = ""
+my_interpret2 = "unknownart2"
 
 test = True
 
@@ -67,6 +57,8 @@ full_song2 = ""
 interpret2 = ""
 full_interpret2 = ""
 
+auto_check_for_is1_is2 = True
+
 currentmsg = ""
 telegram_error = False
 telegram_error_msg = ""
@@ -75,12 +67,15 @@ parse_error_msg = ""
 run_thread = 1
 lastchecktime = ""
 nextchecktime = ""
+starttime = ""
 checkcounter = 0
 errorcounter = 0
+nroftries = 0
 lastwwwparsetime = ""
 fullcurrenttime = ""
 file_name = "./bully_search.txt"
 
+send_false_alarms = False
 
 foundsongslist = []
 
@@ -100,6 +95,8 @@ def telegram_ahoy_reply(body):
     global errorcounter
     global foundsongslist
     global nextchecktime
+    global starttime
+    global auto_check_for_is1_is2
 
     if (re.search("^list$", body,  re.IGNORECASE) is not None) | (re.search("^l$", body,  re.IGNORECASE) is not None):
         soup_page(15)
@@ -151,6 +148,15 @@ def telegram_ahoy_reply(body):
                                             else:
                                                 if re.search("^s", body, re.IGNORECASE):
                                                     message = "Statistics:\n\n"
+                                                    message = message + "Start time: " + starttime + "\n"
+                                                    if auto_check_for_is1_is2:
+                                                        message = message + "Auto getting is1,is2: ON\n"
+                                                    else:
+                                                        message = message + "Auto getting is1,is2: OFF\n"
+                                                    if send_false_alarms:
+                                                        message = message + "Sending false alarms: ON\n"
+                                                    else:
+                                                        message = message + "Sending false alarms: ON\n"
                                                     message = message + "Last overall checktime: " + fullcurrenttime + "\n"
                                                     message = message + "Last automatic checktime: " + lastwwwparsetime + "\n"
                                                     message = message + "Next checktime: " + nextchecktime + "\n"
@@ -162,14 +168,34 @@ def telegram_ahoy_reply(body):
                                                         foundsongslist = []
                                                     else:
                                                         if re.search("^g", body, re.IGNORECASE):
-                                                            message = soup_2songs_page(20)
+                                                            message = soup_2songs_page()
+                                                        else:
+                                                            if re.search("^a", body, re.IGNORECASE):
+                                                                message = "Automatic getting songs1,2 and interprets1,2:\n"
+                                                                if auto_check_for_is1_is2:
+                                                                    auto_check_for_is1_is2 = False
+                                                                    message = message + "SWITCHED OFF now."
+                                                                else:
+                                                                    auto_check_for_is1_is2 = True
+                                                                    message = message + "SWITCHED ON now."
+                                                            else:
+                                                                if re.search("^m", body, re.IGNORECASE):
+                                                                message = "Sending even false alarms:\n"
+                                                                if send_false_alarms:
+                                                                    send_false_alarms = False
+                                                                    message = message + "SWITCHED OFF now."
+                                                                else:
+                                                                    send_false_alarms = True
+                                                                    message = message + "SWITCHED ON now."
+
                                                             else:
                                                                 message = "Send: \n\n'list' or 'l' for latest playlist.\n '?' for songs/title looking for.\n 's1=songname' to change song1.\n"
                                                                 message = message + " 'i1=artist' to change interpret1.\n 's2=songname' to change song2.\n 'i2=artist' to change interpret2.\n"
                                                                 message = message + " 'is1=artist,song to change #1 \n 'is2=artist,song to change #2 \n "
                                                                 message = message + " 'w' write i&s to file.\n 'r' read i&s from file\n"
                                                                 message = message + " 'f' shows found songs.\n 'c' clears list with found songs\n"
-                                                            message = message + " 's' shows statistics.\n 'g' gets song1,2 and interprets1,2"
+                                                                message = message + " 's' shows statistics.\n 'g' gets song1,2 and interprets1,2\n"
+                                                                message = message + " 'a' toggle auto get is1,is2\n 'm' toggle false alarms\n"
 
     return message
 
@@ -218,6 +244,7 @@ def check_for_alarm_case(found_interpret_song_1, found_interpret_song_1_time,fou
     global full_interpret2
     global fullcurrenttime
     global lastchecktime
+    global send_false_alarms
     callermsg = ""
     waittime = wait
     sendmsg = False
@@ -249,7 +276,7 @@ def check_for_alarm_case(found_interpret_song_1, found_interpret_song_1_time,fou
                     callermsg = fullcurrenttime + " -- FEHLALARM! 2.Song lief zuerst. --\n"
                     if sendall is True:
                         # dont send that at night ...
-                        sendmsg = True
+                        sendmsg = send_false_alarms
                 else:
                     waittime = 8 + random.randint(-2, 2)
                     callermsg = fullcurrenttime + " ++ BULLI ERSTER ALARM! ++\n"
@@ -277,12 +304,35 @@ def check_for_alarm_case(found_interpret_song_1, found_interpret_song_1_time,fou
                     callermsg = callermsg + "\n"
                     callermsg = callermsg + "Kein Grund anzurufen.\n"
 
+
                     if sendall is True:
                         # dont send that at night ...
-                        sendmsg = True
+                        sendmsg = send_false_alarms
 
     return(callermsg, waittime, sendmsg)
 
+def current_day_string(currtime):
+
+    wdays = ["So", "Mo", "Di", "Mi" , "Do", "Fr", "Sa"]
+    currtime_weekday = wdays[int(time.strftime("%w"))]
+    currtime_day = time.strftime("%d")
+    currtime_month = time.strftime("%m")
+
+    fullcurrentday = currtime_weekday + "," + currtime_day + "." + currtime_month + " " +currtime
+    #print("Currentday: " + fullcurrentday);
+    return fullcurrentday
+
+
+def current_time_string():
+
+    currtime_hour = time.strftime("%H")
+    currtime_min = time.strftime("%M")
+    currtime_sec = time.strftime("%S")
+
+    fullcurrenttime = currtime_hour + ":" + currtime_min + ":" + currtime_sec
+    #print("Currenttime: " + fullcurrenttime);
+    return fullcurrenttime
+
 
 def soup_page(max_chars):
 
@@ -302,17 +352,14 @@ def soup_page(max_chars):
     global parse_error_msg
     global errorcounter
     global foundsongslist
+    global nroftries
 
     i1 = re.compile(remove_signs_from_string(interpret1), re.IGNORECASE)
     t1 = re.compile(remove_signs_from_string(song1), re.IGNORECASE)
     i2 = re.compile(remove_signs_from_string(interpret2), re.IGNORECASE)
     t2 = re.compile(remove_signs_from_string(song2), re.IGNORECASE)
 
-    currtime_hour = time.strftime("%H")
+    fullcurrenttime = current_day_string(current_time_string())
-    currtime_min = time.strftime("%M")
-    currtime_sec = time.strftime("%S")
-
-    fullcurrenttime = currtime_hour + ":" + currtime_min + ":" + currtime_sec
 
     found_interpret_song_1 = 0
     found_interpret_song_1_time = 0
@@ -332,7 +379,7 @@ def soup_page(max_chars):
             try:
                 soup = BeautifulSoup(page.content, 'html.parser')
                 music_results = soup.find(class_="music_results")
-                music_results_items = music_results.find_all(class_="music_results__content")
+                music_results_items = music_results.find_all(class_="music_results__content", limit = 5)
 
                 for found_item in music_results_items:
                     itemnr = itemnr + 1
@@ -358,7 +405,7 @@ def soup_page(max_chars):
                             full_interpret1 = interpret
                             full_song1 = song
                             found_interpret_song_1 = itemnr
-                            found_interpret_song_1_time = zeit
+                            found_interpret_song_1_time = current_day_string(zeit)
 
                             newelement = found_interpret_song_1_time + " " + full_interpret1 + " '" + full_song1 + "'"
                             if newelement not in foundsongslist:
@@ -373,7 +420,7 @@ def soup_page(max_chars):
                             full_interpret2 = interpret
                             full_song2 = song
                             found_interpret_song_2 = itemnr
-                            found_interpret_song_2_time = zeit
+                            found_interpret_song_2_time = current_day_string(zeit)
 
                             newelement = found_interpret_song_2_time + " " + full_interpret2 + " '" + full_song2 + "'"
                             if newelement not in foundsongslist:
@@ -390,21 +437,25 @@ def soup_page(max_chars):
                 if parse_error == True:
                     parse_error_msg = "PASS - parsing of webpage seems to work properly again ...\n"
                     print(parse_error_msg)
+                    if nroftries > 5:
                         try:
                             send_telegram_err_message(parse_error_msg)
                         except:
                             print("Message couldn't be sent to root.")
                             errorcounter = errorcounter + 1
+                    nroftries = 0
                     parse_error = False
 
             except:
                 if parse_error == False:
                     parse_error_msg = "ERROR - parsing of webpage failed but will try again ...\n"
                     print(parse_error_msg)
+                    if nroftries > 5:
                         try:
                             send_telegram_err_message(parse_error_msg)
                         except:
                             print("Message couldn't be sent to root.")
+                    nroftries = nroftries + 1
                     parse_error = True
                     errorcounter = errorcounter + 1
 
@@ -412,15 +463,19 @@ def soup_page(max_chars):
             if parse_error == False:
                 parse_error_msg = "ERROR - reading of webpage failed but will try again...\n"
                 print(parse_error_msg)
+                if nroftries > 5:
                     try:
                         send_telegram_err_message(parse_error_msg)
                     except:
                         print("Message couldn't be sent to root.")
+                nroftries = nroftries + 1
                 parse_error = True
                 errorcounter = errorcounter + 1
 
     except:
+        if nroftries > 5:
             whatsappmsg = whatsappmsg + "Failed to read webpage - but will retry!!!\n\n"
+        nroftries = nroftries + 1
         errorcounter = errorcounter + 1
         pass
 
@@ -435,7 +490,7 @@ def remove_signs_from_string(string):
     string = string.strip()
     return string
 
-def soup_2songs_page(max_chars):
+def soup_2songs_page():
 
     global interpret1
     global interpret2
@@ -452,17 +507,18 @@ def soup_2songs_page(max_chars):
     global parse_error_msg
     global errorcounter
     global foundsongslist
+    global auto_check_for_is1_is2
+
+
+    whatsappmsg = ""
+    if auto_check_for_is1_is2:
 
         iinterpret1 = interpret1
         iinterpret2 = interpret2
         isong1 = song1
         isong2 = song2
 
-    currtime_hour = time.strftime("%H")
+        fullcurrenttime = current_day_string(current_time_string())
-    currtime_min = time.strftime("%M")
-    currtime_sec = time.strftime("%S")
-
-    fullcurrenttime = currtime_hour + ":" + currtime_min + ":" + currtime_sec
         itemnr = 0
 
         whatsappmsg = " Getting time: " + fullcurrenttime + " Uhr.\n\n"
@@ -475,16 +531,15 @@ def soup_2songs_page(max_chars):
                 try:
                     soup = BeautifulSoup(page.content, 'html.parser')
                     search_results = soup.find(class_="pagecontent")
-                search_results_items = search_results.find_all(class_="text-left")
+                    search_results_items = search_results.find_all("h5")
 
                     is_nr = 1;
                     for found_item in search_results_items:
-                    itemnr = itemnr + 1
                         item_text = found_item.get_text()
-                    if "Bulli" not in item_text:
+                        #print(item_text)
-                        if " - " in item_text:
+                        if ":" in item_text:
                             # now split into interpret and song
-                            isstr = re.search(" *(.+?) - (.+?)$", item_text)
+                            isstr = re.search(" *(.+?) *: *(.+?)$", item_text)
                             if isstr:
                                 if is_nr == 1:
                                     iinterpret1 = isstr.group(1)
@@ -496,14 +551,27 @@ def soup_2songs_page(max_chars):
                                     isong2 = isstr.group(2)
                                     #print("Interpret2 '" + iinterpret2 + "' - Song2 '" + isong2 + "'")
                                     is_nr = is_nr + 1
-            except:
+                    if parse_error == True:
-                if parse_error is False:
+                        parse_error_msg = "PASS - parsing of webpage seems to work properly again ...\n"
-                    parse_error_msg = "ERROR - parsing of webpage failed but will try again ...\n"
                         print(parse_error_msg)
+                        if nroftries > 5:
                             try:
                                 send_telegram_err_message(parse_error_msg)
                             except:
                                 print("Message couldn't be sent to root.")
+                                errorcounter = errorcounter + 1
+                        nroftries = 0
+                        parse_error = False
+                except:
+                    if parse_error is False:
+                        parse_error_msg = "ERROR - parsing of webpage failed but will try again ...\n"
+                        print(parse_error_msg)
+                        if nroftries>5 :
+                            try:
+                                send_telegram_err_message(parse_error_msg)
+                            except:
+                                print("Message couldn't be sent to root.")
+                        nroftries = nroftries + 1
                         parse_error = True
                         errorcounter = errorcounter + 1
 
@@ -511,15 +579,19 @@ def soup_2songs_page(max_chars):
                 if parse_error is False:
                     parse_error_msg = "ERROR - reading of webpage failed but will try again...\n"
                     print(parse_error_msg)
+                    if nroftries >5 :
                         try:
                             send_telegram_err_message(parse_error_msg)
                         except:
                             print("Message couldn't be sent to root.")
+                    nroftries = nroftries + 1
                     parse_error = True
                     errorcounter = errorcounter + 1
 
         except:
+            if nroftries > 5:
                 whatsappmsg = whatsappmsg + "Failed to read webpage - but will retry!!!\n\n"
+            nroftries = nroftries + 1
             errorcounter = errorcounter + 1
             pass
 
@@ -542,6 +614,7 @@ def soup_2songs_page(max_chars):
             whatsappmsg = whatsappmsg + "1: '" + interpret1 + "' - '" + song1 + "'\n"
             whatsappmsg = whatsappmsg + "2: '" + interpret2 + "' - '" + song2 + "'\n"
 
+
     return (whatsappmsg)
 
 def parse_www():
@@ -554,35 +627,48 @@ def parse_www():
     global interpret2
     global lastwwwparsetime
     global nextchecktime
-
+    global lastchecktime
+    global starttime
 
     read_from_file()
 
     # wait between two requests
-    waittime_s = 15
+    waittime_min_s = 15
+    waittime_max_s = 60
+    waittime_s = waittime_min_s
+    waittime_search_s = 600
 
-    nexttime_hour = time.strftime("%H")
+    nextsearchchecktime = datetime.now()
-    nexttime_min  = time.strftime("%M")
-    global lastchecktime
 
-    startime = time.strftime("%H:%M:%S")
+    soup_2songs_page()
-    telegrammsg = "Scan of Bully-page started " + startime + " Uhr.\n\n"
+
+    starttime = current_day_string(current_time_string())
+    telegrammsg = "Scan of Bully-page started " + starttime + " Uhr.\n\n"
     telegrammsg = telegrammsg + "-> Song1:\n '" + interpret1 + "'-'" + song1 + "'\n-> Song2:\n '" + interpret2 + "'-'" + song2 + "'\n"
     print(telegrammsg)
     send_all_telegram_message(telegrammsg)
 
     while run_thread == 1:
 
-        currtime_hour = time.strftime("%H")
+        lastwwwparsetime = current_day_string(current_time_string())
-        currtime_min = time.strftime("%M")
+
-        currtime_sec = time.strftime("%S")
+        if nextsearchchecktime < datetime.now():
-        lastwwwparsetime = currtime_hour + ":" + currtime_min + ":" + currtime_sec
+            soup_2songs_page()
+            nextsearchchecktime = datetime.now() + timedelta(seconds=waittime_search_s)
             
-        soup_2songs_page(40)
         (telegrammsg, resultschanged, found_interpret_song_1, found_interpret_song_1_time, found_interpret_song_2,
          found_interpret_song_2_time) = soup_page(40)
 
-        waittime = waittime_s + random.randint(-10, 10)
+        currtime_min = int(time.strftime("%M"))
+        if (currtime_min >= 30 and currtime_min <= 50) or (currtime_min >= 0 and currtime_min <= 20):
+            waittime_s = waittime_min_s
+            #print("set min waittime: " + str(waittime_s))
+        else:
+            waittime_s = waittime_max_s
+            #print("set max waittime: " + str(waittime_s))
+        
+
+        waittime = waittime_s + random.randint(-10, 5)
 
         if ((lastchecktime > verbose_start) & (lastchecktime < verbose_stop)):
             sendall = True
@@ -618,8 +704,8 @@ def parse_www():
 
         time.sleep(waittime)
 
-    stoptime = time.strftime("%H:%M:%S")
+    stoptime = current_day_string(current_time_string())
-    telegrammsg = "Scan of Bully-page stopped " + stoptime + " Uhr. Bye.\n"
+    telegrammsg = "Scan of Bully-page stopped " + stoptime + " Uhr. Was started at " + starttime + ". Bye.\n"
     print(telegrammsg)
     send_all_telegram_message(telegrammsg)