From 79b836e72e3145ce97fec95faa22918f0643ff89 Mon Sep 17 00:00:00 2001 From: Stefan Bethke Date: Tue, 6 Aug 2013 22:17:16 +0000 Subject: [PATCH] Mechanize durch requests als HTTP-Library ersetzt, und dabei curl rausgeworfen. --- src/tivomirror/tivomirror | 109 ++++++++++++++------------------------ 1 file changed, 40 insertions(+), 69 deletions(-) diff --git a/src/tivomirror/tivomirror b/src/tivomirror/tivomirror index d0be678..fa3dd5d 100755 --- a/src/tivomirror/tivomirror +++ b/src/tivomirror/tivomirror @@ -1,6 +1,6 @@ #!/usr/local/bin/python -# $Schlepperbande: src/tivomirror/tivomirror,v 1.49 2013/02/21 07:06:02 stb Exp $ +# $Schlepperbande: src/tivomirror/tivomirror,v 1.50 2013/03/16 09:54:12 stb Exp $ # # Stefans Script, um die Sendungen vom Tivo runterzuladen und in MPEG4 # zu transkodieren. @@ -11,9 +11,9 @@ import anydbm import cookielib import datetime import getopt -import mechanize import os import re +import requests import signal import shutil import subprocess @@ -54,9 +54,7 @@ includes['The Colbert Report'] = 1 includes['The Daily Show With Jon Stewart'] = 1 includes['The Late Late Show With Craig Ferguson'] = 1 -includes['The 55th Annual Grammy Awards'] = 1; -includes['Live From the Red Carpet: The 2013 Grammy Awards'] = 1 -includes['Countdown to the Red Carpet: The 2013 Grammy Awards'] = 1 +includes['CONSTITUTION USA With Peter Sagal'] = 1; class flushfile(object): @@ -73,11 +71,10 @@ except OSError: pass tmp = "/tmp" -# use mechanize as a HTTP client -browser = mechanize.Browser() -browser.add_password("http://%s/" % host, "tivo", mak) -browser.add_password("https://%s/" % host, "tivo", mak) -browser.set_handle_robots(False) +# prepare global requests sesssion to download the TOC and the episodes +session = requests.session() +session.verify = False +session.auth = requests.auth.HTTPDigestAuth("tivo", mak) def trimDescription(desc): @@ -160,23 +157,24 @@ def makeCookie(name, value): ) +def get_cookie_by_name(cj, name): + return [cookie for cookie in cj if cookie.name == name][0] + + def gettoc(): - global browser, sidcookie + global session - url = "https://" + host + "/TiVoConnect?Command=QueryContainer&Container=%2FNowPlaying&Recurse=Yes&ItemCount=50" - browser.open(url) - response = browser.response() - headers = response.info() - - # work around Tivo's broken Set-cookie header that has an expiry date in - # the past, thus removing the session cookie - sidcookie = response.info().getheader("Set-Cookie") - if sidcookie != None: - sidcookie = re.sub(".*sid=(.*?);.*", "\\1", sidcookie) - #print sidcookie - browser._ua_handlers['_cookies'].cookiejar.set_cookie(makeCookie("sid", sidcookie)) - #print browser._ua_handlers['_cookies'].cookiejar - return response.read() + params = { + 'Command': 'QueryContainer', + 'Container': '/NowPlaying', + 'Recurse': 'Yes', + 'ItemCount': '50' + } + url = "https://{}/TiVoConnect".format(host) + r = session.get(url, params=params) + if r.status_code != 200: + r.raise_for_status() + return r.text def getText(nodelist): @@ -204,57 +202,28 @@ def quit_process(pid): pass -def waitForProcs(pids): - success = True - starttime = time.time() - while len(pids) > 0: - (spid, sse) = os.waitpid(0, os.WNOHANG) - if spid == 0: - if time.time() - starttime > 6 * 60 * 60: - for pid in pids: - os.kill(pid, signal.SIGTERM) - raise TivoException("waiting for download to complete exceeded timeout") - time.sleep(2) - continue - pids.remove(spid) - if os.WIFSIGNALED(sse) \ - or (os.WIFEXITED(sse) and os.WEXITSTATUS(sse) != 0): - if os.WIFSIGNALED(sse): - print "--- process %d was terminated by signal %d" % (spid, os.WTERMSIG(sse)) - elif os.WIFEXITED(sse): - print "--- process %d exited with code %d" % (spid, os.WEXITSTATUS(sse)) - else: - print "--- process %d terminated unsuccessfully" % spid - success = False - for pid in pids: - quit_process(pid) - if not success: - raise TivoException("error executing processes") - pass - - def download(file, url, mak, target): - global sidcookie + global browser, sidcookie #url = re.sub("tivo.lassitu.de:80", "localhost:8888", url) #url = re.sub("tivo.lassitu.de:80", "krokodil-vpn.zs64.net:8888", url) print "--- downloading \"%s\"" % (url) start = time.time() - p_curl = subprocess.Popen(["curl", "--anyauth", "--fail", \ - "--insecure", "--cookie", "sid=%s" % sidcookie, \ - "--silent", "--show-error", \ - "--user-agent", "Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)", \ - "--user", "tivo:%s" % mak, "--url", url ], \ - stdout=subprocess.PIPE) - p_decode = subprocess.Popen(["tivodecode", "--mak", mak, \ - "--no-verify", "--out", target, "-"], stdin=p_curl.stdout) + r = session.get(url, stream=True) try: - waitForProcs([p_curl.pid, p_decode.pid]) - except Exception, e: + p_decode = subprocess.Popen(["tivodecode", "--mak", mak, \ + "--no-verify", "--out", target, "-"], stdin=subprocess.PIPE) + while True: + chunk = r.raw.read(65536) + if chunk: + p_decode.stdin.write(chunk) + else: + break + finally: try: - os.remove(target) - except OSError: + p_decode.stdin.close() + quit_process(p_decode.pid) + except Exception, e: pass - raise size = os.path.getsize(target) if size < 1024: print "error downloading file: too small" @@ -279,11 +248,12 @@ def download_decode(item, mak): try: download(item.file, item.url, mak, target) except Exception, e: + exc_info = sys.exc_info() try: os.remove(target) except Exception, e2: pass - raise e + raise exc_info[1], None, exc_info[2] #tivomp4.transcode(target, mp4, item.ar) try: os.utime(target, (item.time, item.time)) @@ -342,6 +312,7 @@ def mirror(dom, downloaddb): break except TivoException, e: print "Error processing \"%s\": %s" % (item.name, e) + break print "*** Completed %s" % datetime.datetime.now().strftime("%Y-%m-%d %H:%M")