stb/tivomirror
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Mechanize durch requests als HTTP-Library ersetzt, und dabei curl

Browse source 
rausgeworfen.
This commit is contained in:
Stefan Bethke 2013-08-06 22:17:16 +00:00
parent cff554f297
commit 79b836e72e
1 changed files with 40 additions and 69 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

107
src/tivomirror/tivomirror
View file

@ -1,6 +1,6 @@
#!/usr/local/bin/python #!/usr/local/bin/python
# $Schlepperbande: src/tivomirror/tivomirror,v 1.49 2013/02/21 07:06:02 stb Exp $ # $Schlepperbande: src/tivomirror/tivomirror,v 1.50 2013/03/16 09:54:12 stb Exp $
# #
# Stefans Script, um die Sendungen vom Tivo runterzuladen und in MPEG4 # Stefans Script, um die Sendungen vom Tivo runterzuladen und in MPEG4
# zu transkodieren. # zu transkodieren.
@ -11,9 +11,9 @@ import anydbm
import cookielib import cookielib
import datetime import datetime
import getopt import getopt
import mechanize
import os import os
import re import re
import requests
import signal import signal
import shutil import shutil
import subprocess import subprocess
@ -54,9 +54,7 @@ includes['The Colbert Report'] = 1
includes['The Daily Show With Jon Stewart'] = 1 includes['The Daily Show With Jon Stewart'] = 1
includes['The Late Late Show With Craig Ferguson'] = 1 includes['The Late Late Show With Craig Ferguson'] = 1
includes['The 55th Annual Grammy Awards'] = 1; includes['CONSTITUTION USA With Peter Sagal'] = 1;
includes['Live From the Red Carpet: The 2013 Grammy Awards'] = 1
includes['Countdown to the Red Carpet: The 2013 Grammy Awards'] = 1
class flushfile(object): class flushfile(object):
@ -73,11 +71,10 @@ except OSError:
pass pass
tmp = "/tmp" tmp = "/tmp"
# use mechanize as a HTTP client # prepare global requests sesssion to download the TOC and the episodes
browser = mechanize.Browser() session = requests.session()
browser.add_password("http://%s/" % host, "tivo", mak) session.verify = False
browser.add_password("https://%s/" % host, "tivo", mak) session.auth = requests.auth.HTTPDigestAuth("tivo", mak)
browser.set_handle_robots(False)
def trimDescription(desc): def trimDescription(desc):
@ -160,23 +157,24 @@ def makeCookie(name, value):
) )
def get_cookie_by_name(cj, name):
return [cookie for cookie in cj if cookie.name == name][0]
def gettoc(): def gettoc():
global browser, sidcookie global session
url = "https://" + host + "/TiVoConnect?Command=QueryContainer&Container=%2FNowPlaying&Recurse=Yes&ItemCount=50" params = {
browser.open(url) 'Command': 'QueryContainer',
response = browser.response() 'Container': '/NowPlaying',
headers = response.info() 'Recurse': 'Yes',
'ItemCount': '50'
# work around Tivo's broken Set-cookie header that has an expiry date in }
# the past, thus removing the session cookie url = "https://{}/TiVoConnect".format(host)
sidcookie = response.info().getheader("Set-Cookie") r = session.get(url, params=params)
if sidcookie != None: if r.status_code != 200:
sidcookie = re.sub(".*sid=(.*?);.*", "\\1", sidcookie) r.raise_for_status()
#print sidcookie return r.text
browser._ua_handlers['_cookies'].cookiejar.set_cookie(makeCookie("sid", sidcookie))
#print browser._ua_handlers['_cookies'].cookiejar
return response.read()
def getText(nodelist): def getText(nodelist):
@ -204,57 +202,28 @@ def quit_process(pid):
pass pass
def waitForProcs(pids):
success = True
starttime = time.time()
while len(pids) > 0:
(spid, sse) = os.waitpid(0, os.WNOHANG)
if spid == 0:
if time.time() - starttime > 6 * 60 * 60:
for pid in pids:
os.kill(pid, signal.SIGTERM)
raise TivoException("waiting for download to complete exceeded timeout")
time.sleep(2)
continue
pids.remove(spid)
if os.WIFSIGNALED(sse) \
or (os.WIFEXITED(sse) and os.WEXITSTATUS(sse) != 0):
if os.WIFSIGNALED(sse):
print "--- process %d was terminated by signal %d" % (spid, os.WTERMSIG(sse))
elif os.WIFEXITED(sse):
print "--- process %d exited with code %d" % (spid, os.WEXITSTATUS(sse))
else:
print "--- process %d terminated unsuccessfully" % spid
success = False
for pid in pids:
quit_process(pid)
if not success:
raise TivoException("error executing processes")
pass
def download(file, url, mak, target): def download(file, url, mak, target):
global sidcookie global browser, sidcookie
#url = re.sub("tivo.lassitu.de:80", "localhost:8888", url) #url = re.sub("tivo.lassitu.de:80", "localhost:8888", url)
#url = re.sub("tivo.lassitu.de:80", "krokodil-vpn.zs64.net:8888", url) #url = re.sub("tivo.lassitu.de:80", "krokodil-vpn.zs64.net:8888", url)
print "--- downloading \"%s\"" % (url) print "--- downloading \"%s\"" % (url)
start = time.time() start = time.time()
p_curl = subprocess.Popen(["curl", "--anyauth", "--fail", \ r = session.get(url, stream=True)
"--insecure", "--cookie", "sid=%s" % sidcookie, \ try:
"--silent", "--show-error", \
"--user-agent", "Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)", \
"--user", "tivo:%s" % mak, "--url", url ], \
stdout=subprocess.PIPE)
p_decode = subprocess.Popen(["tivodecode", "--mak", mak, \ p_decode = subprocess.Popen(["tivodecode", "--mak", mak, \
"--no-verify", "--out", target, "-"], stdin=p_curl.stdout) "--no-verify", "--out", target, "-"], stdin=subprocess.PIPE)
while True:
chunk = r.raw.read(65536)
if chunk:
p_decode.stdin.write(chunk)
else:
break
finally:
try: try:
waitForProcs([p_curl.pid, p_decode.pid]) p_decode.stdin.close()
quit_process(p_decode.pid)
except Exception, e: except Exception, e:
try:
os.remove(target)
except OSError:
pass pass
raise
size = os.path.getsize(target) size = os.path.getsize(target)
if size < 1024: if size < 1024:
print "error downloading file: too small" print "error downloading file: too small"
@ -279,11 +248,12 @@ def download_decode(item, mak):
try: try:
download(item.file, item.url, mak, target) download(item.file, item.url, mak, target)
except Exception, e: except Exception, e:
exc_info = sys.exc_info()
try: try:
os.remove(target) os.remove(target)
except Exception, e2: except Exception, e2:
pass pass
raise e raise exc_info[1], None, exc_info[2]
#tivomp4.transcode(target, mp4, item.ar) #tivomp4.transcode(target, mp4, item.ar)
try: try:
os.utime(target, (item.time, item.time)) os.utime(target, (item.time, item.time))
@ -342,6 +312,7 @@ def mirror(dom, downloaddb):
break break
except TivoException, e: except TivoException, e:
print "Error processing \"%s\": %s" % (item.name, e) print "Error processing \"%s\": %s" % (item.name, e)
break
print "*** Completed %s" % datetime.datetime.now().strftime("%Y-%m-%d %H:%M") print "*** Completed %s" % datetime.datetime.now().strftime("%Y-%m-%d %H:%M")