diff --git a/tivomirror b/tivomirror index bca02db..8f83304 100755 --- a/tivomirror +++ b/tivomirror @@ -28,7 +28,6 @@ import threading import time import urllib2 import xml.dom.minidom -import tivomp4 host = "tivo.lassitu.de" #host = "wavehh.lassitu.de:30080" @@ -37,6 +36,17 @@ targetdir = "/p2/media/video/TV" gig = 1024.0 * 1024 * 1024 minfree = 10 * gig ignoreepisodetitle = False +tivodecode = "tivodecode" +cookies = "cookies.txt" +proxies=None +#proxies={"http":"http://us:8888","https":"http://us:8888"} + +headers = requests.utils.default_headers() +headers.update( + { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:52.0) Gecko/20100101 Firefox/52.0', + } +) arset = dict() @@ -101,7 +111,7 @@ session = requests.session() session.verify = False session.auth = requests.auth.HTTPDigestAuth("tivo", mak) session.keep_alive = False - +session.proxies = proxies class TimeoutError(Exception): pass @@ -137,6 +147,15 @@ def trimDescription(desc): desc = desc[0:80] return desc +def saveCookies(session, filename): + cj = cookielib.MozillaCookieJar(filename) + for cookie in session.cookies: + logger.debug("storing cookie %s" % (cookie)) + cj.set_cookie(cookie) + logger.debug("Saving cookies to %s" % (cj)) + cj.save(ignore_discard=True, ignore_expires=True) + + class TivoException(Exception): def __init__(self, value): self.value = value @@ -216,7 +235,7 @@ class TivoToc: fd.close() def download_chunk(self, offset): - global session + global session, proxies, headers params = { 'Command': 'QueryContainer', @@ -227,12 +246,13 @@ class TivoToc: } url = "https://{}/TiVoConnect".format(host) logger.debug(" offset %d" % (offset)) - r = session.get(url, params=params, timeout=30, verify=False) + r = session.get(url, params=params, timeout=30, verify=False, proxies=proxies, headers=headers) if r.status_code != 200: r.raise_for_status() return r.text def download(self): + global session offset = 0 itemCount = 1 self.dom = None @@ -249,6 +269,7 @@ class TivoToc: root.appendChild(child.cloneNode(True)) itemCount = int(getElementText(dom.documentElement.childNodes, "ItemCount")) offset += itemCount + saveCookies(session, cookies) return self.dom def getItems(self): @@ -328,26 +349,71 @@ class FdLogger(threading.Thread): self.logger.exception("") +def waitForProcs(pids): + success = True + while len(pids) > 0: + (spid, sse) = os.wait() + pids.remove(spid) + if os.WIFSIGNALED(sse) \ + or (os.WIFEXITED(sse) and os.WEXITSTATUS(sse) != 0): + if os.WIFSIGNALED(sse): + print "--- process %d was terminated by signal %d" % (spid, os.WTERMSIG(sse)) + elif os.WIFEXITED(sse): + print "--- process %d exited with code %d" % (spid, os.WEXITSTATUS(sse)) + else: + print "--- process %d terminated unsuccessfully" % spid + success = False + for pid in pids: + quit_process(pid) + if not success: + raise TivoException("error executing processes") + + @timeout(43200) -#@timeout(7200) -def download(item, mak, target): - global session +def download_curl(item, mak, target): + global cookies + url = item.url + logger.info("--- downloading \"%s\"" % (url)) + p_curl = subprocess.Popen(["curl", "--anyauth", "--fail", \ + "--insecure", "--cookie", cookies, \ + "--silent", "--show-error", \ + "--user", "tivo:%s" % mak, "--url", url ], \ + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + p_decode = subprocess.Popen([tivodecode, "--mak", mak, \ + "--no-verify", "--out", target, "-"], stdin=p_curl.stdout, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + FdLogger(logger, logging.INFO, p_curl.stderr) + FdLogger(logger, logging.INFO, p_decode.stdout) + FdLogger(logger, logging.INFO, p_decode.stderr) + try: + waitForProcs([p_curl.pid, p_decode.pid]) + except Exception, e: + try: + os.remove(target) + except OSError: + pass + raise e + if os.path.getsize(target) < 1024: + print "error transcoding file: too small" + os.remove(target) + raise TivoException("downloaded file is too small") + + +@timeout(43200) +def download_py(item, mak, target): + global session, proxies, headers count = 0 start = time.time() upd = start url = item.url #url = re.sub("tivo.lassitu.de:80", "wavehh.lassitu.de:30080", url) - #url = re.sub("wavehh.lassitu.de:80", "wavehh.lassitu.de:30080", url) - #url = re.sub("tivo.lassitu.de:80", "localhost:8888", url) - #url = re.sub("tivo.lassitu.de:80", "krokodil-vpn.zs64.net:8888", url) logger.info("--- downloading \"%s\"" % (url)) start = time.time() - r = session.get(url, stream=True, verify=False) - #r = session.get(url, stream=True, proxies={"http":"http://wavehh:8888","https":"http://wavehh:8888"}) + r = session.get(url, stream=True, verify=False, proxies=proxies, headers=headers) r.raise_for_status() try: - p_decode = subprocess.Popen(["tivodecode", "--mak", mak, \ + p_decode = subprocess.Popen([tivodecode, "--mak", mak, \ "--no-verify", "--out", target, "-"], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) FdLogger(logger, logging.INFO, p_decode.stdout) @@ -365,11 +431,10 @@ def download(item, mak, target): pass while True: time.sleep(0) # yield to logger threads - chunk = r.raw.read(65536) - if chunk: - p_decode.stdin.write(chunk) - else: + chunk = r.raw.read(256*1024) + if not chunk: break + p_decode.stdin.write(chunk) count += len(chunk) now = time.time() if (now - upd) > 60: @@ -405,7 +470,7 @@ def download(item, mak, target): p_decode.wait() logger.info("tivodecode exited with %s" % (p_decode.returncode)) size = os.path.getsize(target) - if size < 1024: + if size < 1024 or size < item.sourcesize * 0.8: logger.error("error downloading file: too small") os.remove(target) raise TivoException("downloaded file is too small") @@ -413,27 +478,21 @@ def download(item, mak, target): def download_decode(item, mak): target = "%s.mpg" % item.file - mp4 = "%s.mp4" % item.file try: os.makedirs(item.dir) except OSError: pass - if 0 & os.path.exists(target): - logger.info(" reusing existing download file") - else: + try: + download_curl(item, mak, target) + except Exception, e: + exc_info = sys.exc_info() try: - download(item, mak, target) - except Exception, e: - exc_info = sys.exc_info() - try: - os.remove(target) - except Exception, e2: - pass - raise exc_info[1], None, exc_info[2] - #tivomp4.transcode(target, mp4, item.ar) + os.remove(target) + except Exception, e2: + pass + raise exc_info[1], None, exc_info[2] try: os.utime(target, (item.time, item.time)) - #os.utime(mp4, [item.date, item.date]) except Exception, e: logger.error("Problem setting timestamp: %" % (e))