Re-Implement downloading shows with curl.

I suspected the Python requests library of truncating the downloads.
The library works fine, it's the Tivo that is producing the problems.
Keep this code as a reference if we ever need to switch away from
pure python.

tivomirror

@@ -28,7 +28,6 @@ import threading
 import time
 import urllib2
 import xml.dom.minidom
-import tivomp4
 
 host = "tivo.lassitu.de"
 #host = "wavehh.lassitu.de:30080"
@@ -37,6 +36,17 @@ targetdir = "/p2/media/video/TV"
 gig = 1024.0 * 1024 * 1024
 minfree = 10 * gig
 ignoreepisodetitle = False
+tivodecode = "tivodecode"
+cookies = "cookies.txt"
+proxies=None
+#proxies={"http":"http://us:8888","https":"http://us:8888"}
+
+headers = requests.utils.default_headers()
+headers.update(
+    {
+        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:52.0) Gecko/20100101 Firefox/52.0',
+    }
+)
 
 arset = dict()	
 
@@ -101,7 +111,7 @@ session = requests.session()
 session.verify = False
 session.auth = requests.auth.HTTPDigestAuth("tivo", mak)
 session.keep_alive = False
-
+session.proxies = proxies
 
 class TimeoutError(Exception):
     pass
@@ -137,6 +147,15 @@ def trimDescription(desc):
 		desc = desc[0:80]
 	return desc
 
+def saveCookies(session, filename):
+	cj = cookielib.MozillaCookieJar(filename)
+	for cookie in session.cookies:
+		logger.debug("storing cookie %s" % (cookie))
+		cj.set_cookie(cookie)
+	logger.debug("Saving cookies to %s" % (cj))
+	cj.save(ignore_discard=True, ignore_expires=True)
+
+
 class TivoException(Exception):
 	def __init__(self, value):
 		self.value = value
@@ -216,7 +235,7 @@ class TivoToc:
 		fd.close()
 
 	def download_chunk(self, offset):
-		global session
+		global session, proxies, headers
 
 		params = {
 			'Command': 'QueryContainer',
@@ -227,12 +246,13 @@ class TivoToc:
 		}
 		url = "https://{}/TiVoConnect".format(host)
 		logger.debug("    offset %d" % (offset))
-		r = session.get(url, params=params, timeout=30, verify=False)
+		r = session.get(url, params=params, timeout=30, verify=False, proxies=proxies, headers=headers)
 		if r.status_code != 200:
 			r.raise_for_status()
 		return r.text
 
 	def download(self):
+		global session
 		offset = 0
 		itemCount = 1
 		self.dom = None
@@ -249,6 +269,7 @@ class TivoToc:
 						root.appendChild(child.cloneNode(True))
 			itemCount = int(getElementText(dom.documentElement.childNodes, "ItemCount"))
 			offset += itemCount
+		saveCookies(session, cookies)
 		return self.dom
 
 	def getItems(self):
@@ -328,26 +349,71 @@ class FdLogger(threading.Thread):
 			self.logger.exception("")
 
 
+def waitForProcs(pids):
+	success = True
+	while len(pids) > 0:
+		(spid, sse) = os.wait()
+		pids.remove(spid)
+		if os.WIFSIGNALED(sse) \
+				or (os.WIFEXITED(sse) and os.WEXITSTATUS(sse) != 0):
+			if os.WIFSIGNALED(sse):
+				print "--- process %d was terminated by signal %d" % (spid, os.WTERMSIG(sse))
+			elif os.WIFEXITED(sse):
+				print "--- process %d exited with code %d" % (spid, os.WEXITSTATUS(sse))
+			else:
+				print "--- process %d terminated unsuccessfully" % spid
+			success = False
+			for pid in pids:
+				quit_process(pid)
+	if not success:
+		raise TivoException("error executing processes")
+
+
 @timeout(43200)
-#@timeout(7200)
+def download_curl(item, mak, target):
-def download(item, mak, target):
+	global cookies
-	global session
+	url = item.url
+	logger.info("--- downloading \"%s\"" % (url))
+	p_curl = subprocess.Popen(["curl", "--anyauth", "--fail", \
+			"--insecure", "--cookie", cookies, \
+			"--silent", "--show-error", \
+			"--user", "tivo:%s" % mak, "--url", url ], \
+		stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+	p_decode = subprocess.Popen([tivodecode, "--mak", mak, \
+		"--no-verify", "--out", target, "-"], stdin=p_curl.stdout,
+		stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+	FdLogger(logger, logging.INFO, p_curl.stderr)
+	FdLogger(logger, logging.INFO, p_decode.stdout)
+	FdLogger(logger, logging.INFO, p_decode.stderr)
+	try:
+		waitForProcs([p_curl.pid, p_decode.pid])
+	except Exception, e:
+		try:
+			os.remove(target)
+		except OSError:
+			pass
+		raise e
+	if os.path.getsize(target) < 1024:
+		print "error transcoding file: too small"
+		os.remove(target)
+		raise TivoException("downloaded file is too small")
+
+
+@timeout(43200)
+def download_py(item, mak, target):
+	global session, proxies, headers
 	count = 0
 	start = time.time()
 	upd = start
 	url = item.url
 	#url = re.sub("tivo.lassitu.de:80", "wavehh.lassitu.de:30080", url)
-	#url = re.sub("wavehh.lassitu.de:80", "wavehh.lassitu.de:30080", url)
-	#url = re.sub("tivo.lassitu.de:80", "localhost:8888", url)
-	#url = re.sub("tivo.lassitu.de:80", "krokodil-vpn.zs64.net:8888", url)
 	logger.info("--- downloading \"%s\"" % (url))
 	start = time.time()
-	r = session.get(url, stream=True, verify=False)
+	r = session.get(url, stream=True, verify=False, proxies=proxies, headers=headers)
-	#r = session.get(url, stream=True, proxies={"http":"http://wavehh:8888","https":"http://wavehh:8888"})
 	r.raise_for_status()
 
 	try:
-		p_decode = subprocess.Popen(["tivodecode", "--mak", mak, \
+		p_decode = subprocess.Popen([tivodecode, "--mak", mak, \
 			"--no-verify", "--out", target, "-"], stdin=subprocess.PIPE,
 			stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 		FdLogger(logger, logging.INFO, p_decode.stdout)
@@ -365,11 +431,10 @@ def download(item, mak, target):
 			pass
 		while True:
 			time.sleep(0) # yield to logger threads
-			chunk = r.raw.read(65536)
+			chunk = r.raw.read(256*1024)
-			if chunk:
+			if not chunk:
-				p_decode.stdin.write(chunk)
-			else:
 				break
+			p_decode.stdin.write(chunk)
 			count += len(chunk)
 			now = time.time()
 			if (now - upd) > 60:
@@ -405,7 +470,7 @@ def download(item, mak, target):
 		p_decode.wait()
 		logger.info("tivodecode exited with %s" % (p_decode.returncode))
 	size = os.path.getsize(target)
-	if size < 1024:
+	if size < 1024 or size < item.sourcesize * 0.8:
 		logger.error("error downloading file: too small")
 		os.remove(target)
 		raise TivoException("downloaded file is too small")
@@ -413,27 +478,21 @@ def download(item, mak, target):
 
 def download_decode(item, mak):
 	target = "%s.mpg" % item.file
-	mp4 = "%s.mp4" % item.file
 	try:
 		os.makedirs(item.dir)
 	except OSError:
 		pass
-	if 0 & os.path.exists(target):
+	try:
-		logger.info("    reusing existing download file")
+		download_curl(item, mak, target)
-	else:
+	except Exception, e:
+		exc_info = sys.exc_info()
 		try:
-			download(item, mak, target)
+			os.remove(target)
-		except Exception, e:
+		except Exception, e2:
-			exc_info = sys.exc_info()
+			pass
-			try:
+		raise exc_info[1], None, exc_info[2]
-				os.remove(target)
-			except Exception, e2:
-				pass
-			raise exc_info[1], None, exc_info[2]
-	#tivomp4.transcode(target, mp4, item.ar)
 	try:
 		os.utime(target, (item.time, item.time))
-		#os.utime(mp4, [item.date, item.date])
 	except Exception, e:
 		logger.error("Problem setting timestamp: %" % (e))