Import current version from CVS

tivodb

@@ -0,0 +1,36 @@
+#!/usr/local/bin/python
+
+import anydbm
+import getopt
+import operator
+import os
+import sys
+
+def usage():
+	print >>sys.stderr, "usage: dbtool {-a entry|-d entry|-l}"
+
+try:
+	optlist, args = getopt.getopt(sys.argv[1:], "a:d:lk")
+except getopt.GetoptError, err:
+	print >>sys.stderr, str(err)
+	usage()
+	sys.exit(64)
+if len(args) != 0 or len(optlist) != 1:
+	usage()
+	sys.exit(64)
+
+downloaddb = anydbm.open(os.path.expanduser("~") + "/.tivo/downloads.db", "c")
+
+for (o, a) in optlist:
+	if o == "-l":
+		for i in sorted(downloaddb.keys()):
+			print "%s:\t%s" % (i, downloaddb[i])
+	elif o == "-k":
+		for (k, v) in sorted(downloaddb.items(), key=operator.itemgetter(1)):
+			print "%s:\t%s" % (k, v)
+	elif o == "-d":
+		del downloaddb[a]
+	elif o == "-a":
+		downloaddb[a] = "manually added"
+
+downloaddb.close()

tivomirror

@@ -0,0 +1,569 @@
+#!/usr/local/bin/python
+
+# Stefans Script, um die Sendungen vom Tivo runterzuladen und in MPEG4
+# zu transkodieren.
+# Wird auf disklesslibber per Crontab-Eintrag stuendlich gestartet:
+# flock -n /tmp/tivomirror.log -c 'tivomirror >.tivomirror.log 2>&1 </dev/null'
+
+import sys
+reload(sys)  
+sys.setdefaultencoding('utf-8')
+
+import anydbm
+import cookielib
+import datetime
+import getopt
+import errno
+import functools
+import logging
+import logging.handlers
+import os
+import re
+import requests
+import signal
+import shutil
+import subprocess
+import sys
+import threading
+import time
+import urllib2
+import xml.dom.minidom
+import tivomp4
+
+host = "tivo.lassitu.de"
+#host = "wavehh.lassitu.de:30080"
+mak = "7194378159"
+targetdir = "/p2/media/video/TV"
+gig = 1024.0 * 1024 * 1024
+minfree = 10 * gig
+ignoreepisodetitle = False
+
+arset = dict()	
+
+includes = dict()
+includes['Anthony Bourdain Parts Unknown'] = 1
+includes['Better Call Saul'] = 1
+includes['Brooklyn Nine-Nine'] = 1
+includes['Bull'] = 1
+includes['College Football'] = 1
+includes['Conan'] = 1
+includes["Dirk Gently's Holistic Detective Agency"] = 1
+includes['The Expanse'] = 1
+includes['Family Guy'] = 1
+includes['Full Frontal With Samantha Bee'] = 1
+includes['Hot in Cleveland'] = 1
+includes["How It's Made"] = 1
+includes["How Do They Do It?"] = 1
+includes["How We Got to Now With Steven Johnson"] = 1
+includes['Inside Amy Schumer'] = 1
+includes['Join or Die With Craig Ferguson'] = 1
+includes['Last Week Tonight With John Oliver'] = 1
+includes['Louie'] = 1
+includes['Mad Men'] = 1
+includes['Modern Family'] = 1
+includes['MythBusters'] = 1
+includes['MythBusters: The Search'] = 1
+includes['NCIS'] = 1
+includes['NCIS: New Orleans'] = 1
+#includes['NFL Football'] = 1
+includes['Person of Interest'] = 1
+includes['Saturday Night Live'] = 1
+includes['Sesame Street'] = 1
+includes["Somebody's Gotta Do It With Mike Rowe"] = 1
+includes['StarTalk'] = 1
+includes['The Big Bang Theory'] = 1
+includes['The Daily Show With Trevor Noah'] = 1
+includes['The Late Show With Stephen Colbert'] = 1
+#includes['The Late Late Show With James Corden'] = 1
+includes['The Muppets'] = 1
+includes['The X-Files'] = 1
+#includes['The Tonight Show Starring Jimmy Fallon'] = 1
+
+
+
+logger = logging.getLogger('tivomirror')
+logger.setLevel(logging.INFO)
+
+
+class flushfile(object):
+	def __init__(self, f):
+		self.f = f
+	def write(self, x):
+		self.f.write(x)
+		self.f.flush()
+sys.stdout = flushfile(sys.stdout)
+
+tmp = "/tmp"
+
+# prepare global requests sesssion to download the TOC and the episodes
+requests.packages.urllib3.disable_warnings()
+session = requests.session()
+session.verify = False
+session.auth = requests.auth.HTTPDigestAuth("tivo", mak)
+session.keep_alive = False
+
+
+class TimeoutError(Exception):
+    pass
+
+def timeout(seconds=10, error_message=os.strerror(errno.ETIMEDOUT)):
+	def decorator(func):
+		def _handle_timeout(signum, frame):
+			raise TimeoutError(error_message)
+
+		def wrapper(*args, **kwargs):
+			signal.signal(signal.SIGALRM, _handle_timeout)
+			signal.alarm(seconds)
+			try:
+				result = func(*args, **kwargs)
+			finally:
+				signal.alarm(0)
+			return result
+
+		return functools.wraps(func)(wrapper)
+
+	return decorator
+
+
+def trimDescription(desc):
+	desc = desc.strip()
+	i = desc.rfind(". Copyright Tribune Media Services, Inc.");
+	if i > 0:
+		desc = desc[0:i]
+	i = desc.rfind(". * Copyright Rovi, Inc");
+	if i > 0:
+		desc = desc[0:i]
+	if len(desc) > 80:
+		desc = desc[0:80]
+	return desc
+
+class TivoException(Exception):
+	def __init__(self, value):
+		self.value = value
+	def __str__(self):
+		return repr(self.value)
+
+class TivoItem:
+	def __init__(self, i):
+		self.title = getTagText(i, "Title")
+		self.episode = getTagText(i, "EpisodeTitle")
+		self.episodeNumber = getTagText(i, "EpisodeNumber")
+		self.description = trimDescription(getTagText(i, "Description"))
+		d = getTagText(i, "CaptureDate")
+		self.date = datetime.datetime.utcfromtimestamp(int(d, 16))
+		self.time = int(d, base=0)
+		self.datestr = self.date.strftime("%Y%m%d-%H%M")
+		self.url = getTagText(i, "Url")
+		self.inprogress = getTagText(i, "InProgress")
+		self.available = getTagText(i, "Available")
+		self.sourcesize = int(getTagText(i, "SourceSize"))
+		self.highdef = getTagText(i, "HighDefinition")
+		self.ar = 43
+		self.unique = True
+		if arset.has_key(self.title):
+			self.ar = arset[self.title]
+		elif self.highdef == "Yes":
+			self.ar = "hd"
+		if ignoreepisodetitle:
+			self.episode = self.datestr
+		if self.episode == "":
+			if self.description != "":
+				self.episode = self.description
+			else:
+				self.episode = self.datestr
+		self.formatnames()
+	def makeNotUnique(self):
+		self.unique = False
+		self.formatnames()
+	def formatnames(self):
+		if self.episodeNumber and self.episodeNumber != u'0':
+			en = int(self.episodeNumber)
+			if en >= 100:
+				self.name = "%s S%02dE%02d %s" % (self.title, en / 100, en % 100, self.episode)
+			else:
+				self.name = "%s E%s %s" % (self.title, self.episodeNumber, self.episode)
+		elif self.unique:
+			self.name = "%s - %s" % (self.title, self.episode)
+		else:
+			self.name = "%s - %s - %s" % (self.title, self.datestr, self.episode)
+		self.dir = "%s/%s" % (targetdir, re.sub("[:/]", "-", self.title))
+		self.file = "%s/%s" % (self.dir, re.sub("[:/]", "-", self.name))
+		self.name = self.name.encode("utf-8");
+		self.dir = self.dir.encode("utf-8");
+		self.file = self.file.encode("utf-8");
+	def __str__(self):
+		return repr(self.title)
+
+
+class TivoToc:
+	def __init__(self):
+		self.dom = None
+		self.filename = "toc.xml"
+		self.uniquedb = anydbm.open("unique.db", "c")
+		self.items = []
+		pass
+
+	def load(self):
+		fd = open(self.filename, "r")
+		self.dom = xml.dom.minidom.parseString(fd.read())
+		fd.close()
+		return self.dom
+
+	def save(self):
+		fd = open(self.filename, "w")
+		fd.write(self.dom.toprettyxml())
+		fd.close()
+
+	def download_chunk(self, offset):
+		global session
+
+		params = {
+			'Command': 'QueryContainer',
+			'Container': '/NowPlaying',
+			'Recurse': 'Yes',
+			'ItemCount': '50',
+			'AnchorOffset': offset
+		}
+		url = "https://{}/TiVoConnect".format(host)
+		logger.debug("    offset %d" % (offset))
+		r = session.get(url, params=params, timeout=30, verify=False)
+		if r.status_code != 200:
+			r.raise_for_status()
+		return r.text
+
+	def download(self):
+		offset = 0
+		itemCount = 1
+		self.dom = None
+		root = None
+		logger.info("*** Getting listing")
+		while itemCount > 0:
+			dom = xml.dom.minidom.parseString(self.download_chunk(offset))
+			if self.dom == None:
+				self.dom = dom
+				root = self.dom.childNodes.item(0)
+			else:
+				for child in dom.childNodes.item(0).childNodes:
+					if child.nodeName == "Item":
+						root.appendChild(child.cloneNode(True))
+			itemCount = int(getElementText(dom.documentElement.childNodes, "ItemCount"))
+			offset += itemCount
+		return self.dom
+
+	def getItems(self):
+		self.titles = {}
+		for node in self.dom.getElementsByTagName("Item"):
+			item = TivoItem(node)
+			self.items.append(item)
+			if item.title not in self.titles:
+				self.titles[item.title] = []
+			self.titles[item.title].append(item)
+		# see if we have items that end up having an identical name; mark
+		# the program title in uniquedb if that's the case
+		for title in self.titles:
+			names = {}
+			for item in self.titles[title]:
+				if item.name not in names:
+					names[item.name] = []
+				names[item.name].append(item)
+			for name in names:
+				if len(names[name]) > 1:
+					self.uniquedb[title.encode("utf-8")] = "1"
+		if getattr(self.uniquedb, "sync", None) and callable(self.uniquedb.sync):
+			self.uniquedb.sync()
+		for item in self.items:
+			if self.uniquedb.has_key(item.title.encode("utf-8")):
+				item.makeNotUnique()
+		return self.items
+
+
+def getText(nodelist):
+	rc = ""
+	for node in nodelist:
+		if node.nodeType == node.TEXT_NODE:
+			rc = rc + node.data
+	return rc
+
+def getTagText(element, tagname):
+	try:
+		return getText(element.getElementsByTagName(tagname)[0].childNodes)
+	except IndexError:
+		return ""
+
+def getElementText(nodes, name):
+	for node in nodes:
+		if node.nodeType == xml.dom.Node.ELEMENT_NODE and node.nodeName == name:
+			return getText(node.childNodes)
+	return None
+
+def getAvail(dir):
+	s = os.statvfs(dir)
+	return s.f_bsize * s.f_bavail
+
+
+def quit_process(pid):	
+	try:
+		os.kill(pid, signal.SIGQUIT)
+	except OSError:
+		pass
+
+
+class FdLogger(threading.Thread):
+	def __init__(self, logger, lvl, fd):
+		self.logger = logger
+		self.lvl = lvl
+		self.fd = fd
+		threading.Thread.__init__(self)
+		self.daemon = True
+		self.start()
+
+	def run(self):
+		try:
+			# for line in fd buffers, so use this instead
+			for line in iter(self.fd.readline, b''):
+				self.logger.log(self.lvl, ":   %s", line.strip('\n'))
+			self.fd.close()
+		except Exception:
+			self.logger.exception("")
+
+
+@timeout(43200)
+#@timeout(7200)
+def download(item, mak, target):
+	global session
+	count = 0
+	start = time.time()
+	upd = start
+	url = item.url
+	#url = re.sub("tivo.lassitu.de:80", "wavehh.lassitu.de:30080", url)
+	#url = re.sub("wavehh.lassitu.de:80", "wavehh.lassitu.de:30080", url)
+	#url = re.sub("tivo.lassitu.de:80", "localhost:8888", url)
+	#url = re.sub("tivo.lassitu.de:80", "krokodil-vpn.zs64.net:8888", url)
+	logger.info("--- downloading \"%s\"" % (url))
+	start = time.time()
+	r = session.get(url, stream=True, verify=False)
+	#r = session.get(url, stream=True, proxies={"http":"http://wavehh:8888","https":"http://wavehh:8888"})
+	r.raise_for_status()
+
+	try:
+		p_decode = subprocess.Popen(["tivodecode", "--mak", mak, \
+			"--no-verify", "--out", target, "-"], stdin=subprocess.PIPE,
+			stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+		FdLogger(logger, logging.INFO, p_decode.stdout)
+		FdLogger(logger, logging.INFO, p_decode.stderr)
+		def info(signum, frame):
+			upd = time.time()
+			dur = now - start
+			mb = count / 1e6
+			print "%5.1f%% %5.3f GB downloaded in %.0f min, %.3f MB/s" % (
+				100.0 * count / item.sourcesize,
+				mb / 1e3, dur / 60, mb / dur)
+		try:
+			signal.signal(signal.SIGINFO, info)
+		except Exception:
+			pass
+		while True:
+			time.sleep(0) # yield to logger threads
+			chunk = r.raw.read(65536)
+			if chunk:
+				p_decode.stdin.write(chunk)
+			else:
+				break
+			count += len(chunk)
+			now = time.time()
+			if (now - upd) > 60:
+				upd = now
+				dur = now - start
+				mb = count / 1e6
+				logger.debug("    %5.1f%% %5.3f GB downloaded in %.0f min, %.3f MB/s" % (
+					100.0 * count / item.sourcesize,
+					mb / 1e3, dur / 60, mb / dur))
+	except Exception as e:
+		logger.error("problem decoding: %s" % (e))
+		raise
+	finally:
+		try:
+			signal.signal(signal.SIGINFO, signal.SIG_IGN)
+		except Exception:
+			pass
+		elapsed = time.time() - start
+		throughput = count / elapsed
+		logger.info("%5.3fGB transferred in %d:%02d, %.1f MB/s" % (
+			count/1e9, int(elapsed/3600), int(elapsed / 60) % 60, throughput/1e6))
+		try:
+			p_decode.stdin.close()
+			p_decode.poll()
+			if p_decode.returncode == None:
+				time.sleep(1)
+				p_decode.poll()
+				if p_decode.returncode == None:
+					logger.debug("terminating tivodecode")
+					p_decode.terminate()
+		except Exception, e:
+			pass
+		p_decode.wait()
+		logger.info("tivodecode exited with %s" % (p_decode.returncode))
+	size = os.path.getsize(target)
+	if size < 1024:
+		logger.error("error downloading file: too small")
+		os.remove(target)
+		raise TivoException("downloaded file is too small")
+
+
+def download_decode(item, mak):
+	target = "%s.mpg" % item.file
+	mp4 = "%s.mp4" % item.file
+	try:
+		os.makedirs(item.dir)
+	except OSError:
+		pass
+	if 0 & os.path.exists(target):
+		logger.info("    reusing existing download file")
+	else:
+		try:
+			download(item, mak, target)
+		except Exception, e:
+			exc_info = sys.exc_info()
+			try:
+				os.remove(target)
+			except Exception, e2:
+				pass
+			raise exc_info[1], None, exc_info[2]
+	#tivomp4.transcode(target, mp4, item.ar)
+	try:
+		os.utime(target, (item.time, item.time))
+		#os.utime(mp4, [item.date, item.date])
+	except Exception, e:
+		logger.error("Problem setting timestamp: %" % (e))
+
+
+def download_one(item, downloaddb):
+	global logger, mak
+ 	logger.info("*** downloading \"%s\": %.3fGB" % (item.name, item.sourcesize / 1e9))
+	try:
+		download_decode(item, mak)
+		downloaddb[item.name] = item.datestr
+		if getattr(downloaddb, "sync", None) and callable(downloaddb.sync):
+			downloaddb.sync()
+		logger.debug("Sleeping 30 seconds before moving on...")
+		time.sleep(30)
+	except TivoException, e:
+		logger.info("Error processing \"%s\": %s" % (item.name, e))
+
+
+def wantitem(item, downloaddb):
+	if item.inprogress == "Yes":
+		return "recording"
+	if item.available == "No":
+		return "not available"
+	if downloaddb.has_key(item.name):
+		return "already downloaded"
+	#if excludes.has_key(item.title) or excludes.has_key(item.episode) or excludes.has_key(item.name):
+	#	return "excluded"
+	if includes.has_key(item.title) or includes.has_key(item.episode) or includes.has_key(item.name):
+		pass
+	else:
+		return "not included"
+	return ""
+
+
+def mirror(toc, downloaddb, one=False):
+	avail = getAvail(targetdir)
+	if avail < minfree:
+		logger.error("%s: %.1fG available, at least %.1fG needed, stopping" % \
+			(targetdir, avail / gig, minfree / gig))
+		sys.exit(1)
+	
+	items = toc.getItems()
+	logger.info("*** %d shows listed" % (len(items)))
+	for item in items:
+		reason = wantitem(item, downloaddb)
+		if reason != "":
+			logger.debug("*** skipping \"%s\": %s" % (item.name, reason))
+		else:
+			download_one(item, downloaddb)
+			if one:
+				break
+
+
+def download_episode(toc, downloaddb, episode):
+	items = toc.getItems()
+	for item in items:
+		if item.title == episode or item.name == episode or item.episode == episode:
+			download_one(item, downloaddb)
+
+
+def printtoc(toc, downloaddb):
+	items = toc.getItems()
+	print "*** %d shows listed" % (len(items))
+	shows = {}
+	for item in items:
+		if item.title not in shows:
+			shows[item.title] = []
+		shows[item.title].append(item)
+	for title in sorted(shows):
+		for item in sorted(shows[title], key=lambda i: i.name):
+			reason = wantitem(item, downloaddb)
+			if (reason != ""):
+				print "%-7.7s: %s" % (reason, item.name)
+				continue
+			print "*** downloading %s (%.3fGB)" % (item.name, item.sourcesize / 1e9)
+
+
+def main():
+	global ignoreepisodetitle, logger
+	curdir = os.getcwd()
+	os.chdir(os.path.expanduser("~") + "/.tivo")
+	handler = logging.handlers.RotatingFileHandler("tivomirror.log", maxBytes=2*1024*1024, backupCount=5)
+	handler.setFormatter(logging.Formatter(fmt='tivomirror[%d] %%(asctime)s %%(levelname)6.6s %%(message)s' % (os.getpid()),
+		datefmt='%H:%M:%S'))
+	logger.addHandler(handler)
+	downloaddb = anydbm.open("downloads.db", "c")
+	toc = TivoToc()
+	cmd = "list"
+	updateToc = False
+
+	try:
+		options, remainder = getopt.getopt(sys.argv[1:], 'dvuT',
+			['ignoreepisodetitle', 'debug', 'verbose', 'update'])
+
+		for opt, arg in options:
+			if opt in ('-d', '--debug'):
+				logger.setLevel(logging.DEBUG)
+			if opt in ('-v', '--verbose'):
+				handler = logging.StreamHandler()
+				logger.addHandler(handler)
+			if opt in ('-u', '--update'):
+				updateToc = True
+			if opt in ('-T', '--ignoreepisodetitle'):
+				ignoreepisodetitle = True
+
+		if len(remainder) >= 1:
+			cmd = remainder[0]
+
+		if updateToc or cmd == "mirror":
+			toc.download()
+			toc.save()
+		else:
+			toc.load()
+
+		if cmd == "mirror":
+			mirror(toc, downloaddb)
+		elif cmd == "mirrorone":
+			mirror(toc, downloaddb, True)
+		elif cmd == "list":
+			printtoc(toc, downloaddb)
+		elif cmd == "download":
+			download_episode(toc, downloaddb, remainder[1])
+		else:
+			logger.error("invalid command %s" % (cmd))
+			print >>sys.stderr, "invalid command %s" % (cmd)
+			sys.exit(64)
+
+		downloaddb.close()
+	except Exception:
+		logger.exception("")
+	logger.info("*** Completed")
+
+if __name__ == "__main__":
+	main()