Change source of tivodecode

.gitignore

@@ -0,0 +1,2 @@
+.venv
+*~

README.md

@@ -2,15 +2,27 @@
 
 ## Requirements
 
-Python (2.7) needs to be installed; additionally, these packages need to be
+Python 3.4 or newer needs to be installed.
-installed:
-* `py27-yaml`
 
-`tivodecode` needs to be available on the path. [mackworth/tivodecode-ng](https://github.com/mackworth/tivodecode-ng)
+`tivodecode` needs to be available on the path. [wmcbrine/tivodecode-ng](https://github.com/wmcbrine/tivodecode-ng)
 appears to be working well; the original
 [TiVo File Decoder](http://tivodecode.sourceforge.net) has trouble
 decoding some files and fails silently.
 
+## Installing
+
+Create a virtual environment and install the required packages:
+
+* `python -m venv .venv` or `virtualenv .venv`
+* `. .venv/bin/activate`
+* `pip install -r requirements.txt`
+
+## Shell Wrapper
+
+To make it easier to run tivomirror from cron, the shell wrapper
+[`tivomirror`](./tivomirror) will activate the venv and then run
+`tivomirror.py`.
+
 ## Configuration
 
 `tivomirror` reads a config file, by default `~/.tivo/config.yaml`.  The

requirements.txt

@@ -0,0 +1,4 @@
+berkeleydb
+pytz
+requests
+pyyaml

tivodb

@@ -0,0 +1 @@
+wrapper

tivodb.py

@@ -1,33 +1,33 @@
 #!/usr/local/bin/python
 
-import anydbm
+import dbm
 import getopt
 import operator
 import os
 import sys
 
 def usage():
-	print >>sys.stderr, "usage: dbtool {-a entry|-d entry|-l}"
+	print("usage: dbtool {-a entry|-d entry|-l}", file=sys.stderr)
 
 try:
 	optlist, args = getopt.getopt(sys.argv[1:], "a:d:lk")
-except getopt.GetoptError, err:
+except getopt.GetoptError as err:
-	print >>sys.stderr, str(err)
+	print(str(err), file=sys.stderr)
 	usage()
 	sys.exit(64)
 if len(args) != 0 or len(optlist) != 1:
 	usage()
 	sys.exit(64)
 
-downloaddb = anydbm.open(os.path.expanduser("~") + "/.tivo/downloads.db", "c")
+downloaddb = dbm.open(os.path.expanduser("~") + "/.tivo/downloads.db", "c")
 
 for (o, a) in optlist:
 	if o == "-l":
 		for i in sorted(downloaddb.keys()):
-			print "%s:\t%s" % (i, downloaddb[i])
+			print("%s:\t%s" % (i.decode('utf-8'), downloaddb[i].decode('utf-8')))
 	elif o == "-k":
-		for (k, v) in sorted(downloaddb.items(), key=operator.itemgetter(1)):
+		for (k, v) in sorted(list(downloaddb.items()), key=operator.itemgetter(1)):
-			print "%s:\t%s" % (k, v)
+			print("%s:\t%s" % (k, v))
 	elif o == "-d":
 		del downloaddb[a]
 	elif o == "-a":

tivomirror

@@ -0,0 +1 @@
+wrapper

tivomirror.py

@@ -1,17 +1,19 @@
-#!/usr/local/bin/python 
+#!/usr/local/bin/python3.8
 # -*- coding: utf8 -*-
 
 # Download shows from the Tivo
 
 import sys
-reload(sys)
+#import importlib
-sys.setdefaultencoding('utf-8')
+#importlib.reload(sys)
+#sys.setdefaultencoding('utf-8')
 
-import anydbm
+import dbm
-import cookielib
+import http.cookiejar
 import datetime
 import getopt
 import errno
+import fcntl
 import functools
 import logging
 import logging.handlers
@@ -25,15 +27,17 @@ import subprocess
 import sys
 import threading
 import time
-import urllib2
+import urllib.request, urllib.error, urllib.parse
 import xml.dom.minidom
 import yaml
 
-
+from contextlib import contextmanager
+from io import TextIOWrapper
 
 
 class Config:
 	config = '~/.tivo/config.yaml'
+	lockfile = config + '.lock'
 	cookies = "cookies.txt"
 	gig = 1024.0 * 1024 * 1024
 	headers = requests.utils.default_headers()
@@ -109,7 +113,7 @@ class flushfile(object):
 	def write(self, x):
 		self.f.write(x)
 		self.f.flush()
-sys.stdout = flushfile(sys.stdout)
+#sys.stdout = flushfile(sys.stdout)
 
 tmp = "/tmp"
 
@@ -164,7 +168,7 @@ def trimDescription(desc):
 	return desc
 
 def saveCookies(session, filename):
-	cj = cookielib.MozillaCookieJar(filename)
+	cj = http.cookiejar.MozillaCookieJar(filename)
 	for cookie in session.cookies:
 		logger.debug("storing cookie {}".format(cookie))
 		cj.set_cookie(cookie)
@@ -210,10 +214,10 @@ class TivoItem:
 		self.unique = False
 		self.formatnames()
 	def formatnames(self):
-		if self.episodeNumber and self.episodeNumber != u'0':
+		if self.episodeNumber and self.episodeNumber != '0':
 			en = int(self.episodeNumber)
 			if en >= 100:
-				self.name = "{} S{:02d}E{:02d} {}".format(self.title, en / 100, en % 100, self.episode)
+				self.name = "{} S{:02d}E{:02d} {}".format(self.title, int(en / 100), int(en % 100), self.episode)
 			else:
 				self.name = "{} E{} {}".format(self.title, self.episodeNumber, self.episode)
 		elif self.unique:
@@ -222,14 +226,14 @@ class TivoItem:
 			self.name = "{} - {} - {}".format(self.title, self.datestr, self.episode)
 		self.dir  = "{}/{}".format(config.targetdir, re.sub("[:/]", "-", self.title))
 		self.file = "{}/{}".format(self.dir, re.sub("[:/]", "-", self.name))
-		self.name = self.name.encode("utf-8");
+		#self.name = self.name.encode("utf-8");
-		self.dir  = self.dir.encode("utf-8");
+		#self.dir  = self.dir.encode("utf-8");
-		self.file = self.file.encode("utf-8");
+		#self.file = self.file.encode("utf-8");
 	def getPath(self, options):
 		title = self.title
 		if options.short:
 			title = options.short
-		if self.episodeNumber and self.episodeNumber != u'0':
+		if self.episodeNumber and self.episodeNumber != '0':
 			en = int(self.episodeNumber)
 			if en >= 100:
 				name = "{} S{:02d}E{:02d} {}".format(title, en / 100, en % 100, self.episode)
@@ -240,7 +244,8 @@ class TivoItem:
 		else:
 			name = "{} - {} {}".format(title, self.shortdate, self.episode)
 		path = "{}/{}".format(self.dir, re.sub("[:/]", "-", name))
-		return path.encode("utf-8");
+		return path
+		#return path.encode("utf-8");
 	def __str__(self):
 		return repr(self.title)
 
@@ -249,7 +254,7 @@ class TivoToc:
 	def __init__(self):
 		self.dom = None
 		self.filename = "toc.xml"
-		self.uniquedb = anydbm.open("unique.db", "c")
+		self.uniquedb = dbm.open("unique.db", "c")
 		self.items = []
 		pass
 
@@ -319,9 +324,11 @@ class TivoToc:
 					names[item.name] = []
 				names[item.name].append(item)
 			for name in names:
-				utf8title = title.encode("utf-8")
+				if len(names[name]) > 1 and title not in self.uniquedb:
-				if len(names[name]) > 1 and not self.uniquedb.has_key(utf8title):
+					self.uniquedb[title] = "1"
-					self.uniquedb[utf8title] = "1"
+#				utf8title = title.encode("utf-8")
+#				if len(names[name]) > 1 and utf8title not in self.uniquedb:
+#					self.uniquedb[utf8title] = "1"
 		if getattr(self.uniquedb, "sync", None) and callable(self.uniquedb.sync):
 			self.uniquedb.sync()
 		# update all items based on config and uniquedb
@@ -370,12 +377,24 @@ class FdLogger(threading.Thread):
 		try:
 			# for line in fd buffers, so use this instead
 			for line in iter(self.fd.readline, b''):
-				self.logger.log(self.lvl, ":   %s", line.strip('\n'))
+				line = line.strip('\n')
+				if line.strip() != "":
+					self.logger.log(self.lvl, ":   %s", line)
 			self.fd.close()
 		except Exception:
 			self.logger.exception("")
 
 
+@contextmanager
+def exclusive():
+	with open(os.path.expanduser(config.lockfile), 'w') as f:
+		try:
+			fcntl.lockf(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
+		except BlockingIOError as e:
+			raise TivoException('another tivomirror instance is already running')
+		yield 'locked'
+		fcntl.lockf(f, fcntl.LOCK_UN)
+
 @timeout(43200)
 def download_item(item, mak, target):
 	global config
@@ -394,15 +413,15 @@ def download_item(item, mak, target):
 		p_decode = subprocess.Popen([config.tivodecode, "--mak", mak, \
 			"--no-verify", "--out", target, "-"], stdin=subprocess.PIPE,
 			stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-		FdLogger(logger, logging.INFO, p_decode.stdout)
+		FdLogger(logger, logging.INFO, TextIOWrapper(p_decode.stdout))
-		FdLogger(logger, logging.INFO, p_decode.stderr)
+		FdLogger(logger, logging.INFO, TextIOWrapper(p_decode.stderr))
 		def info(signum, frame):
 			upd = time.time()
 			dur = now - start
 			mb = count / 1e6
-			print "{:5.1f}% {:5.3f} GB downloaded in {:.0f} min, {.3f} MB/s".format(
+			print("{:5.1f}% {:5.3f} GB downloaded in {:.0f} min, {:.3f} MB/s".format(
 				100.0 * count / item.sourcesize,
-				mb / 1e3, dur / 60, mb / dur)
+				mb / 1e3, dur / 60, mb / dur))
 		try:
 			signal.signal(signal.SIGINFO, info)
 		except Exception:
@@ -419,7 +438,7 @@ def download_item(item, mak, target):
 				upd = now
 				dur = now - start
 				mb = count / 1e6
-				logger.debug("    {:5.1f}% {:5.3f} GB downloaded in {:.0f} min, {:.3f} MB/s".format(
+				logger.debug("    {:5.1f}% {:5.3f} GB downloaded in {:0.0f} min, {:0.3f} MB/s".format(
 					100.0 * count / item.sourcesize,
 					mb / 1e3, dur / 60, mb / dur))
 	except Exception as e:
@@ -443,7 +462,7 @@ def download_item(item, mak, target):
 				if p_decode.returncode == None:
 					logger.debug("terminating tivodecode")
 					p_decode.terminate()
-		except Exception, e:
+		except Exception as e:
 			pass
 		p_decode.wait()
 		logger.info("tivodecode exited with {}".format(p_decode.returncode))
@@ -462,22 +481,23 @@ def download_decode(item, options, mak):
 		pass
 	try:
 		download_item(item, mak, item.target)
-	except Exception, e:
+	except Exception as e:
 		exc_info = sys.exc_info()
 		try:
 			os.remove(item.target)
-		except Exception, e2:
+		except Exception as e2:
 			pass
-		raise exc_info[1], None, exc_info[2]
+		raise exc_info[1].with_traceback(exc_info[2])
 	try:
 		os.utime(item.target, (item.time, item.time))
-	except Exception, e:
+	except Exception as e:
 		logger.error("Problem setting timestamp: {}".format(e))
 
 
 def download_one(item, downloaddb, options):
 	global config, logger
 	logger.info("*** downloading \"{}\": {:.3f} GB".format(item.name, item.sourcesize / 1e9))
+#	sys.exit(1)
 	try:
 		download_decode(item, options, config.mak)
 		downloaddb[item.name] = item.datestr
@@ -489,11 +509,11 @@ def download_one(item, downloaddb, options):
 				cmd = cmd.format(item=item, options=options, config=config)
 				r = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
 				logger.debug("Post-process  {}: {}".format(cmd, r))
-			except Exception, e:
+			except Exception as e:
 				logger.warn("Error running postprocess command '{}' for item {}: {}".format(cmd, item, e))
 		logger.debug("Sleeping 30 seconds before moving on...")
 		time.sleep(30)
-	except TivoException, e:
+	except TivoException as e:
 		logger.info("Error processing \"{}\": {}".format(item.name, e))
 
 
@@ -502,10 +522,10 @@ def wantitem(item, downloaddb):
 		return "recording"
 	if item.available == "No":
 		return "not available"
-	if downloaddb.has_key(item.name):
+	if item.name in downloaddb:
 		return "already downloaded"
 	for i in (item.title, item.episode, item.name):
-		if IncludeShow.includes.has_key(i):
+		if i in IncludeShow.includes:
 			return IncludeShow.includes[i]
 	return "not included"
 
@@ -517,11 +537,12 @@ def mirror(toc, downloaddb, one=False):
 			(config.targetdir, avail / config.gig, config.minfree / config.gig))
 		sys.exit(1)
 
+	with exclusive() as lock:
 		items = toc.getItems()
 		logger.info("*** {} shows listed".format(len(items)))
 		for item in items:
 			options = wantitem(item, downloaddb)
-		if isinstance(options, basestring):
+			if isinstance(options, str):
 				logger.debug("*** skipping \"{}\": {}".format(item.name, options))
 			else:
 				download_one(item, downloaddb, options)
@@ -530,20 +551,22 @@ def mirror(toc, downloaddb, one=False):
 
 
 def download_episode(toc, downloaddb, episode):
+	with exclusive() as lock:
 		items = toc.getItems()
 		options = {}
 		for item in items:
 			if item.title == episode or item.name == episode or item.episode == episode:
 				for i in (item.title, item.episode, item.name):
-				if IncludeShow.includes.has_key(i):
+					if i in IncludeShow.includes:
 						options = IncludeShow.includes[i]
 						download_one(item, downloaddb, options)
 						return
 
 
 def printtoc(toc, downloaddb):
+	with exclusive() as lock:
 		items = toc.getItems()
-	print "*** {} shows listed".format(len(items))
+		print("*** {} shows listed".format(len(items)))
 		shows = {}
 		for item in items:
 			if item.title not in shows:
@@ -552,16 +575,16 @@ def printtoc(toc, downloaddb):
 		for title in sorted(shows):
 			for item in sorted(shows[title], key=lambda i: i.name):
 				options = wantitem(item, downloaddb)
-			if isinstance(options, basestring):
+				if isinstance(options, str):
-				print "{:>7.7s}: {}".format(options, item.name)
+					print("{:>7.7s}: {}".format(options, item.name))
 					continue
-			print "*** downloading {} ({:.3f} GB)".format(item.name, item.sourcesize / 1e9)
+				print("*** downloading {} ({:.3f} GB)".format(item.name, item.sourcesize / 1e9))
-	print "*** {} shows listed".format(len(items))
+		print("*** {} shows listed".format(len(items)))
 
 
 def usage():
-	print >>sys.stderr, 'usage: tivomirror -dvuT [-c config] cmd'
+	print('usage: tivomirror -dvuT [-c config] cmd', file=sys.stderr)
-	print >>sys.stderr, '    cmd is one of download, list, mirror, mirrorone'
+	print('    cmd is one of download, list, mirror, mirrorone', file=sys.stderr)
 	sys.exit(64)
 
 
@@ -573,7 +596,7 @@ def main():
 	handler.setFormatter(logging.Formatter(fmt='tivomirror[{}] %(asctime)s %(levelname)6.6s %(message)s'.format(os.getpid()),
 		datefmt='%d-%m %H:%M:%S'))
 	logger.addHandler(handler)
-	downloaddb = anydbm.open("downloads.db", "c")
+	downloaddb = dbm.open("downloads.db", "c")
 	toc = TivoToc()
 	cmd = "list"
 	updateToc = False
@@ -619,12 +642,12 @@ def main():
 			download_episode(toc, downloaddb, remainder[1])
 		else:
 			logger.error("invalid command {}".format(cmd))
-			print >>sys.stderr, "invalid command {}".format(cmd)
+			print("invalid command {}".format(cmd), file=sys.stderr)
 			usage()
 
 		downloaddb.close()
 	except getopt.GetoptError as e:
-		print >>sys.stderr, 'Error parsing options: {}'.format(e)
+		print('Error parsing options: {}'.format(e), file=sys.stderr)
 		usage()
 	except Exception:
 		logger.exception("")

wrapper

@@ -0,0 +1,6 @@
+#!/bin/sh
+
+here="$(dirname $0)"
+this="$(basename $0)"
+. ${here}/.venv/bin/activate
+exec python ${here}/${this}.py $@