Refactor config into it's own class and make all options configurable from the config file. Also add descriptions to README

README.md

@@ -2,13 +2,70 @@
 
 ## Requirements
 
-Python needs to be installed; the regular libs appear to be sufficient.
+Python (2.7) needs to be installed; additionally, these packages need to be
+installed:
+* py27-yaml
 
 tivodecode needs to be available on the path. [mackworth/tivodecode-ng](https://github.com/mackworth/tivodecode-ng)
 appears to be working well; the original
 [TiVo File Decoder](http://tivodecode.sourceforge.net) has trouble
 decoding some files and fails silently.
 
-## To Do
+## Configuration
 
-* Add a clever README.
+`tivomirror` reads a config file, by default `~/.tivo/config.yaml`.  The
+config file can contain the following keys:
+* `cookies`: filename of the cookie jar, relative to `~/.tivo`.
+* `host`: hostname of the Tivo.
+* `ignoreepisodetitle`: Only use the series' title; default *false*. See
+  also command line paramter `-i`.
+* `mak`: the Media Access Key for your Tivo account.
+* `minfree`: if there's less space in `targetdir` than these many gigabytes,
+  do not download anything.
+* `proxies`: hash of `http` and `https` proxy URLs to use for talking to the
+  Tivo. See the
+  [Requests](http://docs.python-requests.org/en/master/user/advanced/#proxies)
+  package for details.
+* `shows`: a Hash of series' titles for which episodes should be downloaded. 
+  Can contain an optional sub-hash, with these keys:
+  * `short`: a shorter name for the series, to be used when constructing the
+    file name for an episode to be downloaded.
+* `targetdir`: store downloaded shows here.
+* `tivodecode`: path to tivodecode binary; default `tivodecode`.
+* `useragent`; the user agent to use when talking to the Tivo.
+
+You will need to define at least one `shows` element for tivomirror to
+download anything.
+
+## Command Line Options
+
+`tivomirror` accepts the following command line options:
+* `-c` / `--config`: name of the config file; default `~/.tivo/config.yaml`.
+* `-d`/ `--debug`: print debugging output to the log file at
+  `~/.tivo/tivomirror.log`.
+* `-v` / `--verbose`: print output to stderr as well as to the log file.
+* `-u` / `--update`: load new table of contents irrespective of the age of
+  the current cached copy.
+
+`tivomirror` accepts the following commands:
+* `list`: list all episodes stored on the Tivo, with an indication of:
+  * `download`: episode will be downloaded the next time `mirror` runs.
+  * `already`: the episode was downloaded successfully previously.
+  * `not included': the series has not been selected for download, that is,
+    there's no entry for it in the `shows` hash in the config.
+  * `recording`: this episode is currently being recorded; it can be
+    downloaded after the recording is finished.
+  * `not available`: the Tivo does not make this episode available for
+    download; it might be watchable directly on the Tivo.
+* `mirror`: download all episodes selected through `shows` that haven't been
+  downloaded successfully previously.
+* `mirrorone`: download the first show of all shows to be downloaded, exit
+  after.
+
+## Database Utility
+
+`tivodb` can be used to list, add or remove entries from the download database at
+`~/.tivo/downloads.db`:
+* `-a`: add the named entry to the database.
+* `-d': delete the named entry from the database.
+* `-l`: list all entries in the database.

config.yaml

@@ -4,3 +4,8 @@ shows:
   - "Conan":
   - "Dirk Gently's Holistic Detective Agency":
     short: "Dirk Gently"
+host: "wavehh.lassitu.de:30080"
+proxies:
+  http: "http://us.lassitu.de:8888"
+  https: "http://us.lassitu.de:8888"
+targetdir: /data/downloads

tivomirror.py

@@ -1,13 +1,10 @@
 #!/usr/local/bin/python
 # -*- coding: utf8 -*-
 
-# Stefans Script, um die Sendungen vom Tivo runterzuladen und in MPEG4
+# Download shows from the Tivo
-# zu transkodieren.
-# Wird auf disklesslibber per Crontab-Eintrag stuendlich gestartet:
-# flock -n /tmp/tivomirror.log -c 'tivomirror >.tivomirror.log 2>&1 </dev/null'
 
 import sys
-reload(sys)  
+reload(sys)
 sys.setdefaultencoding('utf-8')
 
 import anydbm
@@ -32,25 +29,55 @@ import urllib2
 import xml.dom.minidom
 import yaml
 
-host = "tivo.lassitu.de"
-#host = "wavehh.lassitu.de:30080"
-mak = "7194378159"
-targetdir = "/p2/media/video/TV"
-gig = 1024.0 * 1024 * 1024
-minfree = 10 * gig
-ignoreepisodetitle = False
-tivodecode = "tivodecode"
-cookies = "cookies.txt"
-proxies=None
-#proxies={"http":"http://us:8888","https":"http://us:8888"}
-config = '~/.tivo/config.yaml'
 
-headers = requests.utils.default_headers()
+
-headers.update(
+
-    {
+class Config:
-        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:52.0) Gecko/20100101 Firefox/52.0',
+	config = '~/.tivo/config.yaml'
-    }
+	cookies = "cookies.txt"
-)
+	gig = 1024.0 * 1024 * 1024
+	headers = requests.utils.default_headers()
+	host = "tivo.lassitu.de"
+	ignoreepisodetitle = False
+	mak = "7194378159"
+	minfree = 10 * gig
+	proxies = None
+	targetdir = "/p2/media/video/TV"
+	tivodecode = "tivodecode"
+	useragent = 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:52.0) Gecko/20100101 Firefox/52.0'
+
+	def __init__(self, file=None):
+		if file:
+			self.config = file
+		self.load(self.config)
+
+		self.headers.update({ 'User-Agent': self.useragent })
+		requests.packages.urllib3.disable_warnings()
+		self.session = requests.session()
+		self.session.verify = False
+		self.session.auth = requests.auth.HTTPDigestAuth("tivo", self.mak)
+		self.session.keep_alive = False
+		self.session.proxies = self.proxies
+
+	def load(self, file):
+		file = os.path.expanduser(file)
+		with open(file, 'r') as f:
+			y = yaml.load(f)
+
+		for show in y['shows']:
+			for key in show:
+				value = show[key]
+				if value and 'short' in value:
+					IncludeShow(key, value['short'])
+				else:
+					IncludeShow(key)
+		for key in y:
+			setattr(self, key, y[key])
+
+	def __repr__(self):
+		return "Config options for tivomirror (singleton)"
+
+config = None
 
 class IncludeShow:
 	includes = dict()
@@ -79,12 +106,6 @@ sys.stdout = flushfile(sys.stdout)
 tmp = "/tmp"
 
 # prepare global requests sesssion to download the TOC and the episodes
-requests.packages.urllib3.disable_warnings()
-session = requests.session()
-session.verify = False
-session.auth = requests.auth.HTTPDigestAuth("tivo", mak)
-session.keep_alive = False
-session.proxies = proxies
 
 
 def roundTime(dt=None, roundTo=60):
@@ -125,6 +146,9 @@ def trimDescription(desc):
 	if i > 0:
 		desc = desc[0:i]
 	i = desc.rfind(". * Copyright Rovi, Inc");
+	if i > 0:
+		desc = desc[0:i]
+	i = desc.rfind(". Copyright Rovi, Inc");
 	if i > 0:
 		desc = desc[0:i]
 	if len(desc) > 80:
@@ -166,7 +190,7 @@ class TivoItem:
 		self.sourcesize = int(getTagText(i, "SourceSize"))
 		self.highdef = getTagText(i, "HighDefinition")
 		self.unique = True
-		if ignoreepisodetitle:
+		if config.ignoreepisodetitle:
 			self.episode = self.datestr
 		if self.episode == "":
 			if self.description != "":
@@ -188,7 +212,7 @@ class TivoItem:
 			self.name = "{} - {}".format(self.title, self.episode)
 		else:
 			self.name = "{} - {} - {}".format(self.title, self.datestr, self.episode)
-		self.dir  = "{}/{}".format(targetdir, re.sub("[:/]", "-", self.title))
+		self.dir  = "{}/{}".format(config.targetdir, re.sub("[:/]", "-", self.title))
 		self.file = "{}/{}".format(self.dir, re.sub("[:/]", "-", self.name))
 		self.name = self.name.encode("utf-8");
 		self.dir  = self.dir.encode("utf-8");
@@ -233,7 +257,7 @@ class TivoToc:
 		fd.close()
 
 	def download_chunk(self, offset):
-		global session, proxies, headers
+		global config
 
 		params = {
 			'Command': 'QueryContainer',
@@ -242,15 +266,15 @@ class TivoToc:
 			'ItemCount': '50',
 			'AnchorOffset': offset
 		}
-		url = "https://{}/TiVoConnect".format(host)
+		url = "https://{}/TiVoConnect".format(config.host)
 		logger.debug("    offset {}".format(offset))
-		r = session.get(url, params=params, timeout=30, verify=False, proxies=proxies, headers=headers)
+		r = config.session.get(url, params=params, timeout=30, verify=False, proxies=config.proxies, headers=config.headers)
 		if r.status_code != 200:
 			r.raise_for_status()
 		return r.text
 
 	def download(self):
-		global session
+		global config
 		offset = 0
 		itemCount = 1
 		self.dom = None
@@ -267,7 +291,7 @@ class TivoToc:
 						root.appendChild(child.cloneNode(True))
 			itemCount = int(getElementText(dom.documentElement.childNodes, "ItemCount"))
 			offset += itemCount
-		saveCookies(session, cookies)
+		saveCookies(config.session, config.cookies)
 		return self.dom
 
 	def getItems(self):
@@ -342,7 +366,7 @@ class FdLogger(threading.Thread):
 
 @timeout(43200)
 def download_item(item, mak, target):
-	global session, proxies, headers
+	global config
 	count = 0
 	start = time.time()
 	upd = start
@@ -351,11 +375,11 @@ def download_item(item, mak, target):
 	logger.info("--- downloading \"{}\"".format(url))
 	logger.info("    {}".format(target))
 	start = time.time()
-	r = session.get(url, stream=True, verify=False, proxies=proxies, headers=headers)
+	r = config.session.get(url, stream=True, verify=False, proxies=config.proxies, headers=config.headers)
 	r.raise_for_status()
 
 	try:
-		p_decode = subprocess.Popen([tivodecode, "--mak", mak, \
+		p_decode = subprocess.Popen([config.tivodecode, "--mak", mak, \
 			"--no-verify", "--out", target, "-"], stdin=subprocess.PIPE,
 			stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 		FdLogger(logger, logging.INFO, p_decode.stdout)
@@ -440,10 +464,10 @@ def download_decode(item, options, mak):
 
 
 def download_one(item, downloaddb, options):
-	global logger, mak
+	global logger
 	logger.info("*** downloading \"{}\": {:.3f} GB".format(item.name, item.sourcesize / 1e9))
 	try:
-		download_decode(item, options, mak)
+		download_decode(item, options, config.mak)
 		downloaddb[item.name] = item.datestr
 		if getattr(downloaddb, "sync", None) and callable(downloaddb.sync):
 			downloaddb.sync()
@@ -467,10 +491,10 @@ def wantitem(item, downloaddb):
 
 
 def mirror(toc, downloaddb, one=False):
-	avail = getAvail(targetdir)
+	avail = getAvail(config.targetdir)
-	if avail < minfree:
+	if avail < config.minfree:
 		logger.error("{}: {:.1f} GB available, at least {:.1f} GB needed, stopping".format\
-			(targetdir, avail / gig, minfree / gig))
+			(config.targetdir, avail / config.gig, config.minfree / config.gig))
 		sys.exit(1)
 	
 	items = toc.getItems()
@@ -507,12 +531,13 @@ def printtoc(toc, downloaddb):
 				print "{:>7.7s}: {}".format(options, item.name)
 				continue
 			print "*** downloading {} ({:.3f} GB)".format(item.name, item.sourcesize / 1e9)
+	print "*** {} shows listed".format(len(items))
 
 
 def main():
-	global config, ignoreepisodetitle, logger
+	global config, logger
 	curdir = os.getcwd()
-	os.chdir(os.path.expanduser("~") + "/.tivo")
+	os.chdir(os.path.expanduser("~/.tivo"))
 	handler = logging.handlers.RotatingFileHandler("tivomirror.log", maxBytes=2*1024*1024, backupCount=5)
 	handler.setFormatter(logging.Formatter(fmt='tivomirror[{}] %(asctime)s %(levelname)6.6s %(message)s'.format(os.getpid()),
 		datefmt='%H:%M:%S'))
@@ -521,6 +546,7 @@ def main():
 	toc = TivoToc()
 	cmd = "list"
 	updateToc = False
+	conffile = None
 
 	try:
 		options, remainder = getopt.getopt(sys.argv[1:], 'c:dvuT',
@@ -528,7 +554,7 @@ def main():
 
 		for opt, arg in options:
 			if opt in ('-c', '--config'):
-				config = arg
+				conffile = arg
 			if opt in ('-d', '--debug'):
 				logger.setLevel(logging.DEBUG)
 			if opt in ('-v', '--verbose'):
@@ -537,19 +563,9 @@ def main():
 			if opt in ('-u', '--update'):
 				updateToc = True
 			if opt in ('-T', '--ignoreepisodetitle'):
-				ignoreepisodetitle = True
+				config.ignoreepisodetitle = True
 
-		config = os.path.expanduser(config)
+		config = Config(conffile)
-		with open(config, 'r') as ymlfile:
-			y = yaml.load(ymlfile)
-		
-		for show in y['shows']:
-			for key in show:
-				value = show[key]
-				if value and 'short' in value:
-					IncludeShow(key, value['short'])
-				else:
-					IncludeShow(key)
 
 		if len(remainder) >= 1:
 			cmd = remainder[0]