stb/tivomirror
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update to Python 3

Browse source
This commit is contained in:
Stefan Bethke 2021-08-28 23:33:58 +02:00
parent 3e92f9b39b
commit 40caba68fc
5 changed files with 72 additions and 55 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored Normal file
View file

@ -0,0 +1 @@
.venv

4
requirements.txt Normal file
View file

@ -0,0 +1,4 @@
berkeleydb
pytz
requests
pyyaml

16
tivodb.py
View file

@ -1,33 +1,33 @@
#!/usr/local/bin/python #!/usr/local/bin/python
import anydbm import dbm
import getopt import getopt
import operator import operator
import os import os
import sys import sys
def usage(): def usage():
print >>sys.stderr, "usage: dbtool {-a entry|-d entry|-l}" print("usage: dbtool {-a entry|-d entry|-l}", file=sys.stderr)
try: try:
optlist, args = getopt.getopt(sys.argv[1:], "a:d:lk") optlist, args = getopt.getopt(sys.argv[1:], "a:d:lk")
except getopt.GetoptError, err: except getopt.GetoptError as err:
print >>sys.stderr, str(err) print(str(err), file=sys.stderr)
usage() usage()
sys.exit(64) sys.exit(64)
if len(args) != 0 or len(optlist) != 1: if len(args) != 0 or len(optlist) != 1:
usage() usage()
sys.exit(64) sys.exit(64)
downloaddb = anydbm.open(os.path.expanduser("~") + "/.tivo/downloads.db", "c") downloaddb = dbm.open(os.path.expanduser("~") + "/.tivo/downloads.db", "c")
for (o, a) in optlist: for (o, a) in optlist:
if o == "-l": if o == "-l":
for i in sorted(downloaddb.keys()): for i in sorted(downloaddb.keys()):
print "%s:\t%s" % (i, downloaddb[i]) print("%s:\t%s" % (i.decode('utf-8'), downloaddb[i].decode('utf-8')))
elif o == "-k": elif o == "-k":
for (k, v) in sorted(downloaddb.items(), key=operator.itemgetter(1)): for (k, v) in sorted(list(downloaddb.items()), key=operator.itemgetter(1)):
print "%s:\t%s" % (k, v) print("%s:\t%s" % (k, v))
elif o == "-d": elif o == "-d":
del downloaddb[a] del downloaddb[a]
elif o == "-a": elif o == "-a":

5
tivomirror Executable file
View file

@ -0,0 +1,5 @@
#!/bin/sh
here="$(dirname $0)"
. ${here}/.venv/bin/activate
exec python ${here}/tivomirror.py $@

101
tivomirror.py
View file

@ -1,14 +1,15 @@
#!/usr/local/bin/python #!/usr/local/bin/python3.8
# -*- coding: utf8 -*- # -*- coding: utf8 -*-
# Download shows from the Tivo # Download shows from the Tivo
import sys import sys
reload(sys) #import importlib
sys.setdefaultencoding('utf-8') #importlib.reload(sys)
#sys.setdefaultencoding('utf-8')
import anydbm import dbm
import cookielib import http.cookiejar
import datetime import datetime
import getopt import getopt
import errno import errno
@ -25,11 +26,11 @@ import subprocess
import sys import sys
import threading import threading
import time import time
import urllib2 import urllib.request, urllib.error, urllib.parse
import xml.dom.minidom import xml.dom.minidom
import yaml import yaml
from io import TextIOWrapper
class Config: class Config:
@ -109,7 +110,7 @@ class flushfile(object):
def write(self, x): def write(self, x):
self.f.write(x) self.f.write(x)
self.f.flush() self.f.flush()
sys.stdout = flushfile(sys.stdout) #sys.stdout = flushfile(sys.stdout)
tmp = "/tmp" tmp = "/tmp"
@ -164,7 +165,7 @@ def trimDescription(desc):
return desc return desc
def saveCookies(session, filename): def saveCookies(session, filename):
cj = cookielib.MozillaCookieJar(filename) cj = http.cookiejar.MozillaCookieJar(filename)
for cookie in session.cookies: for cookie in session.cookies:
logger.debug("storing cookie {}".format(cookie)) logger.debug("storing cookie {}".format(cookie))
cj.set_cookie(cookie) cj.set_cookie(cookie)
@ -210,10 +211,10 @@ class TivoItem:
self.unique = False self.unique = False
self.formatnames() self.formatnames()
def formatnames(self): def formatnames(self):
if self.episodeNumber and self.episodeNumber != u'0': if self.episodeNumber and self.episodeNumber != '0':
en = int(self.episodeNumber) en = int(self.episodeNumber)
if en >= 100: if en >= 100:
self.name = "{} S{:02d}E{:02d} {}".format(self.title, en / 100, en % 100, self.episode) self.name = "{} S{:02d}E{:02d} {}".format(self.title, int(en / 100), int(en % 100), self.episode)
else: else:
self.name = "{} E{} {}".format(self.title, self.episodeNumber, self.episode) self.name = "{} E{} {}".format(self.title, self.episodeNumber, self.episode)
elif self.unique: elif self.unique:
@ -222,14 +223,14 @@ class TivoItem:
self.name = "{} - {} - {}".format(self.title, self.datestr, self.episode) self.name = "{} - {} - {}".format(self.title, self.datestr, self.episode)
self.dir = "{}/{}".format(config.targetdir, re.sub("[:/]", "-", self.title)) self.dir = "{}/{}".format(config.targetdir, re.sub("[:/]", "-", self.title))
self.file = "{}/{}".format(self.dir, re.sub("[:/]", "-", self.name)) self.file = "{}/{}".format(self.dir, re.sub("[:/]", "-", self.name))
self.name = self.name.encode("utf-8"); #self.name = self.name.encode("utf-8");
self.dir = self.dir.encode("utf-8"); #self.dir = self.dir.encode("utf-8");
self.file = self.file.encode("utf-8"); #self.file = self.file.encode("utf-8");
def getPath(self, options): def getPath(self, options):
title = self.title title = self.title
if options.short: if options.short:
title = options.short title = options.short
if self.episodeNumber and self.episodeNumber != u'0': if self.episodeNumber and self.episodeNumber != '0':
en = int(self.episodeNumber) en = int(self.episodeNumber)
if en >= 100: if en >= 100:
name = "{} S{:02d}E{:02d} {}".format(title, en / 100, en % 100, self.episode) name = "{} S{:02d}E{:02d} {}".format(title, en / 100, en % 100, self.episode)
@ -240,7 +241,8 @@ class TivoItem:
else: else:
name = "{} - {} {}".format(title, self.shortdate, self.episode) name = "{} - {} {}".format(title, self.shortdate, self.episode)
path = "{}/{}".format(self.dir, re.sub("[:/]", "-", name)) path = "{}/{}".format(self.dir, re.sub("[:/]", "-", name))
return path.encode("utf-8"); return path
#return path.encode("utf-8");
def __str__(self): def __str__(self):
return repr(self.title) return repr(self.title)
@ -249,7 +251,7 @@ class TivoToc:
def __init__(self): def __init__(self):
self.dom = None self.dom = None
self.filename = "toc.xml" self.filename = "toc.xml"
self.uniquedb = anydbm.open("unique.db", "c") self.uniquedb = dbm.open("unique.db", "c")
self.items = [] self.items = []
pass pass
@ -319,9 +321,11 @@ class TivoToc:
names[item.name] = [] names[item.name] = []
names[item.name].append(item) names[item.name].append(item)
for name in names: for name in names:
utf8title = title.encode("utf-8") if len(names[name]) > 1 and title not in self.uniquedb:
if len(names[name]) > 1 and not self.uniquedb.has_key(utf8title): self.uniquedb[title] = "1"
self.uniquedb[utf8title] = "1" # utf8title = title.encode("utf-8")
# if len(names[name]) > 1 and utf8title not in self.uniquedb:
# self.uniquedb[utf8title] = "1"
if getattr(self.uniquedb, "sync", None) and callable(self.uniquedb.sync): if getattr(self.uniquedb, "sync", None) and callable(self.uniquedb.sync):
self.uniquedb.sync() self.uniquedb.sync()
# update all items based on config and uniquedb # update all items based on config and uniquedb
@ -370,7 +374,9 @@ class FdLogger(threading.Thread):
try: try:
# for line in fd buffers, so use this instead # for line in fd buffers, so use this instead
for line in iter(self.fd.readline, b''): for line in iter(self.fd.readline, b''):
self.logger.log(self.lvl, ": %s", line.strip('\n')) line = line.strip('\n')
if line.strip() != "":
self.logger.log(self.lvl, ": %s", line)
self.fd.close() self.fd.close()
except Exception: except Exception:
self.logger.exception("") self.logger.exception("")
@ -394,15 +400,15 @@ def download_item(item, mak, target):
p_decode = subprocess.Popen([config.tivodecode, "--mak", mak, \ p_decode = subprocess.Popen([config.tivodecode, "--mak", mak, \
"--no-verify", "--out", target, "-"], stdin=subprocess.PIPE, "--no-verify", "--out", target, "-"], stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout=subprocess.PIPE, stderr=subprocess.PIPE)
FdLogger(logger, logging.INFO, p_decode.stdout) FdLogger(logger, logging.INFO, TextIOWrapper(p_decode.stdout))
FdLogger(logger, logging.INFO, p_decode.stderr) FdLogger(logger, logging.INFO, TextIOWrapper(p_decode.stderr))
def info(signum, frame): def info(signum, frame):
upd = time.time() upd = time.time()
dur = now - start dur = now - start
mb = count / 1e6 mb = count / 1e6
print "{:5.1f}% {:5.3f} GB downloaded in {:.0f} min, {.3f} MB/s".format( print("{:5.1f}% {:5.3f} GB downloaded in {:.0f} min, {:.3f} MB/s".format(
100.0 * count / item.sourcesize, 100.0 * count / item.sourcesize,
mb / 1e3, dur / 60, mb / dur) mb / 1e3, dur / 60, mb / dur))
try: try:
signal.signal(signal.SIGINFO, info) signal.signal(signal.SIGINFO, info)
except Exception: except Exception:
@ -419,7 +425,7 @@ def download_item(item, mak, target):
upd = now upd = now
dur = now - start dur = now - start
mb = count / 1e6 mb = count / 1e6
logger.debug(" {:5.1f}% {:5.3f} GB downloaded in {:.0f} min, {:.3f} MB/s".format( logger.debug(" {:5.1f}% {:5.3f} GB downloaded in {:0.0f} min, {:0.3f} MB/s".format(
100.0 * count / item.sourcesize, 100.0 * count / item.sourcesize,
mb / 1e3, dur / 60, mb / dur)) mb / 1e3, dur / 60, mb / dur))
except Exception as e: except Exception as e:
@ -443,7 +449,7 @@ def download_item(item, mak, target):
if p_decode.returncode == None: if p_decode.returncode == None:
logger.debug("terminating tivodecode") logger.debug("terminating tivodecode")
p_decode.terminate() p_decode.terminate()
except Exception, e: except Exception as e:
pass pass
p_decode.wait() p_decode.wait()
logger.info("tivodecode exited with {}".format(p_decode.returncode)) logger.info("tivodecode exited with {}".format(p_decode.returncode))
@ -462,22 +468,23 @@ def download_decode(item, options, mak):
pass pass
try: try:
download_item(item, mak, item.target) download_item(item, mak, item.target)
except Exception, e: except Exception as e:
exc_info = sys.exc_info() exc_info = sys.exc_info()
try: try:
os.remove(item.target) os.remove(item.target)
except Exception, e2: except Exception as e2:
pass pass
raise exc_info[1], None, exc_info[2] raise exc_info[1].with_traceback(exc_info[2])
try: try:
os.utime(item.target, (item.time, item.time)) os.utime(item.target, (item.time, item.time))
except Exception, e: except Exception as e:
logger.error("Problem setting timestamp: {}".format(e)) logger.error("Problem setting timestamp: {}".format(e))
def download_one(item, downloaddb, options): def download_one(item, downloaddb, options):
global config, logger global config, logger
logger.info("*** downloading \"{}\": {:.3f} GB".format(item.name, item.sourcesize / 1e9)) logger.info("*** downloading \"{}\": {:.3f} GB".format(item.name, item.sourcesize / 1e9))
# sys.exit(1)
try: try:
download_decode(item, options, config.mak) download_decode(item, options, config.mak)
downloaddb[item.name] = item.datestr downloaddb[item.name] = item.datestr
@ -489,11 +496,11 @@ def download_one(item, downloaddb, options):
cmd = cmd.format(item=item, options=options, config=config) cmd = cmd.format(item=item, options=options, config=config)
r = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) r = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
logger.debug("Post-process {}: {}".format(cmd, r)) logger.debug("Post-process {}: {}".format(cmd, r))
except Exception, e: except Exception as e:
logger.warn("Error running postprocess command '{}' for item {}: {}".format(cmd, item, e)) logger.warn("Error running postprocess command '{}' for item {}: {}".format(cmd, item, e))
logger.debug("Sleeping 30 seconds before moving on...") logger.debug("Sleeping 30 seconds before moving on...")
time.sleep(30) time.sleep(30)
except TivoException, e: except TivoException as e:
logger.info("Error processing \"{}\": {}".format(item.name, e)) logger.info("Error processing \"{}\": {}".format(item.name, e))
@ -502,10 +509,10 @@ def wantitem(item, downloaddb):
return "recording" return "recording"
if item.available == "No": if item.available == "No":
return "not available" return "not available"
if downloaddb.has_key(item.name): if item.name in downloaddb:
return "already downloaded" return "already downloaded"
for i in (item.title, item.episode, item.name): for i in (item.title, item.episode, item.name):
if IncludeShow.includes.has_key(i): if i in IncludeShow.includes:
return IncludeShow.includes[i] return IncludeShow.includes[i]
return "not included" return "not included"
@ -521,7 +528,7 @@ def mirror(toc, downloaddb, one=False):
logger.info("*** {} shows listed".format(len(items))) logger.info("*** {} shows listed".format(len(items)))
for item in items: for item in items:
options = wantitem(item, downloaddb) options = wantitem(item, downloaddb)
if isinstance(options, basestring): if isinstance(options, str):
logger.debug("*** skipping \"{}\": {}".format(item.name, options)) logger.debug("*** skipping \"{}\": {}".format(item.name, options))
else: else:
download_one(item, downloaddb, options) download_one(item, downloaddb, options)
@ -535,7 +542,7 @@ def download_episode(toc, downloaddb, episode):
for item in items: for item in items:
if item.title == episode or item.name == episode or item.episode == episode: if item.title == episode or item.name == episode or item.episode == episode:
for i in (item.title, item.episode, item.name): for i in (item.title, item.episode, item.name):
if IncludeShow.includes.has_key(i): if i in IncludeShow.includes:
options = IncludeShow.includes[i] options = IncludeShow.includes[i]
download_one(item, downloaddb, options) download_one(item, downloaddb, options)
return return
@ -543,7 +550,7 @@ def download_episode(toc, downloaddb, episode):
def printtoc(toc, downloaddb): def printtoc(toc, downloaddb):
items = toc.getItems() items = toc.getItems()
print "*** {} shows listed".format(len(items)) print("*** {} shows listed".format(len(items)))
shows = {} shows = {}
for item in items: for item in items:
if item.title not in shows: if item.title not in shows:
@ -552,16 +559,16 @@ def printtoc(toc, downloaddb):
for title in sorted(shows): for title in sorted(shows):
for item in sorted(shows[title], key=lambda i: i.name): for item in sorted(shows[title], key=lambda i: i.name):
options = wantitem(item, downloaddb) options = wantitem(item, downloaddb)
if isinstance(options, basestring): if isinstance(options, str):
print "{:>7.7s}: {}".format(options, item.name) print("{:>7.7s}: {}".format(options, item.name))
continue continue
print "*** downloading {} ({:.3f} GB)".format(item.name, item.sourcesize / 1e9) print("*** downloading {} ({:.3f} GB)".format(item.name, item.sourcesize / 1e9))
print "*** {} shows listed".format(len(items)) print("*** {} shows listed".format(len(items)))
def usage(): def usage():
print >>sys.stderr, 'usage: tivomirror -dvuT [-c config] cmd' print('usage: tivomirror -dvuT [-c config] cmd', file=sys.stderr)
print >>sys.stderr, ' cmd is one of download, list, mirror, mirrorone' print(' cmd is one of download, list, mirror, mirrorone', file=sys.stderr)
sys.exit(64) sys.exit(64)
@ -573,7 +580,7 @@ def main():
handler.setFormatter(logging.Formatter(fmt='tivomirror[{}] %(asctime)s %(levelname)6.6s %(message)s'.format(os.getpid()), handler.setFormatter(logging.Formatter(fmt='tivomirror[{}] %(asctime)s %(levelname)6.6s %(message)s'.format(os.getpid()),
datefmt='%d-%m %H:%M:%S')) datefmt='%d-%m %H:%M:%S'))
logger.addHandler(handler) logger.addHandler(handler)
downloaddb = anydbm.open("downloads.db", "c") downloaddb = dbm.open("downloads.db", "c")
toc = TivoToc() toc = TivoToc()
cmd = "list" cmd = "list"
updateToc = False updateToc = False
@ -619,12 +626,12 @@ def main():
download_episode(toc, downloaddb, remainder[1]) download_episode(toc, downloaddb, remainder[1])
else: else:
logger.error("invalid command {}".format(cmd)) logger.error("invalid command {}".format(cmd))
print >>sys.stderr, "invalid command {}".format(cmd) print("invalid command {}".format(cmd), file=sys.stderr)
usage() usage()
downloaddb.close() downloaddb.close()
except getopt.GetoptError as e: except getopt.GetoptError as e:
print >>sys.stderr, 'Error parsing options: {}'.format(e) print('Error parsing options: {}'.format(e), file=sys.stderr)
usage() usage()
except Exception: except Exception:
logger.exception("") logger.exception("")