stb/tivomirror
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Compare commits

base: stb:master
Branches Tags
stb:master
stb:python3
stb:2016
..
compare: stb:python3
Branches Tags
stb:master
stb:python3
stb:2016

1 commit
master ... python3

Author SHA1 Message Date
Stefan Bethke
823129f1d8 WIP 2020-11-25 21:26:48 +01:00
8 changed files with 73 additions and 127 deletions
Show stats Expand all files Collapse all files

2
.gitignore vendored
View file

@ -1,2 +0,0 @@
.venv
*~

20
README.md
View file

@ -2,27 +2,15 @@
## Requirements ## Requirements
Python 3.4 or newer needs to be installed. Python (2.7) needs to be installed; additionally, these packages need to be
installed:
* `py27-yaml`
`tivodecode` needs to be available on the path. [wmcbrine/tivodecode-ng](https://github.com/wmcbrine/tivodecode-ng) `tivodecode` needs to be available on the path. [mackworth/tivodecode-ng](https://github.com/mackworth/tivodecode-ng)
appears to be working well; the original appears to be working well; the original
[TiVo File Decoder](http://tivodecode.sourceforge.net) has trouble [TiVo File Decoder](http://tivodecode.sourceforge.net) has trouble
decoding some files and fails silently. decoding some files and fails silently.
## Installing
Create a virtual environment and install the required packages:
* `python -m venv .venv` or `virtualenv .venv`
* `. .venv/bin/activate`
* `pip install -r requirements.txt`
## Shell Wrapper
To make it easier to run tivomirror from cron, the shell wrapper
[`tivomirror`](./tivomirror) will activate the venv and then run
`tivomirror.py`.
## Configuration ## Configuration
`tivomirror` reads a config file, by default `~/.tivo/config.yaml`. The `tivomirror` reads a config file, by default `~/.tivo/config.yaml`. The

4
requirements.txt
View file

@ -1,4 +0,0 @@
berkeleydb
pytz
requests
pyyaml

1
tivodb
View file

@ -1 +0,0 @@
wrapper

16
tivodb.py
View file

@ -1,33 +1,33 @@
#!/usr/local/bin/python #!/usr/local/bin/python
import dbm import anydbm
import getopt import getopt
import operator import operator
import os import os
import sys import sys
def usage(): def usage():
print("usage: dbtool {-a entry|-d entry|-l}", file=sys.stderr) print >>sys.stderr, "usage: dbtool {-a entry|-d entry|-l}"
try: try:
optlist, args = getopt.getopt(sys.argv[1:], "a:d:lk") optlist, args = getopt.getopt(sys.argv[1:], "a:d:lk")
except getopt.GetoptError as err: except getopt.GetoptError, err:
print(str(err), file=sys.stderr) print >>sys.stderr, str(err)
usage() usage()
sys.exit(64) sys.exit(64)
if len(args) != 0 or len(optlist) != 1: if len(args) != 0 or len(optlist) != 1:
usage() usage()
sys.exit(64) sys.exit(64)
downloaddb = dbm.open(os.path.expanduser("~") + "/.tivo/downloads.db", "c") downloaddb = anydbm.open(os.path.expanduser("~") + "/.tivo/downloads.db", "c")
for (o, a) in optlist: for (o, a) in optlist:
if o == "-l": if o == "-l":
for i in sorted(downloaddb.keys()): for i in sorted(downloaddb.keys()):
print("%s:\t%s" % (i.decode('utf-8'), downloaddb[i].decode('utf-8'))) print "%s:\t%s" % (i, downloaddb[i])
elif o == "-k": elif o == "-k":
for (k, v) in sorted(list(downloaddb.items()), key=operator.itemgetter(1)): for (k, v) in sorted(downloaddb.items(), key=operator.itemgetter(1)):
print("%s:\t%s" % (k, v)) print "%s:\t%s" % (k, v)
elif o == "-d": elif o == "-d":
del downloaddb[a] del downloaddb[a]
elif o == "-a": elif o == "-a":

1
tivomirror
View file

@ -1 +0,0 @@
wrapper

86
tivomirror.py
View file

@ -1,19 +1,13 @@
#!/usr/local/bin/python3.8 #!/usr/local/bin/python
# -*- coding: utf8 -*- # -*- coding: utf8 -*-
# Download shows from the Tivo # Download shows from the Tivo
import sys
#import importlib
#importlib.reload(sys)
#sys.setdefaultencoding('utf-8')
import dbm import dbm
import http.cookiejar import http.cookiejar
import datetime import datetime
import getopt import getopt
import errno import errno
import fcntl
import functools import functools
import logging import logging
import logging.handlers import logging.handlers
@ -27,17 +21,15 @@ import subprocess
import sys import sys
import threading import threading
import time import time
import urllib.request, urllib.error, urllib.parse import urllib
import xml.dom.minidom import xml.dom.minidom
import yaml import yaml
from contextlib import contextmanager
from io import TextIOWrapper
class Config: class Config:
config = '~/.tivo/config.yaml' config = '~/.tivo/config.yaml'
lockfile = config + '.lock'
cookies = "cookies.txt" cookies = "cookies.txt"
gig = 1024.0 * 1024 * 1024 gig = 1024.0 * 1024 * 1024
headers = requests.utils.default_headers() headers = requests.utils.default_headers()
@ -113,7 +105,7 @@ class flushfile(object):
def write(self, x): def write(self, x):
self.f.write(x) self.f.write(x)
self.f.flush() self.f.flush()
#sys.stdout = flushfile(sys.stdout) sys.stdout = flushfile(sys.stdout)
tmp = "/tmp" tmp = "/tmp"
@ -214,10 +206,10 @@ class TivoItem:
self.unique = False self.unique = False
self.formatnames() self.formatnames()
def formatnames(self): def formatnames(self):
if self.episodeNumber and self.episodeNumber != '0': if self.episodeNumber and self.episodeNumber != u'0':
en = int(self.episodeNumber) en = int(self.episodeNumber)
if en >= 100: if en >= 100:
self.name = "{} S{:02d}E{:02d} {}".format(self.title, int(en / 100), int(en % 100), self.episode) self.name = "{} S{:02d}E{:02d} {}".format(self.title, en / 100, en % 100, self.episode)
else: else:
self.name = "{} E{} {}".format(self.title, self.episodeNumber, self.episode) self.name = "{} E{} {}".format(self.title, self.episodeNumber, self.episode)
elif self.unique: elif self.unique:
@ -226,14 +218,14 @@ class TivoItem:
self.name = "{} - {} - {}".format(self.title, self.datestr, self.episode) self.name = "{} - {} - {}".format(self.title, self.datestr, self.episode)
self.dir = "{}/{}".format(config.targetdir, re.sub("[:/]", "-", self.title)) self.dir = "{}/{}".format(config.targetdir, re.sub("[:/]", "-", self.title))
self.file = "{}/{}".format(self.dir, re.sub("[:/]", "-", self.name)) self.file = "{}/{}".format(self.dir, re.sub("[:/]", "-", self.name))
#self.name = self.name.encode("utf-8"); self.name = self.name.encode("utf-8");
#self.dir = self.dir.encode("utf-8"); self.dir = self.dir.encode("utf-8");
#self.file = self.file.encode("utf-8"); self.file = self.file.encode("utf-8");
def getPath(self, options): def getPath(self, options):
title = self.title title = self.title
if options.short: if options.short:
title = options.short title = options.short
if self.episodeNumber and self.episodeNumber != '0': if self.episodeNumber and self.episodeNumber != u'0':
en = int(self.episodeNumber) en = int(self.episodeNumber)
if en >= 100: if en >= 100:
name = "{} S{:02d}E{:02d} {}".format(title, en / 100, en % 100, self.episode) name = "{} S{:02d}E{:02d} {}".format(title, en / 100, en % 100, self.episode)
@ -244,8 +236,7 @@ class TivoItem:
else: else:
name = "{} - {} {}".format(title, self.shortdate, self.episode) name = "{} - {} {}".format(title, self.shortdate, self.episode)
path = "{}/{}".format(self.dir, re.sub("[:/]", "-", name)) path = "{}/{}".format(self.dir, re.sub("[:/]", "-", name))
return path return path.encode("utf-8");
#return path.encode("utf-8");
def __str__(self): def __str__(self):
return repr(self.title) return repr(self.title)
@ -324,11 +315,9 @@ class TivoToc:
names[item.name] = [] names[item.name] = []
names[item.name].append(item) names[item.name].append(item)
for name in names: for name in names:
if len(names[name]) > 1 and title not in self.uniquedb: utf8title = title.encode("utf-8")
self.uniquedb[title] = "1" if len(names[name]) > 1 and not self.uniquedb.has_key(utf8title):
# utf8title = title.encode("utf-8") self.uniquedb[utf8title] = "1"
# if len(names[name]) > 1 and utf8title not in self.uniquedb:
# self.uniquedb[utf8title] = "1"
if getattr(self.uniquedb, "sync", None) and callable(self.uniquedb.sync): if getattr(self.uniquedb, "sync", None) and callable(self.uniquedb.sync):
self.uniquedb.sync() self.uniquedb.sync()
# update all items based on config and uniquedb # update all items based on config and uniquedb
@ -377,24 +366,12 @@ class FdLogger(threading.Thread):
try: try:
# for line in fd buffers, so use this instead # for line in fd buffers, so use this instead
for line in iter(self.fd.readline, b''): for line in iter(self.fd.readline, b''):
line = line.strip('\n') self.logger.log(self.lvl, ": %s", line.strip('\n'))
if line.strip() != "":
self.logger.log(self.lvl, ": %s", line)
self.fd.close() self.fd.close()
except Exception: except Exception:
self.logger.exception("") self.logger.exception("")
@contextmanager
def exclusive():
with open(os.path.expanduser(config.lockfile), 'w') as f:
try:
fcntl.lockf(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
except BlockingIOError as e:
raise TivoException('another tivomirror instance is already running')
yield 'locked'
fcntl.lockf(f, fcntl.LOCK_UN)
@timeout(43200) @timeout(43200)
def download_item(item, mak, target): def download_item(item, mak, target):
global config global config
@ -413,13 +390,13 @@ def download_item(item, mak, target):
p_decode = subprocess.Popen([config.tivodecode, "--mak", mak, \ p_decode = subprocess.Popen([config.tivodecode, "--mak", mak, \
"--no-verify", "--out", target, "-"], stdin=subprocess.PIPE, "--no-verify", "--out", target, "-"], stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout=subprocess.PIPE, stderr=subprocess.PIPE)
FdLogger(logger, logging.INFO, TextIOWrapper(p_decode.stdout)) FdLogger(logger, logging.INFO, p_decode.stdout)
FdLogger(logger, logging.INFO, TextIOWrapper(p_decode.stderr)) FdLogger(logger, logging.INFO, p_decode.stderr)
def info(signum, frame): def info(signum, frame):
upd = time.time() upd = time.time()
dur = now - start dur = now - start
mb = count / 1e6 mb = count / 1e6
print("{:5.1f}% {:5.3f} GB downloaded in {:.0f} min, {:.3f} MB/s".format( print("{:5.1f}% {:5.3f} GB downloaded in {:.0f} min, {.3f} MB/s".format(
100.0 * count / item.sourcesize, 100.0 * count / item.sourcesize,
mb / 1e3, dur / 60, mb / dur)) mb / 1e3, dur / 60, mb / dur))
try: try:
@ -438,7 +415,7 @@ def download_item(item, mak, target):
upd = now upd = now
dur = now - start dur = now - start
mb = count / 1e6 mb = count / 1e6
logger.debug(" {:5.1f}% {:5.3f} GB downloaded in {:0.0f} min, {:0.3f} MB/s".format( logger.debug(" {:5.1f}% {:5.3f} GB downloaded in {:.0f} min, {:.3f} MB/s".format(
100.0 * count / item.sourcesize, 100.0 * count / item.sourcesize,
mb / 1e3, dur / 60, mb / dur)) mb / 1e3, dur / 60, mb / dur))
except Exception as e: except Exception as e:
@ -482,12 +459,11 @@ def download_decode(item, options, mak):
try: try:
download_item(item, mak, item.target) download_item(item, mak, item.target)
except Exception as e: except Exception as e:
exc_info = sys.exc_info()
try: try:
os.remove(item.target) os.remove(item.target)
except Exception as e2: except Exception as e2:
pass pass
raise exc_info[1].with_traceback(exc_info[2]) raise e
try: try:
os.utime(item.target, (item.time, item.time)) os.utime(item.target, (item.time, item.time))
except Exception as e: except Exception as e:
@ -497,7 +473,6 @@ def download_decode(item, options, mak):
def download_one(item, downloaddb, options): def download_one(item, downloaddb, options):
global config, logger global config, logger
logger.info("*** downloading \"{}\": {:.3f} GB".format(item.name, item.sourcesize / 1e9)) logger.info("*** downloading \"{}\": {:.3f} GB".format(item.name, item.sourcesize / 1e9))
# sys.exit(1)
try: try:
download_decode(item, options, config.mak) download_decode(item, options, config.mak)
downloaddb[item.name] = item.datestr downloaddb[item.name] = item.datestr
@ -522,10 +497,10 @@ def wantitem(item, downloaddb):
return "recording" return "recording"
if item.available == "No": if item.available == "No":
return "not available" return "not available"
if item.name in downloaddb: if downloaddb.has_key(item.name):
return "already downloaded" return "already downloaded"
for i in (item.title, item.episode, item.name): for i in (item.title, item.episode, item.name):
if i in IncludeShow.includes: if IncludeShow.includes.has_key(i):
return IncludeShow.includes[i] return IncludeShow.includes[i]
return "not included" return "not included"
@ -537,12 +512,11 @@ def mirror(toc, downloaddb, one=False):
(config.targetdir, avail / config.gig, config.minfree / config.gig)) (config.targetdir, avail / config.gig, config.minfree / config.gig))
sys.exit(1) sys.exit(1)
with exclusive() as lock:
items = toc.getItems() items = toc.getItems()
logger.info("*** {} shows listed".format(len(items))) logger.info("*** {} shows listed".format(len(items)))
for item in items: for item in items:
options = wantitem(item, downloaddb) options = wantitem(item, downloaddb)
if isinstance(options, str): if isinstance(options, basestring):
logger.debug("*** skipping \"{}\": {}".format(item.name, options)) logger.debug("*** skipping \"{}\": {}".format(item.name, options))
else: else:
download_one(item, downloaddb, options) download_one(item, downloaddb, options)
@ -551,20 +525,18 @@ def mirror(toc, downloaddb, one=False):
def download_episode(toc, downloaddb, episode): def download_episode(toc, downloaddb, episode):
with exclusive() as lock:
items = toc.getItems() items = toc.getItems()
options = {} options = {}
for item in items: for item in items:
if item.title == episode or item.name == episode or item.episode == episode: if item.title == episode or item.name == episode or item.episode == episode:
for i in (item.title, item.episode, item.name): for i in (item.title, item.episode, item.name):
if i in IncludeShow.includes: if IncludeShow.includes.has_key(i):
options = IncludeShow.includes[i] options = IncludeShow.includes[i]
download_one(item, downloaddb, options) download_one(item, downloaddb, options)
return return
def printtoc(toc, downloaddb): def printtoc(toc, downloaddb):
with exclusive() as lock:
items = toc.getItems() items = toc.getItems()
print("*** {} shows listed".format(len(items))) print("*** {} shows listed".format(len(items)))
shows = {} shows = {}
@ -575,7 +547,7 @@ def printtoc(toc, downloaddb):
for title in sorted(shows): for title in sorted(shows):
for item in sorted(shows[title], key=lambda i: i.name): for item in sorted(shows[title], key=lambda i: i.name):
options = wantitem(item, downloaddb) options = wantitem(item, downloaddb)
if isinstance(options, str): if isinstance(options, basestring):
print("{:>7.7s}: {}".format(options, item.name)) print("{:>7.7s}: {}".format(options, item.name))
continue continue
print("*** downloading {} ({:.3f} GB)".format(item.name, item.sourcesize / 1e9)) print("*** downloading {} ({:.3f} GB)".format(item.name, item.sourcesize / 1e9))
@ -583,8 +555,8 @@ def printtoc(toc, downloaddb):
def usage(): def usage():
print('usage: tivomirror -dvuT [-c config] cmd', file=sys.stderr) print >>sys.stderr, 'usage: tivomirror -dvuT [-c config] cmd'
print(' cmd is one of download, list, mirror, mirrorone', file=sys.stderr) print >>sys.stderr, ' cmd is one of download, list, mirror, mirrorone'
sys.exit(64) sys.exit(64)
@ -642,12 +614,12 @@ def main():
download_episode(toc, downloaddb, remainder[1]) download_episode(toc, downloaddb, remainder[1])
else: else:
logger.error("invalid command {}".format(cmd)) logger.error("invalid command {}".format(cmd))
print("invalid command {}".format(cmd), file=sys.stderr) print >>sys.stderr, "invalid command {}".format(cmd)
usage() usage()
downloaddb.close() downloaddb.close()
except getopt.GetoptError as e: except getopt.GetoptError as e:
print('Error parsing options: {}'.format(e), file=sys.stderr) print >>sys.stderr, 'Error parsing options: {}'.format(e)
usage() usage()
except Exception: except Exception:
logger.exception("") logger.exception("")

6
wrapper
View file

@ -1,6 +0,0 @@
#!/bin/sh
here="$(dirname $0)"
this="$(basename $0)"
. ${here}/.venv/bin/activate
exec python ${here}/${this}.py $@