stb/tivomirror
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Compare commits

base: stb:master
Branches Tags
stb:master
stb:python3
stb:2016
..
compare: stb:python3
Branches Tags
stb:master
stb:python3
stb:2016

1 commit
master ... python3

Author SHA1 Message Date
Stefan Bethke
823129f1d8 WIP 2020-11-25 21:26:48 +01:00
8 changed files with 73 additions and 127 deletions
Show stats Expand all files Collapse all files

2
.gitignore vendored
View file

@ -1,2 +0,0 @@
.venv
*~

20
README.md
View file

@ -2,27 +2,15 @@
## Requirements
Python 3.4 or newer needs to be installed.
Python (2.7) needs to be installed; additionally, these packages need to be
installed:
* `py27-yaml`
`tivodecode` needs to be available on the path. [wmcbrine/tivodecode-ng](https://github.com/wmcbrine/tivodecode-ng)
`tivodecode` needs to be available on the path. [mackworth/tivodecode-ng](https://github.com/mackworth/tivodecode-ng)
appears to be working well; the original
[TiVo File Decoder](http://tivodecode.sourceforge.net) has trouble
decoding some files and fails silently.
## Installing
Create a virtual environment and install the required packages:
* `python -m venv .venv` or `virtualenv .venv`
* `. .venv/bin/activate`
* `pip install -r requirements.txt`
## Shell Wrapper
To make it easier to run tivomirror from cron, the shell wrapper
[`tivomirror`](./tivomirror) will activate the venv and then run
`tivomirror.py`.
## Configuration
`tivomirror` reads a config file, by default `~/.tivo/config.yaml`. The

4
requirements.txt
View file

@ -1,4 +0,0 @@
berkeleydb
pytz
requests
pyyaml

1
tivodb
View file

@ -1 +0,0 @@
wrapper

16
tivodb.py
View file

@ -1,33 +1,33 @@
#!/usr/local/bin/python
import dbm
import anydbm
import getopt
import operator
import os
import sys
def usage():
print("usage: dbtool {-a entry|-d entry|-l}", file=sys.stderr)
print >>sys.stderr, "usage: dbtool {-a entry|-d entry|-l}"
try:
optlist, args = getopt.getopt(sys.argv[1:], "a:d:lk")
except getopt.GetoptError as err:
print(str(err), file=sys.stderr)
except getopt.GetoptError, err:
print >>sys.stderr, str(err)
usage()
sys.exit(64)
if len(args) != 0 or len(optlist) != 1:
usage()
sys.exit(64)
downloaddb = dbm.open(os.path.expanduser("~") + "/.tivo/downloads.db", "c")
downloaddb = anydbm.open(os.path.expanduser("~") + "/.tivo/downloads.db", "c")
for (o, a) in optlist:
if o == "-l":
for i in sorted(downloaddb.keys()):
print("%s:\t%s" % (i.decode('utf-8'), downloaddb[i].decode('utf-8')))
print "%s:\t%s" % (i, downloaddb[i])
elif o == "-k":
for (k, v) in sorted(list(downloaddb.items()), key=operator.itemgetter(1)):
print("%s:\t%s" % (k, v))
for (k, v) in sorted(downloaddb.items(), key=operator.itemgetter(1)):
print "%s:\t%s" % (k, v)
elif o == "-d":
del downloaddb[a]
elif o == "-a":

1
tivomirror
View file

@ -1 +0,0 @@
wrapper

150
tivomirror.py
View file

@ -1,19 +1,13 @@
#!/usr/local/bin/python3.8
#!/usr/local/bin/python
# -*- coding: utf8 -*-
# Download shows from the Tivo
import sys
#import importlib
#importlib.reload(sys)
#sys.setdefaultencoding('utf-8')
import dbm
import http.cookiejar
import datetime
import getopt
import errno
import fcntl
import functools
import logging
import logging.handlers
@ -27,17 +21,15 @@ import subprocess
import sys
import threading
import time
import urllib.request, urllib.error, urllib.parse
import urllib
import xml.dom.minidom
import yaml
from contextlib import contextmanager
from io import TextIOWrapper
class Config:
config = '~/.tivo/config.yaml'
lockfile = config + '.lock'
cookies = "cookies.txt"
gig = 1024.0 * 1024 * 1024
headers = requests.utils.default_headers()
@ -113,7 +105,7 @@ class flushfile(object):
def write(self, x):
self.f.write(x)
self.f.flush()
#sys.stdout = flushfile(sys.stdout)
sys.stdout = flushfile(sys.stdout)
tmp = "/tmp"
@ -214,10 +206,10 @@ class TivoItem:
self.unique = False
self.formatnames()
def formatnames(self):
if self.episodeNumber and self.episodeNumber != '0':
if self.episodeNumber and self.episodeNumber != u'0':
en = int(self.episodeNumber)
if en >= 100:
self.name = "{} S{:02d}E{:02d} {}".format(self.title, int(en / 100), int(en % 100), self.episode)
self.name = "{} S{:02d}E{:02d} {}".format(self.title, en / 100, en % 100, self.episode)
else:
self.name = "{} E{} {}".format(self.title, self.episodeNumber, self.episode)
elif self.unique:
@ -226,14 +218,14 @@ class TivoItem:
self.name = "{} - {} - {}".format(self.title, self.datestr, self.episode)
self.dir = "{}/{}".format(config.targetdir, re.sub("[:/]", "-", self.title))
self.file = "{}/{}".format(self.dir, re.sub("[:/]", "-", self.name))
#self.name = self.name.encode("utf-8");
#self.dir = self.dir.encode("utf-8");
#self.file = self.file.encode("utf-8");
self.name = self.name.encode("utf-8");
self.dir = self.dir.encode("utf-8");
self.file = self.file.encode("utf-8");
def getPath(self, options):
title = self.title
if options.short:
title = options.short
if self.episodeNumber and self.episodeNumber != '0':
if self.episodeNumber and self.episodeNumber != u'0':
en = int(self.episodeNumber)
if en >= 100:
name = "{} S{:02d}E{:02d} {}".format(title, en / 100, en % 100, self.episode)
@ -244,8 +236,7 @@ class TivoItem:
else:
name = "{} - {} {}".format(title, self.shortdate, self.episode)
path = "{}/{}".format(self.dir, re.sub("[:/]", "-", name))
return path
#return path.encode("utf-8");
return path.encode("utf-8");
def __str__(self):
return repr(self.title)
@ -324,11 +315,9 @@ class TivoToc:
names[item.name] = []
names[item.name].append(item)
for name in names:
if len(names[name]) > 1 and title not in self.uniquedb:
self.uniquedb[title] = "1"
# utf8title = title.encode("utf-8")
# if len(names[name]) > 1 and utf8title not in self.uniquedb:
# self.uniquedb[utf8title] = "1"
utf8title = title.encode("utf-8")
if len(names[name]) > 1 and not self.uniquedb.has_key(utf8title):
self.uniquedb[utf8title] = "1"
if getattr(self.uniquedb, "sync", None) and callable(self.uniquedb.sync):
self.uniquedb.sync()
# update all items based on config and uniquedb
@ -377,24 +366,12 @@ class FdLogger(threading.Thread):
try:
# for line in fd buffers, so use this instead
for line in iter(self.fd.readline, b''):
line = line.strip('\n')
if line.strip() != "":
self.logger.log(self.lvl, ": %s", line)
self.logger.log(self.lvl, ": %s", line.strip('\n'))
self.fd.close()
except Exception:
self.logger.exception("")
@contextmanager
def exclusive():
with open(os.path.expanduser(config.lockfile), 'w') as f:
try:
fcntl.lockf(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
except BlockingIOError as e:
raise TivoException('another tivomirror instance is already running')
yield 'locked'
fcntl.lockf(f, fcntl.LOCK_UN)
@timeout(43200)
def download_item(item, mak, target):
global config
@ -413,13 +390,13 @@ def download_item(item, mak, target):
p_decode = subprocess.Popen([config.tivodecode, "--mak", mak, \
"--no-verify", "--out", target, "-"], stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
FdLogger(logger, logging.INFO, TextIOWrapper(p_decode.stdout))
FdLogger(logger, logging.INFO, TextIOWrapper(p_decode.stderr))
FdLogger(logger, logging.INFO, p_decode.stdout)
FdLogger(logger, logging.INFO, p_decode.stderr)
def info(signum, frame):
upd = time.time()
dur = now - start
mb = count / 1e6
print("{:5.1f}% {:5.3f} GB downloaded in {:.0f} min, {:.3f} MB/s".format(
print("{:5.1f}% {:5.3f} GB downloaded in {:.0f} min, {.3f} MB/s".format(
100.0 * count / item.sourcesize,
mb / 1e3, dur / 60, mb / dur))
try:
@ -438,7 +415,7 @@ def download_item(item, mak, target):
upd = now
dur = now - start
mb = count / 1e6
logger.debug(" {:5.1f}% {:5.3f} GB downloaded in {:0.0f} min, {:0.3f} MB/s".format(
logger.debug(" {:5.1f}% {:5.3f} GB downloaded in {:.0f} min, {:.3f} MB/s".format(
100.0 * count / item.sourcesize,
mb / 1e3, dur / 60, mb / dur))
except Exception as e:
@ -482,12 +459,11 @@ def download_decode(item, options, mak):
try:
download_item(item, mak, item.target)
except Exception as e:
exc_info = sys.exc_info()
try:
os.remove(item.target)
except Exception as e2:
pass
raise exc_info[1].with_traceback(exc_info[2])
raise e
try:
os.utime(item.target, (item.time, item.time))
except Exception as e:
@ -497,7 +473,6 @@ def download_decode(item, options, mak):
def download_one(item, downloaddb, options):
global config, logger
logger.info("*** downloading \"{}\": {:.3f} GB".format(item.name, item.sourcesize / 1e9))
# sys.exit(1)
try:
download_decode(item, options, config.mak)
downloaddb[item.name] = item.datestr
@ -522,10 +497,10 @@ def wantitem(item, downloaddb):
return "recording"
if item.available == "No":
return "not available"
if item.name in downloaddb:
if downloaddb.has_key(item.name):
return "already downloaded"
for i in (item.title, item.episode, item.name):
if i in IncludeShow.includes:
if IncludeShow.includes.has_key(i):
return IncludeShow.includes[i]
return "not included"
@ -536,55 +511,52 @@ def mirror(toc, downloaddb, one=False):
logger.error("{}: {:.1f} GB available, at least {:.1f} GB needed, stopping".format\
(config.targetdir, avail / config.gig, config.minfree / config.gig))
sys.exit(1)
with exclusive() as lock:
items = toc.getItems()
logger.info("*** {} shows listed".format(len(items)))
for item in items:
options = wantitem(item, downloaddb)
if isinstance(options, str):
logger.debug("*** skipping \"{}\": {}".format(item.name, options))
else:
download_one(item, downloaddb, options)
if one:
break
items = toc.getItems()
logger.info("*** {} shows listed".format(len(items)))
for item in items:
options = wantitem(item, downloaddb)
if isinstance(options, basestring):
logger.debug("*** skipping \"{}\": {}".format(item.name, options))
else:
download_one(item, downloaddb, options)
if one:
break
def download_episode(toc, downloaddb, episode):
with exclusive() as lock:
items = toc.getItems()
options = {}
for item in items:
if item.title == episode or item.name == episode or item.episode == episode:
for i in (item.title, item.episode, item.name):
if i in IncludeShow.includes:
options = IncludeShow.includes[i]
download_one(item, downloaddb, options)
return
items = toc.getItems()
options = {}
for item in items:
if item.title == episode or item.name == episode or item.episode == episode:
for i in (item.title, item.episode, item.name):
if IncludeShow.includes.has_key(i):
options = IncludeShow.includes[i]
download_one(item, downloaddb, options)
return
def printtoc(toc, downloaddb):
with exclusive() as lock:
items = toc.getItems()
print("*** {} shows listed".format(len(items)))
shows = {}
for item in items:
if item.title not in shows:
shows[item.title] = []
shows[item.title].append(item)
for title in sorted(shows):
for item in sorted(shows[title], key=lambda i: i.name):
options = wantitem(item, downloaddb)
if isinstance(options, str):
print("{:>7.7s}: {}".format(options, item.name))
continue
print("*** downloading {} ({:.3f} GB)".format(item.name, item.sourcesize / 1e9))
print("*** {} shows listed".format(len(items)))
items = toc.getItems()
print("*** {} shows listed".format(len(items)))
shows = {}
for item in items:
if item.title not in shows:
shows[item.title] = []
shows[item.title].append(item)
for title in sorted(shows):
for item in sorted(shows[title], key=lambda i: i.name):
options = wantitem(item, downloaddb)
if isinstance(options, basestring):
print("{:>7.7s}: {}".format(options, item.name))
continue
print("*** downloading {} ({:.3f} GB)".format(item.name, item.sourcesize / 1e9))
print("*** {} shows listed".format(len(items)))
def usage():
print('usage: tivomirror -dvuT [-c config] cmd', file=sys.stderr)
print(' cmd is one of download, list, mirror, mirrorone', file=sys.stderr)
print >>sys.stderr, 'usage: tivomirror -dvuT [-c config] cmd'
print >>sys.stderr, ' cmd is one of download, list, mirror, mirrorone'
sys.exit(64)
@ -642,12 +614,12 @@ def main():
download_episode(toc, downloaddb, remainder[1])
else:
logger.error("invalid command {}".format(cmd))
print("invalid command {}".format(cmd), file=sys.stderr)
print >>sys.stderr, "invalid command {}".format(cmd)
usage()
downloaddb.close()
except getopt.GetoptError as e:
print('Error parsing options: {}'.format(e), file=sys.stderr)
print >>sys.stderr, 'Error parsing options: {}'.format(e)
usage()
except Exception:
logger.exception("")

6
wrapper
View file

@ -1,6 +0,0 @@
#!/bin/sh
here="$(dirname $0)"
this="$(basename $0)"
. ${here}/.venv/bin/activate
exec python ${here}/${this}.py $@