Compare commits
5 commits
Author | SHA1 | Date | |
---|---|---|---|
3c1fe23bee | |||
448ef57b40 | |||
53420ffc31 | |||
a70203ae95 | |||
40caba68fc |
8 changed files with 127 additions and 73 deletions
2
.gitignore
vendored
Normal file
2
.gitignore
vendored
Normal file
|
@ -0,0 +1,2 @@
|
||||||
|
.venv
|
||||||
|
*~
|
20
README.md
20
README.md
|
@ -2,15 +2,27 @@
|
||||||
|
|
||||||
## Requirements
|
## Requirements
|
||||||
|
|
||||||
Python (2.7) needs to be installed; additionally, these packages need to be
|
Python 3.4 or newer needs to be installed.
|
||||||
installed:
|
|
||||||
* `py27-yaml`
|
|
||||||
|
|
||||||
`tivodecode` needs to be available on the path. [mackworth/tivodecode-ng](https://github.com/mackworth/tivodecode-ng)
|
`tivodecode` needs to be available on the path. [wmcbrine/tivodecode-ng](https://github.com/wmcbrine/tivodecode-ng)
|
||||||
appears to be working well; the original
|
appears to be working well; the original
|
||||||
[TiVo File Decoder](http://tivodecode.sourceforge.net) has trouble
|
[TiVo File Decoder](http://tivodecode.sourceforge.net) has trouble
|
||||||
decoding some files and fails silently.
|
decoding some files and fails silently.
|
||||||
|
|
||||||
|
## Installing
|
||||||
|
|
||||||
|
Create a virtual environment and install the required packages:
|
||||||
|
|
||||||
|
* `python -m venv .venv` or `virtualenv .venv`
|
||||||
|
* `. .venv/bin/activate`
|
||||||
|
* `pip install -r requirements.txt`
|
||||||
|
|
||||||
|
## Shell Wrapper
|
||||||
|
|
||||||
|
To make it easier to run tivomirror from cron, the shell wrapper
|
||||||
|
[`tivomirror`](./tivomirror) will activate the venv and then run
|
||||||
|
`tivomirror.py`.
|
||||||
|
|
||||||
## Configuration
|
## Configuration
|
||||||
|
|
||||||
`tivomirror` reads a config file, by default `~/.tivo/config.yaml`. The
|
`tivomirror` reads a config file, by default `~/.tivo/config.yaml`. The
|
||||||
|
|
4
requirements.txt
Normal file
4
requirements.txt
Normal file
|
@ -0,0 +1,4 @@
|
||||||
|
berkeleydb
|
||||||
|
pytz
|
||||||
|
requests
|
||||||
|
pyyaml
|
1
tivodb
Symbolic link
1
tivodb
Symbolic link
|
@ -0,0 +1 @@
|
||||||
|
wrapper
|
16
tivodb.py
16
tivodb.py
|
@ -1,33 +1,33 @@
|
||||||
#!/usr/local/bin/python
|
#!/usr/local/bin/python
|
||||||
|
|
||||||
import anydbm
|
import dbm
|
||||||
import getopt
|
import getopt
|
||||||
import operator
|
import operator
|
||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
def usage():
|
def usage():
|
||||||
print >>sys.stderr, "usage: dbtool {-a entry|-d entry|-l}"
|
print("usage: dbtool {-a entry|-d entry|-l}", file=sys.stderr)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
optlist, args = getopt.getopt(sys.argv[1:], "a:d:lk")
|
optlist, args = getopt.getopt(sys.argv[1:], "a:d:lk")
|
||||||
except getopt.GetoptError, err:
|
except getopt.GetoptError as err:
|
||||||
print >>sys.stderr, str(err)
|
print(str(err), file=sys.stderr)
|
||||||
usage()
|
usage()
|
||||||
sys.exit(64)
|
sys.exit(64)
|
||||||
if len(args) != 0 or len(optlist) != 1:
|
if len(args) != 0 or len(optlist) != 1:
|
||||||
usage()
|
usage()
|
||||||
sys.exit(64)
|
sys.exit(64)
|
||||||
|
|
||||||
downloaddb = anydbm.open(os.path.expanduser("~") + "/.tivo/downloads.db", "c")
|
downloaddb = dbm.open(os.path.expanduser("~") + "/.tivo/downloads.db", "c")
|
||||||
|
|
||||||
for (o, a) in optlist:
|
for (o, a) in optlist:
|
||||||
if o == "-l":
|
if o == "-l":
|
||||||
for i in sorted(downloaddb.keys()):
|
for i in sorted(downloaddb.keys()):
|
||||||
print "%s:\t%s" % (i, downloaddb[i])
|
print("%s:\t%s" % (i.decode('utf-8'), downloaddb[i].decode('utf-8')))
|
||||||
elif o == "-k":
|
elif o == "-k":
|
||||||
for (k, v) in sorted(downloaddb.items(), key=operator.itemgetter(1)):
|
for (k, v) in sorted(list(downloaddb.items()), key=operator.itemgetter(1)):
|
||||||
print "%s:\t%s" % (k, v)
|
print("%s:\t%s" % (k, v))
|
||||||
elif o == "-d":
|
elif o == "-d":
|
||||||
del downloaddb[a]
|
del downloaddb[a]
|
||||||
elif o == "-a":
|
elif o == "-a":
|
||||||
|
|
1
tivomirror
Symbolic link
1
tivomirror
Symbolic link
|
@ -0,0 +1 @@
|
||||||
|
wrapper
|
|
@ -1,13 +1,19 @@
|
||||||
#!/usr/local/bin/python
|
#!/usr/local/bin/python3.8
|
||||||
# -*- coding: utf8 -*-
|
# -*- coding: utf8 -*-
|
||||||
|
|
||||||
# Download shows from the Tivo
|
# Download shows from the Tivo
|
||||||
|
|
||||||
|
import sys
|
||||||
|
#import importlib
|
||||||
|
#importlib.reload(sys)
|
||||||
|
#sys.setdefaultencoding('utf-8')
|
||||||
|
|
||||||
import dbm
|
import dbm
|
||||||
import http.cookiejar
|
import http.cookiejar
|
||||||
import datetime
|
import datetime
|
||||||
import getopt
|
import getopt
|
||||||
import errno
|
import errno
|
||||||
|
import fcntl
|
||||||
import functools
|
import functools
|
||||||
import logging
|
import logging
|
||||||
import logging.handlers
|
import logging.handlers
|
||||||
|
@ -21,15 +27,17 @@ import subprocess
|
||||||
import sys
|
import sys
|
||||||
import threading
|
import threading
|
||||||
import time
|
import time
|
||||||
import urllib
|
import urllib.request, urllib.error, urllib.parse
|
||||||
import xml.dom.minidom
|
import xml.dom.minidom
|
||||||
import yaml
|
import yaml
|
||||||
|
|
||||||
|
from contextlib import contextmanager
|
||||||
|
from io import TextIOWrapper
|
||||||
|
|
||||||
|
|
||||||
class Config:
|
class Config:
|
||||||
config = '~/.tivo/config.yaml'
|
config = '~/.tivo/config.yaml'
|
||||||
|
lockfile = config + '.lock'
|
||||||
cookies = "cookies.txt"
|
cookies = "cookies.txt"
|
||||||
gig = 1024.0 * 1024 * 1024
|
gig = 1024.0 * 1024 * 1024
|
||||||
headers = requests.utils.default_headers()
|
headers = requests.utils.default_headers()
|
||||||
|
@ -105,7 +113,7 @@ class flushfile(object):
|
||||||
def write(self, x):
|
def write(self, x):
|
||||||
self.f.write(x)
|
self.f.write(x)
|
||||||
self.f.flush()
|
self.f.flush()
|
||||||
sys.stdout = flushfile(sys.stdout)
|
#sys.stdout = flushfile(sys.stdout)
|
||||||
|
|
||||||
tmp = "/tmp"
|
tmp = "/tmp"
|
||||||
|
|
||||||
|
@ -206,10 +214,10 @@ class TivoItem:
|
||||||
self.unique = False
|
self.unique = False
|
||||||
self.formatnames()
|
self.formatnames()
|
||||||
def formatnames(self):
|
def formatnames(self):
|
||||||
if self.episodeNumber and self.episodeNumber != u'0':
|
if self.episodeNumber and self.episodeNumber != '0':
|
||||||
en = int(self.episodeNumber)
|
en = int(self.episodeNumber)
|
||||||
if en >= 100:
|
if en >= 100:
|
||||||
self.name = "{} S{:02d}E{:02d} {}".format(self.title, en / 100, en % 100, self.episode)
|
self.name = "{} S{:02d}E{:02d} {}".format(self.title, int(en / 100), int(en % 100), self.episode)
|
||||||
else:
|
else:
|
||||||
self.name = "{} E{} {}".format(self.title, self.episodeNumber, self.episode)
|
self.name = "{} E{} {}".format(self.title, self.episodeNumber, self.episode)
|
||||||
elif self.unique:
|
elif self.unique:
|
||||||
|
@ -218,14 +226,14 @@ class TivoItem:
|
||||||
self.name = "{} - {} - {}".format(self.title, self.datestr, self.episode)
|
self.name = "{} - {} - {}".format(self.title, self.datestr, self.episode)
|
||||||
self.dir = "{}/{}".format(config.targetdir, re.sub("[:/]", "-", self.title))
|
self.dir = "{}/{}".format(config.targetdir, re.sub("[:/]", "-", self.title))
|
||||||
self.file = "{}/{}".format(self.dir, re.sub("[:/]", "-", self.name))
|
self.file = "{}/{}".format(self.dir, re.sub("[:/]", "-", self.name))
|
||||||
self.name = self.name.encode("utf-8");
|
#self.name = self.name.encode("utf-8");
|
||||||
self.dir = self.dir.encode("utf-8");
|
#self.dir = self.dir.encode("utf-8");
|
||||||
self.file = self.file.encode("utf-8");
|
#self.file = self.file.encode("utf-8");
|
||||||
def getPath(self, options):
|
def getPath(self, options):
|
||||||
title = self.title
|
title = self.title
|
||||||
if options.short:
|
if options.short:
|
||||||
title = options.short
|
title = options.short
|
||||||
if self.episodeNumber and self.episodeNumber != u'0':
|
if self.episodeNumber and self.episodeNumber != '0':
|
||||||
en = int(self.episodeNumber)
|
en = int(self.episodeNumber)
|
||||||
if en >= 100:
|
if en >= 100:
|
||||||
name = "{} S{:02d}E{:02d} {}".format(title, en / 100, en % 100, self.episode)
|
name = "{} S{:02d}E{:02d} {}".format(title, en / 100, en % 100, self.episode)
|
||||||
|
@ -236,7 +244,8 @@ class TivoItem:
|
||||||
else:
|
else:
|
||||||
name = "{} - {} {}".format(title, self.shortdate, self.episode)
|
name = "{} - {} {}".format(title, self.shortdate, self.episode)
|
||||||
path = "{}/{}".format(self.dir, re.sub("[:/]", "-", name))
|
path = "{}/{}".format(self.dir, re.sub("[:/]", "-", name))
|
||||||
return path.encode("utf-8");
|
return path
|
||||||
|
#return path.encode("utf-8");
|
||||||
def __str__(self):
|
def __str__(self):
|
||||||
return repr(self.title)
|
return repr(self.title)
|
||||||
|
|
||||||
|
@ -315,9 +324,11 @@ class TivoToc:
|
||||||
names[item.name] = []
|
names[item.name] = []
|
||||||
names[item.name].append(item)
|
names[item.name].append(item)
|
||||||
for name in names:
|
for name in names:
|
||||||
utf8title = title.encode("utf-8")
|
if len(names[name]) > 1 and title not in self.uniquedb:
|
||||||
if len(names[name]) > 1 and not self.uniquedb.has_key(utf8title):
|
self.uniquedb[title] = "1"
|
||||||
self.uniquedb[utf8title] = "1"
|
# utf8title = title.encode("utf-8")
|
||||||
|
# if len(names[name]) > 1 and utf8title not in self.uniquedb:
|
||||||
|
# self.uniquedb[utf8title] = "1"
|
||||||
if getattr(self.uniquedb, "sync", None) and callable(self.uniquedb.sync):
|
if getattr(self.uniquedb, "sync", None) and callable(self.uniquedb.sync):
|
||||||
self.uniquedb.sync()
|
self.uniquedb.sync()
|
||||||
# update all items based on config and uniquedb
|
# update all items based on config and uniquedb
|
||||||
|
@ -366,12 +377,24 @@ class FdLogger(threading.Thread):
|
||||||
try:
|
try:
|
||||||
# for line in fd buffers, so use this instead
|
# for line in fd buffers, so use this instead
|
||||||
for line in iter(self.fd.readline, b''):
|
for line in iter(self.fd.readline, b''):
|
||||||
self.logger.log(self.lvl, ": %s", line.strip('\n'))
|
line = line.strip('\n')
|
||||||
|
if line.strip() != "":
|
||||||
|
self.logger.log(self.lvl, ": %s", line)
|
||||||
self.fd.close()
|
self.fd.close()
|
||||||
except Exception:
|
except Exception:
|
||||||
self.logger.exception("")
|
self.logger.exception("")
|
||||||
|
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def exclusive():
|
||||||
|
with open(os.path.expanduser(config.lockfile), 'w') as f:
|
||||||
|
try:
|
||||||
|
fcntl.lockf(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||||
|
except BlockingIOError as e:
|
||||||
|
raise TivoException('another tivomirror instance is already running')
|
||||||
|
yield 'locked'
|
||||||
|
fcntl.lockf(f, fcntl.LOCK_UN)
|
||||||
|
|
||||||
@timeout(43200)
|
@timeout(43200)
|
||||||
def download_item(item, mak, target):
|
def download_item(item, mak, target):
|
||||||
global config
|
global config
|
||||||
|
@ -390,13 +413,13 @@ def download_item(item, mak, target):
|
||||||
p_decode = subprocess.Popen([config.tivodecode, "--mak", mak, \
|
p_decode = subprocess.Popen([config.tivodecode, "--mak", mak, \
|
||||||
"--no-verify", "--out", target, "-"], stdin=subprocess.PIPE,
|
"--no-verify", "--out", target, "-"], stdin=subprocess.PIPE,
|
||||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||||
FdLogger(logger, logging.INFO, p_decode.stdout)
|
FdLogger(logger, logging.INFO, TextIOWrapper(p_decode.stdout))
|
||||||
FdLogger(logger, logging.INFO, p_decode.stderr)
|
FdLogger(logger, logging.INFO, TextIOWrapper(p_decode.stderr))
|
||||||
def info(signum, frame):
|
def info(signum, frame):
|
||||||
upd = time.time()
|
upd = time.time()
|
||||||
dur = now - start
|
dur = now - start
|
||||||
mb = count / 1e6
|
mb = count / 1e6
|
||||||
print("{:5.1f}% {:5.3f} GB downloaded in {:.0f} min, {.3f} MB/s".format(
|
print("{:5.1f}% {:5.3f} GB downloaded in {:.0f} min, {:.3f} MB/s".format(
|
||||||
100.0 * count / item.sourcesize,
|
100.0 * count / item.sourcesize,
|
||||||
mb / 1e3, dur / 60, mb / dur))
|
mb / 1e3, dur / 60, mb / dur))
|
||||||
try:
|
try:
|
||||||
|
@ -415,7 +438,7 @@ def download_item(item, mak, target):
|
||||||
upd = now
|
upd = now
|
||||||
dur = now - start
|
dur = now - start
|
||||||
mb = count / 1e6
|
mb = count / 1e6
|
||||||
logger.debug(" {:5.1f}% {:5.3f} GB downloaded in {:.0f} min, {:.3f} MB/s".format(
|
logger.debug(" {:5.1f}% {:5.3f} GB downloaded in {:0.0f} min, {:0.3f} MB/s".format(
|
||||||
100.0 * count / item.sourcesize,
|
100.0 * count / item.sourcesize,
|
||||||
mb / 1e3, dur / 60, mb / dur))
|
mb / 1e3, dur / 60, mb / dur))
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
@ -459,11 +482,12 @@ def download_decode(item, options, mak):
|
||||||
try:
|
try:
|
||||||
download_item(item, mak, item.target)
|
download_item(item, mak, item.target)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
exc_info = sys.exc_info()
|
||||||
try:
|
try:
|
||||||
os.remove(item.target)
|
os.remove(item.target)
|
||||||
except Exception as e2:
|
except Exception as e2:
|
||||||
pass
|
pass
|
||||||
raise e
|
raise exc_info[1].with_traceback(exc_info[2])
|
||||||
try:
|
try:
|
||||||
os.utime(item.target, (item.time, item.time))
|
os.utime(item.target, (item.time, item.time))
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
@ -473,6 +497,7 @@ def download_decode(item, options, mak):
|
||||||
def download_one(item, downloaddb, options):
|
def download_one(item, downloaddb, options):
|
||||||
global config, logger
|
global config, logger
|
||||||
logger.info("*** downloading \"{}\": {:.3f} GB".format(item.name, item.sourcesize / 1e9))
|
logger.info("*** downloading \"{}\": {:.3f} GB".format(item.name, item.sourcesize / 1e9))
|
||||||
|
# sys.exit(1)
|
||||||
try:
|
try:
|
||||||
download_decode(item, options, config.mak)
|
download_decode(item, options, config.mak)
|
||||||
downloaddb[item.name] = item.datestr
|
downloaddb[item.name] = item.datestr
|
||||||
|
@ -497,10 +522,10 @@ def wantitem(item, downloaddb):
|
||||||
return "recording"
|
return "recording"
|
||||||
if item.available == "No":
|
if item.available == "No":
|
||||||
return "not available"
|
return "not available"
|
||||||
if downloaddb.has_key(item.name):
|
if item.name in downloaddb:
|
||||||
return "already downloaded"
|
return "already downloaded"
|
||||||
for i in (item.title, item.episode, item.name):
|
for i in (item.title, item.episode, item.name):
|
||||||
if IncludeShow.includes.has_key(i):
|
if i in IncludeShow.includes:
|
||||||
return IncludeShow.includes[i]
|
return IncludeShow.includes[i]
|
||||||
return "not included"
|
return "not included"
|
||||||
|
|
||||||
|
@ -512,11 +537,12 @@ def mirror(toc, downloaddb, one=False):
|
||||||
(config.targetdir, avail / config.gig, config.minfree / config.gig))
|
(config.targetdir, avail / config.gig, config.minfree / config.gig))
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
|
with exclusive() as lock:
|
||||||
items = toc.getItems()
|
items = toc.getItems()
|
||||||
logger.info("*** {} shows listed".format(len(items)))
|
logger.info("*** {} shows listed".format(len(items)))
|
||||||
for item in items:
|
for item in items:
|
||||||
options = wantitem(item, downloaddb)
|
options = wantitem(item, downloaddb)
|
||||||
if isinstance(options, basestring):
|
if isinstance(options, str):
|
||||||
logger.debug("*** skipping \"{}\": {}".format(item.name, options))
|
logger.debug("*** skipping \"{}\": {}".format(item.name, options))
|
||||||
else:
|
else:
|
||||||
download_one(item, downloaddb, options)
|
download_one(item, downloaddb, options)
|
||||||
|
@ -525,18 +551,20 @@ def mirror(toc, downloaddb, one=False):
|
||||||
|
|
||||||
|
|
||||||
def download_episode(toc, downloaddb, episode):
|
def download_episode(toc, downloaddb, episode):
|
||||||
|
with exclusive() as lock:
|
||||||
items = toc.getItems()
|
items = toc.getItems()
|
||||||
options = {}
|
options = {}
|
||||||
for item in items:
|
for item in items:
|
||||||
if item.title == episode or item.name == episode or item.episode == episode:
|
if item.title == episode or item.name == episode or item.episode == episode:
|
||||||
for i in (item.title, item.episode, item.name):
|
for i in (item.title, item.episode, item.name):
|
||||||
if IncludeShow.includes.has_key(i):
|
if i in IncludeShow.includes:
|
||||||
options = IncludeShow.includes[i]
|
options = IncludeShow.includes[i]
|
||||||
download_one(item, downloaddb, options)
|
download_one(item, downloaddb, options)
|
||||||
return
|
return
|
||||||
|
|
||||||
|
|
||||||
def printtoc(toc, downloaddb):
|
def printtoc(toc, downloaddb):
|
||||||
|
with exclusive() as lock:
|
||||||
items = toc.getItems()
|
items = toc.getItems()
|
||||||
print("*** {} shows listed".format(len(items)))
|
print("*** {} shows listed".format(len(items)))
|
||||||
shows = {}
|
shows = {}
|
||||||
|
@ -547,7 +575,7 @@ def printtoc(toc, downloaddb):
|
||||||
for title in sorted(shows):
|
for title in sorted(shows):
|
||||||
for item in sorted(shows[title], key=lambda i: i.name):
|
for item in sorted(shows[title], key=lambda i: i.name):
|
||||||
options = wantitem(item, downloaddb)
|
options = wantitem(item, downloaddb)
|
||||||
if isinstance(options, basestring):
|
if isinstance(options, str):
|
||||||
print("{:>7.7s}: {}".format(options, item.name))
|
print("{:>7.7s}: {}".format(options, item.name))
|
||||||
continue
|
continue
|
||||||
print("*** downloading {} ({:.3f} GB)".format(item.name, item.sourcesize / 1e9))
|
print("*** downloading {} ({:.3f} GB)".format(item.name, item.sourcesize / 1e9))
|
||||||
|
@ -555,8 +583,8 @@ def printtoc(toc, downloaddb):
|
||||||
|
|
||||||
|
|
||||||
def usage():
|
def usage():
|
||||||
print >>sys.stderr, 'usage: tivomirror -dvuT [-c config] cmd'
|
print('usage: tivomirror -dvuT [-c config] cmd', file=sys.stderr)
|
||||||
print >>sys.stderr, ' cmd is one of download, list, mirror, mirrorone'
|
print(' cmd is one of download, list, mirror, mirrorone', file=sys.stderr)
|
||||||
sys.exit(64)
|
sys.exit(64)
|
||||||
|
|
||||||
|
|
||||||
|
@ -614,12 +642,12 @@ def main():
|
||||||
download_episode(toc, downloaddb, remainder[1])
|
download_episode(toc, downloaddb, remainder[1])
|
||||||
else:
|
else:
|
||||||
logger.error("invalid command {}".format(cmd))
|
logger.error("invalid command {}".format(cmd))
|
||||||
print >>sys.stderr, "invalid command {}".format(cmd)
|
print("invalid command {}".format(cmd), file=sys.stderr)
|
||||||
usage()
|
usage()
|
||||||
|
|
||||||
downloaddb.close()
|
downloaddb.close()
|
||||||
except getopt.GetoptError as e:
|
except getopt.GetoptError as e:
|
||||||
print >>sys.stderr, 'Error parsing options: {}'.format(e)
|
print('Error parsing options: {}'.format(e), file=sys.stderr)
|
||||||
usage()
|
usage()
|
||||||
except Exception:
|
except Exception:
|
||||||
logger.exception("")
|
logger.exception("")
|
||||||
|
|
6
wrapper
Executable file
6
wrapper
Executable file
|
@ -0,0 +1,6 @@
|
||||||
|
#!/bin/sh
|
||||||
|
|
||||||
|
here="$(dirname $0)"
|
||||||
|
this="$(basename $0)"
|
||||||
|
. ${here}/.venv/bin/activate
|
||||||
|
exec python ${here}/${this}.py $@
|
Loading…
Reference in a new issue