Compare commits
1 commit
Author | SHA1 | Date | |
---|---|---|---|
823129f1d8 |
8 changed files with 73 additions and 127 deletions
2
.gitignore
vendored
2
.gitignore
vendored
|
@ -1,2 +0,0 @@
|
||||||
.venv
|
|
||||||
*~
|
|
20
README.md
20
README.md
|
@ -2,27 +2,15 @@
|
||||||
|
|
||||||
## Requirements
|
## Requirements
|
||||||
|
|
||||||
Python 3.4 or newer needs to be installed.
|
Python (2.7) needs to be installed; additionally, these packages need to be
|
||||||
|
installed:
|
||||||
|
* `py27-yaml`
|
||||||
|
|
||||||
`tivodecode` needs to be available on the path. [wmcbrine/tivodecode-ng](https://github.com/wmcbrine/tivodecode-ng)
|
`tivodecode` needs to be available on the path. [mackworth/tivodecode-ng](https://github.com/mackworth/tivodecode-ng)
|
||||||
appears to be working well; the original
|
appears to be working well; the original
|
||||||
[TiVo File Decoder](http://tivodecode.sourceforge.net) has trouble
|
[TiVo File Decoder](http://tivodecode.sourceforge.net) has trouble
|
||||||
decoding some files and fails silently.
|
decoding some files and fails silently.
|
||||||
|
|
||||||
## Installing
|
|
||||||
|
|
||||||
Create a virtual environment and install the required packages:
|
|
||||||
|
|
||||||
* `python -m venv .venv` or `virtualenv .venv`
|
|
||||||
* `. .venv/bin/activate`
|
|
||||||
* `pip install -r requirements.txt`
|
|
||||||
|
|
||||||
## Shell Wrapper
|
|
||||||
|
|
||||||
To make it easier to run tivomirror from cron, the shell wrapper
|
|
||||||
[`tivomirror`](./tivomirror) will activate the venv and then run
|
|
||||||
`tivomirror.py`.
|
|
||||||
|
|
||||||
## Configuration
|
## Configuration
|
||||||
|
|
||||||
`tivomirror` reads a config file, by default `~/.tivo/config.yaml`. The
|
`tivomirror` reads a config file, by default `~/.tivo/config.yaml`. The
|
||||||
|
|
|
@ -1,4 +0,0 @@
|
||||||
berkeleydb
|
|
||||||
pytz
|
|
||||||
requests
|
|
||||||
pyyaml
|
|
1
tivodb
1
tivodb
|
@ -1 +0,0 @@
|
||||||
wrapper
|
|
16
tivodb.py
16
tivodb.py
|
@ -1,33 +1,33 @@
|
||||||
#!/usr/local/bin/python
|
#!/usr/local/bin/python
|
||||||
|
|
||||||
import dbm
|
import anydbm
|
||||||
import getopt
|
import getopt
|
||||||
import operator
|
import operator
|
||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
def usage():
|
def usage():
|
||||||
print("usage: dbtool {-a entry|-d entry|-l}", file=sys.stderr)
|
print >>sys.stderr, "usage: dbtool {-a entry|-d entry|-l}"
|
||||||
|
|
||||||
try:
|
try:
|
||||||
optlist, args = getopt.getopt(sys.argv[1:], "a:d:lk")
|
optlist, args = getopt.getopt(sys.argv[1:], "a:d:lk")
|
||||||
except getopt.GetoptError as err:
|
except getopt.GetoptError, err:
|
||||||
print(str(err), file=sys.stderr)
|
print >>sys.stderr, str(err)
|
||||||
usage()
|
usage()
|
||||||
sys.exit(64)
|
sys.exit(64)
|
||||||
if len(args) != 0 or len(optlist) != 1:
|
if len(args) != 0 or len(optlist) != 1:
|
||||||
usage()
|
usage()
|
||||||
sys.exit(64)
|
sys.exit(64)
|
||||||
|
|
||||||
downloaddb = dbm.open(os.path.expanduser("~") + "/.tivo/downloads.db", "c")
|
downloaddb = anydbm.open(os.path.expanduser("~") + "/.tivo/downloads.db", "c")
|
||||||
|
|
||||||
for (o, a) in optlist:
|
for (o, a) in optlist:
|
||||||
if o == "-l":
|
if o == "-l":
|
||||||
for i in sorted(downloaddb.keys()):
|
for i in sorted(downloaddb.keys()):
|
||||||
print("%s:\t%s" % (i.decode('utf-8'), downloaddb[i].decode('utf-8')))
|
print "%s:\t%s" % (i, downloaddb[i])
|
||||||
elif o == "-k":
|
elif o == "-k":
|
||||||
for (k, v) in sorted(list(downloaddb.items()), key=operator.itemgetter(1)):
|
for (k, v) in sorted(downloaddb.items(), key=operator.itemgetter(1)):
|
||||||
print("%s:\t%s" % (k, v))
|
print "%s:\t%s" % (k, v)
|
||||||
elif o == "-d":
|
elif o == "-d":
|
||||||
del downloaddb[a]
|
del downloaddb[a]
|
||||||
elif o == "-a":
|
elif o == "-a":
|
||||||
|
|
|
@ -1 +0,0 @@
|
||||||
wrapper
|
|
150
tivomirror.py
150
tivomirror.py
|
@ -1,19 +1,13 @@
|
||||||
#!/usr/local/bin/python3.8
|
#!/usr/local/bin/python
|
||||||
# -*- coding: utf8 -*-
|
# -*- coding: utf8 -*-
|
||||||
|
|
||||||
# Download shows from the Tivo
|
# Download shows from the Tivo
|
||||||
|
|
||||||
import sys
|
|
||||||
#import importlib
|
|
||||||
#importlib.reload(sys)
|
|
||||||
#sys.setdefaultencoding('utf-8')
|
|
||||||
|
|
||||||
import dbm
|
import dbm
|
||||||
import http.cookiejar
|
import http.cookiejar
|
||||||
import datetime
|
import datetime
|
||||||
import getopt
|
import getopt
|
||||||
import errno
|
import errno
|
||||||
import fcntl
|
|
||||||
import functools
|
import functools
|
||||||
import logging
|
import logging
|
||||||
import logging.handlers
|
import logging.handlers
|
||||||
|
@ -27,17 +21,15 @@ import subprocess
|
||||||
import sys
|
import sys
|
||||||
import threading
|
import threading
|
||||||
import time
|
import time
|
||||||
import urllib.request, urllib.error, urllib.parse
|
import urllib
|
||||||
import xml.dom.minidom
|
import xml.dom.minidom
|
||||||
import yaml
|
import yaml
|
||||||
|
|
||||||
from contextlib import contextmanager
|
|
||||||
from io import TextIOWrapper
|
|
||||||
|
|
||||||
|
|
||||||
class Config:
|
class Config:
|
||||||
config = '~/.tivo/config.yaml'
|
config = '~/.tivo/config.yaml'
|
||||||
lockfile = config + '.lock'
|
|
||||||
cookies = "cookies.txt"
|
cookies = "cookies.txt"
|
||||||
gig = 1024.0 * 1024 * 1024
|
gig = 1024.0 * 1024 * 1024
|
||||||
headers = requests.utils.default_headers()
|
headers = requests.utils.default_headers()
|
||||||
|
@ -113,7 +105,7 @@ class flushfile(object):
|
||||||
def write(self, x):
|
def write(self, x):
|
||||||
self.f.write(x)
|
self.f.write(x)
|
||||||
self.f.flush()
|
self.f.flush()
|
||||||
#sys.stdout = flushfile(sys.stdout)
|
sys.stdout = flushfile(sys.stdout)
|
||||||
|
|
||||||
tmp = "/tmp"
|
tmp = "/tmp"
|
||||||
|
|
||||||
|
@ -214,10 +206,10 @@ class TivoItem:
|
||||||
self.unique = False
|
self.unique = False
|
||||||
self.formatnames()
|
self.formatnames()
|
||||||
def formatnames(self):
|
def formatnames(self):
|
||||||
if self.episodeNumber and self.episodeNumber != '0':
|
if self.episodeNumber and self.episodeNumber != u'0':
|
||||||
en = int(self.episodeNumber)
|
en = int(self.episodeNumber)
|
||||||
if en >= 100:
|
if en >= 100:
|
||||||
self.name = "{} S{:02d}E{:02d} {}".format(self.title, int(en / 100), int(en % 100), self.episode)
|
self.name = "{} S{:02d}E{:02d} {}".format(self.title, en / 100, en % 100, self.episode)
|
||||||
else:
|
else:
|
||||||
self.name = "{} E{} {}".format(self.title, self.episodeNumber, self.episode)
|
self.name = "{} E{} {}".format(self.title, self.episodeNumber, self.episode)
|
||||||
elif self.unique:
|
elif self.unique:
|
||||||
|
@ -226,14 +218,14 @@ class TivoItem:
|
||||||
self.name = "{} - {} - {}".format(self.title, self.datestr, self.episode)
|
self.name = "{} - {} - {}".format(self.title, self.datestr, self.episode)
|
||||||
self.dir = "{}/{}".format(config.targetdir, re.sub("[:/]", "-", self.title))
|
self.dir = "{}/{}".format(config.targetdir, re.sub("[:/]", "-", self.title))
|
||||||
self.file = "{}/{}".format(self.dir, re.sub("[:/]", "-", self.name))
|
self.file = "{}/{}".format(self.dir, re.sub("[:/]", "-", self.name))
|
||||||
#self.name = self.name.encode("utf-8");
|
self.name = self.name.encode("utf-8");
|
||||||
#self.dir = self.dir.encode("utf-8");
|
self.dir = self.dir.encode("utf-8");
|
||||||
#self.file = self.file.encode("utf-8");
|
self.file = self.file.encode("utf-8");
|
||||||
def getPath(self, options):
|
def getPath(self, options):
|
||||||
title = self.title
|
title = self.title
|
||||||
if options.short:
|
if options.short:
|
||||||
title = options.short
|
title = options.short
|
||||||
if self.episodeNumber and self.episodeNumber != '0':
|
if self.episodeNumber and self.episodeNumber != u'0':
|
||||||
en = int(self.episodeNumber)
|
en = int(self.episodeNumber)
|
||||||
if en >= 100:
|
if en >= 100:
|
||||||
name = "{} S{:02d}E{:02d} {}".format(title, en / 100, en % 100, self.episode)
|
name = "{} S{:02d}E{:02d} {}".format(title, en / 100, en % 100, self.episode)
|
||||||
|
@ -244,8 +236,7 @@ class TivoItem:
|
||||||
else:
|
else:
|
||||||
name = "{} - {} {}".format(title, self.shortdate, self.episode)
|
name = "{} - {} {}".format(title, self.shortdate, self.episode)
|
||||||
path = "{}/{}".format(self.dir, re.sub("[:/]", "-", name))
|
path = "{}/{}".format(self.dir, re.sub("[:/]", "-", name))
|
||||||
return path
|
return path.encode("utf-8");
|
||||||
#return path.encode("utf-8");
|
|
||||||
def __str__(self):
|
def __str__(self):
|
||||||
return repr(self.title)
|
return repr(self.title)
|
||||||
|
|
||||||
|
@ -324,11 +315,9 @@ class TivoToc:
|
||||||
names[item.name] = []
|
names[item.name] = []
|
||||||
names[item.name].append(item)
|
names[item.name].append(item)
|
||||||
for name in names:
|
for name in names:
|
||||||
if len(names[name]) > 1 and title not in self.uniquedb:
|
utf8title = title.encode("utf-8")
|
||||||
self.uniquedb[title] = "1"
|
if len(names[name]) > 1 and not self.uniquedb.has_key(utf8title):
|
||||||
# utf8title = title.encode("utf-8")
|
self.uniquedb[utf8title] = "1"
|
||||||
# if len(names[name]) > 1 and utf8title not in self.uniquedb:
|
|
||||||
# self.uniquedb[utf8title] = "1"
|
|
||||||
if getattr(self.uniquedb, "sync", None) and callable(self.uniquedb.sync):
|
if getattr(self.uniquedb, "sync", None) and callable(self.uniquedb.sync):
|
||||||
self.uniquedb.sync()
|
self.uniquedb.sync()
|
||||||
# update all items based on config and uniquedb
|
# update all items based on config and uniquedb
|
||||||
|
@ -377,24 +366,12 @@ class FdLogger(threading.Thread):
|
||||||
try:
|
try:
|
||||||
# for line in fd buffers, so use this instead
|
# for line in fd buffers, so use this instead
|
||||||
for line in iter(self.fd.readline, b''):
|
for line in iter(self.fd.readline, b''):
|
||||||
line = line.strip('\n')
|
self.logger.log(self.lvl, ": %s", line.strip('\n'))
|
||||||
if line.strip() != "":
|
|
||||||
self.logger.log(self.lvl, ": %s", line)
|
|
||||||
self.fd.close()
|
self.fd.close()
|
||||||
except Exception:
|
except Exception:
|
||||||
self.logger.exception("")
|
self.logger.exception("")
|
||||||
|
|
||||||
|
|
||||||
@contextmanager
|
|
||||||
def exclusive():
|
|
||||||
with open(os.path.expanduser(config.lockfile), 'w') as f:
|
|
||||||
try:
|
|
||||||
fcntl.lockf(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
||||||
except BlockingIOError as e:
|
|
||||||
raise TivoException('another tivomirror instance is already running')
|
|
||||||
yield 'locked'
|
|
||||||
fcntl.lockf(f, fcntl.LOCK_UN)
|
|
||||||
|
|
||||||
@timeout(43200)
|
@timeout(43200)
|
||||||
def download_item(item, mak, target):
|
def download_item(item, mak, target):
|
||||||
global config
|
global config
|
||||||
|
@ -413,13 +390,13 @@ def download_item(item, mak, target):
|
||||||
p_decode = subprocess.Popen([config.tivodecode, "--mak", mak, \
|
p_decode = subprocess.Popen([config.tivodecode, "--mak", mak, \
|
||||||
"--no-verify", "--out", target, "-"], stdin=subprocess.PIPE,
|
"--no-verify", "--out", target, "-"], stdin=subprocess.PIPE,
|
||||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||||
FdLogger(logger, logging.INFO, TextIOWrapper(p_decode.stdout))
|
FdLogger(logger, logging.INFO, p_decode.stdout)
|
||||||
FdLogger(logger, logging.INFO, TextIOWrapper(p_decode.stderr))
|
FdLogger(logger, logging.INFO, p_decode.stderr)
|
||||||
def info(signum, frame):
|
def info(signum, frame):
|
||||||
upd = time.time()
|
upd = time.time()
|
||||||
dur = now - start
|
dur = now - start
|
||||||
mb = count / 1e6
|
mb = count / 1e6
|
||||||
print("{:5.1f}% {:5.3f} GB downloaded in {:.0f} min, {:.3f} MB/s".format(
|
print("{:5.1f}% {:5.3f} GB downloaded in {:.0f} min, {.3f} MB/s".format(
|
||||||
100.0 * count / item.sourcesize,
|
100.0 * count / item.sourcesize,
|
||||||
mb / 1e3, dur / 60, mb / dur))
|
mb / 1e3, dur / 60, mb / dur))
|
||||||
try:
|
try:
|
||||||
|
@ -438,7 +415,7 @@ def download_item(item, mak, target):
|
||||||
upd = now
|
upd = now
|
||||||
dur = now - start
|
dur = now - start
|
||||||
mb = count / 1e6
|
mb = count / 1e6
|
||||||
logger.debug(" {:5.1f}% {:5.3f} GB downloaded in {:0.0f} min, {:0.3f} MB/s".format(
|
logger.debug(" {:5.1f}% {:5.3f} GB downloaded in {:.0f} min, {:.3f} MB/s".format(
|
||||||
100.0 * count / item.sourcesize,
|
100.0 * count / item.sourcesize,
|
||||||
mb / 1e3, dur / 60, mb / dur))
|
mb / 1e3, dur / 60, mb / dur))
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
@ -482,12 +459,11 @@ def download_decode(item, options, mak):
|
||||||
try:
|
try:
|
||||||
download_item(item, mak, item.target)
|
download_item(item, mak, item.target)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
exc_info = sys.exc_info()
|
|
||||||
try:
|
try:
|
||||||
os.remove(item.target)
|
os.remove(item.target)
|
||||||
except Exception as e2:
|
except Exception as e2:
|
||||||
pass
|
pass
|
||||||
raise exc_info[1].with_traceback(exc_info[2])
|
raise e
|
||||||
try:
|
try:
|
||||||
os.utime(item.target, (item.time, item.time))
|
os.utime(item.target, (item.time, item.time))
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
@ -497,7 +473,6 @@ def download_decode(item, options, mak):
|
||||||
def download_one(item, downloaddb, options):
|
def download_one(item, downloaddb, options):
|
||||||
global config, logger
|
global config, logger
|
||||||
logger.info("*** downloading \"{}\": {:.3f} GB".format(item.name, item.sourcesize / 1e9))
|
logger.info("*** downloading \"{}\": {:.3f} GB".format(item.name, item.sourcesize / 1e9))
|
||||||
# sys.exit(1)
|
|
||||||
try:
|
try:
|
||||||
download_decode(item, options, config.mak)
|
download_decode(item, options, config.mak)
|
||||||
downloaddb[item.name] = item.datestr
|
downloaddb[item.name] = item.datestr
|
||||||
|
@ -522,10 +497,10 @@ def wantitem(item, downloaddb):
|
||||||
return "recording"
|
return "recording"
|
||||||
if item.available == "No":
|
if item.available == "No":
|
||||||
return "not available"
|
return "not available"
|
||||||
if item.name in downloaddb:
|
if downloaddb.has_key(item.name):
|
||||||
return "already downloaded"
|
return "already downloaded"
|
||||||
for i in (item.title, item.episode, item.name):
|
for i in (item.title, item.episode, item.name):
|
||||||
if i in IncludeShow.includes:
|
if IncludeShow.includes.has_key(i):
|
||||||
return IncludeShow.includes[i]
|
return IncludeShow.includes[i]
|
||||||
return "not included"
|
return "not included"
|
||||||
|
|
||||||
|
@ -536,55 +511,52 @@ def mirror(toc, downloaddb, one=False):
|
||||||
logger.error("{}: {:.1f} GB available, at least {:.1f} GB needed, stopping".format\
|
logger.error("{}: {:.1f} GB available, at least {:.1f} GB needed, stopping".format\
|
||||||
(config.targetdir, avail / config.gig, config.minfree / config.gig))
|
(config.targetdir, avail / config.gig, config.minfree / config.gig))
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
with exclusive() as lock:
|
items = toc.getItems()
|
||||||
items = toc.getItems()
|
logger.info("*** {} shows listed".format(len(items)))
|
||||||
logger.info("*** {} shows listed".format(len(items)))
|
for item in items:
|
||||||
for item in items:
|
options = wantitem(item, downloaddb)
|
||||||
options = wantitem(item, downloaddb)
|
if isinstance(options, basestring):
|
||||||
if isinstance(options, str):
|
logger.debug("*** skipping \"{}\": {}".format(item.name, options))
|
||||||
logger.debug("*** skipping \"{}\": {}".format(item.name, options))
|
else:
|
||||||
else:
|
download_one(item, downloaddb, options)
|
||||||
download_one(item, downloaddb, options)
|
if one:
|
||||||
if one:
|
break
|
||||||
break
|
|
||||||
|
|
||||||
|
|
||||||
def download_episode(toc, downloaddb, episode):
|
def download_episode(toc, downloaddb, episode):
|
||||||
with exclusive() as lock:
|
items = toc.getItems()
|
||||||
items = toc.getItems()
|
options = {}
|
||||||
options = {}
|
for item in items:
|
||||||
for item in items:
|
if item.title == episode or item.name == episode or item.episode == episode:
|
||||||
if item.title == episode or item.name == episode or item.episode == episode:
|
for i in (item.title, item.episode, item.name):
|
||||||
for i in (item.title, item.episode, item.name):
|
if IncludeShow.includes.has_key(i):
|
||||||
if i in IncludeShow.includes:
|
options = IncludeShow.includes[i]
|
||||||
options = IncludeShow.includes[i]
|
download_one(item, downloaddb, options)
|
||||||
download_one(item, downloaddb, options)
|
return
|
||||||
return
|
|
||||||
|
|
||||||
|
|
||||||
def printtoc(toc, downloaddb):
|
def printtoc(toc, downloaddb):
|
||||||
with exclusive() as lock:
|
items = toc.getItems()
|
||||||
items = toc.getItems()
|
print("*** {} shows listed".format(len(items)))
|
||||||
print("*** {} shows listed".format(len(items)))
|
shows = {}
|
||||||
shows = {}
|
for item in items:
|
||||||
for item in items:
|
if item.title not in shows:
|
||||||
if item.title not in shows:
|
shows[item.title] = []
|
||||||
shows[item.title] = []
|
shows[item.title].append(item)
|
||||||
shows[item.title].append(item)
|
for title in sorted(shows):
|
||||||
for title in sorted(shows):
|
for item in sorted(shows[title], key=lambda i: i.name):
|
||||||
for item in sorted(shows[title], key=lambda i: i.name):
|
options = wantitem(item, downloaddb)
|
||||||
options = wantitem(item, downloaddb)
|
if isinstance(options, basestring):
|
||||||
if isinstance(options, str):
|
print("{:>7.7s}: {}".format(options, item.name))
|
||||||
print("{:>7.7s}: {}".format(options, item.name))
|
continue
|
||||||
continue
|
print("*** downloading {} ({:.3f} GB)".format(item.name, item.sourcesize / 1e9))
|
||||||
print("*** downloading {} ({:.3f} GB)".format(item.name, item.sourcesize / 1e9))
|
print("*** {} shows listed".format(len(items)))
|
||||||
print("*** {} shows listed".format(len(items)))
|
|
||||||
|
|
||||||
|
|
||||||
def usage():
|
def usage():
|
||||||
print('usage: tivomirror -dvuT [-c config] cmd', file=sys.stderr)
|
print >>sys.stderr, 'usage: tivomirror -dvuT [-c config] cmd'
|
||||||
print(' cmd is one of download, list, mirror, mirrorone', file=sys.stderr)
|
print >>sys.stderr, ' cmd is one of download, list, mirror, mirrorone'
|
||||||
sys.exit(64)
|
sys.exit(64)
|
||||||
|
|
||||||
|
|
||||||
|
@ -642,12 +614,12 @@ def main():
|
||||||
download_episode(toc, downloaddb, remainder[1])
|
download_episode(toc, downloaddb, remainder[1])
|
||||||
else:
|
else:
|
||||||
logger.error("invalid command {}".format(cmd))
|
logger.error("invalid command {}".format(cmd))
|
||||||
print("invalid command {}".format(cmd), file=sys.stderr)
|
print >>sys.stderr, "invalid command {}".format(cmd)
|
||||||
usage()
|
usage()
|
||||||
|
|
||||||
downloaddb.close()
|
downloaddb.close()
|
||||||
except getopt.GetoptError as e:
|
except getopt.GetoptError as e:
|
||||||
print('Error parsing options: {}'.format(e), file=sys.stderr)
|
print >>sys.stderr, 'Error parsing options: {}'.format(e)
|
||||||
usage()
|
usage()
|
||||||
except Exception:
|
except Exception:
|
||||||
logger.exception("")
|
logger.exception("")
|
||||||
|
|
6
wrapper
6
wrapper
|
@ -1,6 +0,0 @@
|
||||||
#!/bin/sh
|
|
||||||
|
|
||||||
here="$(dirname $0)"
|
|
||||||
this="$(basename $0)"
|
|
||||||
. ${here}/.venv/bin/activate
|
|
||||||
exec python ${here}/${this}.py $@
|
|
Loading…
Reference in a new issue