From 549de4082ca844f92b881693b19407dc07a12686 Mon Sep 17 00:00:00 2001 From: John-Mark Gurney Date: Tue, 30 Aug 2022 15:23:30 -0700 Subject: [PATCH] add container support, handle when a fileobject exists, but no metadata The later happens after importing a container. --- ui/fixtures/cmd.container.json | 38 ++++++++++ ui/medashare/cli.py | 134 ++++++++++++++++++++++++++++----- 2 files changed, 155 insertions(+), 17 deletions(-) create mode 100644 ui/fixtures/cmd.container.json diff --git a/ui/fixtures/cmd.container.json b/ui/fixtures/cmd.container.json new file mode 100644 index 0000000..770113e --- /dev/null +++ b/ui/fixtures/cmd.container.json @@ -0,0 +1,38 @@ +[ +{ + "title": "gen ident", + "cmd": [ "genident", "name=A Test User" ], + "exit": 0 +}, +{ + "special": "setup bittorrent files" +}, +{ + "title": "add metadata before import", + "cmd": [ "modify", "+foo=bar", "somedir/fileb.txt" ] +}, +{ + "title": "import partial container", + "cmd": [ "container", "somedir.torrent" ], + "stderr": "Warning, incomple/invalid files, not added:\n\t'filea.txt'\n\t'filec.txt'\n\t'filee.txt'\n" +}, +{ + "special": "verify store object cnt", + "comment": "should only have one container and three files, and a metadata", + "count": 5 +}, +{ + "title": "verify correct files imported", + "cmd": [ "dump" ], + "stdout_re": "fileb.txt.*file.*\n.*foo.*bar.*cc06808cbbee0510331aa97974132e8dc296aeb795be229d064bae784b0a87a5cf4281d82e8c99271b75db2148f08a026c1a60ed9cabdb8cac6d24242dac4063.*\n.*filed.txt.*file.*\n.*filef.txt.*file.*\n.*fileb.txt.*filed.txt.*filef.txt.*cc06808cbbee0510331aa97974132e8dc296aeb795be229d064bae784b0a87a5cf4281d82e8c99271b75db2148f08a026c1.*7831bd05e23877e08a97362bab2ad7bcc7d08d8f841f42e8dee545781792b987aa7637f12cec399e261f798c10d3475add0db7de2643af86a346b6b451a69ec4.*be688838ca8686e5c90689bf2ab585cef1137c.*container.*magnet:\\?xt=urn:btih:501cf3bd4797f49fd7a624e8a9a8ce5cccceb602&dn=somedir" +}, +{ + "title": "add metadata after import", + "cmd": [ "modify", "+foo=bar", "somedir/filed.txt" ] +}, +{ + "special": "verify store object cnt", + "comment": "should only have one container and three files, and a metadata", + "count": 6 +} +] diff --git a/ui/medashare/cli.py b/ui/medashare/cli.py index 0abe77d..2c78dd0 100644 --- a/ui/medashare/cli.py +++ b/ui/medashare/cli.py @@ -11,12 +11,16 @@ from unittest import mock from .hostid import hostuuid +from .btv import _TestCases as bttestcase, validate_file + import base64 import base58 +from .btv import bencode import copy import datetime import functools import hashlib +import importlib import io import itertools import json @@ -310,6 +314,11 @@ class Persona(object): self._created_by_ref = self._identity.uuid + def Container(self, *args, **kwargs): + kwargs['created_by_ref'] = self.uuid + + return self.sign(Container(*args, **kwargs)) + def MetaData(self, *args, **kwargs): kwargs['created_by_ref'] = self.uuid @@ -557,10 +566,7 @@ class ObjectStore(object): def by_id(self, id): '''Look up an object by it's UUID.''' - if not isinstance(id, uuid.UUID): - uid = uuid.UUID(id) - else: - uid = id + uid = _makeuuid(id) return self._uuids[uid] @@ -570,6 +576,40 @@ class ObjectStore(object): h = self.makehash(hash, strict=False) return self._hashes[h] + def get_metadata(self, fname, persona): + '''Get all MetaData objects for fname, or create one if + not found. + + If a FileObject is not present, one will be created. + + A Persona must be passed in to create the FileObject and + MetaData objects as needed. + + Note: if a new MetaData object is created, it is not + stored in the database automatically. It is expected that + it will be modified and then saved, so call ObjectStore.loadobj + with it to save it. + ''' + + try: + fobj = self.by_file(fname, ('file',))[0] + #print('x:', repr(objs), file=_real_stderr) + except KeyError: + #print('b:', repr(fname), file=_real_stderr) + + fobj = persona.by_file(fname) + #print('c:', repr(fobj), file=_real_stderr) + + self.loadobj(fobj) + + # we now have the fobj, get the metadata for it. + try: + objs = self.by_file(fname) + except KeyError: + objs = [ persona.MetaData(hashes=fobj.hashes) ] + + return objs + def by_file(self, fname, types=('metadata', )): '''Return a metadata object for the file named fname. @@ -582,6 +622,7 @@ class ObjectStore(object): fid = FileObject.make_id(fname) + #print('bf:', repr(fid), file=_real_stderr) fobj = self.by_id(fid) fobj.verify() @@ -660,11 +701,15 @@ class FileObject(MDBase): mtimets = datetime.datetime.fromtimestamp(s.st_mtime, tz=datetime.timezone.utc).timestamp() + #print(repr(self), repr(s), s.st_mtime, file=_real_stderr) if self.mtime.timestamp() != mtimets or \ self.size != s.st_size: raise ValueError('file %s has changed' % repr(self.filename)) +class Container(MDBase): + _type = 'container' + def enumeratedir(_dir, created_by_ref): '''Enumerate all the files and directories (not recursive) in _dir. @@ -772,22 +817,16 @@ def cmd_modify(options): dels = [ x[1:] for x in props if x[0] == '-' ] for i in options.files: - # Get MetaData + #print('a:', repr(i), file=_real_stderr) + try: - objs = objstr.by_file(i) - #print('x:', repr(objs), file=_real_stderr) - except KeyError: - try: - fobj = persona.by_file(i) - except FileNotFoundError: - print('ERROR: file not found: %s, or invalid tag specification.' % repr(i), file=sys.stderr) - sys.exit(1) + objs = objstr.get_metadata(i, persona) + #print('d:', repr(i), repr(objs), file=_real_stderr) + except FileNotFoundError: + print('ERROR: file not found: %s, or invalid tag specification.' % repr(i), file=sys.stderr) + sys.exit(1) - objstr.loadobj(fobj) - objs = [ persona.MetaData(hashes=fobj.hashes) ] - #print('y:', repr(objs), file=_real_stderr) - #print('b:', repr(i), repr(objs), file=_real_stderr) for j in objs: #print('c:', repr(j), file=_real_stderr) # make into key/values @@ -873,6 +912,47 @@ def cmd_list(options): write_objstore(options, objstr) +def cmd_container(options): + persona, objstr = get_objstore(options) + + for i in options.files: + good, bad = validate_file(i) + + if bad: + print('Warning, incomple/invalid files, not added:', + file=sys.stderr) + print('\n'.join('\t%s' % + repr(str(pathlib.Path(*x.parts[1:]))) for x in + sorted(bad)), file=sys.stderr) + + files = [] + hashes = [] + for j in sorted(good): + files.append(str(pathlib.PosixPath(*j.parts[1:]))) + try: + fobj = objstr.by_file(j, ('file',))[0] + except: + # XXX - old file w/ incorrect hash may be + # left behind. + fobj = persona.by_file(j) + objstr.loadobj(fobj) + + # XXX - ensure only one is added? + hashes.extend(fobj.hashes) + + with open(i, 'rb') as fp: + torrent = bencode.bdecode(fp.read()) + bencodedinfo = bencode.bencode(torrent['info']) + infohash = hashlib.sha1(bencodedinfo).hexdigest() + # XXX - not entirely happy w/ URI + cont = persona.Container(files=files, hashes=hashes, + uri='magnet:?xt=urn:btih:%s&dn=%s' % (infohash, + torrent['info']['name'].decode('utf-8'))) + + objstr.loadobj(cont) + + write_objstore(options, objstr) + def cmd_import(options): persona, objstr = get_objstore(options) @@ -942,6 +1022,11 @@ def main(): help='files to modify') parser_list.set_defaults(func=cmd_list) + parser_container = subparsers.add_parser('container', help='file is examined as a container and the internal files imported as entries') + parser_container.add_argument('files', nargs='+', + help='files to modify') + parser_container.set_defaults(func=cmd_container) + parser_dump = subparsers.add_parser('dump', help='dump all the objects') parser_dump.set_defaults(func=cmd_dump) @@ -1447,6 +1532,21 @@ class _TestCases(unittest.TestCase): objst = ObjectStore.load(storefname) uniqobjs = len(set((x['uuid'] for x in objst))) self.assertEqual(len(list(objst)), uniqobjs) + elif special == 'setup bittorrent files': + # copy in the torrent file + tor = importlib.resources.files('medashare.btv') + tor = tor / 'fixtures' / 'somedir.torrent' + shutil.copy(tor, self.tempdir) + + # partly recreate files + missingfiles = bttestcase.origfiledata.copy() + + missingfiles.update(bttestcase.badfiles) + + sd = self.tempdir / bttestcase.dirname + sd.mkdir() + + bttestcase.make_files(sd, missingfiles) else: # pragma: no cover raise ValueError('unhandled special: %s' % repr(special))