From 5c3ec4fc7835799e375fc76bcb42689153dfe2ea Mon Sep 17 00:00:00 2001 From: John-Mark Gurney Date: Tue, 6 Sep 2022 13:55:11 -0700 Subject: [PATCH] big update, fix a couple bugs, and add an interactive mode make sure tags/properties are unique... support tuples in asn1coder, for the tag cache make sure that only one of each uuid is stored in hashes, and that it is the last modified version. for containers, since we're uniquified them, just replace them.. --- ui/medashare/cli.py | 298 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 294 insertions(+), 4 deletions(-) diff --git a/ui/medashare/cli.py b/ui/medashare/cli.py index cce22b5..ef346f4 100644 --- a/ui/medashare/cli.py +++ b/ui/medashare/cli.py @@ -32,6 +32,7 @@ import re import shutil import socket import string +import subprocess import sys import tempfile import unittest @@ -198,12 +199,17 @@ class MDBase(object): obj = copy.deepcopy(self._obj) common = self._common_names | self._common_optional + uniquify = set() for k, v in args: if k in common: obj[k] = v else: + uniquify.add(k) obj.setdefault(k, []).append(v) + for k in uniquify: + obj[k] = list(set(obj[k])) + for i in dels: del obj[i] @@ -282,6 +288,8 @@ class Identity(MDBase): def _trytodict(o): if isinstance(o, uuid.UUID): return 'bytes', o.bytes + if isinstance(o, tuple): + return 'list', o try: return 'dict', o.__to_dict__() except Exception: # pragma: no cover @@ -603,7 +611,9 @@ class ObjectStore(object): else: for j in hashes: h = self.makehash(j) - self._hashes.setdefault(h, []).append(obj) + d = self._hashes.setdefault(h, {}) + if obj.uuid not in d or obj.modified > d[obj.uuid].modified: + d[obj.uuid] = obj @classmethod def load(cls, fname): @@ -634,7 +644,7 @@ class ObjectStore(object): for j in obj.hashes: h = self.makehash(j) - self._hashes[h].remove(obj) + del self._hashes[h][obj.uuid] def by_id(self, id): '''Look up an object by it's UUID.''' @@ -647,7 +657,7 @@ class ObjectStore(object): '''Look up an object by it's hash value.''' h = self.makehash(hash, strict=False) - return self._hashes[h] + return list(self._hashes[h].values()) def get_metadata(self, fname, persona, create_metadata=True): '''Get all MetaData objects for fname, or create one if @@ -834,6 +844,55 @@ def enumeratedir(_dir, created_by_ref): created_by_ref) for x in sorted(os.listdir(_dir)) if not os.path.isdir(os.path.join(_dir, x)) ] +class TagCache: + def __init__(self, tags=(), count=10): + self._cache = dict((x, None) for x in tags) + self._count = 10 + + def add(self, tag): + try: + del self._cache[tag] + except KeyError: + pass + + self._cache[tag] = None + + if len(self._cache) > self._count: + del self._cache[next(iter(a.keys()))] + + def tags(self): + return sorted(self._cache.keys()) + + @classmethod + def load(cls, fname): + try: + with open(fname, 'rb') as fp: + cache = _asn1coder.loads(fp.read()) + except (FileNotFoundError, IndexError): + # IndexError when file exists, but is invalid + return cls() + + # fix up + cache['tags'] = [ tuple(x) for x in cache['tags'] ] + + return cls(**cache) + + def store(self, fname): + cache = dict(tags=list(self._cache.keys())) + + with open(fname, 'wb') as fp: + fp.write(_asn1coder.dumps(cache)) + +def get_cache(options): + cachefname = os.path.expanduser('~/.medashare_cache.pasn1') + + return TagCache.load(cachefname) + +def write_cache(options, cache): + cachefname = os.path.expanduser('~/.medashare_cache.pasn1') + + cache.store(cachefname) + def get_objstore(options): persona = get_persona(options) storefname = os.path.expanduser('~/.medashare_store.pasn1') @@ -1031,6 +1090,209 @@ def cmd_hosts(options): write_objstore(options, objstr) +def getnextfile(files, idx): + origidx = idx + + maxstart = max(0, len(files) - 10) + idx = min(maxstart, idx) + + while True: + startidx = min(max(10, idx - 10), maxstart) + _debprint(len(files), startidx) + subset = files[startidx:min(len(files), idx + 10)] + selfile = -1 if origidx < startidx or origidx >= startidx + 20 else origidx - startidx + for i, f in enumerate(subset): + print('%2d)%1s%s' % (i + 1, '*' if i == selfile else '', repr(str(f)))) + + print('P) Previous page') + print('N) Next page') + print('A) Abort') + print('Selection:') + inp = sys.stdin.readline().strip() + + if inp.lower() == 'p': + idx = max(10, idx - 19) + continue + if inp.lower() == 'n': + idx = min(maxstart, idx + 19) + continue + if inp.lower() == 'a': + return origidx + + try: + inp = int(inp) + except ValueError: + print('Invalid selection.') + continue + + if inp < 1 or inp > len(subset): + print('Invalid selection.') + continue + + return startidx - 1 + inp + +def checkforfile(objstr, curfile, ask=False): + try: + fobj = objstr.by_file(curfile, ('file',)) + except (ValueError, KeyError): + if not ask: + return + + while True: + print('file unknown, hash(y/n)?') + inp = sys.stdin.readline().strip().lower() + if inp == 'n': + return + if inp == 'y': + break + + try: + fobj = persona.by_file(curfile) + except (FileNotFoundError, KeyError) as e: + print('ERROR: file not found: %s' % repr(curfile), file=sys.stderr) + return + else: + objstr.loadobj(fobj) + + return fobj + +def cmd_interactive(options): + persona, objstr = get_objstore(options) + + cache = get_cache(options) + + files = [ pathlib.Path(x) for x in options.files ] + + autoskip = True + + idx = 0 + if not files: + files = sorted(pathlib.Path('.').iterdir()) + + while True: + curfile = files[idx] + + fobj = checkforfile(objstr, curfile, not autoskip) + + if fobj is None and autoskip and idx > 0 and idx < len(files) - 1: + # if we are auto skipping, and within range, continue + if inp == '1': + idx = max(0, idx - 1) + continue + if inp == '2': + idx = min(len(files) - 1, idx + 1) + continue + + print('Current: %s' % repr(str(curfile))) + + if fobj is None: + print('No file object for this file.') + else: + try: + objs = objstr.by_file(curfile) + except KeyError: + print('No tags or metadata object for this file.') + else: + for k, v in _iterdictlist(objs[0]): + if k in { 'sig', 'hashes' }: + continue + print('%s:\t%s' % (k, v)) + + if idx == 0: + print('1) No previous file') + else: + print('1) Previous: %s' % repr(str(files[idx - 1]))) + + if idx + 1 == len(files): + print('2) No next file') + else: + print('2) Next: %s' % repr(str(files[idx + 1]))) + + print('3) List files') + print('4) Browse directory of file.') + print('5) Browse original list of files.') + print('6) Add new tag.') + print('7) Open file.') + print('8) Turn auto skip %s' % 'off' if autoskip else 'on') + + tags = sorted(cache.tags()) + + for pos, (tag, value) in enumerate(tags): + print('%s) %s=%s' % (string.ascii_lowercase[pos], tag, value)) + + print('Q) Save and quit') + + print('Select option: ') + + inp = sys.stdin.readline().strip() + + if inp == '1': + idx = max(0, idx - 1) + continue + if inp == '2': + idx = min(len(files) - 1, idx + 1) + continue + if inp == '3': + idx = getnextfile(files, idx) + continue + if inp == '4': + files = sorted(curfile.parent.iterdir()) + try: + idx = files.index(curfile) + except ValueError: + print('WARNING: File no longer present.') + idx = 0 + continue + if inp == '5': + files = [ pathlib.Path(x) for x in options.files ] + try: + idx = files.index(curfile) + except ValueError: + print('WARNING: File not present.') + idx = 0 + continue + if inp == '6': + print('Tag?') + try: + tag, value = sys.stdin.readline().strip().split('=', 1) + except ValueError: + print('Invalid tag, no "=".') + else: + cache.add((tag, value)) + metadata = objstr.get_metadata(curfile, persona)[0] + + metadata = metadata.new_version((tag, value)) + + objstr.loadobj(metadata) + + continue + if inp == '7': + subprocess.run(('open', curfile)) + continue + + if inp.lower() == 'q': + break + + try: + i = string.ascii_lowercase.index(inp.lower()) + cache.add(tags[i]) + except (ValueError, IndexError): + pass + else: + metadata = objstr.get_metadata(curfile, persona)[0] + + metadata = metadata.new_version(tags[i]) + + objstr.loadobj(metadata) + + continue + + print('Invalid selection.') + + write_objstore(options, objstr) + + write_cache(options, cache) + def cmd_dump(options): persona, objstr = get_objstore(options) @@ -1134,7 +1396,7 @@ def cmd_container(options): try: cont = objstr.by_id(Container.make_id(uri)) - cont = cont.new_version(*kwargs.items(), dels=() if bad + cont = cont.new_version(dels=() if bad else ('incomplete',), replaces=kwargs.items()) except KeyError: cont = persona.Container(**kwargs) @@ -1238,6 +1500,11 @@ def main(): help='mapping to add, host|hostuuid:path host|hostuuid:path') parser_mapping.set_defaults(func=cmd_mapping) + parser_interactive = subparsers.add_parser('interactive', help='start in interactive mode') + parser_interactive.add_argument('files', nargs='*', + help='files to work with') + parser_interactive.set_defaults(func=cmd_interactive) + parser_dump = subparsers.add_parser('dump', help='dump all the objects') parser_dump.set_defaults(func=cmd_dump) @@ -1399,6 +1666,12 @@ class _TestCases(unittest.TestCase): # that invalid attribute access raises correct exception self.assertRaises(AttributeError, getattr, md, 'somerandombogusattribute') + # that when readding an attribute that already exists + md3 = md2.new_version(('dc:creator', 'Jim Bob',)) + + # that only one exists + self.assertEqual(md3['dc:creator'], [ 'Jim Bob' ]) + def test_mdbase_encode_decode(self): # that an object baseobj = { @@ -1689,6 +1962,23 @@ class _TestCases(unittest.TestCase): self.assertRaises(KeyError, objst.by_file, '/dev/null') + # that when a metadata object + mdouuid = 'c9a1d1e2-3109-4efd-8948-577dc15e44e7' + origobj = objst.by_id(mdouuid) + + # is updated: + obj = origobj.new_version(('foo', 'bar')) + + # and stored + objst.loadobj(obj) + + # that it is the new one + self.assertEqual(obj, objst.by_id(mdouuid)) + + # and that the old one isn't present anymore in by file + lst = objst.by_hash('91751cee0a1ab8414400238a761411daa29643ab4b8243e9a91649e25be53ada') + self.assertNotIn(origobj, lst) + # XXX make sure that object store contains fileobject # Tests to add: