diff --git a/ui/Makefile b/ui/Makefile index cfebcd6..5e48429 100644 --- a/ui/Makefile +++ b/ui/Makefile @@ -1,7 +1,7 @@ VIRTUALENV?=python3 -m venv MODULES=medashare -test: fixtures/sample.data.pasn1 fixtures/sample.persona.pasn1 fixtures/sample.mtree +test: fixtures/sample.data.sqlite3 fixtures/sample.persona.pasn1 fixtures/sample.mtree (. ./p/bin/activate && \ ((find fixtures -type f; find $(MODULES) -type f) | entr sh -c 'python3 -m coverage run -m unittest --failfast $(MODULES).tests && coverage report -m --omit=p/\*')) @@ -10,7 +10,7 @@ env: (. ./p/bin/activate && \ pip install -r requirements.txt) -fixtures/sample.data.pasn1 fixtures/sample.persona.pasn1: fixtures/genfixtures.py +fixtures/sample.data.sqlite3 fixtures/sample.persona.pasn1: fixtures/genfixtures.py (. ./p/bin/activate && cd fixtures && PYTHONPATH=.. python3 genfixtures.py ) fixtures/sample.mtree: fixtures/mtree.dir diff --git a/ui/fixtures/cmd.basic.json b/ui/fixtures/cmd.basic.json index c3a6106..32af8ae 100644 --- a/ui/fixtures/cmd.basic.json +++ b/ui/fixtures/cmd.basic.json @@ -206,7 +206,14 @@ "title": "dump is correct", "cmd": [ "dump" ], "exit": 0, - "stdout_re": "{.*name.*Changed Name.*type.*identity.*}\n{.*filename.*newfile.txt.*hashes.*90f8342520f0ac57fb5a779f5d331c2fa87aa40f8799940257f9ba619940951e67143a8d746535ed0284924b2b7bc1478f095198800ba96d01847d7b56ca465c.*size.*19.*type.*file.*}\n{.*foo.*bar=baz.*hashes.*90f8342520f0ac57fb5a779f5d331c2fa87aa40f8799940257f9ba619940951e67143a8d746535ed0284924b2b7bc1478f095198800ba96d01847d7b56ca465c.*type.*metadata.*}\n{.*filename.*test.txt.*90f8342520f0ac57fb5a779f5d331c2fa87aa40f8799940257f9ba619940951e67143a8d746535ed0284924b2b7bc1478f095198800ba96d01847d7b56ca465c.*size.*19.*type.*file.*}\n{.*filename.*newfile.txt.*90f8342520f0ac57fb5a779f5d331c2fa87aa40f8799940257f9ba619940951e67143a8d746535ed0284924b2b7bc1478f095198800ba96d01847d7b56ca465c.*size.*19.*type.*file.*}\n{.*filename.*newfile.txt.*90f8342520f0ac57fb5a779f5d331c2fa87aa40f8799940257f9ba619940951e67143a8d746535ed0284924b2b7bc1478f095198800ba96d01847d7b56ca465c.*size.*19.*type.*file.*}\n" + "stdout_check": [ + { "name": "Changed Name", "type": "identity" }, + { "filename": "newfile.txt", "hashes": [ "sha512:90f8342520f0ac57fb5a779f5d331c2fa87aa40f8799940257f9ba619940951e67143a8d746535ed0284924b2b7bc1478f095198800ba96d01847d7b56ca465c" ], "size": 19, "type": "file" }, + { "foo": [ "bar=baz" ], "hashes": [ "sha512:90f8342520f0ac57fb5a779f5d331c2fa87aa40f8799940257f9ba619940951e67143a8d746535ed0284924b2b7bc1478f095198800ba96d01847d7b56ca465c" ], "type": "metadata" }, + { "filename": "test.txt", "hashes": [ "sha512:90f8342520f0ac57fb5a779f5d331c2fa87aa40f8799940257f9ba619940951e67143a8d746535ed0284924b2b7bc1478f095198800ba96d01847d7b56ca465c" ], "size": 19, "type": "file" }, + { "filename": "newfile.txt", "hashes": [ "sha512:90f8342520f0ac57fb5a779f5d331c2fa87aa40f8799940257f9ba619940951e67143a8d746535ed0284924b2b7bc1478f095198800ba96d01847d7b56ca465c" ], "size": 19, "type": "file" }, + { "filename": "newfile.txt", "hashes": [ "sha512:b0551b2fb5d045a74d36a08ac49aea66790ea4fb5e84f9326a32db44fc78ca0676e65a8d1f0d98f62589eeaef105f303c81f07f3f862ad3bace7960fe59de4d5" ], "size": 17, "type": "file" } + ] }, { "title": "that import can be done", diff --git a/ui/fixtures/cmd.container.json b/ui/fixtures/cmd.container.json index 0605ea5..206eca3 100644 --- a/ui/fixtures/cmd.container.json +++ b/ui/fixtures/cmd.container.json @@ -23,7 +23,7 @@ "count": 5 }, { - "title": "verify correct files imported", + "title": "verify correct files imported a", "cmd": [ "dump" ], "stdout_re": "fileb.txt.*file.*\n.*foo.*bar.*cc06808cbbee0510331aa97974132e8dc296aeb795be229d064bae784b0a87a5cf4281d82e8c99271b75db2148f08a026c1a60ed9cabdb8cac6d24242dac4063.*\n.*filed.txt.*file.*\n.*filef.txt.*file.*\n.*fileb.txt.*filed.txt.*filef.txt.*cc06808cbbee0510331aa97974132e8dc296aeb795be229d064bae784b0a87a5cf4281d82e8c99271b75db2148f08a026c1.*7831bd05e23877e08a97362bab2ad7bcc7d08d8f841f42e8dee545781792b987aa7637f12cec399e261f798c10d3475add0db7de2643af86a346b6b451a69ec4.*be688838ca8686e5c90689bf2ab585cef1137c.*incomplete.*true.*container.*magnet:\\?xt=urn:btih:501cf3bd4797f49fd7a624e8a9a8ce5cccceb602&dn=somedir" }, @@ -45,9 +45,25 @@ "cmd": [ "container", "somedir.torrent" ] }, { - "title": "verify correct files imported", + "title": "verify correct files imported b", "cmd": [ "dump" ], - "stdout_re": ".*\n.*fileb.txt.*file.*\n.*foo.*bar.*cc06808cbbee0510331aa97974132e8dc296aeb795be229d064bae784b0a87a5cf4281d82e8c99271b75db2148f08a026c1a60ed9cabdb8cac6d24242dac4063.*\n.*filed.txt.*file.*\n.*filef.txt.*file.*\n.*filea.txt.*fileb.txt.*filec.txt.*filed.txt.*filee.txt.*filef.txt.*0cf9180a764aba863a67b6d72f0918bc131c6772642cb2dce5a34f0a702f9470ddc2bf125c12198b1995c233c34b4afd346c54a2334c350a948a51b6e8b4e6b6.*cc06808cbbee0510331aa97974132e8dc296aeb795be229d064bae784b0a87a5cf4281d82e8c99271b75db2148f08a026c1.*7831bd05e23877e08a97362bab2ad7bcc7d08d8f841f42e8dee545781792b987aa7637f12cec399e261f798c10d3475add0db7de2643af86a346b6b451a69ec4.*be688838ca8686e5c90689bf2ab585cef1137c.*container.*magnet:\\?xt=urn:btih:501cf3bd4797f49fd7a624e8a9a8ce5cccceb602&dn=somedir" + "stdout_check": [ + { "type": "identity" }, + { "foo": [ "bar" ], "hashes": [ + "sha512:cc06808cbbee0510331aa97974132e8dc296aeb795be229d064bae784b0a87a5cf4281d82e8c99271b75db2148f08a026c1a60ed9cabdb8cac6d24242dac4063" ] }, + { "foo": [ "bar" ], "hashes": [ + "sha512:7831bd05e23877e08a97362bab2ad7bcc7d08d8f841f42e8dee545781792b987aa7637f12cec399e261f798c10d3475add0db7de2643af86a346b6b451a69ec4" ] }, + { "filename": "filea.txt", "type": "file" }, + { "filename": "fileb.txt", "type": "file" }, + { "filename": "filec.txt", "type": "file" }, + { "filename": "filed.txt", "type": "file" }, + { "filename": "filee.txt", "type": "file" }, + { "filename": "filef.txt", "type": "file" }, + { "files": [ "filea.txt", "fileb.txt", "filec.txt", "filed.txt", "filee.txt", "filef/filef.txt" ], + "hashes": [ "sha512:0cf9180a764aba863a67b6d72f0918bc131c6772642cb2dce5a34f0a702f9470ddc2bf125c12198b1995c233c34b4afd346c54a2334c350a948a51b6e8b4e6b6", "sha512:cc06808cbbee0510331aa97974132e8dc296aeb795be229d064bae784b0a87a5cf4281d82e8c99271b75db2148f08a026c1a60ed9cabdb8cac6d24242dac4063", "sha512:cb9eb9ec6c2cd9b0d9451e6b179d91e24906a3123be5e5f18e182be09fab30ad6f5de391bb3cf53933d3a1ca29fdd68d23e17c49fbc1a9117c8ab08154c7df30", "sha512:7831bd05e23877e08a97362bab2ad7bcc7d08d8f841f42e8dee545781792b987aa7637f12cec399e261f798c10d3475add0db7de2643af86a346b6b451a69ec4", "sha512:b09a577f24fb7a6f4b3ea641b2b67120e187b605ef27db97bef178457d9002bec846435a205466e327e5ab151ab1b350b5ac1c9f97e48333cec84fecec3b7037", "sha512:be688838ca8686e5c90689bf2ab585cef1137c999b48c70b92f67a5c34dc15697b5d11c982ed6d71be1e1e7f7b4e0733884aa97c3f7a339a8ed03577cf74be09" ], + "type": "container", + "uri": "magnet:?xt=urn:btih:501cf3bd4797f49fd7a624e8a9a8ce5cccceb602&dn=somedir" } + ] }, { "special": "verify store object cnt", diff --git a/ui/fixtures/cmd.mapping.json b/ui/fixtures/cmd.mapping.json index 595a6dd..7c80ba0 100644 --- a/ui/fixtures/cmd.mapping.json +++ b/ui/fixtures/cmd.mapping.json @@ -37,7 +37,7 @@ "title": "that a mapping with unknown host errors", "cmd": [ "mapping", "--create", "ceaa4862-dd00-41ba-9787-7480ec1b267a:/foo", "efdb5d9c-d123-4b30-aaa8-45a9ea8f6053:/bar" ], "exit": 1, - "stderr": "ERROR: Unable to find host 'ceaa4862-dd00-41ba-9787-7480ec1b267a'\n" + "stderr": "ERROR: Unable to find host ceaa4862-dd00-41ba-9787-7480ec1b267a\n" }, { "title": "that a host mapping that isn't absolute errors", diff --git a/ui/fixtures/genfixtures.py b/ui/fixtures/genfixtures.py index 3de1e6b..6ebdc23 100644 --- a/ui/fixtures/genfixtures.py +++ b/ui/fixtures/genfixtures.py @@ -1,12 +1,13 @@ import pasn1 -import cli +from medashare import cli import datetime import uuid persona = cli.Persona() persona.generate_key() cbr = persona.get_identity().uuid -objst = cli.ObjectStore(cbr) +storename = 'sample.data.sqlite3' +objst = cli.ObjectStore.load(storename, cbr) list(map(objst.loadobj, [ { @@ -21,5 +22,4 @@ list(map(objst.loadobj, ] )) -objst.store('sample.data.pasn1') persona.store('sample.persona.pasn1') diff --git a/ui/medashare/cli.py b/ui/medashare/cli.py index ef346f4..f889d7c 100644 --- a/ui/medashare/cli.py +++ b/ui/medashare/cli.py @@ -1,5 +1,31 @@ #!/usr/bin/env python +import sys +import logging + +# useful for debugging when stderr is redirected/captured +_real_stderr = sys.stderr +_sql_verbose = False + +if False: + _handler = logging.StreamHandler(_real_stderr) + _handler.setLevel(logging.DEBUG) + import sqlalchemy + logging.getLogger('sqlalchemy').addHandler(_handler) + logging.getLogger('sqlalchemy.engine').setLevel(logging.DEBUG) + +def _debprint(*args): # pragma: no cover + import traceback, sys, os.path + st = traceback.extract_stack(limit=2)[0] + + sep = '' + if args: + sep = ':' + + print('%s:%d%s' % (os.path.basename(st.filename), st.lineno, sep), + *args, file=_real_stderr) + sys.stderr.flush() + #import pdb, sys; mypdb = pdb.Pdb(stdout=sys.stderr); mypdb.set_trace() from cryptography.hazmat.primitives.asymmetric.ed448 import Ed448PrivateKey, \ @@ -10,6 +36,7 @@ from cryptography.hazmat.primitives.serialization import Encoding, \ from unittest import mock from .hostid import hostuuid +from . import orm from .btv import _TestCases as bttestcase, validate_file @@ -31,6 +58,8 @@ import pasn1 import re import shutil import socket +from sqlalchemy import create_engine, select, func, delete +from sqlalchemy.orm import sessionmaker import string import subprocess import sys @@ -42,21 +71,6 @@ import uuid _NAMESPACE_MEDASHARE_PATH = uuid.UUID('f6f36b62-3770-4a68-bc3d-dc3e31e429e6') _NAMESPACE_MEDASHARE_CONTAINER = uuid.UUID('890a9d5c-0626-4de1-ab05-9e14947391eb') -# useful for debugging when stderr is redirected/captured -_real_stderr = sys.stderr - -def _debprint(*args): # pragma: no cover - import traceback, sys, os.path - st = traceback.extract_stack(limit=2)[0] - - sep = '' - if args: - sep = ':' - - print('%s:%d%s' % (os.path.basename(st.filename), st.lineno, sep), - *args, file=_real_stderr) - sys.stderr.flush() - _defaulthash = 'sha512' _validhashes = set([ 'sha256', 'sha512' ]) _hashlengths = { len(getattr(hashlib, x)().hexdigest()): x for x in @@ -78,196 +92,9 @@ def _iterdictlist(obj, **kwargs): else: yield k, v -def _makeuuid(s): - if isinstance(s, uuid.UUID): - return s - - if isinstance(s, bytes): - return uuid.UUID(bytes=s) - else: - return uuid.UUID(s) - -def _makedatetime(s): - if isinstance(s, datetime.datetime): - return s - - return datetime.datetime.strptime(s, '%Y-%m-%dT%H:%M:%S.%fZ').replace( - tzinfo=datetime.timezone.utc) - -def _makebytes(s): - if isinstance(s, bytes): - return s - - return base64.urlsafe_b64decode(s) - -# XXX - known issue, store is not atomic/safe, overwrites in place instead of -# renames - -# XXX - add validation -# XXX - how to add singletons -class MDBase(object): - '''This is a simple wrapper that turns a JSON object into a pythonesc - object where attribute accesses work.''' - - _type = 'invalid' - - _generated_properties = { - 'uuid': uuid.uuid4, - 'modified': lambda: datetime.datetime.now( - tz=datetime.timezone.utc), - } - - # When decoding, the decoded value should be passed to this function - # to get the correct type - _instance_properties = { - 'uuid': _makeuuid, - 'modified': _makedatetime, - 'created_by_ref': _makeuuid, - #'parent_refs': lambda x: [ _makeuuid(y) for y in x ], - 'sig': _makebytes, - } - - # Override on a per subclass basis - _class_instance_properties = { - } - - _common_properties = [ 'type', 'created_by_ref' ] # XXX - add lang? - _common_optional = set(('parent_refs', 'sig')) - _common_names = set(_common_properties + list( - _generated_properties.keys())) - _common_names_list = _common_properties + list( - _generated_properties.keys()) - - def __init__(self, obj={}, **kwargs): - obj = copy.deepcopy(obj) - obj.update(kwargs) - - if self._type == MDBase._type: - raise ValueError('call MDBase.create_obj instead so correct class is used.') - - if 'type' in obj and obj['type'] != self._type: - raise ValueError( - 'trying to create the wrong type of object, got: %s, expected: %s' % - (repr(obj['type']), repr(self._type))) - - if 'type' not in obj: - obj['type'] = self._type - - for x in self._common_properties: - if x not in obj: - raise ValueError('common property %s not present' % repr(x)) - - for x, fun in itertools.chain( - self._instance_properties.items(), - self._class_instance_properties.items()): - if x in obj: - obj[x] = fun(obj[x]) - - for x, fun in self._generated_properties.items(): - if x not in obj: - obj[x] = fun() - - self._obj = obj - - @classmethod - def create_obj(cls, obj): - '''Using obj as a base, create an instance of MDBase of the - correct type. - - If the correct type is not found, a ValueError is raised.''' - - if isinstance(obj, cls): - obj = obj._obj - - ty = obj['type'] - - for i in MDBase.__subclasses__(): - if i._type == ty: - return i(obj) - else: - raise ValueError('Unable to find class for type %s' % - repr(ty)) - - def new_version(self, *args, dels=(), replaces=()): - '''For each k, v pair, add the property k as an additional one - (or new one if first), with the value v. - - Any key in dels is removed. - - Any k, v pair in replaces, replaces the entire key.''' - - obj = copy.deepcopy(self._obj) - - common = self._common_names | self._common_optional - uniquify = set() - for k, v in args: - if k in common: - obj[k] = v - else: - uniquify.add(k) - obj.setdefault(k, []).append(v) - - for k in uniquify: - obj[k] = list(set(obj[k])) - - for i in dels: - del obj[i] - - for k, v in replaces: - obj[k] = v - - del obj['modified'] - - return self.create_obj(obj) - - def __repr__(self): # pragma: no cover - return '%s(%s)' % (self.__class__.__name__, repr(self._obj)) - - def __getattr__(self, k): - try: - return self._obj[k] - except KeyError: - raise AttributeError(k) - - def __setattr__(self, k, v): - if k[0] == '_': # direct attribute - self.__dict__[k] = v - else: - self._obj[k] = v - - def __getitem__(self, k): - return self._obj[k] - - def __to_dict__(self): - '''Returns an internal object. If modification is necessary, - make sure to .copy() it first.''' - - return self._obj - - def __eq__(self, o): - return self._obj == o +from .utils import _makeuuid, _makedatetime, _asn1coder - def __contains__(self, k): - return k in self._obj - - def items(self, skipcommon=True): - return [ (k, v) for k, v in self._obj.items() if - not skipcommon or k not in self._common_names ] - - def encode(self, meth='asn1'): - if meth == 'asn1': - return _asn1coder.dumps(self) - - return _jsonencoder.encode(self._obj) - - @classmethod - def decode(cls, s, meth='asn1'): - if meth == 'asn1': - obj = _asn1coder.loads(s) - else: - obj = json.loads(s) - - return cls.create_obj(obj) +from .mdb import MDBase class MetaData(MDBase): _type = 'metadata' @@ -285,48 +112,6 @@ class Identity(MDBase): _common_names = set(_common_properties + list( MDBase._generated_properties.keys())) -def _trytodict(o): - if isinstance(o, uuid.UUID): - return 'bytes', o.bytes - if isinstance(o, tuple): - return 'list', o - try: - return 'dict', o.__to_dict__() - except Exception: # pragma: no cover - raise TypeError('unable to find __to_dict__ on %s: %s' % - (type(o), repr(o))) - -class CanonicalCoder(pasn1.ASN1DictCoder): - def enc_dict(self, obj, **kwargs): - class FakeIter: - def items(self): - return iter(sorted(obj.items())) - - return pasn1.ASN1DictCoder.enc_dict(self, FakeIter(), **kwargs) - -_asn1coder = CanonicalCoder(coerce=_trytodict) - -class _JSONEncoder(json.JSONEncoder): - def default(self, o): - if isinstance(o, uuid.UUID): - return str(o) - elif isinstance(o, datetime.datetime): - o = o.astimezone(datetime.timezone.utc) - return o.strftime('%Y-%m-%dT%H:%M:%S.%fZ') - elif isinstance(o, bytes): - return base64.urlsafe_b64encode(o).decode('US-ASCII') - - return json.JSONEncoder.default(self, o) - -_jsonencoder = _JSONEncoder() - -class _TestJSONEncoder(unittest.TestCase): - def test_defaultfailure(self): - class Foo: - pass - - self.assertRaises(TypeError, _jsonencoder.encode, Foo()) - class Persona(object): '''The object that represents a persona, or identity. It will create the proper identity object, serialize for saving keys, @@ -493,23 +278,34 @@ class Persona(object): return self.sign(fobj) class ObjectStore(object): - '''A container to store for the various Metadata objects.''' + '''A container to store the various MetaData objects.''' # The _uuids property contains both the UUIDv4 for objects, and # looking up the UUIDv5 for FileObjects. - def __init__(self, created_by_ref): + def __init__(self, engine, created_by_ref): + orm.Base.metadata.create_all(engine) + + self._engine = engine + self._ses = sessionmaker(engine) + self._created_by_ref = created_by_ref - self._uuids = {} - self._hashes = {} - self._hostuuids = {} - self._hostmappings = [] def get_host(self, hostuuid): - return self._hostuuids[hostuuid] + hostuuid = _makeuuid(hostuuid) + + with self._ses() as session: + a = session.get(orm.HostTable, hostuuid) + + if a is None: + raise KeyError(hostuuid) + + return self._by_id(a.objid, session) def get_hosts(self): - return self._hostuuids.values() + with self._ses() as session: + for i in session.query(orm.HostTable.objid).all(): + yield self._by_id(i.objid, session) @staticmethod def makehash(hashstr, strict=True): @@ -545,119 +341,173 @@ class ObjectStore(object): raise ValueError def __len__(self): - return len(self._uuids) + with self._ses() as session: + return list(session.query(func.count( + orm.MetaDataObject.uuid)))[0][0] def __iter__(self): - seen = set() - for i in self._uuids.values(): - if i['uuid'] in seen: - continue + with self._ses() as session: + for i in session.query(orm.MetaDataObject.data).all(): + yield i.data - yield i - seen.add(i['uuid']) + @classmethod + def load(cls, fname, cbr): + engine = create_engine("sqlite+pysqlite:///%s" % fname, + echo=_sql_verbose, future=True) + + return cls(engine, cbr) def store(self, fname): '''Write out the objects in the store to the file named fname.''' - # eliminate objs stored by multiple uuids (FileObjects) - objs = { id(x): x for x in self._uuids.values() } + pass - with open(fname, 'wb') as fp: - obj = { - 'created_by_ref': self._created_by_ref, - 'objects': list(objs.values()), - } - fp.write(_asn1coder.dumps(obj)) + def _add_uuidv5(self, id, obj, session): + session.execute(delete(orm.UUIDv5Table).where( + orm.UUIDv5Table.uuid == id)) - def loadobj(self, obj): - '''Load obj into the data store.''' + o = orm.UUIDv5Table(uuid=id, objid=obj.uuid) + session.add(o) - obj = MDBase.create_obj(obj) + def _lock(self, session): + '''Function to issue a write to "lock" the database transaction.''' - self._uuids[obj.uuid] = obj + res = list(session.scalars(select(orm.Dummy).where( + orm.Dummy.id == 1))) - if obj.type == 'file': - objid = _makeuuid(obj.id) - if objid in self._uuids: - # pick which obj - oldobj = self._uuids[objid] - if oldobj.modified > obj.modified: - del self._uuids[obj.uuid] - obj = oldobj - else: - # get ride of old obj - del self._uuids[oldobj.uuid] - - self._uuids[_makeuuid(obj.id)] = obj - elif obj.type == 'container': - self._uuids[obj.make_id(obj.uri)] = obj - elif obj.type == 'host': - self._uuids[obj.hostuuid] = obj - self._hostuuids[obj.hostuuid] = obj - elif obj.type == 'mapping': - hostid = _makeuuid(hostuuid()) - - maps = [ (lambda a, b: (uuid.UUID(a), pathlib.Path(b).resolve()))(*x.split(':', 1)) for x in obj.mapping ] - for idx, (id, path) in enumerate(maps): - if hostid == id: - # add other to mapping - other = tuple(maps[(idx + 1) % 2]) - self._hostmappings.append((path, ) + other) - try: - hashes = obj.hashes - except AttributeError: - pass + if res: + session.delete(res[0]) else: - for j in hashes: - h = self.makehash(j) - d = self._hashes.setdefault(h, {}) - if obj.uuid not in d or obj.modified > d[obj.uuid].modified: - d[obj.uuid] = obj - - @classmethod - def load(cls, fname): - '''Load objects from the provided file name. + d = orm.Dummy(id=1) + session.add(d) - Basic validation will be done on the objects in the file. - - The objects will be accessible via other methods.''' + def loadobj(self, obj): + '''Load obj into the data store.''' - with open(fname, 'rb') as fp: - objs = _asn1coder.loads(fp.read()) + obj = MDBase.create_obj(obj) - obj = cls(objs['created_by_ref']) - for i in objs['objects']: - obj.loadobj(i) + with self._ses() as session: + self._lock(session) + + oldobj = session.get(orm.MetaDataObject, obj.uuid) + #if oldobj.modified > obj.modified: + # return + if oldobj is not None: + session.delete(oldobj) + + sobj = orm.MetaDataObject(uuid=obj.uuid, + modified=obj.modified, data=obj) + session.add(sobj) + + if obj.type == 'file': + objid = _makeuuid(obj.id) + oldobj = self._by_id(objid, session) + if oldobj is not None: + # pick which obj + if oldobj.modified > obj.modified: + session.delete(session.get( + orm.MetaDataObject, + obj.uuid)) + obj = oldobj + else: + # get ride of old obj + session.delete(session.get( + orm.MetaDataObject, + oldobj.uuid)) + + self._add_uuidv5(obj.id, obj, session) + elif obj.type == 'container': + self._add_uuidv5(obj.make_id(obj.uri), obj, + session) + elif obj.type == 'host': + o = orm.HostTable(hostid=_makeuuid( + obj.hostuuid), objid=obj.uuid) + session.add(o) + elif obj.type == 'mapping': + hostid = _makeuuid(hostuuid()) + + maps = [ (lambda a, b: orm.HostMapping( + hostid=uuid.UUID(a), objid=obj.uuid))( + *x.split(':', 1)) for x in obj.mapping ] + session.add_all(maps) + try: + hashes = obj.hashes + except AttributeError: + pass + else: + for j in hashes: + h = self.makehash(j) + r = session.get(orm.HashTable, + dict(hash=h, uuid=obj.uuid)) + if r is None: + session.add(orm.HashTable( + hash=h, uuid=obj.uuid)) - return obj + session.commit() def drop_uuid(self, uuid): uuid = _makeuuid(uuid) - obj = self.by_id(uuid) + with self._ses() as session: + obj = session.get(orm.MetaDataObject, uuid) + session.delete(obj) - del self._uuids[uuid] + obj = obj.data - if obj.type == 'file': - del self._uuids[obj.id] + if obj.type == 'file': + session.execute(delete(orm.UUIDv5Table).where( + orm.UUIDv5Table.uuid == obj.id)) + + for j in obj.hashes: + h = self.makehash(j) + session.execute(delete(orm.HashTable).where( + orm.HashTable.hash == h and + orm.HashTable.uuid == obj.uuid)) - for j in obj.hashes: - h = self.makehash(j) - del self._hashes[h][obj.uuid] + session.commit() def by_id(self, id): '''Look up an object by it's UUID.''' - uid = _makeuuid(id) + id = _makeuuid(id) - return self._uuids[uid] + with self._ses() as session: + res = self._by_id(id, session) + if res is None: + raise KeyError(id) + + return res + + def _by_id(self, id, session): + if id.version == 5: + res = session.get(orm.UUIDv5Table, id) + if res is None: + return + + id = res.objid + + res = session.get(orm.MetaDataObject, id) + if res is None: + return + + return res.data def by_hash(self, hash): '''Look up an object by it's hash value.''' h = self.makehash(hash, strict=False) - return list(self._hashes[h].values()) + + r = [] + with self._ses() as session: + # XXX - convert to union/join query + for i in session.scalars(select( + orm.HashTable.uuid).where(orm.HashTable.hash == h)): + v = self._by_id(i, session) + if v is not None: + r.append(v) + + return r def get_metadata(self, fname, persona, create_metadata=True): '''Get all MetaData objects for fname, or create one if @@ -695,6 +545,30 @@ class ObjectStore(object): return objs + def _get_hostmappings(self): + '''Returns the tuple (lclpath, hostid, rempath) for all + the mappings for this hostid.''' + + hostid = _makeuuid(hostuuid()) + + res = [] + with self._ses() as session: + # XXX - view + for i in session.scalars(select(orm.HostMapping).where( + orm.HostMapping.hostid == hostid)): + obj = self._by_id(i.objid, session) + + maps = [ (lambda a, b: (uuid.UUID(a), + pathlib.Path(b).resolve()))(*x.split(':', + 1)) for x in obj.mapping ] + for idx, (id, path) in enumerate(maps): + if hostid == id: + # add other to mapping + other = tuple(maps[(idx + 1) % + 2]) + res.append((path, ) + other) + return res + def by_file(self, fname, types=('metadata', )): '''Return a metadata object for the file named fname. @@ -717,11 +591,15 @@ class ObjectStore(object): except KeyError: # check mappings fname = pathlib.Path(fname).resolve() - for lclpath, hostid, rempath in self._hostmappings: + for lclpath, hostid, rempath in self._get_hostmappings(): if fname.parts[:len(lclpath.parts)] == lclpath.parts: try: - rempath = pathlib.Path(*rempath.parts + fname.parts[len(lclpath.parts):]) - fid = FileObject.make_id(rempath, hostid) + rempath = pathlib.Path( + *rempath.parts + + fname.parts[len( + lclpath.parts):]) + fid = FileObject.make_id( + rempath, hostid) fobj = self.by_id(fid) lclfile = fname break @@ -895,18 +773,18 @@ def write_cache(options, cache): def get_objstore(options): persona = get_persona(options) - storefname = os.path.expanduser('~/.medashare_store.pasn1') - try: - objstr = ObjectStore.load(storefname) - except FileNotFoundError: - objstr = ObjectStore(persona.get_identity().uuid) + + storefname = os.path.expanduser('~/.medashare_store.sqlite3') + + engine = create_engine("sqlite+pysqlite:///%s" % storefname, + echo=_sql_verbose, future=True) + + objstr = ObjectStore(engine, persona.get_identity().uuid) return persona, objstr def write_objstore(options, objstr): - storefname = os.path.expanduser('~/.medashare_store.pasn1') - - objstr.store(storefname) + pass def get_persona(options): identfname = os.path.expanduser('~/.medashare_identity.pasn1') @@ -1058,7 +936,7 @@ def cmd_mapping(options): [ objstr.get_host(x[0]) for x in parts ] except KeyError as e: print('ERROR: Unable to find host %s' % - repr(e.args[0]), file=sys.stderr) + str(e.args[0]), file=sys.stderr) sys.exit(1) m = persona.Mapping(mapping=[ ':'.join(x) for x in parts ]) @@ -1100,7 +978,8 @@ def getnextfile(files, idx): startidx = min(max(10, idx - 10), maxstart) _debprint(len(files), startidx) subset = files[startidx:min(len(files), idx + 10)] - selfile = -1 if origidx < startidx or origidx >= startidx + 20 else origidx - startidx + selfile = -1 if origidx < startidx or origidx >= startidx + \ + 20 else origidx - startidx for i, f in enumerate(subset): print('%2d)%1s%s' % (i + 1, '*' if i == selfile else '', repr(str(f)))) @@ -1405,17 +1284,23 @@ def cmd_container(options): write_objstore(options, objstr) -def cmd_import(options): - persona, objstr = get_objstore(options) +def _json_objstream(fp): + inp = fp.read() jd = json.JSONDecoder() - inp = sys.stdin.read() - while inp: inp = inp.strip() jobj, endpos = jd.raw_decode(inp) + yield jobj + + inp = inp[endpos:] + +def cmd_import(options): + persona, objstr = get_objstore(options) + + for jobj in _json_objstream(sys.stdin): if options.sign: cbr = _makeuuid(jobj['created_by_ref']) if cbr != persona.uuid: @@ -1433,8 +1318,6 @@ def cmd_import(options): objstr.loadobj(obj) - inp = inp[endpos:] - write_objstore(options, objstr) def cmd_drop(options): @@ -1466,18 +1349,21 @@ def main(): help='add the arg as metadata for the identity, tag=[value]') parser_i.set_defaults(func=cmd_ident) - parser_pubkey = subparsers.add_parser('pubkey', help='print public key of identity') + parser_pubkey = subparsers.add_parser('pubkey', + help='print public key of identity') parser_pubkey.set_defaults(func=cmd_pubkey) # used so that - isn't treated as an option - parser_mod = subparsers.add_parser('modify', help='modify tags on file(s)', prefix_chars='@') + parser_mod = subparsers.add_parser('modify', + help='modify tags on file(s)', prefix_chars='@') parser_mod.add_argument('modtagvalues', nargs='+', help='add (+) or delete (-) the tag=[value], for the specified files') parser_mod.add_argument('files', nargs='+', help='files to modify') parser_mod.set_defaults(func=cmd_modify) - parser_auto = subparsers.add_parser('auto', help='automatic detection of file properties') + parser_auto = subparsers.add_parser('auto', + help='automatic detection of file properties') parser_auto.add_argument('files', nargs='+', help='files to modify') parser_auto.set_defaults(func=cmd_auto) @@ -1487,20 +1373,24 @@ def main(): help='files to modify') parser_list.set_defaults(func=cmd_list) - parser_container = subparsers.add_parser('container', help='file is examined as a container and the internal files imported as entries') + parser_container = subparsers.add_parser('container', + help='file is examined as a container and the internal files imported as entries') parser_container.add_argument('files', nargs='+', help='files to modify') parser_container.set_defaults(func=cmd_container) - parser_hosts = subparsers.add_parser('hosts', help='dump all the hosts, self is always first') + parser_hosts = subparsers.add_parser('hosts', + help='dump all the hosts, self is always first') parser_hosts.set_defaults(func=cmd_hosts) - parser_mapping = subparsers.add_parser('mapping', help='list mappings, or create a mapping') + parser_mapping = subparsers.add_parser('mapping', + help='list mappings, or create a mapping') parser_mapping.add_argument('--create', dest='mapping', nargs=2, help='mapping to add, host|hostuuid:path host|hostuuid:path') parser_mapping.set_defaults(func=cmd_mapping) - parser_interactive = subparsers.add_parser('interactive', help='start in interactive mode') + parser_interactive = subparsers.add_parser('interactive', + help='start in interactive mode') parser_interactive.add_argument('files', nargs='*', help='files to work with') parser_interactive.set_defaults(func=cmd_interactive) @@ -1508,12 +1398,14 @@ def main(): parser_dump = subparsers.add_parser('dump', help='dump all the objects') parser_dump.set_defaults(func=cmd_dump) - parser_import = subparsers.add_parser('import', help='import objects encoded as json') + parser_import = subparsers.add_parser('import', + help='import objects encoded as json') parser_import.add_argument('--sign', action='store_true', help='import as new identity, and sign objects (if created_by_ref is different, new uuid is created)') parser_import.set_defaults(func=cmd_import) - parser_drop = subparsers.add_parser('drop', help='drop the object specified by UUID') + parser_drop = subparsers.add_parser('drop', + help='drop the object specified by UUID') parser_drop.add_argument('uuids', nargs='+', help='UUID of object to drop') parser_drop.set_defaults(func=cmd_drop) @@ -1566,10 +1458,12 @@ class _TestCases(unittest.TestCase): self.basetempdir = d self.tempdir = d / 'subdir' - self.persona = Persona.load(os.path.join('fixtures', 'sample.persona.pasn1')) + self.persona = Persona.load(os.path.join('fixtures', + 'sample.persona.pasn1')) self.created_by_ref = self.persona.get_identity().uuid shutil.copytree(self.fixtures / 'testfiles', self.tempdir) + shutil.copy(self.fixtures / 'sample.data.sqlite3', self.tempdir) self.oldcwd = os.getcwd() @@ -1582,7 +1476,11 @@ class _TestCases(unittest.TestCase): def test_fileobject(self): os.chdir(self.tempdir) - objst = ObjectStore(self.created_by_ref) + engine = create_engine( + "sqlite+pysqlite:///memdb1?mode=memory&cache=shared", + echo=_sql_verbose, future=True) + + objst = ObjectStore(engine, self.created_by_ref) a = self.persona.by_file('test.txt') @@ -1597,11 +1495,14 @@ class _TestCases(unittest.TestCase): objst.loadobj(a) + #_debprint('a:', repr(a)) + #_debprint('by_id:', objst.by_id(a.uuid)) + # write out the store objst.store('teststore.pasn1') # load it back in - objstr = ObjectStore.load('teststore.pasn1') + objstr = ObjectStore(engine, self.created_by_ref) a = objstr.by_id(a['uuid']) @@ -1622,8 +1523,10 @@ class _TestCases(unittest.TestCase): def test_mdbase(self): self.assertRaises(ValueError, MDBase, created_by_ref='') - self.assertRaises(ValueError, MDBase.create_obj, { 'type': 'unknosldkfj' }) - self.assertRaises(ValueError, MDBase.create_obj, { 'type': 'metadata' }) + self.assertRaises(ValueError, MDBase.create_obj, + { 'type': 'unknosldkfj' }) + self.assertRaises(ValueError, MDBase.create_obj, + { 'type': 'metadata' }) baseobj = { 'type': 'metadata', @@ -1664,7 +1567,8 @@ class _TestCases(unittest.TestCase): self.assertEqual(md3.sig, fvalue) # that invalid attribute access raises correct exception - self.assertRaises(AttributeError, getattr, md, 'somerandombogusattribute') + self.assertRaises(AttributeError, getattr, md, + 'somerandombogusattribute') # that when readding an attribute that already exists md3 = md2.new_version(('dc:creator', 'Jim Bob',)) @@ -1758,10 +1662,12 @@ class _TestCases(unittest.TestCase): # XXX - make sure works w/ relative dirs files = enumeratedir(os.path.relpath(self.tempdir), self.created_by_ref) - self.assertEqual(oldid, files[1].id) + self.assertEqual(files[2].filename, 'test.txt') + self.assertEqual(oldid, files[2].id) def test_mdbaseoverlay(self): - objst = ObjectStore(self.created_by_ref) + engine = create_engine("sqlite+pysqlite:///:memory:", echo=_sql_verbose, future=True) + objst = ObjectStore(engine, self.created_by_ref) # that a base object bid = uuid.uuid4() @@ -1905,8 +1811,12 @@ class _TestCases(unittest.TestCase): persona.verify(mdobj) def test_objectstore(self): - persona = Persona.load(os.path.join('fixtures', 'sample.persona.pasn1')) - objst = ObjectStore.load(os.path.join('fixtures', 'sample.data.pasn1')) + persona = self.persona + objst = ObjectStore.load(self.tempdir / 'sample.data.sqlite3', + persona.get_identity().uuid) + + lst = objst.by_hash('91751cee0a1ab8414400238a761411daa29643ab4b8243e9a91649e25be53ada') + self.assertEqual(len(lst), 1) objst.loadobj({ 'type': 'metadata', @@ -1931,24 +1841,26 @@ class _TestCases(unittest.TestCase): self.assertEqual(r.uuid, uuid.UUID('3e466e06-45de-4ecc-84ba-2d2a3d970e96')) self.assertEqual(r['dc:creator'], [ 'John-Mark Gurney' ]) - # test storing the object store - fname = 'testfile.pasn1' - objst.store(fname) + # XXX do we care anymore? + if False: + # test storing the object store + fname = 'testfile.sqlite3' + objst.store(fname) - with open(fname, 'rb') as fp: - objs = _asn1coder.loads(fp.read()) + with open(fname, 'rb') as fp: + objs = _asn1coder.loads(fp.read()) - os.unlink(fname) + os.unlink(fname) - self.assertEqual(len(objs), len(objst)) + self.assertEqual(len(objs), len(objst)) - self.assertEqual(objs['created_by_ref'], self.created_by_ref.bytes) + self.assertEqual(objs['created_by_ref'], self.created_by_ref.bytes) - # make sure that the read back data matches - for i in objs['objects']: - i['created_by_ref'] = uuid.UUID(bytes=i['created_by_ref']) - i['uuid'] = uuid.UUID(bytes=i['uuid']) - self.assertEqual(objst.by_id(i['uuid']), i) + # make sure that the read back data matches + for i in objs['objects']: + i['created_by_ref'] = uuid.UUID(bytes=i['created_by_ref']) + i['uuid'] = uuid.UUID(bytes=i['uuid']) + self.assertEqual(objst.by_id(i['uuid']), i) # that a file testfname = os.path.join(self.tempdir, 'test.txt') @@ -1958,7 +1870,6 @@ class _TestCases(unittest.TestCase): # can be found self.assertEqual(objst.by_file(testfname), [ byid ]) - self.assertEqual(objst.by_file(testfname), [ byid ]) self.assertRaises(KeyError, objst.by_file, '/dev/null') @@ -1984,6 +1895,27 @@ class _TestCases(unittest.TestCase): # Tests to add: # Non-duplicates when same metadata is located by multiple hashes. + def objcompare(self, fullobjs, partialobjs): + fullobjs = list(fullobjs) + #_debprint('objs:', repr(fullobjs)) + self.assertEqual(len(fullobjs), len(partialobjs)) + + missing = [] + for i in partialobjs: + for idx, j in enumerate(fullobjs): + cmpobj = dict((k, v) for k, v in j.items() if k in set(i.keys())) + if cmpobj == i: + break + else: # pragma: no cover + missing.append(i) + continue + + fullobjs.pop(idx) + + if missing: # pragma: no cover + _debprint('remaining objs:', repr(fullobjs)) + self.fail('Unable to find objects %s in dump' % missing) + def run_command_file(self, f): with open(f) as fp: cmds = json.load(fp) @@ -1994,7 +1926,7 @@ class _TestCases(unittest.TestCase): # setup path mapping def expandusermock(arg): - if arg == '~/.medashare_store.pasn1': + if arg == '~/.medashare_store.sqlite3': return storefname elif arg == '~/.medashare_identity.pasn1': return identfname @@ -2030,9 +1962,9 @@ class _TestCases(unittest.TestCase): with open(newtestfname, 'w') as fp: fp.write('some new contents') elif special == 'verify store object cnt': - with open(storefname, 'rb') as fp: - pasn1obj = pasn1.loads(fp.read()) - objcnt = len(pasn1obj['objects']) + objst = ObjectStore.load(storefname, None) + objcnt = len(objst) + self.assertEqual(objcnt, len(list(objst))) self.assertEqual(objcnt, cmd['count']) elif special == 'set hostid': hostidpatch = mock.patch(__name__ + '.hostuuid') @@ -2040,7 +1972,7 @@ class _TestCases(unittest.TestCase): hostidpatch.start().return_value = hid patches.append(hostidpatch) elif special == 'iter is unique': - objst = ObjectStore.load(storefname) + objst = ObjectStore.load(storefname, None) uniqobjs = len(set((x['uuid'] for x in objst))) self.assertEqual(len(list(objst)), uniqobjs) elif special == 'setup bittorrent files': @@ -2106,14 +2038,21 @@ class _TestCases(unittest.TestCase): # with the correct output self.maxDiff = None + outeq = cmd.get('stdout') outnre = cmd.get('stdout_nre') outre = cmd.get('stdout_re') + outcheck = cmd.get('stdout_check') + # python3 -c 'import ast, sys; print(ast.literal_eval(sys.stdin.read()))' << EOF | jq '.' if outnre: self.assertNotRegex(stdout.getvalue(), outnre) - elif outre: + if outre: self.assertRegex(stdout.getvalue(), outre) - else: - self.assertEqual(stdout.getvalue(), cmd.get('stdout', '')) + if outeq: + self.assertEqual(stdout.getvalue(), outeq) + if outcheck: + stdout.seek(0) + self.objcompare(_json_objstream(stdout), outcheck) + self.assertEqual(stderr.getvalue(), cmd.get('stderr', '')) @@ -2123,6 +2062,7 @@ class _TestCases(unittest.TestCase): for i in patches: i.stop() + #@unittest.skip('temp') def test_cmds(self): cmds = sorted(self.fixtures.glob('cmd.*.json')) @@ -2141,15 +2081,12 @@ class _TestCases(unittest.TestCase): # created. # setup object store - storefname = os.path.join(self.tempdir, 'storefname') - identfname = os.path.join(self.tempdir, 'identfname') - - # XXX part of the problem - shutil.copy(os.path.join('fixtures', 'sample.data.pasn1'), storefname) + storefname = self.tempdir / 'storefname' + identfname = self.tempdir / 'identfname' # setup path mapping def expandusermock(arg): - if arg == '~/.medashare_store.pasn1': + if arg == '~/.medashare_store.sqlite3': return storefname elif arg == '~/.medashare_identity.pasn1': return identfname diff --git a/ui/medashare/mdb.py b/ui/medashare/mdb.py new file mode 100644 index 0000000..937f244 --- /dev/null +++ b/ui/medashare/mdb.py @@ -0,0 +1,197 @@ +import base64 +import copy +import datetime +import itertools +import json +import unittest +import uuid + +from .utils import _makeuuid, _makedatetime, _makebytes, _asn1coder + +class _JSONEncoder(json.JSONEncoder): + def default(self, o): + if isinstance(o, uuid.UUID): + return str(o) + elif isinstance(o, datetime.datetime): + o = o.astimezone(datetime.timezone.utc) + return o.strftime('%Y-%m-%dT%H:%M:%S.%fZ') + elif isinstance(o, bytes): + return base64.urlsafe_b64encode(o).decode('US-ASCII') + + return json.JSONEncoder.default(self, o) + +_jsonencoder = _JSONEncoder() + +class _TestJSONEncoder(unittest.TestCase): + def test_defaultfailure(self): + class Foo: + pass + + self.assertRaises(TypeError, _jsonencoder.encode, Foo()) + +# XXX - add validation +# XXX - how to add singletons +class MDBase(object): + '''This is a simple wrapper that turns a JSON object into a pythonesc + object where attribute accesses work.''' + + _type = 'invalid' + + _generated_properties = { + 'uuid': uuid.uuid4, + 'modified': lambda: datetime.datetime.now( + tz=datetime.timezone.utc), + } + + # When decoding, the decoded value should be passed to this function + # to get the correct type + _instance_properties = { + 'uuid': _makeuuid, + 'modified': _makedatetime, + 'created_by_ref': _makeuuid, + 'parent_refs': lambda x: [ _makeuuid(y) for y in x ], + 'sig': _makebytes, + } + + # Override on a per subclass basis + _class_instance_properties = { + } + + _common_properties = [ 'type', 'created_by_ref' ] # XXX - add lang? + _common_optional = set(('parent_refs', 'sig')) + _common_names = set(_common_properties + list( + _generated_properties.keys())) + _common_names_list = _common_properties + list( + _generated_properties.keys()) + + def __init__(self, obj={}, **kwargs): + obj = copy.deepcopy(obj) + obj.update(kwargs) + + if self._type == MDBase._type: + raise ValueError('call MDBase.create_obj instead so correct class is used.') + + if 'type' in obj and obj['type'] != self._type: + raise ValueError( + 'trying to create the wrong type of object, got: %s, expected: %s' % + (repr(obj['type']), repr(self._type))) + + if 'type' not in obj: + obj['type'] = self._type + + for x in self._common_properties: + if x not in obj: + raise ValueError('common property %s not present' % repr(x)) + + for x, fun in itertools.chain( + self._instance_properties.items(), + self._class_instance_properties.items()): + if x in obj: + obj[x] = fun(obj[x]) + + for x, fun in self._generated_properties.items(): + if x not in obj: + obj[x] = fun() + + self._obj = obj + + @classmethod + def create_obj(cls, obj): + '''Using obj as a base, create an instance of MDBase of the + correct type. + + If the correct type is not found, a ValueError is raised.''' + + if isinstance(obj, cls): + obj = obj._obj + + ty = obj['type'] + + for i in MDBase.__subclasses__(): + if i._type == ty: + return i(obj) + else: + raise ValueError('Unable to find class for type %s' % + repr(ty)) + + def new_version(self, *args, dels=(), replaces=()): + '''For each k, v pair, add the property k as an additional one + (or new one if first), with the value v. + + Any key in dels is removed. + + Any k, v pair in replaces, replaces the entire key.''' + + obj = copy.deepcopy(self._obj) + + common = self._common_names | self._common_optional + uniquify = set() + for k, v in args: + if k in common: + obj[k] = v + else: + uniquify.add(k) + obj.setdefault(k, []).append(v) + + for k in uniquify: + obj[k] = list(set(obj[k])) + + for i in dels: + del obj[i] + + for k, v in replaces: + obj[k] = v + + del obj['modified'] + + return self.create_obj(obj) + + def __repr__(self): # pragma: no cover + return '%s(%s)' % (self.__class__.__name__, repr(self._obj)) + + def __getattr__(self, k): + try: + return self._obj[k] + except KeyError: + raise AttributeError(k) + + def __setattr__(self, k, v): + if k[0] == '_': # direct attribute + self.__dict__[k] = v + else: + self._obj[k] = v + + def __getitem__(self, k): + return self._obj[k] + + def __to_dict__(self): + '''Returns an internal object. If modification is necessary, + make sure to .copy() it first.''' + + return self._obj + + def __eq__(self, o): + return self._obj == o + + def __contains__(self, k): + return k in self._obj + + def items(self, skipcommon=True): + return [ (k, v) for k, v in self._obj.items() if + not skipcommon or k not in self._common_names ] + + def encode(self, meth='asn1'): + if meth == 'asn1': + return _asn1coder.dumps(self) + + return _jsonencoder.encode(self._obj) + + @classmethod + def decode(cls, s, meth='asn1'): + if meth == 'asn1': + obj = _asn1coder.loads(s) + else: + obj = json.loads(s) + + return cls.create_obj(obj) + diff --git a/ui/medashare/orm.py b/ui/medashare/orm.py new file mode 100644 index 0000000..3cdf6e5 --- /dev/null +++ b/ui/medashare/orm.py @@ -0,0 +1,72 @@ +import uuid +from sqlalchemy import Table, Column, DateTime, String, Integer, LargeBinary +from sqlalchemy import types +from sqlalchemy.orm import declarative_base +from .cli import _debprint +from .mdb import MDBase + +Base = declarative_base() + +class MDBaseType(types.TypeDecorator): + impl = LargeBinary + + cache_ok = True + + def process_bind_param(self, value, dialect): + return value.encode() + + def process_result_value(self, value, dialect): + return MDBase.decode(value) + +class UUID(types.TypeDecorator): + impl = String(32) + + cache_ok = True + + def process_bind_param(self, value, dialect): + return value.hex + + def process_result_value(self, value, dialect): + return uuid.UUID(hex=value) + +class Dummy(Base): + __tablename__ = 'dummy' + + id = Column(Integer, primary_key=True) + +class UUIDv5Table(Base): + __tablename__ = 'uuidv5_index' + + uuid = Column(UUID, primary_key=True) + objid = Column(UUID) + +class HostMapping(Base): + __tablename__ = 'hostmapping' + + hostid = Column(UUID, primary_key=True) + objid = Column(UUID, primary_key=True) + + # https://stackoverflow.com/questions/57685385/how-to-avoid-inserting-duplicate-data-when-inserting-data-into-sqlite3-database + #UniqueConstraint('hostid', 'objid', on conflict ignore) + +class HostTable(Base): + __tablename__ = 'hosttable' + + hostid = Column(UUID, primary_key=True) + objid = Column(UUID) + +class HashTable(Base): + __tablename__ = 'hash_index' + + hash = Column(String, primary_key=True) + uuid = Column(UUID, primary_key=True) + +class MetaDataObject(Base): + __tablename__ = 'metadata_objects' + + uuid = Column(UUID, primary_key=True) + modified = Column(DateTime) + data = Column(MDBaseType) + + def __repr__(self): + return 'MetaDataObject(uuid=%s, modified=%s, data=%s)' % (repr(self.uuid), repr(self.modified), repr(self.data)) diff --git a/ui/medashare/tests.py b/ui/medashare/tests.py index ebadd83..263e75c 100644 --- a/ui/medashare/tests.py +++ b/ui/medashare/tests.py @@ -1,6 +1,6 @@ from .btv import _TestCases as btv_test_cases from .btv.bencode import _TestCases as bencode_test_cases +from .mdb import _TestJSONEncoder from .cli import _TestCononicalCoder, _TestCases as cli_test_cases -from .cli import _TestJSONEncoder from .mtree import Test from .server import _TestCases, _TestPostConfig diff --git a/ui/medashare/utils.py b/ui/medashare/utils.py new file mode 100644 index 0000000..8c09292 --- /dev/null +++ b/ui/medashare/utils.py @@ -0,0 +1,47 @@ +import base64 +import datetime +import pasn1 +import uuid + +def _makeuuid(s): + if isinstance(s, uuid.UUID): + return s + + if isinstance(s, bytes): + return uuid.UUID(bytes=s) + else: + return uuid.UUID(s) + +def _makedatetime(s): + if isinstance(s, datetime.datetime): + return s + + return datetime.datetime.strptime(s, '%Y-%m-%dT%H:%M:%S.%fZ').replace( + tzinfo=datetime.timezone.utc) + +def _makebytes(s): + if isinstance(s, bytes): + return s + + return base64.urlsafe_b64decode(s) + +def _trytodict(o): + if isinstance(o, uuid.UUID): + return 'bytes', o.bytes + if isinstance(o, tuple): + return 'list', o + try: + return 'dict', o.__to_dict__() + except Exception: # pragma: no cover + raise TypeError('unable to find __to_dict__ on %s: %s' % + (type(o), repr(o))) + +class CanonicalCoder(pasn1.ASN1DictCoder): + def enc_dict(self, obj, **kwargs): + class FakeIter: + def items(self): + return iter(sorted(obj.items())) + + return pasn1.ASN1DictCoder.enc_dict(self, FakeIter(), **kwargs) + +_asn1coder = CanonicalCoder(coerce=_trytodict) diff --git a/ui/setup.py b/ui/setup.py index 4d069ae..ba2f069 100644 --- a/ui/setup.py +++ b/ui/setup.py @@ -25,6 +25,7 @@ setup( 'fastapi', 'fastapi_restful', 'httpx', + 'SQLAlchemy', 'hypercorn', # option, for server only? 'orm', 'pasn1 @ git+https://www.funkthat.com/gitea/jmg/pasn1.git@c6c64510b42292557ace2b77272eb32cb647399d#egg=pasn1',