convert the main data store to SQLite3...

one minor issue is that the identity object gets dump'd twice..

ui/Makefile

@@ -1,7 +1,7 @@
 VIRTUALENV?=python3 -m venv
 MODULES=medashare
 
-test: fixtures/sample.data.pasn1 fixtures/sample.persona.pasn1 fixtures/sample.mtree
+test: fixtures/sample.data.sqlite3 fixtures/sample.persona.pasn1 fixtures/sample.mtree
 	(. ./p/bin/activate && \
 		((find fixtures -type f; find $(MODULES) -type f) | entr sh -c 'python3 -m coverage run -m unittest --failfast $(MODULES).tests && coverage report -m --omit=p/\*'))
 
@@ -10,7 +10,7 @@ env:
 	(. ./p/bin/activate && \
 		pip install -r requirements.txt)
 
-fixtures/sample.data.pasn1 fixtures/sample.persona.pasn1: fixtures/genfixtures.py
+fixtures/sample.data.sqlite3 fixtures/sample.persona.pasn1: fixtures/genfixtures.py
 	(. ./p/bin/activate && cd fixtures && PYTHONPATH=.. python3 genfixtures.py )
 
 fixtures/sample.mtree: fixtures/mtree.dir

ui/fixtures/cmd.basic.json

@@ -206,7 +206,14 @@
 	"title": "dump is correct",
 	"cmd": [ "dump" ],
 	"exit": 0,
-	"stdout_re": "{.*name.*Changed Name.*type.*identity.*}\n{.*filename.*newfile.txt.*hashes.*90f8342520f0ac57fb5a779f5d331c2fa87aa40f8799940257f9ba619940951e67143a8d746535ed0284924b2b7bc1478f095198800ba96d01847d7b56ca465c.*size.*19.*type.*file.*}\n{.*foo.*bar=baz.*hashes.*90f8342520f0ac57fb5a779f5d331c2fa87aa40f8799940257f9ba619940951e67143a8d746535ed0284924b2b7bc1478f095198800ba96d01847d7b56ca465c.*type.*metadata.*}\n{.*filename.*test.txt.*90f8342520f0ac57fb5a779f5d331c2fa87aa40f8799940257f9ba619940951e67143a8d746535ed0284924b2b7bc1478f095198800ba96d01847d7b56ca465c.*size.*19.*type.*file.*}\n{.*filename.*newfile.txt.*90f8342520f0ac57fb5a779f5d331c2fa87aa40f8799940257f9ba619940951e67143a8d746535ed0284924b2b7bc1478f095198800ba96d01847d7b56ca465c.*size.*19.*type.*file.*}\n{.*filename.*newfile.txt.*90f8342520f0ac57fb5a779f5d331c2fa87aa40f8799940257f9ba619940951e67143a8d746535ed0284924b2b7bc1478f095198800ba96d01847d7b56ca465c.*size.*19.*type.*file.*}\n"
+	"stdout_check": [
+		{ "name": "Changed Name", "type": "identity" },
+		{ "filename": "newfile.txt", "hashes": [ "sha512:90f8342520f0ac57fb5a779f5d331c2fa87aa40f8799940257f9ba619940951e67143a8d746535ed0284924b2b7bc1478f095198800ba96d01847d7b56ca465c" ], "size": 19, "type": "file" },
+		{ "foo": [ "bar=baz" ], "hashes": [ "sha512:90f8342520f0ac57fb5a779f5d331c2fa87aa40f8799940257f9ba619940951e67143a8d746535ed0284924b2b7bc1478f095198800ba96d01847d7b56ca465c" ], "type": "metadata" },
+		{ "filename": "test.txt", "hashes": [ "sha512:90f8342520f0ac57fb5a779f5d331c2fa87aa40f8799940257f9ba619940951e67143a8d746535ed0284924b2b7bc1478f095198800ba96d01847d7b56ca465c" ], "size": 19, "type": "file" },
+		{ "filename": "newfile.txt", "hashes": [ "sha512:90f8342520f0ac57fb5a779f5d331c2fa87aa40f8799940257f9ba619940951e67143a8d746535ed0284924b2b7bc1478f095198800ba96d01847d7b56ca465c" ], "size": 19, "type": "file" },
+		{ "filename": "newfile.txt", "hashes": [ "sha512:b0551b2fb5d045a74d36a08ac49aea66790ea4fb5e84f9326a32db44fc78ca0676e65a8d1f0d98f62589eeaef105f303c81f07f3f862ad3bace7960fe59de4d5" ], "size": 17, "type": "file" }
+	]
 },
 {
 	"title": "that import can be done",

ui/fixtures/cmd.container.json

@@ -23,7 +23,7 @@
         "count": 5
 },
 {
-	"title": "verify correct files imported",
+	"title": "verify correct files imported a",
 	"cmd": [ "dump" ],
 	"stdout_re": "fileb.txt.*file.*\n.*foo.*bar.*cc06808cbbee0510331aa97974132e8dc296aeb795be229d064bae784b0a87a5cf4281d82e8c99271b75db2148f08a026c1a60ed9cabdb8cac6d24242dac4063.*\n.*filed.txt.*file.*\n.*filef.txt.*file.*\n.*fileb.txt.*filed.txt.*filef.txt.*cc06808cbbee0510331aa97974132e8dc296aeb795be229d064bae784b0a87a5cf4281d82e8c99271b75db2148f08a026c1.*7831bd05e23877e08a97362bab2ad7bcc7d08d8f841f42e8dee545781792b987aa7637f12cec399e261f798c10d3475add0db7de2643af86a346b6b451a69ec4.*be688838ca8686e5c90689bf2ab585cef1137c.*incomplete.*true.*container.*magnet:\\?xt=urn:btih:501cf3bd4797f49fd7a624e8a9a8ce5cccceb602&dn=somedir"
 },
@@ -45,9 +45,25 @@
 	"cmd": [ "container", "somedir.torrent" ]
 },
 {
-	"title": "verify correct files imported",
+	"title": "verify correct files imported b",
 	"cmd": [ "dump" ],
-	"stdout_re": ".*\n.*fileb.txt.*file.*\n.*foo.*bar.*cc06808cbbee0510331aa97974132e8dc296aeb795be229d064bae784b0a87a5cf4281d82e8c99271b75db2148f08a026c1a60ed9cabdb8cac6d24242dac4063.*\n.*filed.txt.*file.*\n.*filef.txt.*file.*\n.*filea.txt.*fileb.txt.*filec.txt.*filed.txt.*filee.txt.*filef.txt.*0cf9180a764aba863a67b6d72f0918bc131c6772642cb2dce5a34f0a702f9470ddc2bf125c12198b1995c233c34b4afd346c54a2334c350a948a51b6e8b4e6b6.*cc06808cbbee0510331aa97974132e8dc296aeb795be229d064bae784b0a87a5cf4281d82e8c99271b75db2148f08a026c1.*7831bd05e23877e08a97362bab2ad7bcc7d08d8f841f42e8dee545781792b987aa7637f12cec399e261f798c10d3475add0db7de2643af86a346b6b451a69ec4.*be688838ca8686e5c90689bf2ab585cef1137c.*container.*magnet:\\?xt=urn:btih:501cf3bd4797f49fd7a624e8a9a8ce5cccceb602&dn=somedir"
+	"stdout_check": [
+		{ "type": "identity" },
+		{ "foo": [ "bar" ], "hashes": [
+		  "sha512:cc06808cbbee0510331aa97974132e8dc296aeb795be229d064bae784b0a87a5cf4281d82e8c99271b75db2148f08a026c1a60ed9cabdb8cac6d24242dac4063" ] },
+		{ "foo": [ "bar" ], "hashes": [
+		  "sha512:7831bd05e23877e08a97362bab2ad7bcc7d08d8f841f42e8dee545781792b987aa7637f12cec399e261f798c10d3475add0db7de2643af86a346b6b451a69ec4" ] },
+		{ "filename": "filea.txt", "type": "file" },
+		{ "filename": "fileb.txt", "type": "file" },
+		{ "filename": "filec.txt", "type": "file" },
+		{ "filename": "filed.txt", "type": "file" },
+		{ "filename": "filee.txt", "type": "file" },
+		{ "filename": "filef.txt", "type": "file" },
+		{ "files": [ "filea.txt", "fileb.txt", "filec.txt", "filed.txt", "filee.txt", "filef/filef.txt" ],
+		  "hashes": [ "sha512:0cf9180a764aba863a67b6d72f0918bc131c6772642cb2dce5a34f0a702f9470ddc2bf125c12198b1995c233c34b4afd346c54a2334c350a948a51b6e8b4e6b6", "sha512:cc06808cbbee0510331aa97974132e8dc296aeb795be229d064bae784b0a87a5cf4281d82e8c99271b75db2148f08a026c1a60ed9cabdb8cac6d24242dac4063", "sha512:cb9eb9ec6c2cd9b0d9451e6b179d91e24906a3123be5e5f18e182be09fab30ad6f5de391bb3cf53933d3a1ca29fdd68d23e17c49fbc1a9117c8ab08154c7df30", "sha512:7831bd05e23877e08a97362bab2ad7bcc7d08d8f841f42e8dee545781792b987aa7637f12cec399e261f798c10d3475add0db7de2643af86a346b6b451a69ec4", "sha512:b09a577f24fb7a6f4b3ea641b2b67120e187b605ef27db97bef178457d9002bec846435a205466e327e5ab151ab1b350b5ac1c9f97e48333cec84fecec3b7037", "sha512:be688838ca8686e5c90689bf2ab585cef1137c999b48c70b92f67a5c34dc15697b5d11c982ed6d71be1e1e7f7b4e0733884aa97c3f7a339a8ed03577cf74be09" ],
+		  "type": "container",
+		  "uri": "magnet:?xt=urn:btih:501cf3bd4797f49fd7a624e8a9a8ce5cccceb602&dn=somedir" }
+	]
 },
 {
         "special": "verify store object cnt",

ui/fixtures/cmd.mapping.json

@@ -37,7 +37,7 @@
 	"title": "that a mapping with unknown host errors",
 	"cmd": [ "mapping", "--create", "ceaa4862-dd00-41ba-9787-7480ec1b267a:/foo", "efdb5d9c-d123-4b30-aaa8-45a9ea8f6053:/bar" ],
 	"exit": 1,
-	"stderr": "ERROR: Unable to find host 'ceaa4862-dd00-41ba-9787-7480ec1b267a'\n"
+	"stderr": "ERROR: Unable to find host ceaa4862-dd00-41ba-9787-7480ec1b267a\n"
 },
 {
 	"title": "that a host mapping that isn't absolute errors",

ui/fixtures/genfixtures.py

@@ -1,12 +1,13 @@
 import pasn1
-import cli
+from medashare import cli
 import datetime
 import uuid
 
 persona = cli.Persona()
 persona.generate_key()
 cbr = persona.get_identity().uuid
-objst = cli.ObjectStore(cbr)
+storename = 'sample.data.sqlite3'
+objst = cli.ObjectStore.load(storename, cbr)
 list(map(objst.loadobj,
 	[
 		{
@@ -21,5 +22,4 @@ list(map(objst.loadobj,
 	]
 ))
 
-objst.store('sample.data.pasn1')
 persona.store('sample.persona.pasn1')

ui/medashare/mdb.py

@@ -0,0 +1,197 @@
+import base64
+import copy
+import datetime
+import itertools
+import json
+import unittest
+import uuid
+
+from .utils import _makeuuid, _makedatetime, _makebytes, _asn1coder
+
+class _JSONEncoder(json.JSONEncoder):
+	def default(self, o):
+		if isinstance(o, uuid.UUID):
+			return str(o)
+		elif isinstance(o, datetime.datetime):
+			o = o.astimezone(datetime.timezone.utc)
+			return o.strftime('%Y-%m-%dT%H:%M:%S.%fZ')
+		elif isinstance(o, bytes):
+			return base64.urlsafe_b64encode(o).decode('US-ASCII')
+
+		return json.JSONEncoder.default(self, o)
+
+_jsonencoder = _JSONEncoder()
+
+class _TestJSONEncoder(unittest.TestCase):
+	def test_defaultfailure(self):
+		class Foo:
+			pass
+
+		self.assertRaises(TypeError, _jsonencoder.encode, Foo())
+
+# XXX - add validation
+# XXX - how to add singletons
+class MDBase(object):
+	'''This is a simple wrapper that turns a JSON object into a pythonesc
+	object where attribute accesses work.'''
+
+	_type = 'invalid'
+
+	_generated_properties = {
+		'uuid': uuid.uuid4,
+		'modified': lambda: datetime.datetime.now(
+		    tz=datetime.timezone.utc),
+	}
+
+	# When decoding, the decoded value should be passed to this function
+	# to get the correct type
+	_instance_properties = {
+		'uuid': _makeuuid,
+		'modified': _makedatetime,
+		'created_by_ref': _makeuuid,
+		'parent_refs': lambda x: [ _makeuuid(y) for y in x ],
+		'sig': _makebytes,
+	}
+
+	# Override on a per subclass basis
+	_class_instance_properties = {
+	}
+
+	_common_properties = [ 'type', 'created_by_ref' ] # XXX - add lang?
+	_common_optional = set(('parent_refs', 'sig'))
+	_common_names = set(_common_properties + list(
+	    _generated_properties.keys()))
+	_common_names_list = _common_properties + list(
+	    _generated_properties.keys())
+
+	def __init__(self, obj={}, **kwargs):
+		obj = copy.deepcopy(obj)
+		obj.update(kwargs)
+
+		if self._type == MDBase._type:
+			raise ValueError('call MDBase.create_obj instead so correct class is used.')
+
+		if 'type' in obj and obj['type'] != self._type:
+			raise ValueError(
+			    'trying to create the wrong type of object, got: %s, expected: %s' %
+			    (repr(obj['type']), repr(self._type)))
+
+		if 'type' not in obj:
+			obj['type'] = self._type
+
+		for x in self._common_properties:
+			if x not in obj:
+				raise ValueError('common property %s not present' % repr(x))
+
+		for x, fun in itertools.chain(
+		    self._instance_properties.items(),
+		    self._class_instance_properties.items()):
+			if x in obj:
+				obj[x] = fun(obj[x])
+
+		for x, fun in self._generated_properties.items():
+			if x not in obj:
+				obj[x] = fun()
+
+		self._obj = obj
+
+	@classmethod
+	def create_obj(cls, obj):
+		'''Using obj as a base, create an instance of MDBase of the
+		correct type.
+
+		If the correct type is not found, a ValueError is raised.'''
+
+		if isinstance(obj, cls):
+			obj = obj._obj
+
+		ty = obj['type']
+
+		for i in MDBase.__subclasses__():
+			if i._type == ty:
+				return i(obj)
+		else:
+			raise ValueError('Unable to find class for type %s' %
+			    repr(ty))
+
+	def new_version(self, *args, dels=(), replaces=()):
+		'''For each k, v pair, add the property k as an additional one
+		(or new one if first), with the value v.
+
+		Any key in dels is removed.
+
+		Any k, v pair in replaces, replaces the entire key.'''
+
+		obj = copy.deepcopy(self._obj)
+
+		common = self._common_names | self._common_optional
+		uniquify = set()
+		for k, v in args:
+			if k in common:
+				obj[k] = v
+			else:
+				uniquify.add(k)
+				obj.setdefault(k, []).append(v)
+
+		for k in uniquify:
+			obj[k] = list(set(obj[k]))
+
+		for i in dels:
+			del obj[i]
+
+		for k, v in replaces:
+			obj[k] = v
+
+		del obj['modified']
+
+		return self.create_obj(obj)
+
+	def __repr__(self): # pragma: no cover
+		return '%s(%s)' % (self.__class__.__name__, repr(self._obj))
+
+	def __getattr__(self, k):
+		try:
+			return self._obj[k]
+		except KeyError:
+			raise AttributeError(k)
+
+	def __setattr__(self, k, v):
+		if k[0] == '_': # direct attribute
+			self.__dict__[k] = v
+		else:
+			self._obj[k] = v
+
+	def __getitem__(self, k):
+		return self._obj[k]
+
+	def __to_dict__(self):
+		'''Returns an internal object.  If modification is necessary,
+		make sure to .copy() it first.'''
+
+		return self._obj
+
+	def __eq__(self, o):
+		return self._obj == o
+
+	def __contains__(self, k):
+		return k in self._obj
+
+	def items(self, skipcommon=True):
+		return [ (k, v) for k, v in self._obj.items() if
+		    not skipcommon or k not in self._common_names ]
+
+	def encode(self, meth='asn1'):
+		if meth == 'asn1':
+			return _asn1coder.dumps(self)
+
+		return _jsonencoder.encode(self._obj)
+
+	@classmethod
+	def decode(cls, s, meth='asn1'):
+		if meth == 'asn1':
+			obj = _asn1coder.loads(s)
+		else:
+			obj = json.loads(s)
+
+		return cls.create_obj(obj)
+

ui/medashare/orm.py

@@ -0,0 +1,72 @@
+import uuid
+from sqlalchemy import Table, Column, DateTime, String, Integer, LargeBinary
+from sqlalchemy import types
+from sqlalchemy.orm import declarative_base
+from .cli import _debprint
+from .mdb import MDBase
+
+Base = declarative_base()
+
+class MDBaseType(types.TypeDecorator):
+	impl = LargeBinary
+
+	cache_ok = True
+
+	def process_bind_param(self, value, dialect):
+		return value.encode()
+
+	def process_result_value(self, value, dialect):
+		return MDBase.decode(value)
+
+class UUID(types.TypeDecorator):
+	impl = String(32)
+
+	cache_ok = True
+
+	def process_bind_param(self, value, dialect):
+		return value.hex
+
+	def process_result_value(self, value, dialect):
+		return uuid.UUID(hex=value)
+
+class Dummy(Base):
+	__tablename__ = 'dummy'
+
+	id = Column(Integer, primary_key=True)
+
+class UUIDv5Table(Base):
+	__tablename__ = 'uuidv5_index'
+
+	uuid = Column(UUID, primary_key=True)
+	objid = Column(UUID)
+
+class HostMapping(Base):
+	__tablename__ = 'hostmapping'
+
+	hostid = Column(UUID, primary_key=True)
+	objid = Column(UUID, primary_key=True)
+
+	# https://stackoverflow.com/questions/57685385/how-to-avoid-inserting-duplicate-data-when-inserting-data-into-sqlite3-database
+	#UniqueConstraint('hostid', 'objid', on conflict ignore)
+
+class HostTable(Base):
+	__tablename__ = 'hosttable'
+
+	hostid = Column(UUID, primary_key=True)
+	objid = Column(UUID)
+
+class HashTable(Base):
+	__tablename__ = 'hash_index'
+
+	hash = Column(String, primary_key=True)
+	uuid = Column(UUID, primary_key=True)
+
+class MetaDataObject(Base):
+	__tablename__ = 'metadata_objects'
+
+	uuid = Column(UUID, primary_key=True)
+	modified = Column(DateTime)
+	data = Column(MDBaseType)
+
+	def __repr__(self):
+		return 'MetaDataObject(uuid=%s, modified=%s, data=%s)' % (repr(self.uuid), repr(self.modified), repr(self.data))

ui/medashare/tests.py

@@ -1,6 +1,6 @@
 from .btv import _TestCases as btv_test_cases
 from .btv.bencode import _TestCases as bencode_test_cases
+from .mdb import _TestJSONEncoder
 from .cli import _TestCononicalCoder, _TestCases as cli_test_cases
-from .cli import _TestJSONEncoder
 from .mtree import Test
 from .server import _TestCases, _TestPostConfig

ui/medashare/utils.py

@@ -0,0 +1,47 @@
+import base64
+import datetime
+import pasn1
+import uuid
+
+def _makeuuid(s):
+	if isinstance(s, uuid.UUID):
+		return s
+
+	if isinstance(s, bytes):
+		return uuid.UUID(bytes=s)
+	else:
+		return uuid.UUID(s)
+
+def _makedatetime(s):
+	if isinstance(s, datetime.datetime):
+		return s
+
+	return datetime.datetime.strptime(s, '%Y-%m-%dT%H:%M:%S.%fZ').replace(
+	    tzinfo=datetime.timezone.utc)
+
+def _makebytes(s):
+	if isinstance(s, bytes):
+		return s
+
+	return base64.urlsafe_b64decode(s)
+
+def _trytodict(o):
+	if isinstance(o, uuid.UUID):
+		return 'bytes', o.bytes
+	if isinstance(o, tuple):
+		return 'list', o
+	try:
+		return 'dict', o.__to_dict__()
+	except Exception:	# pragma: no cover
+		raise TypeError('unable to find __to_dict__ on %s: %s' %
+		    (type(o), repr(o)))
+
+class CanonicalCoder(pasn1.ASN1DictCoder):
+	def enc_dict(self, obj, **kwargs):
+		class FakeIter:
+			def items(self):
+				return iter(sorted(obj.items()))
+
+		return pasn1.ASN1DictCoder.enc_dict(self, FakeIter(), **kwargs)
+
+_asn1coder = CanonicalCoder(coerce=_trytodict)

ui/setup.py

@@ -25,6 +25,7 @@ setup(
         'fastapi',
         'fastapi_restful',
         'httpx',
+	'SQLAlchemy',
         'hypercorn', # option, for server only?
         'orm',
 	'pasn1 @ git+https://www.funkthat.com/gitea/jmg/pasn1.git@c6c64510b42292557ace2b77272eb32cb647399d#egg=pasn1',