Import python modules by their hash.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

825 lines
22 KiB

  1. # Copyright 2020 John-Mark Gurney.
  2. # All rights reserved.
  3. #
  4. # Redistribution and use in source and binary forms, with or without
  5. # modification, are permitted provided that the following conditions
  6. # are met:
  7. # 1. Redistributions of source code must retain the above copyright
  8. # notice, this list of conditions and the following disclaimer.
  9. # 2. Redistributions in binary form must reproduce the above copyright
  10. # notice, this list of conditions and the following disclaimer in the
  11. # documentation and/or other materials provided with the distribution.
  12. #
  13. # THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  14. # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  15. # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  16. # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  17. # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  18. # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  19. # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  20. # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  21. # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  22. # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  23. # SUCH DAMAGE.
  24. import configparser
  25. import contextlib
  26. import filecmp
  27. import functools
  28. import glob
  29. import hashlib
  30. import importlib.resources
  31. import mock
  32. import os.path
  33. import pathlib
  34. import shutil
  35. import sys
  36. import tempfile
  37. import urllib.request
  38. from importlib.abc import MetaPathFinder, Loader
  39. from importlib.machinery import ModuleSpec
  40. __author__ = 'John-Mark Gurney'
  41. __copyright__ = 'Copyright 2020 John-Mark Gurney. All rights reserved.'
  42. __license__ = '2-clause BSD license'
  43. __version__ = '0.1.0.dev'
  44. def _printanyexc(f): # pragma: no cover
  45. '''Prints any exception that gets raised by the wrapped function.'''
  46. @functools.wraps(f)
  47. def wrapper(*args, **kwargs):
  48. try:
  49. return f(*args, **kwargs)
  50. except Exception:
  51. import traceback
  52. traceback.print_exc()
  53. raise
  54. return wrapper
  55. @contextlib.contextmanager
  56. def tempset(obj, key, value):
  57. '''A context (with) manager for changing the value of an item in a
  58. dictionary, and restoring it after the with block.
  59. Example usage:
  60. ```
  61. d = dict(a=5, b=10)
  62. with tempset(d, 'a', 15):
  63. print(repr(d['a'])
  64. print(repr(d['a'])
  65. ```
  66. '''
  67. try:
  68. dodelitem = False
  69. if key in obj:
  70. oldvalue = obj[key]
  71. else:
  72. dodelitem = True
  73. obj[key] = value
  74. yield
  75. finally:
  76. if not dodelitem:
  77. obj[key] = oldvalue
  78. else:
  79. del obj[key]
  80. @contextlib.contextmanager
  81. def tempattrset(obj, key, value):
  82. '''A context (with) manager for changing the value of an attribute
  83. of an object, and restoring it after the with block.
  84. If the attribute does not exist, it will be deleted afterward.
  85. Example usage:
  86. ```
  87. with tempattrset(someobj, 'a', 15):
  88. print(repr(someobj.a)
  89. print(repr(someobj.a)
  90. ```
  91. '''
  92. try:
  93. dodelattr = False
  94. if hasattr(obj, key):
  95. oldvalue = getattr(obj, key)
  96. else:
  97. dodelattr = True
  98. setattr(obj, key, value)
  99. yield
  100. finally:
  101. if not dodelattr:
  102. setattr(obj, key, oldvalue)
  103. else:
  104. delattr(obj, key)
  105. def urlfetch(url):
  106. with urllib.request.urlopen(url) as req:
  107. if req.status // 100 != 2:
  108. raise RuntimeError('bad fetch')
  109. return req.read()
  110. class HTTPSCAS(object):
  111. @classmethod
  112. def fromconfig(cls, conf):
  113. return cls()
  114. def fetch_data(self, url):
  115. if url.scheme != 'https':
  116. raise ValueError('cannot handle scheme %s' %
  117. repr(url.scheme))
  118. url = urllib.parse.urlunparse(url)
  119. return urlfetch(url)
  120. class IPFSCAS(object):
  121. gwurl = 'https://gateway.ipfs.io/ipfs/'
  122. gwurl = 'https://cloudflare-ipfs.com/ipfs/'
  123. def __init__(self, gw=None):
  124. if gw is not None:
  125. self.gwurl = gw
  126. @classmethod
  127. def fromconfig(cls, conf):
  128. return cls(conf.get('gateway', None))
  129. def make_url(self, url):
  130. return urllib.parse.urljoin(self.gwurl, url.netloc)
  131. def fetch_data(self, url):
  132. if url.scheme != 'ipfs':
  133. raise ValueError('cannot handle scheme %s' %
  134. repr(url.scheme))
  135. gwurl = self.make_url(url)
  136. return urlfetch(gwurl)
  137. class FileDirCAS(object):
  138. '''A file loader for CAS that operates on a directory. It looks
  139. at files, caches their hash, and loads them upon request.'''
  140. def __init__(self, path):
  141. self._path = pathlib.Path(path)
  142. self._path.mkdir(exist_ok=True)
  143. self._hashes = {}
  144. @classmethod
  145. def fromconfig(cls, conf):
  146. return cls(os.path.expanduser(conf['path']))
  147. def refresh_dir(self):
  148. '''Internal method to refresh the internal cache of
  149. hashes.'''
  150. for i in glob.glob(os.path.join(self._path, '*.py')):
  151. _, hash = self.read_hash_file(i)
  152. self._hashes[hash] = i
  153. @staticmethod
  154. def read_hash_file(fname):
  155. '''Helper function that will read the file at fname, and
  156. return the tuple of it's contents and it's hash.'''
  157. with open(fname, 'rb') as fp:
  158. data = fp.read()
  159. hash = hashlib.sha256(data).hexdigest()
  160. return data, hash
  161. def is_package(self, hash):
  162. '''Decode the provided hash, and decide if it's a package
  163. or not.'''
  164. return False
  165. def fetch_data(self, url):
  166. '''Given the URL (must be a hash URL), return the code for
  167. it.'''
  168. self.refresh_dir()
  169. hashurl = url
  170. if hashurl.scheme != 'hash' or hashurl.netloc != 'sha256':
  171. raise ValueError('invalid hash url')
  172. hash = hashurl.path[1:]
  173. fname = self._hashes[hash]
  174. data, fhash = self.read_hash_file(fname)
  175. if fhash != hash:
  176. raise ValueError('file no longer matches hash on disk')
  177. return data
  178. class CASFinder(MetaPathFinder, Loader):
  179. '''Overall class for using Content Addressable Storage to load
  180. Python modules into your code. It contains code to dispatch to
  181. the various loaders to attempt to load the hash.'''
  182. def __init__(self):
  183. self._loaders = []
  184. self._aliases = {}
  185. if [ x for x in sys.meta_path if
  186. isinstance(x, self.__class__) ]:
  187. raise RuntimeError(
  188. 'cannot register more than on CASFinder')
  189. sys.meta_path.append(self)
  190. def __enter__(self):
  191. return self
  192. def __exit__(self, exc_type, exc_value, traceback):
  193. self.disconnect()
  194. def load_aliases(self, data):
  195. self._aliases.update(self._parsealiases(data))
  196. def load_mod_aliases(self, name):
  197. '''Load the aliases from the module with the passed in name.'''
  198. aliases = importlib.resources.read_text(sys.modules[name],
  199. 'cas_aliases.txt')
  200. self.load_aliases(aliases)
  201. @staticmethod
  202. def _makebasichashurl(url):
  203. try:
  204. hashurl = urllib.parse.urlparse(url)
  205. except AttributeError:
  206. hashurl = url
  207. return urllib.parse.urlunparse(hashurl[:3] + ('', '', ''))
  208. @classmethod
  209. def _parsealiases(cls, data):
  210. ret = {}
  211. lines = data.split('\n')
  212. for i in lines:
  213. if not i:
  214. continue
  215. name, hash = i.split()
  216. ret.setdefault(name, []).append(hash)
  217. # split out the hashes
  218. for items in list(ret.values()):
  219. lst = [ x for x in items if
  220. not x.startswith('hash://') ]
  221. for h in [ x for x in items if
  222. x.startswith('hash://') ]:
  223. h = cls._makebasichashurl(h)
  224. ret[h] = lst
  225. return ret
  226. def disconnect(self):
  227. '''Disconnect this Finder from being used to load modules.
  228. As this claims an entire namespace, only the first loaded
  229. one will work, and any others will be hidden until the
  230. first one is disconnected.
  231. This can be used w/ a with block to automatically
  232. disconnect when no longer needed. This is mostly useful
  233. for testing.'''
  234. try:
  235. sys.meta_path.remove(self)
  236. except ValueError:
  237. pass
  238. def register(self, loader):
  239. '''Register a loader w/ this finder. This will attempt
  240. to load the hash passed to it. It is also (currently)
  241. responsible for executing the code in the module.'''
  242. self._loaders.append(loader)
  243. # MetaPathFinder methods
  244. def find_spec(self, fullname, path, target=None):
  245. if path is None:
  246. ms = ModuleSpec(fullname, self, is_package=True)
  247. else:
  248. parts = fullname.split('.')
  249. ver, typ, arg = parts[1].split('_')
  250. if typ == 'f':
  251. # make hash url:
  252. hashurl = ('hash://sha256/%s' %
  253. bytes.fromhex(arg).hex())
  254. hashurl = urllib.parse.urlparse(hashurl)
  255. for l in self._loaders:
  256. ispkg = l.is_package(hashurl)
  257. break
  258. else:
  259. return None
  260. else:
  261. # an alias
  262. for i in self._aliases[arg]:
  263. hashurl = urllib.parse.urlparse(i)
  264. if hashurl.scheme == 'hash':
  265. break
  266. else:
  267. raise ValueError('unable to find base hash url for alias %s' % repr(arg))
  268. # fix up the full name:
  269. fullname = 'cas.v1_f_%s' % hashurl.path[1:]
  270. ms = ModuleSpec(fullname, self, is_package=False,
  271. loader_state=(hashurl,))
  272. return ms
  273. def invalidate_caches(self):
  274. return None
  275. # Loader methods
  276. def exec_module(self, module):
  277. if module.__name__ == 'cas':
  278. pass
  279. else:
  280. (url,) = module.__spec__.loader_state
  281. for load in self._loaders:
  282. try:
  283. data = load.fetch_data(url)
  284. break
  285. except Exception:
  286. pass
  287. else:
  288. for url in self._aliases[
  289. self._makebasichashurl(url)]:
  290. url = urllib.parse.urlparse(url)
  291. for load in self._loaders:
  292. try:
  293. data = load.fetch_data(url)
  294. break
  295. except Exception:
  296. pass
  297. else:
  298. continue
  299. break
  300. else:
  301. raise ValueError('unable to find loader for url %s' % repr(urllib.parse.urlunparse(url)))
  302. exec(data, module.__dict__)
  303. _supportedmodules = {
  304. 'https': HTTPSCAS.fromconfig,
  305. 'ipfs': IPFSCAS.fromconfig,
  306. 'cache': FileDirCAS.fromconfig,
  307. }
  308. def defaultinit(casf):
  309. basedir = pathlib.Path.home() / '.casimport'
  310. basedir.mkdir(exist_ok=True)
  311. conffile = pathlib.Path(os.environ.get('CASIMPORT_CONF', basedir / 'casimport.conf'))
  312. if not conffile.exists():
  313. import casimport
  314. with importlib.resources.path(casimport,
  315. 'default.conf') as defconf:
  316. shutil.copy(defconf, conffile)
  317. cp = configparser.ConfigParser()
  318. cp.read(conffile)
  319. modorder = [ x.strip() for x in
  320. cp['casimport']['module_prio'].split(',') ]
  321. for i in modorder:
  322. modfun = _supportedmodules[cp[i]['type']]
  323. casf.register(modfun(cp[i]))
  324. cachedir = basedir / 'cache'
  325. cachedir.mkdir(parents=True, exist_ok=True)
  326. casf.register(FileDirCAS(cachedir))
  327. casf.register(IPFSCAS())
  328. casf.register(HTTPSCAS())
  329. # The global version
  330. _casfinder = CASFinder()
  331. load_mod_aliases = _casfinder.load_mod_aliases
  332. defaultinit(_casfinder)
  333. import unittest
  334. class TestHelpers(unittest.TestCase):
  335. def test_testset(self):
  336. origobj = object()
  337. d = dict(a=origobj, b=10)
  338. # that when we temporarily set it
  339. with tempset(d, 'a', 15):
  340. # the new value is there
  341. self.assertEqual(d['a'], 15)
  342. # and that the original object is restored
  343. self.assertIs(d['a'], origobj)
  344. def test_testattrset(self):
  345. class TestObj(object):
  346. pass
  347. testobj = TestObj()
  348. # that when we temporarily set it
  349. with tempattrset(testobj, 'a', 15):
  350. # the new value is there
  351. self.assertEqual(testobj.a, 15)
  352. # and that there is no object
  353. self.assertFalse(hasattr(testobj, 'a'))
  354. origobj = object()
  355. newobj = object()
  356. testobj.b = origobj
  357. # that when we temporarily set it
  358. with tempattrset(testobj, 'b', newobj):
  359. # the new value is there
  360. self.assertIs(testobj.b, newobj)
  361. # and the original value is restored
  362. self.assertIs(testobj.b, origobj)
  363. class Test(unittest.TestCase):
  364. def setUp(self):
  365. # clear out the default casfinder if there is one
  366. self.old_meta_path = sys.meta_path
  367. sys.meta_path = [ x for x in sys.meta_path if
  368. not isinstance(x, CASFinder) ]
  369. # setup temporary directory
  370. d = pathlib.Path(os.path.realpath(tempfile.mkdtemp()))
  371. self.basetempdir = d
  372. self.tempdir = d / 'subdir'
  373. self.tempdir.mkdir()
  374. self.fixtures = \
  375. pathlib.Path(__file__).parent.parent / 'fixtures'
  376. def tearDown(self):
  377. # restore environment
  378. sys.meta_path = self.old_meta_path
  379. importlib.invalidate_caches()
  380. # clean up sys.modules
  381. [ sys.modules.pop(x) for x in list(sys.modules.keys()) if
  382. x == 'cas' or x.startswith('cas.') ]
  383. shutil.rmtree(self.basetempdir)
  384. self.tempdir = None
  385. def test_filedircas(self):
  386. cachedir = self.tempdir / 'cache'
  387. fd = FileDirCAS(cachedir)
  388. self.assertTrue(cachedir.exists())
  389. def test_filedircas_limit_refresh(self):
  390. # XXX - only refresh when the dir has changed, and each
  391. # file has changed
  392. pass
  393. def test_casimport(self):
  394. # That a CASFinder
  395. f = CASFinder()
  396. # make sure that we can't import anything at first
  397. with self.assertRaises(ImportError):
  398. import cas.v1_f_2398472398
  399. # when registering the fixtures directory
  400. f.register(FileDirCAS(self.fixtures))
  401. # can import the function
  402. from cas.v1_f_330884aa2febb5e19fb7194ec6a69ed11dd3d77122f1a5175ee93e73cf0161c3 import hello
  403. name = 'Olof'
  404. # and run the code
  405. self.assertEqual(hello(name), 'hello ' + name)
  406. # and when finished, can disconnect
  407. f.disconnect()
  408. # and is no longer in the meta_path
  409. self.assertNotIn(f, sys.meta_path)
  410. # and when disconnected as second time, nothing happens
  411. f.disconnect()
  412. def test_conforder(self):
  413. conf = self.fixtures / 'ordertest.conf'
  414. # that w/ a custom config
  415. with tempset(os.environ, 'CASIMPORT_CONF', str(conf)), \
  416. CASFinder() as f:
  417. # that is gets loaded
  418. defaultinit(f)
  419. # and that the first loader is the HTTPSCAS
  420. self.assertIsInstance(f._loaders[0], HTTPSCAS)
  421. # and that the second loader is the IPFSCAS
  422. self.assertIsInstance(f._loaders[1], IPFSCAS)
  423. def test_ipfsgwfromconfig(self):
  424. gwurl = 'https://www.example.com/somepath/'
  425. ipfsconf = dict(gateway=gwurl)
  426. ipfsobj = IPFSCAS.fromconfig(ipfsconf)
  427. self.assertEqual(ipfsobj.make_url(urllib.parse.urlparse('ipfs://someobj')),
  428. 'https://www.example.com/somepath/someobj')
  429. def test_defaultinit(self):
  430. temphome = self.tempdir / 'home'
  431. temphome.mkdir()
  432. defcachedir = temphome / '.casimport' / 'cache'
  433. # testing w/ default config
  434. with tempset(os.environ, 'HOME', str(temphome)):
  435. with CASFinder() as f:
  436. # Setup the defaults
  437. defaultinit(f)
  438. # That the default.conf file got copied over.
  439. filecmp.cmp(defcachedir.parent /
  440. 'casimport.conf',
  441. pathlib.Path(__file__).parent / 'default.conf')
  442. # that the cache got created
  443. self.assertTrue(defcachedir.is_dir())
  444. # and that when hello.py is copied to the cache
  445. shutil.copy(self.fixtures / 'hello.py', defcachedir)
  446. # it can be imported
  447. from cas.v1_f_330884aa2febb5e19fb7194ec6a69ed11dd3d77122f1a5175ee93e73cf0161c3 import hello
  448. # and that the second loader is the IPFSCAS
  449. self.assertIsInstance(f._loaders[1], IPFSCAS)
  450. # and that the third loader is the HTTPSCAS
  451. self.assertIsInstance(f._loaders[2], HTTPSCAS)
  452. with CASFinder() as f:
  453. defaultinit(f)
  454. # and that a new CASFinder can still find it
  455. from cas.v1_f_330884aa2febb5e19fb7194ec6a69ed11dd3d77122f1a5175ee93e73cf0161c3 import hello
  456. def test_multiplecas(self):
  457. # that once we have one
  458. with CASFinder() as f:
  459. # if we try to create a second, it fails
  460. self.assertRaises(RuntimeError, CASFinder)
  461. def test_parsealiases(self):
  462. with open(self.fixtures / 'randpkg' / 'cas_aliases.txt') as fp:
  463. aliasdata = fp.read()
  464. res = CASFinder._parsealiases(aliasdata)
  465. self.assertEqual(res, {
  466. 'hello': [
  467. 'hash://sha256/330884aa2febb5e19fb7194ec6a69ed11dd3d77122f1a5175ee93e73cf0161c3?type=text/x-python',
  468. 'ipfs://bafkreibtbcckul7lwxqz7nyzj3dknhwrdxj5o4jc6gsroxxjhzz46albym',
  469. 'https://www.funkthat.com/gitea/jmg/casimport/raw/commit/753e64f53c73d9d1afc4d8a617edb9d3542dcea2/fixtures/hello.py',
  470. ],
  471. 'hash://sha256/330884aa2febb5e19fb7194ec6a69ed11dd3d77122f1a5175ee93e73cf0161c3': [
  472. 'ipfs://bafkreibtbcckul7lwxqz7nyzj3dknhwrdxj5o4jc6gsroxxjhzz46albym',
  473. 'https://www.funkthat.com/gitea/jmg/casimport/raw/commit/753e64f53c73d9d1afc4d8a617edb9d3542dcea2/fixtures/hello.py',
  474. ],
  475. })
  476. def test_aliasmulti(self):
  477. # setup the cache
  478. cachedir = self.tempdir / 'cache'
  479. cachedir.mkdir()
  480. with CASFinder() as f, \
  481. tempattrset(sys.modules[__name__],
  482. 'load_mod_aliases', f.load_mod_aliases):
  483. f.register(FileDirCAS(cachedir))
  484. # and that hello.py is in the cache
  485. shutil.copy(self.fixtures / 'hello.py',
  486. cachedir)
  487. # and that the aliases are loaded
  488. with open(self.fixtures / 'randpkg' / 'cas_aliases.txt') as fp:
  489. f.load_aliases(fp.read())
  490. # that when we load the alias first
  491. from cas.v1_a_hello import hello as hello_alias
  492. # and then load the same module via hash
  493. from cas.v1_f_330884aa2febb5e19fb7194ec6a69ed11dd3d77122f1a5175ee93e73cf0161c3 import hello as hello_hash
  494. # they are the same
  495. self.assertIs(hello_alias, hello_hash)
  496. def test_aliasimports(self):
  497. # setup the cache
  498. cachedir = self.tempdir / 'cache'
  499. cachedir.mkdir()
  500. # add the test module's path
  501. fixdir = str(self.fixtures)
  502. sys.path.append(fixdir)
  503. try:
  504. with CASFinder() as f, \
  505. tempattrset(sys.modules[__name__],
  506. 'load_mod_aliases', f.load_mod_aliases):
  507. f.register(FileDirCAS(cachedir))
  508. # and that hello.py is in the cache
  509. shutil.copy(self.fixtures / 'hello.py',
  510. cachedir)
  511. self.assertNotIn('randpkg', sys.modules)
  512. # that the import is successful
  513. import randpkg
  514. # and pulled in the method
  515. self.assertTrue(hasattr(randpkg, 'hello'))
  516. del sys.modules['randpkg']
  517. finally:
  518. sys.path.remove(fixdir)
  519. def test_aliasipfsimports(self):
  520. # add the test module's path
  521. fixdir = str(self.fixtures)
  522. sys.path.append(fixdir)
  523. # that a fake ipfsloader
  524. with open(self.fixtures / 'hello.py') as fp:
  525. # that returns the correct data
  526. fakedata = fp.read()
  527. def fakeload(url, fd=fakedata):
  528. if url.scheme != 'ipfs' or url.netloc != 'bafkreibtbcckul7lwxqz7nyzj3dknhwrdxj5o4jc6gsroxxjhzz46albym':
  529. raise ValueError
  530. return fd
  531. fakeipfsloader = mock.MagicMock()
  532. fakeipfsloader.fetch_data = fakeload
  533. try:
  534. with CASFinder() as f, \
  535. tempattrset(sys.modules[__name__], 'load_mod_aliases',
  536. f.load_mod_aliases):
  537. f.register(fakeipfsloader)
  538. self.assertNotIn('randpkg', sys.modules)
  539. # that the import is successful
  540. import randpkg
  541. # and pulled in the method
  542. self.assertTrue(hasattr(randpkg, 'hello'))
  543. del sys.modules['randpkg']
  544. finally:
  545. sys.path.remove(fixdir)
  546. @mock.patch('urllib.request.urlopen')
  547. def test_ipfscasloader(self, uomock):
  548. # prep return test data
  549. with open(self.fixtures / 'hello.py') as fp:
  550. # that returns the correct data
  551. ipfsdata = fp.read()
  552. # that the ipfs CAS loader
  553. ipfs = IPFSCAS()
  554. # that the request is successfull
  555. uomock.return_value.__enter__.return_value.status = 200
  556. # and returns the correct data
  557. uomock.return_value.__enter__.return_value.read.return_value = ipfsdata
  558. # that when called
  559. hashurl = urllib.parse.urlparse('ipfs://bafkreibtbcckul7lwxqz7nyzj3dknhwrdxj5o4jc6gsroxxjhzz46albym')
  560. data = ipfs.fetch_data(hashurl)
  561. # it opens the correct url
  562. uomock.assert_called_with('https://cloudflare-ipfs.com/ipfs/bafkreibtbcckul7lwxqz7nyzj3dknhwrdxj5o4jc6gsroxxjhzz46albym')
  563. # and returns the correct data
  564. self.assertEqual(data, ipfsdata)
  565. with self.assertRaises(ValueError):
  566. # that a hash url fails
  567. ipfs.fetch_data(urllib.parse.urlparse('hash://sha256/asldfkj'))
  568. # that when the request fails
  569. uomock.return_value.__enter__.return_value.status = 400
  570. # it raises a RuntimeError
  571. with self.assertRaises(RuntimeError):
  572. ipfs.fetch_data(hashurl)
  573. # Note: mostly copied from above, test_ipfscasloader
  574. @mock.patch('urllib.request.urlopen')
  575. def test_httpscasloader(self, uomock):
  576. # prep return test data
  577. with open(self.fixtures / 'hello.py') as fp:
  578. # that returns the correct data
  579. httpsdata = fp.read()
  580. # that the https CAS loader
  581. httpsldr = HTTPSCAS()
  582. # that the request is successfull
  583. uomock.return_value.__enter__.return_value.status = 200
  584. # and returns the correct data
  585. uomock.return_value.__enter__.return_value.read.return_value = httpsdata
  586. # that when called
  587. hashurl = urllib.parse.urlparse('https://www.funkthat.com/gitea/jmg/casimport/raw/commit/753e64f53c73d9d1afc4d8a617edb9d3542dcea2/fixtures/hello.py')
  588. data = httpsldr.fetch_data(hashurl)
  589. # it opens the correct url
  590. uomock.assert_called_with('https://www.funkthat.com/gitea/jmg/casimport/raw/commit/753e64f53c73d9d1afc4d8a617edb9d3542dcea2/fixtures/hello.py')
  591. # and returns the correct data
  592. self.assertEqual(data, httpsdata)
  593. with self.assertRaises(ValueError):
  594. # that a hash url fails
  595. httpsldr.fetch_data(urllib.parse.urlparse('hash://sha256/asldfkj'))
  596. # that when the request fails
  597. uomock.return_value.__enter__.return_value.status = 400
  598. # it raises a RuntimeError
  599. with self.assertRaises(RuntimeError):
  600. httpsldr.fetch_data(hashurl)
  601. @unittest.skip('todo')
  602. def test_overlappingaliases(self):
  603. # make sure that an aliases file is consistent and does not
  604. # override other urls. That is that any hashes are
  605. # consistent, and that they have at least one root hash that
  606. # is the same, and will be used for fetching.
  607. #
  608. # Likely will also have to deal w/ an issue where two
  609. # aliases share sha256, and a third shares sha512, which in
  610. # this case, BOTH hashse have to be checked.
  611. pass
  612. @unittest.skip('todo')
  613. def test_loaderpriority(self):
  614. # XXX - write test to allow you to specify the priority of
  615. # a loader, to ensure that cache stays at top.
  616. # Maybe also think of a way to say local/remote, because
  617. # some loaders may be "more local" than others, like using
  618. # a local ipfs gateway makes more sense than hitting a
  619. # public gateway
  620. pass
  621. def test_filecorruption(self):
  622. cachedir = self.tempdir / 'cachedir'
  623. cachedir.mkdir()
  624. # that an existing file
  625. shutil.copy(self.fixtures / 'hello.py', cachedir)
  626. # is in the cache
  627. fdcas = FileDirCAS(cachedir)
  628. # that when refresh is surpressed
  629. fdcas.refresh_dir = lambda: None
  630. # and has a bogus hash
  631. fdcas._hashes['0000'] = cachedir / 'hello.py'
  632. # that when read raises an exception
  633. with self.assertRaises(ValueError):
  634. fdcas.fetch_data(urllib.parse.urlparse('hash://sha256/0000'))
  635. # that when passed an invalid url
  636. with self.assertRaises(ValueError):
  637. fdcas.fetch_data(urllib.parse.urlparse('https://sha256/0000'))