Import python modules by their hash.
選択できるのは25トピックまでです。 トピックは、先頭が英数字で、英数字とダッシュ('-')を使用した35文字以内のものにしてください。

936 行
24 KiB

  1. # Copyright 2020 John-Mark Gurney.
  2. # All rights reserved.
  3. #
  4. # Redistribution and use in source and binary forms, with or without
  5. # modification, are permitted provided that the following conditions
  6. # are met:
  7. # 1. Redistributions of source code must retain the above copyright
  8. # notice, this list of conditions and the following disclaimer.
  9. # 2. Redistributions in binary form must reproduce the above copyright
  10. # notice, this list of conditions and the following disclaimer in the
  11. # documentation and/or other materials provided with the distribution.
  12. #
  13. # THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  14. # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  15. # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  16. # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  17. # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  18. # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  19. # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  20. # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  21. # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  22. # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  23. # SUCH DAMAGE.
  24. import configparser
  25. import contextlib
  26. import filecmp
  27. import functools
  28. import glob
  29. import hashlib
  30. import importlib.resources
  31. import mock
  32. import os.path
  33. import pathlib
  34. import shutil
  35. import sys
  36. import tempfile
  37. import urllib.request
  38. from importlib.abc import MetaPathFinder, Loader
  39. from importlib.machinery import ModuleSpec
  40. from urllib.parse import urlparse
  41. __author__ = 'John-Mark Gurney'
  42. __copyright__ = 'Copyright 2020 John-Mark Gurney. All rights reserved.'
  43. __license__ = '2-clause BSD license'
  44. __version__ = '0.1.0.dev'
  45. def _printanyexc(f): # pragma: no cover
  46. '''Prints any exception that gets raised by the wrapped function.'''
  47. @functools.wraps(f)
  48. def wrapper(*args, **kwargs):
  49. try:
  50. return f(*args, **kwargs)
  51. except Exception:
  52. import traceback
  53. traceback.print_exc()
  54. raise
  55. return wrapper
  56. @contextlib.contextmanager
  57. def tempset(obj, key, value):
  58. '''A context (with) manager for changing the value of an item in a
  59. dictionary, and restoring it after the with block.
  60. Example usage:
  61. ```
  62. d = dict(a=5, b=10)
  63. with tempset(d, 'a', 15):
  64. print(repr(d['a'])
  65. print(repr(d['a'])
  66. ```
  67. '''
  68. try:
  69. dodelitem = False
  70. if key in obj:
  71. oldvalue = obj[key]
  72. else:
  73. dodelitem = True
  74. obj[key] = value
  75. yield
  76. finally:
  77. if not dodelitem:
  78. obj[key] = oldvalue
  79. else:
  80. del obj[key]
  81. @contextlib.contextmanager
  82. def tempattrset(obj, key, value):
  83. '''A context (with) manager for changing the value of an attribute
  84. of an object, and restoring it after the with block.
  85. If the attribute does not exist, it will be deleted afterward.
  86. Example usage:
  87. ```
  88. with tempattrset(someobj, 'a', 15):
  89. print(repr(someobj.a)
  90. print(repr(someobj.a)
  91. ```
  92. '''
  93. try:
  94. dodelattr = False
  95. if hasattr(obj, key):
  96. oldvalue = getattr(obj, key)
  97. else:
  98. dodelattr = True
  99. setattr(obj, key, value)
  100. yield
  101. finally:
  102. if not dodelattr:
  103. setattr(obj, key, oldvalue)
  104. else:
  105. delattr(obj, key)
  106. def urlfetch(url):
  107. with urllib.request.urlopen(url) as req:
  108. if req.status // 100 != 2:
  109. raise RuntimeError('bad fetch')
  110. return req.read()
  111. class HTTPSCAS(object):
  112. @classmethod
  113. def fromconfig(cls, conf):
  114. return cls()
  115. def fetch_data(self, url):
  116. if url.scheme != 'https':
  117. raise ValueError('cannot handle scheme %s' %
  118. repr(url.scheme))
  119. url = urllib.parse.urlunparse(url)
  120. return urlfetch(url)
  121. class IPFSCAS(object):
  122. gwurl = 'https://gateway.ipfs.io/ipfs/'
  123. gwurl = 'https://cloudflare-ipfs.com/ipfs/'
  124. def __init__(self, gw=None):
  125. if gw is not None:
  126. self.gwurl = gw
  127. @classmethod
  128. def fromconfig(cls, conf):
  129. return cls(conf.get('gateway', None))
  130. def make_url(self, url):
  131. return urllib.parse.urljoin(self.gwurl, url.netloc)
  132. def fetch_data(self, url):
  133. if url.scheme != 'ipfs':
  134. raise ValueError('cannot handle scheme %s' %
  135. repr(url.scheme))
  136. gwurl = self.make_url(url)
  137. return urlfetch(gwurl)
  138. class FileDirCAS(object):
  139. '''A file loader for CAS that operates on a directory. It looks
  140. at files, caches their hash, and loads them upon request.'''
  141. def __init__(self, path):
  142. self._path = pathlib.Path(path)
  143. self._path.mkdir(exist_ok=True)
  144. self._hashes = {}
  145. @classmethod
  146. def fromconfig(cls, conf):
  147. return cls(os.path.expanduser(conf['path']))
  148. def refresh_dir(self):
  149. '''Internal method to refresh the internal cache of
  150. hashes.'''
  151. for i in glob.glob(os.path.join(self._path, '?/*.py')):
  152. _, hash = self.read_hash_file(i)
  153. self._hashes[hash] = i
  154. def add_file(self, fname):
  155. '''Note: this is primarily a testing function.'''
  156. with open(fname, 'rb') as fp:
  157. data = fp.read()
  158. hash = hashlib.sha256(data).hexdigest()
  159. self.write_cache(hash, data)
  160. def write_cache(self, hash, data):
  161. d = self._path / hash[0]
  162. d.mkdir()
  163. with open(d / (hash + '.py'), 'wb') as fp:
  164. fp.write(data)
  165. @staticmethod
  166. def read_hash_file(fname):
  167. '''Helper function that will read the file at fname, and
  168. return the tuple of it's contents and it's hash.'''
  169. with open(fname, 'rb') as fp:
  170. data = fp.read()
  171. hash = hashlib.sha256(data).hexdigest()
  172. return data, hash
  173. def is_package(self, hash):
  174. '''Decode the provided hash, and decide if it's a package
  175. or not.'''
  176. return False
  177. def fetch_data(self, url):
  178. '''Given the URL (must be a hash URL), return the code for
  179. it.'''
  180. self.refresh_dir()
  181. hashurl = url
  182. if hashurl.scheme != 'hash' or hashurl.netloc != 'sha256':
  183. raise ValueError('invalid hash url')
  184. hash = hashurl.path[1:]
  185. fname = self._hashes[hash]
  186. data, fhash = self.read_hash_file(fname)
  187. if fhash != hash:
  188. raise ValueError('file no longer matches hash on disk')
  189. return data
  190. class CASFinder(MetaPathFinder, Loader):
  191. '''Overall class for using Content Addressable Storage to load
  192. Python modules into your code. It contains code to dispatch to
  193. the various loaders to attempt to load the hash.'''
  194. def __init__(self):
  195. self._loaders = []
  196. self._aliases = {}
  197. self._writer = None
  198. if [ x for x in sys.meta_path if
  199. isinstance(x, self.__class__) ]:
  200. raise RuntimeError(
  201. 'cannot register more than on CASFinder')
  202. sys.meta_path.append(self)
  203. def __enter__(self):
  204. return self
  205. def __exit__(self, exc_type, exc_value, traceback):
  206. self.disconnect()
  207. def register_write(self, wr):
  208. self._writer = wr
  209. def load_aliases(self, data):
  210. self._aliases.update(self._parsealiases(data))
  211. def load_mod_aliases(self, name):
  212. '''Load the aliases from the module with the passed in name.'''
  213. aliases = importlib.resources.read_text(sys.modules[name],
  214. 'cas_aliases.txt')
  215. self.load_aliases(aliases)
  216. @staticmethod
  217. def _makebasichashurl(url):
  218. try:
  219. hashurl = urlparse(url)
  220. except AttributeError:
  221. hashurl = url
  222. return urllib.parse.urlunparse(hashurl[:3] + ('', '', ''))
  223. @classmethod
  224. def _parsealiases(cls, data):
  225. ret = {}
  226. lines = data.split('\n')
  227. for i in lines:
  228. if not i:
  229. continue
  230. name, hash = i.split()
  231. ret.setdefault(name, []).append(hash)
  232. # split out the hashes
  233. for items in list(ret.values()):
  234. lst = [ x for x in items if
  235. not x.startswith('hash://') ]
  236. for h in [ x for x in items if
  237. x.startswith('hash://') ]:
  238. h = cls._makebasichashurl(h)
  239. ret[h] = lst
  240. return ret
  241. def disconnect(self):
  242. '''Disconnect this Finder from being used to load modules.
  243. As this claims an entire namespace, only the first loaded
  244. one will work, and any others will be hidden until the
  245. first one is disconnected.
  246. This can be used w/ a with block to automatically
  247. disconnect when no longer needed. This is mostly useful
  248. for testing.'''
  249. try:
  250. sys.meta_path.remove(self)
  251. except ValueError:
  252. pass
  253. def register(self, loader):
  254. '''Register a loader w/ this finder. This will attempt
  255. to load the hash passed to it. It is also (currently)
  256. responsible for executing the code in the module.'''
  257. self._loaders.append(loader)
  258. # MetaPathFinder methods
  259. def find_spec(self, fullname, path, target=None):
  260. aliases_to_add = ()
  261. if path is None:
  262. ms = ModuleSpec(fullname, self, is_package=True)
  263. else:
  264. parts = fullname.split('.')
  265. ver, typ, arg = parts[1].split('_')
  266. if typ == 'f':
  267. # make hash url:
  268. hashurl = ('hash://sha256/%s' %
  269. bytes.fromhex(arg).hex())
  270. hashurl = urlparse(hashurl)
  271. for l in self._loaders:
  272. ispkg = l.is_package(hashurl)
  273. break
  274. else:
  275. return None
  276. else:
  277. # an alias
  278. for i in self._aliases[arg]:
  279. hashurl = urlparse(i)
  280. if hashurl.scheme == 'hash':
  281. break
  282. else:
  283. raise ValueError('unable to find base hash url for alias %s' % repr(arg))
  284. # fix up the full name:
  285. aliases_to_add = (fullname,)
  286. fullname = 'cas.v1_f_%s' % hashurl.path[1:]
  287. ms = ModuleSpec(fullname, self, is_package=False,
  288. loader_state=(hashurl,) + aliases_to_add)
  289. return ms
  290. def invalidate_caches(self):
  291. return None
  292. # Loader methods
  293. def exec_module(self, module):
  294. if module.__name__ == 'cas':
  295. return
  296. (url, *aliases) = module.__spec__.loader_state
  297. for load in self._loaders:
  298. try:
  299. data = load.fetch_data(url)
  300. break
  301. except Exception:
  302. pass
  303. else:
  304. for url in self._aliases[
  305. self._makebasichashurl(url)]:
  306. url = urlparse(url)
  307. for load in self._loaders:
  308. try:
  309. data = load.fetch_data(url)
  310. break
  311. except Exception:
  312. pass
  313. else:
  314. continue
  315. break
  316. else:
  317. raise ValueError('unable to find loader for url %s' % repr(urllib.parse.urlunparse(url)))
  318. hash = hashlib.sha256(data).hexdigest()
  319. wrtr = self._writer
  320. if wrtr is not None:
  321. wrtr.write_cache(hash, data)
  322. exec(data, module.__dict__)
  323. # we were successful, add the aliases
  324. for i in aliases:
  325. sys.modules[i] = module
  326. _supportedmodules = {
  327. 'https': HTTPSCAS.fromconfig,
  328. 'ipfs': IPFSCAS.fromconfig,
  329. 'cache': FileDirCAS.fromconfig,
  330. }
  331. def defaultinit(casf):
  332. basedir = pathlib.Path.home() / '.casimport'
  333. basedir.mkdir(exist_ok=True)
  334. conffile = pathlib.Path(os.environ.get('CASIMPORT_CONF', basedir /
  335. 'casimport.conf'))
  336. if not conffile.exists():
  337. import casimport
  338. with importlib.resources.path(casimport,
  339. 'default.conf') as defconf:
  340. shutil.copy(defconf, conffile)
  341. cp = configparser.ConfigParser()
  342. cp.read(conffile)
  343. modorder = [ x.strip() for x in
  344. cp['casimport']['module_prio'].split(',') ]
  345. mods = {}
  346. for i in modorder:
  347. modfun = _supportedmodules[cp[i]['type']]
  348. mod = modfun(cp[i])
  349. mods[i] = mod
  350. casf.register(mod)
  351. try:
  352. casf.register_write(mods[cp['casimport']['write_cache']])
  353. except KeyError:
  354. pass
  355. def main():
  356. # Thoughts on supported features:
  357. # automatic pinning/adding to ipfs
  358. # automatic github/gitea url generation
  359. # automatic alias generation
  360. #
  361. pass
  362. # The global version
  363. _casfinder = CASFinder()
  364. load_mod_aliases = _casfinder.load_mod_aliases
  365. defaultinit(_casfinder)
  366. import unittest
  367. class TestHelpers(unittest.TestCase):
  368. def test_testset(self):
  369. origobj = object()
  370. d = dict(a=origobj, b=10)
  371. # that when we temporarily set it
  372. with tempset(d, 'a', 15):
  373. # the new value is there
  374. self.assertEqual(d['a'], 15)
  375. # and that the original object is restored
  376. self.assertIs(d['a'], origobj)
  377. def test_testattrset(self):
  378. class TestObj(object):
  379. pass
  380. testobj = TestObj()
  381. # that when we temporarily set it
  382. with tempattrset(testobj, 'a', 15):
  383. # the new value is there
  384. self.assertEqual(testobj.a, 15)
  385. # and that there is no object
  386. self.assertFalse(hasattr(testobj, 'a'))
  387. origobj = object()
  388. newobj = object()
  389. testobj.b = origobj
  390. # that when we temporarily set it
  391. with tempattrset(testobj, 'b', newobj):
  392. # the new value is there
  393. self.assertIs(testobj.b, newobj)
  394. # and the original value is restored
  395. self.assertIs(testobj.b, origobj)
  396. class Test(unittest.TestCase):
  397. def setUp(self):
  398. # clear out the default casfinder if there is one
  399. self.old_meta_path = sys.meta_path
  400. sys.meta_path = [ x for x in sys.meta_path if
  401. not isinstance(x, CASFinder) ]
  402. # setup temporary directory
  403. d = pathlib.Path(os.path.realpath(tempfile.mkdtemp()))
  404. self.basetempdir = d
  405. self.tempdir = d / 'subdir'
  406. self.tempdir.mkdir()
  407. self.fixtures = \
  408. pathlib.Path(__file__).parent.parent / 'fixtures'
  409. def tearDown(self):
  410. # restore environment
  411. sys.meta_path = self.old_meta_path
  412. importlib.invalidate_caches()
  413. # clean up sys.modules
  414. [ sys.modules.pop(x) for x in list(sys.modules.keys()) if
  415. x == 'cas' or x.startswith('cas.') ]
  416. shutil.rmtree(self.basetempdir)
  417. self.tempdir = None
  418. def test_filedircas(self):
  419. cachedir = self.tempdir / 'cache'
  420. fd = FileDirCAS(cachedir)
  421. self.assertTrue(cachedir.exists())
  422. with open(self.fixtures / 'hello.py', 'rb') as fp:
  423. # that when hello.py's data
  424. hellodata = fp.read()
  425. # is hashed
  426. hellohash = hashlib.sha256(hellodata).hexdigest()
  427. # and written to the cache
  428. fd.write_cache(hellohash, hellodata)
  429. # that the file exists in the correct place
  430. self.assertTrue((cachedir / hellohash[0] /
  431. (hellohash + '.py')).exists())
  432. # and that when fetched
  433. data = fd.fetch_data(urlparse('hash://sha256/%s' % hellohash))
  434. # it matches what was passed
  435. self.assertEqual(data, hellodata)
  436. def test_filedircas_limit_refresh(self):
  437. # XXX - only refresh when the dir has changed, and each
  438. # file has changed
  439. pass
  440. def test_casimport(self):
  441. # That a CASFinder
  442. f = CASFinder()
  443. cachedir = self.tempdir / 'cache'
  444. cachedir.mkdir()
  445. # make sure that we can't import anything at first
  446. with self.assertRaises(ImportError):
  447. import cas.v1_f_2398472398
  448. # when registering the cache directory
  449. fdc = FileDirCAS(cachedir)
  450. f.register(fdc)
  451. # and adding the hello.py file
  452. fdc.add_file(self.fixtures / 'hello.py')
  453. # can import the function
  454. from cas.v1_f_330884aa2febb5e19fb7194ec6a69ed11dd3d77122f1a5175ee93e73cf0161c3 import hello
  455. name = 'Olof'
  456. # and run the code
  457. self.assertEqual(hello(name), 'hello ' + name)
  458. # and when finished, can disconnect
  459. f.disconnect()
  460. # and is no longer in the meta_path
  461. self.assertNotIn(f, sys.meta_path)
  462. # and when disconnected as second time, nothing happens
  463. f.disconnect()
  464. def test_conforder(self):
  465. conf = self.fixtures / 'ordertest.conf'
  466. # that w/ a custom config
  467. with tempset(os.environ, 'CASIMPORT_CONF', str(conf)), \
  468. CASFinder() as f:
  469. # that is gets loaded
  470. defaultinit(f)
  471. # and that the first loader is the HTTPSCAS
  472. self.assertIsInstance(f._loaders[0], HTTPSCAS)
  473. # and that the second loader is the IPFSCAS
  474. self.assertIsInstance(f._loaders[1], IPFSCAS)
  475. def test_writecache(self):
  476. conf = self.fixtures / 'writecache.conf'
  477. # that w/ a custom config
  478. with tempset(os.environ, 'CASIMPORT_CONF', str(conf)), \
  479. CASFinder() as f:
  480. # that is gets loaded
  481. defaultinit(f)
  482. # that the first loader is the cache
  483. self.assertIsInstance(f._loaders[0], FileDirCAS)
  484. # and that the stored writer matches
  485. self.assertIs(f._writer, f._loaders[0])
  486. def test_ipfsgwfromconfig(self):
  487. gwurl = 'https://www.example.com/somepath/'
  488. ipfsconf = dict(gateway=gwurl)
  489. ipfsobj = IPFSCAS.fromconfig(ipfsconf)
  490. self.assertEqual(ipfsobj.make_url(urlparse('ipfs://someobj')),
  491. 'https://www.example.com/somepath/someobj')
  492. def test_defaultinit(self):
  493. temphome = self.tempdir / 'home'
  494. temphome.mkdir()
  495. defcachedir = temphome / '.casimport' / 'cache'
  496. # testing w/ default config
  497. with tempset(os.environ, 'HOME', str(temphome)):
  498. with CASFinder() as f:
  499. # Setup the defaults
  500. defaultinit(f)
  501. # That the default.conf file got copied over.
  502. filecmp.cmp(defcachedir.parent /
  503. 'casimport.conf',
  504. pathlib.Path(__file__).parent /
  505. 'default.conf')
  506. # that the cache got created
  507. self.assertTrue(defcachedir.is_dir())
  508. # and that when hello.py is added to the cache
  509. f._loaders[0].add_file(self.fixtures /
  510. 'hello.py')
  511. # it can be imported
  512. from cas.v1_f_330884aa2febb5e19fb7194ec6a69ed11dd3d77122f1a5175ee93e73cf0161c3 import hello
  513. # and that the second loader is the IPFSCAS
  514. self.assertIsInstance(f._loaders[1], IPFSCAS)
  515. # and that the third loader is the HTTPSCAS
  516. self.assertIsInstance(f._loaders[2], HTTPSCAS)
  517. # and that these are the only ones loaded
  518. self.assertEqual(len(f._loaders), 3)
  519. with CASFinder() as f:
  520. defaultinit(f)
  521. # and that a new CASFinder can still find it
  522. from cas.v1_f_330884aa2febb5e19fb7194ec6a69ed11dd3d77122f1a5175ee93e73cf0161c3 import hello
  523. def test_multiplecas(self):
  524. # that once we have one
  525. with CASFinder() as f:
  526. # if we try to create a second, it fails
  527. self.assertRaises(RuntimeError, CASFinder)
  528. def test_parsealiases(self):
  529. with open(self.fixtures / 'randpkg' / 'cas_aliases.txt') as fp:
  530. aliasdata = fp.read()
  531. res = CASFinder._parsealiases(aliasdata)
  532. self.assertEqual(res, {
  533. 'hello': [
  534. 'hash://sha256/330884aa2febb5e19fb7194ec6a69ed11dd3d77122f1a5175ee93e73cf0161c3?type=text/x-python',
  535. 'ipfs://bafkreibtbcckul7lwxqz7nyzj3dknhwrdxj5o4jc6gsroxxjhzz46albym',
  536. 'https://www.funkthat.com/gitea/jmg/casimport/raw/commit/753e64f53c73d9d1afc4d8a617edb9d3542dcea2/fixtures/hello.py',
  537. ],
  538. 'hash://sha256/330884aa2febb5e19fb7194ec6a69ed11dd3d77122f1a5175ee93e73cf0161c3': [
  539. 'ipfs://bafkreibtbcckul7lwxqz7nyzj3dknhwrdxj5o4jc6gsroxxjhzz46albym',
  540. 'https://www.funkthat.com/gitea/jmg/casimport/raw/commit/753e64f53c73d9d1afc4d8a617edb9d3542dcea2/fixtures/hello.py',
  541. ],
  542. })
  543. def test_aliasmulti(self):
  544. # setup the cache
  545. cachedir = self.tempdir / 'cache'
  546. cachedir.mkdir()
  547. with CASFinder() as f, \
  548. tempattrset(sys.modules[__name__],
  549. 'load_mod_aliases', f.load_mod_aliases):
  550. fdc = FileDirCAS(cachedir)
  551. f.register(fdc)
  552. # and that hello.py is in the cache
  553. fdc.add_file(self.fixtures / 'hello.py')
  554. # and that the aliases are loaded
  555. with open(self.fixtures / 'randpkg' /
  556. 'cas_aliases.txt') as fp:
  557. f.load_aliases(fp.read())
  558. # that when we load the alias first
  559. from cas.v1_a_hello import hello as hello_alias
  560. # and then load the same module via hash
  561. from cas.v1_f_330884aa2febb5e19fb7194ec6a69ed11dd3d77122f1a5175ee93e73cf0161c3 import hello as hello_hash
  562. # they are the same
  563. self.assertIs(hello_alias, hello_hash)
  564. # that when we reimport the alias
  565. from cas.v1_a_hello import hello as hello_alias
  566. # it is still the same
  567. self.assertIs(hello_alias, hello_hash)
  568. # and when we reimport the hash
  569. from cas.v1_f_330884aa2febb5e19fb7194ec6a69ed11dd3d77122f1a5175ee93e73cf0161c3 import hello as hello_hash
  570. # it is still the same
  571. self.assertIs(hello_alias, hello_hash)
  572. def test_aliasimports(self):
  573. # setup the cache
  574. cachedir = self.tempdir / 'cache'
  575. cachedir.mkdir()
  576. # add the test module's path
  577. fixdir = str(self.fixtures)
  578. sys.path.append(fixdir)
  579. try:
  580. with CASFinder() as f, \
  581. tempattrset(sys.modules[__name__],
  582. 'load_mod_aliases', f.load_mod_aliases):
  583. fdc = FileDirCAS(cachedir)
  584. f.register(fdc)
  585. # and that hello.py is in the cache
  586. fdc.add_file(self.fixtures / 'hello.py')
  587. self.assertNotIn('randpkg', sys.modules)
  588. # that the import is successful
  589. import randpkg
  590. # and pulled in the method
  591. self.assertTrue(hasattr(randpkg, 'hello'))
  592. del sys.modules['randpkg']
  593. finally:
  594. sys.path.remove(fixdir)
  595. def test_aliasipfsimports(self):
  596. # add the test module's path
  597. fixdir = str(self.fixtures)
  598. sys.path.append(fixdir)
  599. # that a fake ipfsloader
  600. with open(self.fixtures / 'hello.py', 'rb') as fp:
  601. # that returns the correct data
  602. fakedata = fp.read()
  603. def fakeload(url, fd=fakedata):
  604. if url.scheme != 'ipfs' or url.netloc != 'bafkreibtbcckul7lwxqz7nyzj3dknhwrdxj5o4jc6gsroxxjhzz46albym':
  605. raise ValueError
  606. return fd
  607. fakeipfsloader = mock.MagicMock()
  608. fakeipfsloader.fetch_data = fakeload
  609. try:
  610. with CASFinder() as f, \
  611. tempattrset(sys.modules[__name__],
  612. 'load_mod_aliases', f.load_mod_aliases):
  613. # that a cache dir
  614. cachedir = self.tempdir / 'cache'
  615. fd = FileDirCAS(cachedir)
  616. # when registered
  617. f.register(fd)
  618. # as a writer
  619. f.register_write(fd)
  620. # and the fake ipfs module
  621. f.register(fakeipfsloader)
  622. # and the pkg we are about to load, isn't
  623. self.assertNotIn('randpkg', sys.modules)
  624. # that the import is successful
  625. import randpkg
  626. # and pulled in the method
  627. self.assertTrue(hasattr(randpkg, 'hello'))
  628. # that the cache was written to, and matches
  629. hash = hashlib.sha256(fakedata).hexdigest()
  630. filecmp.cmp(cachedir / '3' / (hash + '.py'),
  631. self.fixtures / 'hello.py')
  632. # clean up
  633. del sys.modules['randpkg']
  634. finally:
  635. sys.path.remove(fixdir)
  636. @mock.patch('urllib.request.urlopen')
  637. def test_ipfscasloader(self, uomock):
  638. # prep return test data
  639. with open(self.fixtures / 'hello.py') as fp:
  640. # that returns the correct data
  641. ipfsdata = fp.read()
  642. # that the ipfs CAS loader
  643. ipfs = IPFSCAS()
  644. # that the request is successfull
  645. uomock.return_value.__enter__.return_value.status = 200
  646. # and returns the correct data
  647. uomock.return_value.__enter__.return_value.read. \
  648. return_value = ipfsdata
  649. # that when called
  650. hashurl = urlparse('ipfs://bafkreibtbcckul7lwxqz7nyzj3dknhwrdxj5o4jc6gsroxxjhzz46albym')
  651. data = ipfs.fetch_data(hashurl)
  652. # it opens the correct url
  653. uomock.assert_called_with('https://cloudflare-ipfs.com/ipfs/bafkreibtbcckul7lwxqz7nyzj3dknhwrdxj5o4jc6gsroxxjhzz46albym')
  654. # and returns the correct data
  655. self.assertEqual(data, ipfsdata)
  656. with self.assertRaises(ValueError):
  657. # that a hash url fails
  658. ipfs.fetch_data(urlparse('hash://sha256/asldfkj'))
  659. # that when the request fails
  660. uomock.return_value.__enter__.return_value.status = 400
  661. # it raises a RuntimeError
  662. with self.assertRaises(RuntimeError):
  663. ipfs.fetch_data(hashurl)
  664. # Note: mostly copied from above, test_ipfscasloader
  665. @mock.patch('urllib.request.urlopen')
  666. def test_httpscasloader(self, uomock):
  667. # prep return test data
  668. with open(self.fixtures / 'hello.py') as fp:
  669. # that returns the correct data
  670. httpsdata = fp.read()
  671. # that the https CAS loader
  672. httpsldr = HTTPSCAS()
  673. # that the request is successfull
  674. uomock.return_value.__enter__.return_value.status = 200
  675. # and returns the correct data
  676. uomock.return_value.__enter__.return_value.read. \
  677. return_value = httpsdata
  678. # that when called
  679. hashurl = urlparse('https://www.funkthat.com/gitea/jmg/casimport/raw/commit/753e64f53c73d9d1afc4d8a617edb9d3542dcea2/fixtures/hello.py')
  680. data = httpsldr.fetch_data(hashurl)
  681. # it opens the correct url
  682. uomock.assert_called_with('https://www.funkthat.com/gitea/jmg/casimport/raw/commit/753e64f53c73d9d1afc4d8a617edb9d3542dcea2/fixtures/hello.py')
  683. # and returns the correct data
  684. self.assertEqual(data, httpsdata)
  685. with self.assertRaises(ValueError):
  686. # that a hash url fails
  687. httpsldr.fetch_data(urlparse('hash://sha256/asldfkj'))
  688. # that when the request fails
  689. uomock.return_value.__enter__.return_value.status = 400
  690. # it raises a RuntimeError
  691. with self.assertRaises(RuntimeError):
  692. httpsldr.fetch_data(hashurl)
  693. @unittest.skip('todo')
  694. def test_overlappingaliases(self):
  695. # make sure that an aliases file is consistent and does not
  696. # override other urls. That is that any hashes are
  697. # consistent, and that they have at least one root hash that
  698. # is the same, and will be used for fetching.
  699. #
  700. # Likely will also have to deal w/ an issue where two
  701. # aliases share sha256, and a third shares sha512, which in
  702. # this case, BOTH hashse have to be checked.
  703. pass
  704. def test_filecorruption(self):
  705. cachedir = self.tempdir / 'cachedir'
  706. cachedir.mkdir()
  707. # that an existing file
  708. shutil.copy(self.fixtures / 'hello.py', cachedir)
  709. # is in the cache
  710. fdcas = FileDirCAS(cachedir)
  711. # that when refresh is surpressed
  712. fdcas.refresh_dir = lambda: None
  713. # and has a bogus hash
  714. fdcas._hashes['0000'] = cachedir / 'hello.py'
  715. # that when read raises an exception
  716. with self.assertRaises(ValueError):
  717. fdcas.fetch_data(urlparse('hash://sha256/0000'))
  718. # that when passed an invalid url
  719. with self.assertRaises(ValueError):
  720. fdcas.fetch_data(urlparse('https://sha256/0000'))