Package instant :: Module cache
[hide private]
[frames] | no frames]

Source Code for Module instant.cache

  1  """This module contains helper functions for working with the module cache. 
  2   
  3  Example operations: 
  4    - modulename = modulename_from_checksum(checksum) 
  5    - modulename = modulename_from_checksum(compute_checksum(signature)) 
  6    - module = import_module_directly(path, modulename) 
  7    - module = import_module(modulename) 
  8    - module = import_module(checksum) 
  9    - module = import_module(compute_checksum(signature)) 
 10    - modules = cached_modules() 
 11    - modules = cached_modules(cache_dir) 
 12  """ 
 13   
 14  import os, sys, re 
 15  from output import instant_warning, instant_assert, instant_debug 
 16  from paths import get_default_cache_dir, validate_cache_dir 
 17  from signatures import compute_checksum 
 18  from locking import get_lock, release_lock 
 19   
 20  # TODO: We could make this an argument, but it's used indirectly several places so take care. 
 21  _modulename_prefix = "instant_module_" 
22 -def modulename_from_checksum(checksum):
23 "Construct a module name from a checksum for use in cache." 24 return _modulename_prefix + checksum
25 26
27 -def checksum_from_modulename(modulename):
28 "Construct a module name from a checksum for use in cache." 29 return modulename.remove(_modulename_prefix)
30 31
32 -def import_module_directly(path, modulename):
33 "Import a module with the given module name that resides in the given path." 34 sys.path.insert(0, path) 35 try: 36 module = __import__(modulename) 37 except: 38 instant_warning("In instant.import_module_directly: Failed to import module '%s' from '%s'." % (modulename, path)) 39 module = None 40 finally: 41 sys.path.pop(0) 42 return module
43 44 45 _memory_cache = {}
46 -def memory_cached_module(moduleid):
47 "Returns the cached module if found." 48 module = _memory_cache.get(moduleid, None) 49 instant_debug("Found '%s' in memory cache with key '%r'." % (module, moduleid)) 50 return module
51 52
53 -def place_module_in_memory_cache(moduleid, module):
54 "Place a compiled module in cache with given id." 55 _memory_cache[moduleid] = module 56 instant_debug("Added module '%s' to cache with key '%r'." % (module, moduleid))
57 58
59 -def is_valid_module_name(name):
60 NAMELENGTHLIMIT = 100 61 return len(name) < NAMELENGTHLIMIT and bool(re.search(r"^[a-zA-Z_][\w]*$", name))
62 63
64 -def import_and_cache_module(path, modulename, moduleids):
65 module = import_module_directly(path, modulename) 66 instant_assert(module is not None, "Failed to import module found in cache. Modulename: '%s'; Path: '%s'." % (modulename, path)) 67 for moduleid in moduleids: 68 place_module_in_memory_cache(moduleid, module) 69 return module
70 71
72 -def check_memory_cache(moduleid):
73 # Check memory cache first with the given moduleid 74 moduleids = [moduleid] 75 module = memory_cached_module(moduleid) 76 if module: return module, moduleids 77 78 # Get signature from moduleid if it isn't a string, 79 # and check memory cache again 80 if hasattr(moduleid, "signature"): 81 moduleid = moduleid.signature() 82 instant_debug("In instant.check_memory_cache: Got signature "\ 83 "'%s' from moduleid.signature()." % moduleid) 84 module = memory_cached_module(moduleid) 85 if module: 86 for moduleid in moduleids: 87 place_module_in_memory_cache(moduleid, module) 88 return module, moduleids 89 moduleids.append(moduleid) 90 91 # Construct a filename from the checksum of moduleid if it 92 # isn't already a valid name, and check memory cache again 93 if not is_valid_module_name(moduleid): 94 moduleid = modulename_from_checksum(compute_checksum(moduleid)) 95 instant_debug("In instant.check_memory_cache: Constructed module name "\ 96 "'%s' from moduleid '%s'." % (moduleid, moduleids[-1])) 97 module = memory_cached_module(moduleid) 98 if module: return module, moduleids 99 moduleids.append(moduleid) 100 101 instant_debug("In instant.check_memory_cache: Failed to find module.") 102 return None, moduleids
103 104
105 -def check_disk_cache(modulename, cache_dir, moduleids):
106 # Get file lock to avoid race conditions in cache 107 lock = get_lock(cache_dir, modulename) 108 109 # Ensure a valid cache_dir 110 cache_dir = validate_cache_dir(cache_dir) 111 112 # Check on disk, in current directory and cache directory 113 for path in (os.getcwd(), cache_dir): 114 if os.path.isdir(os.path.join(path, modulename)): 115 # Found existing directory, try to import and place in memory cache 116 module = import_and_cache_module(path, modulename, moduleids) 117 if module: 118 instant_debug("In instant.check_disk_cache: Imported module "\ 119 "'%s' from '%s'." % (modulename, path)) 120 release_lock(lock) 121 return module 122 else: 123 instant_debug("In instant.check_disk_cache: Failed to imported "\ 124 "module '%s' from '%s'." % (modulename, path)) 125 126 # All attempts failed 127 instant_debug("In instant.check_disk_cache: Can't import module with modulename "\ 128 "%r using cache directory %r." % (modulename, cache_dir)) 129 release_lock(lock) 130 return None
131 132
133 -def import_module(moduleid, cache_dir=None):
134 """Import module from cache given its moduleid and an optional cache directory. 135 136 The moduleid can be either 137 - the module name 138 - a signature string, of which a checksum is taken to look up in the cache 139 - a checksum string, which is used directly to look up in the cache 140 - a hashable non-string object with a function moduleid.signature() which is used to get a signature string 141 The hashable object is used to look up in the memory cache before signature() is called. 142 If the module is found on disk, it is placed in the memory cache. 143 """ 144 # Look for module in memory cache 145 module, moduleids = check_memory_cache(moduleid) 146 if module: return module 147 148 # Look for module in disk cache 149 modulename = moduleids[-1] 150 return check_disk_cache(modulename, cache_dir, moduleids)
151 152
153 -def cached_modules(cache_dir=None):
154 "Return a list with the names of all cached modules." 155 cache_dir = validate_cache_dir(cache_dir) 156 return os.listdir(cache_dir)
157