# -*- coding: utf-8 -*- # Import python libs import os import importlib.machinery import inspect import logging import secrets import sys # Import pop libs import pop.dirs import pop.scanner import pop.loader import pop.exc import pop.contract import pop.verify from typing import Any, Dict, List, Tuple, Iterator from types import ModuleType EXT_SUFFIXES = tuple(importlib.machinery.EXTENSION_SUFFIXES) log = logging.getLogger(__name__) def ex_path(path: str) -> List[str]: """ Take a path that is sent to the Sub and expand it if it is a string or not """ if path is None: return [] elif isinstance(path, str): return path.split(",") elif isinstance(path, list): return path return [] class Hub: """ The redistributed pop central hub. All components of the system are rooted to the Hub. """ def __init__(self): self._subs = {} self._sub_alias = {} self._dynamic = {} self._dscan = False # Add the pop sub to the hub, this should always use pypath and # Should never be made dynamic. This is a core system sub and should # NOT be app-merged self._subs["pop"] = Sub(self, "pop", pypath="pop.mods.pop") self._iter_subs = sorted(self._subs.keys()) self._iter_ind = 0 # Set up the conf OPT structure so it is always available self.OPT = {} def __getstate__(self) -> Dict: return dict(_subs=self._subs) def __setstate__(self, state: Dict): self.__dict__.update(state) def __iter__(self) -> Iterator["Sub"]: def iter(subs: Dict[str, Sub]): for sub in sorted(subs.keys()): yield subs[sub] return iter(self._subs) def _resolve_this(self, levels: int) -> "Hub": """ This function allows for hub to pop introspective calls. This should only ever be called from within a hub module, otherwise it should stack trace, or return heaven knows what... :param levels: The number of frames to search for a hub reference """ if hasattr( sys, "_getframe" ): # implementation detail of CPython, speeds up things by 100x. desired_frame = sys._getframe(3) contracted = desired_frame.f_locals["self"] else: call_frame = inspect.stack(0)[3] contracted = call_frame[0].f_locals["self"] ref = contracted.ref.split(".") # (0=module, 1=module's parent etc.) level_offset = levels - 1 traversed = self for i in range(len(ref) - level_offset): traversed = getattr(traversed, ref[i]) return traversed def _remove_subsystem(self, subname: str) -> bool: """ Remove the named subsystem :param subname: The name of a subsystem to remove :return True if the subsystem was successfully removed, else False """ if subname in self._subs: # Remove the subsystem self._subs.pop(subname) # reset the iterator self._iter_subs = sorted(self._subs.keys()) self._iter_ind = 0 return True return False def _scan_dynamic(self): """ Refresh the dynamic roots data used for loading app merge module roots """ self._dynamic = pop.dirs.dynamic_dirs() self._dscan = True def __getattr__(self, item: str): if item.startswith("_"): if item == item[0] * len(item): return self._resolve_this(len(item)) else: return self.__getattribute__(item) if "." in item: return self.pop.ref.last(item) if item in self._subs: return self._subs[item] elif item in self._sub_alias: resolved = self._sub_alias[item] if resolved in self._subs: return self._subs[resolved] return self.__getattribute__(item) class Sub: """ The pop object that contains the loaded module data """ def __init__( self, hub: Hub, subname: str, root: Hub or Sub = None, pypath: List[str] or str = None, static: List[str] or str = None, contracts_pypath: List[str] or str = None, contracts_static: List[str] or str = None, default_contracts: List[str] or str = None, virtual: bool = True, dyne_name: str = None, omit_start: Tuple[str] = ("_",), omit_end: Tuple[str] = (), omit_func: bool = False, omit_class: bool = False, omit_vars: bool = False, mod_basename: str = "", stop_on_failures: bool = False, init: bool = True, is_contract: bool = False, sub_virtual: bool = True, recursive_contracts_static=None, default_recursive_contracts=None, ): """ :param hub: The redistributed pop central hub :param subname: The name that the sub is going to take on the hub if nothing else is passed, it is used as the pypath (TODO make it the dyne_name not the pypath) :param pypath: One or many python paths which will be imported :param static: Directories that can be explicitly passed :param contracts_pypath: Load additional contract paths :param contracts_static: Load additional contract paths from a specific directory :param default_contracts: Specifies that a specific contract plugin will be applied as a default to all plugins :param virtual: Toggle whether or not to process __virtual__ functions :param dyne_name: The dynamic name to use to look up paths to find plugins -- linked to conf.py :param omit_start: Allows you to pass in a tuple of characters that would omit the loading of any object I.E. Any function starting with an underscore will not be loaded onto a plugin (You should probably never change this) :param omit_end:Allows you to pass in a tuple of characters that would omit the loading of an object (You should probably never change this) :param omit_func: bool: Don't load any functions :param omit_class: bool: Don't load any classes :param omit_vars: bool: Don't load any vars :param mod_basename: str: Manipulate the location in sys.modules that the plugin will be loaded to. Allow plugins to be loaded into a separate namespace. :param stop_on_failures: If any module fails to load for any reason, stacktrace and do not continue loading this sub :param init: bool: determine whether or not we process __init__ functions :param is_contract: Specify whether or not this sub is a contract :param sub_virtual: bool: Recursively ignore this sub and it's subs """ self._iter_ind = 0 self._hub = hub self._root = root or hub self._subs = {} self._alias = [] self._sub_alias = {} self._subname = subname self._pypath = ex_path(pypath) self._static = ex_path(static) self._contracts_pypath = ex_path(contracts_pypath) self._contracts_static = ex_path(contracts_static) self._recursive_contracts_static = ex_path(recursive_contracts_static) if isinstance(default_contracts, str): default_contracts = [default_contracts] if isinstance(default_recursive_contracts, str): default_recursive_contracts = [default_recursive_contracts] self._default_recursive_contracts = default_recursive_contracts or [] self._default_contracts = default_contracts or () self._dyne_name = dyne_name self._virtual = virtual self._omit_start = omit_start self._sub_virtual = sub_virtual self._omit_end = omit_end self._omit_func = omit_func self._omit_class = omit_class self._omit_vars = omit_vars self._mod_basename = mod_basename self._stop_on_failures = stop_on_failures self._is_contract = is_contract self._process_init = init self._prepare() def _prepare(self): self._dirs = pop.dirs.dir_list( self._subname, "mods", self._pypath, self._static, ) if self._dyne_name: self._load_dyne() self._contract_dirs = pop.dirs.dir_list( self._subname, "contracts", self._contracts_pypath, self._contracts_static, ) self._contract_dirs.extend(pop.dirs.inline_dirs(self._dirs, "contracts")) self._recursive_contract_dirs = pop.dirs.dir_list( self._subname, "recursive_contracts", [], self._recursive_contracts_static, ) self._recursive_contract_dirs.extend( pop.dirs.inline_dirs(self._dirs, "recursive_contracts") ) if self._contract_dirs: self._contracts = Sub( self._hub, f"{self._subname}.contracts", static=self._contract_dirs, is_contract=True, ) else: self._contracts = None if self._recursive_contract_dirs: self._recursive_contracts = Sub( self._hub, f"{self._subname}.recursive_contracts", static=self._recursive_contract_dirs, is_contract=True, ) else: self._recursive_contracts = getattr( self._root, "_recursive_contracts", None ) self._name_root = self._load_name_root() self._scan = pop.scanner.scan(self._dirs) self._loaded = {} self._vmap = {} self._load_errors = {} self._loaded_all = False def _load_dyne(self): """ Load up the dynamic dirs for this sub """ if not self._hub._dscan: self._hub._scan_dynamic() for path in self._hub._dynamic.get(self._dyne_name, {}).get("paths", []): self._dirs.append(path) def _load_name_root(self): """ Generate the root of the name to be used to apply to the loaded modules """ if self._pypath: return self._pypath[0] elif self._dirs: return secrets.token_hex() def __getstate__(self): return dict( _hub=self._hub, _subname=self._subname, _pypath=self._pypath, _static=self._static, _contracts_pypath=self._contracts_pypath, _contracts_static=self._contracts_static, _default_contracts=self._default_contracts, _virtual=self._virtual, _omit_start=self._omit_start, _omit_end=self._omit_end, _omit_func=self._omit_func, _omit_class=self._omit_class, _omit_vars=self._omit_vars, _mod_basename=self._mod_basename, _stop_on_failures=self._stop_on_failures, ) def __setstate__(self, state: Dict): self.__dict__.update(state) self._prepare() def __getattr__(self, item: str): """ If the item should be loaded, load it, else serve it """ if item.startswith("_"): return self.__getattribute__(item) if "." in item: return self._hub.pop.ref.last(f"{self._subname}.{item}") if item in self._loaded: ret = self._loaded[item] # If this previously errored on load, try it again, # it might be ready to load now if isinstance(ret, pop.loader.LoadError): ret = self._find_mod(item) if isinstance(ret, pop.loader.LoadError): # If this is still a LoadError, process it self._process_load_error(ret) return ret elif item in self._subs: return self._subs[item] elif item in self._sub_alias: resolved = self._sub_alias[item] if resolved in self._subs: return self._subs[resolved] mod = self._find_mod(item) if mod is None: raise AttributeError(f"'{self._subname}' has no attribute '{item}'") return mod def __contains__(self, item: str): try: return hasattr(self, item) except pop.exc.PopLookupError: return False def __iter__(self) -> Iterator["Sub"]: self._load_all() def iter(loaded): for l in sorted(loaded.keys()): yield loaded[l] return iter(self._loaded) def __next__(self) -> "Sub": self._load_all() if self._iter_ind == len(self._iter_keys): self._iter_ind = 0 raise StopIteration self._iter_ind += 1 return self._loaded[self._iter_keys[self._iter_ind - 1]] def _sub_init(self): """ Run load init.py for the sub, running '__init__' function if present """ self._find_mod("init", match_only=True) def _process_load_error( self, mod: ModuleType, skip_full_stop: bool = False ) -> bool: if not isinstance(mod, pop.loader.LoadError): # This is not a LoadError, return now! return False if mod.edict["verror"]: error = "{0[msg]}: {0[verror]}".format(mod()) if skip_full_stop is False and self._stop_on_failures is True: raise pop.exc.PopError(error) log.info(error) return False error = "{0[msg]}: {0[exception]!r}".format(mod()) if mod.traceback: error += "\n" + mod.traceback if skip_full_stop is False and self._stop_on_failures is True: raise pop.exc.PopError(error) if mod.traceback: log.warning(error) else: log.info(error) return True def _find_mod(self, item: str, match_only: bool = False) -> Dict: """ Find the module named item :param item: The module to search for (then load) from any scanned interface :param match_only: return the loaded module :return a loaded mod_dict """ for iface in self._scan: for bname in self._scan[iface]: if os.path.basename(bname) == item: self._load_item(iface, bname) if item in self._loaded: return self._loaded[item] if not match_only: for iface in self._scan: for bname in self._scan[iface]: if self._scan[iface][bname].get("loaded"): continue self._load_item(iface, bname) if item in self._loaded: return self._loaded[item] # Let's see if the module being lookup is in the load errors dictionary if item in self._load_errors: # Return the LoadError return self._load_errors[item] def _load_item(self, iface: str, bname: str): """ Load the named basename :param iface: A scanned directory type :param bname: The base name of the python path of a module """ if iface not in self._scan: raise pop.exc.PopLoadError( "Bad call to load item, no iface {}".format(iface) ) if bname not in self._scan[iface]: raise pop.exc.PopLoadError( "Bad call to load item, no bname {} in iface {}".format(bname, iface) ) # The mname is the name to give the module in python's sys.modules # This name must be unique for every loaded module, so we use the full # module path sans the file extention mname = self._scan[iface][bname]["path"].replace(os.sep, ".") mname = mname[mname.index(".") + 1 : mname.rindex(".")].strip(".") mod = pop.loader.load_mod(mname, iface, self._scan[iface][bname]["path"],) if self._process_load_error(mod): self._load_errors[os.path.basename(bname)] = mod return self._prep_mod(mod, iface, bname) def _process_vret(self, vret: Dict[str, Any]) -> bool: """ :param vret: The return from a __virtual__ or __sub_virtual__ function :return: True if there was an error, else false """ if "error" in vret: # Virtual Errors should not full stop pop self._process_load_error(vret["error"], skip_full_stop=True) # Store the LoadError under the __virtualname__ if defined self._load_errors[vret["vname"]] = vret["error"] return True else: return False def _prep_mod(self, mod: ModuleType, iface: str, bname: str): """ Prepare the module! :param mod: A python module containing data :param iface: A scanned directory type :param bname: The base name of the python path of a module """ if not self._sub_virtual: return else: vret = pop.loader.load_sub_virtual(self._hub, self._virtual, mod, bname) if self._process_vret(vret): self._sub_virtual = False return vret = pop.loader.load_virtual(self._hub, self._virtual, mod, bname) if self._process_vret(vret): return contracts = pop.contract.load_contract( self._contracts, self._default_contracts, mod, vret["name"] ) recursive_contracts = set( pop.contract.load_contract( self._recursive_contracts, self._default_recursive_contracts, mod, vret["name"], ) ) if getattr(self._root, "_recursive_contracts", None): recursive_contracts.update( pop.contract.load_contract( self._root._recursive_contracts, self._root._default_recursive_contracts, mod, vret["name"], ) ) recursive_contracts = list(recursive_contracts) name = vret["name"] if name.endswith(EXT_SUFFIXES): for ext in EXT_SUFFIXES: if name.endswith(ext): name = name.split(ext)[0] break mod_dict = pop.loader.prep_loaded_mod( self, mod, name, contracts, recursive_contracts ) if name != "init": pop.verify.contract(self._hub, contracts + recursive_contracts, mod_dict) self._loaded[name] = mod_dict self._vmap[mod.__file__] = name # Let's mark the module as loaded self._scan[iface][bname]["loaded"] = True if self._process_init: # Now that the module has been added to the sub, call mod_init pop.loader.mod_init(self, mod, name) def _load_all(self): """ Load all modules found during the scan. .. attention:: This completely disables the lazy loader behavior of pop """ if self._loaded_all is True: return for iface in self._scan: for bname in self._scan[iface]: if self._scan[iface][bname].get("loaded"): continue self._load_item(iface, bname) self._loaded_all = True