first commit
This commit is contained in:
528
python3-vckonline/lib/python3.8/site-packages/pop/hub.py
Normal file
528
python3-vckonline/lib/python3.8/site-packages/pop/hub.py
Normal file
@@ -0,0 +1,528 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Import python libs
|
||||
import os
|
||||
import importlib.machinery
|
||||
import inspect
|
||||
import logging
|
||||
import secrets
|
||||
import sys
|
||||
|
||||
# Import pop libs
|
||||
import pop.dirs
|
||||
import pop.scanner
|
||||
import pop.loader
|
||||
import pop.exc
|
||||
import pop.contract
|
||||
import pop.verify
|
||||
|
||||
from typing import Any, Dict, List, Tuple, Iterator
|
||||
from types import ModuleType
|
||||
|
||||
EXT_SUFFIXES = tuple(importlib.machinery.EXTENSION_SUFFIXES)
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def ex_path(path: str) -> List[str]:
|
||||
"""
|
||||
Take a path that is sent to the Sub and expand it if it is a string or not
|
||||
"""
|
||||
if path is None:
|
||||
return []
|
||||
elif isinstance(path, str):
|
||||
return path.split(",")
|
||||
elif isinstance(path, list):
|
||||
return path
|
||||
return []
|
||||
|
||||
|
||||
class Hub:
|
||||
"""
|
||||
The redistributed pop central hub. All components of the system are
|
||||
rooted to the Hub.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self._subs = {}
|
||||
self._sub_alias = {}
|
||||
self._dynamic = {}
|
||||
self._dscan = False
|
||||
# Add the pop sub to the hub, this should always use pypath and
|
||||
# Should never be made dynamic. This is a core system sub and should
|
||||
# NOT be app-merged
|
||||
self._subs["pop"] = Sub(self, "pop", pypath="pop.mods.pop")
|
||||
self._iter_subs = sorted(self._subs.keys())
|
||||
self._iter_ind = 0
|
||||
# Set up the conf OPT structure so it is always available
|
||||
self.OPT = {}
|
||||
|
||||
def __getstate__(self) -> Dict:
|
||||
return dict(_subs=self._subs)
|
||||
|
||||
def __setstate__(self, state: Dict):
|
||||
self.__dict__.update(state)
|
||||
|
||||
def __iter__(self) -> Iterator["Sub"]:
|
||||
def iter(subs: Dict[str, Sub]):
|
||||
for sub in sorted(subs.keys()):
|
||||
yield subs[sub]
|
||||
|
||||
return iter(self._subs)
|
||||
|
||||
def _resolve_this(self, levels: int) -> "Hub":
|
||||
"""
|
||||
This function allows for hub to pop introspective calls.
|
||||
This should only ever be called from within a hub module, otherwise
|
||||
it should stack trace, or return heaven knows what...
|
||||
:param levels: The number of frames to search for a hub reference
|
||||
"""
|
||||
if hasattr(
|
||||
sys, "_getframe"
|
||||
): # implementation detail of CPython, speeds up things by 100x.
|
||||
desired_frame = sys._getframe(3)
|
||||
contracted = desired_frame.f_locals["self"]
|
||||
else:
|
||||
call_frame = inspect.stack(0)[3]
|
||||
contracted = call_frame[0].f_locals["self"]
|
||||
ref = contracted.ref.split(".")
|
||||
|
||||
# (0=module, 1=module's parent etc.)
|
||||
level_offset = levels - 1
|
||||
traversed = self
|
||||
for i in range(len(ref) - level_offset):
|
||||
traversed = getattr(traversed, ref[i])
|
||||
return traversed
|
||||
|
||||
def _remove_subsystem(self, subname: str) -> bool:
|
||||
"""
|
||||
Remove the named subsystem
|
||||
:param subname: The name of a subsystem to remove
|
||||
:return True if the subsystem was successfully removed, else False
|
||||
"""
|
||||
if subname in self._subs:
|
||||
# Remove the subsystem
|
||||
self._subs.pop(subname)
|
||||
# reset the iterator
|
||||
self._iter_subs = sorted(self._subs.keys())
|
||||
self._iter_ind = 0
|
||||
return True
|
||||
return False
|
||||
|
||||
def _scan_dynamic(self):
|
||||
"""
|
||||
Refresh the dynamic roots data used for loading app merge module roots
|
||||
"""
|
||||
self._dynamic = pop.dirs.dynamic_dirs()
|
||||
self._dscan = True
|
||||
|
||||
def __getattr__(self, item: str):
|
||||
if item.startswith("_"):
|
||||
if item == item[0] * len(item):
|
||||
return self._resolve_this(len(item))
|
||||
else:
|
||||
return self.__getattribute__(item)
|
||||
if "." in item:
|
||||
return self.pop.ref.last(item)
|
||||
if item in self._subs:
|
||||
return self._subs[item]
|
||||
elif item in self._sub_alias:
|
||||
resolved = self._sub_alias[item]
|
||||
if resolved in self._subs:
|
||||
return self._subs[resolved]
|
||||
return self.__getattribute__(item)
|
||||
|
||||
|
||||
class Sub:
|
||||
"""
|
||||
The pop object that contains the loaded module data
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
hub: Hub,
|
||||
subname: str,
|
||||
root: Hub or Sub = None,
|
||||
pypath: List[str] or str = None,
|
||||
static: List[str] or str = None,
|
||||
contracts_pypath: List[str] or str = None,
|
||||
contracts_static: List[str] or str = None,
|
||||
default_contracts: List[str] or str = None,
|
||||
virtual: bool = True,
|
||||
dyne_name: str = None,
|
||||
omit_start: Tuple[str] = ("_",),
|
||||
omit_end: Tuple[str] = (),
|
||||
omit_func: bool = False,
|
||||
omit_class: bool = False,
|
||||
omit_vars: bool = False,
|
||||
mod_basename: str = "",
|
||||
stop_on_failures: bool = False,
|
||||
init: bool = True,
|
||||
is_contract: bool = False,
|
||||
sub_virtual: bool = True,
|
||||
recursive_contracts_static=None,
|
||||
default_recursive_contracts=None,
|
||||
):
|
||||
"""
|
||||
:param hub: The redistributed pop central hub
|
||||
:param subname: The name that the sub is going to take on the hub
|
||||
if nothing else is passed, it is used as the pypath (TODO make it the dyne_name not the pypath)
|
||||
:param pypath: One or many python paths which will be imported
|
||||
:param static: Directories that can be explicitly passed
|
||||
:param contracts_pypath: Load additional contract paths
|
||||
:param contracts_static: Load additional contract paths from a specific directory
|
||||
:param default_contracts: Specifies that a specific contract plugin will be applied as a default to all plugins
|
||||
:param virtual: Toggle whether or not to process __virtual__ functions
|
||||
:param dyne_name: The dynamic name to use to look up paths to find plugins -- linked to conf.py
|
||||
:param omit_start: Allows you to pass in a tuple of characters that would omit the loading of any object
|
||||
I.E. Any function starting with an underscore will not be loaded onto a plugin
|
||||
(You should probably never change this)
|
||||
:param omit_end:Allows you to pass in a tuple of characters that would omit the loading of an object
|
||||
(You should probably never change this)
|
||||
:param omit_func: bool: Don't load any functions
|
||||
:param omit_class: bool: Don't load any classes
|
||||
:param omit_vars: bool: Don't load any vars
|
||||
:param mod_basename: str: Manipulate the location in sys.modules that the plugin will be loaded to.
|
||||
Allow plugins to be loaded into a separate namespace.
|
||||
:param stop_on_failures: If any module fails to load for any reason, stacktrace and do not continue loading this sub
|
||||
:param init: bool: determine whether or not we process __init__ functions
|
||||
:param is_contract: Specify whether or not this sub is a contract
|
||||
:param sub_virtual: bool: Recursively ignore this sub and it's subs
|
||||
"""
|
||||
self._iter_ind = 0
|
||||
self._hub = hub
|
||||
self._root = root or hub
|
||||
self._subs = {}
|
||||
self._alias = []
|
||||
self._sub_alias = {}
|
||||
self._subname = subname
|
||||
self._pypath = ex_path(pypath)
|
||||
self._static = ex_path(static)
|
||||
self._contracts_pypath = ex_path(contracts_pypath)
|
||||
self._contracts_static = ex_path(contracts_static)
|
||||
self._recursive_contracts_static = ex_path(recursive_contracts_static)
|
||||
if isinstance(default_contracts, str):
|
||||
default_contracts = [default_contracts]
|
||||
if isinstance(default_recursive_contracts, str):
|
||||
default_recursive_contracts = [default_recursive_contracts]
|
||||
self._default_recursive_contracts = default_recursive_contracts or []
|
||||
self._default_contracts = default_contracts or ()
|
||||
self._dyne_name = dyne_name
|
||||
self._virtual = virtual
|
||||
self._omit_start = omit_start
|
||||
self._sub_virtual = sub_virtual
|
||||
self._omit_end = omit_end
|
||||
self._omit_func = omit_func
|
||||
self._omit_class = omit_class
|
||||
self._omit_vars = omit_vars
|
||||
self._mod_basename = mod_basename
|
||||
self._stop_on_failures = stop_on_failures
|
||||
self._is_contract = is_contract
|
||||
self._process_init = init
|
||||
self._prepare()
|
||||
|
||||
def _prepare(self):
|
||||
self._dirs = pop.dirs.dir_list(
|
||||
self._subname, "mods", self._pypath, self._static,
|
||||
)
|
||||
if self._dyne_name:
|
||||
self._load_dyne()
|
||||
self._contract_dirs = pop.dirs.dir_list(
|
||||
self._subname, "contracts", self._contracts_pypath, self._contracts_static,
|
||||
)
|
||||
self._contract_dirs.extend(pop.dirs.inline_dirs(self._dirs, "contracts"))
|
||||
self._recursive_contract_dirs = pop.dirs.dir_list(
|
||||
self._subname, "recursive_contracts", [], self._recursive_contracts_static,
|
||||
)
|
||||
self._recursive_contract_dirs.extend(
|
||||
pop.dirs.inline_dirs(self._dirs, "recursive_contracts")
|
||||
)
|
||||
|
||||
if self._contract_dirs:
|
||||
self._contracts = Sub(
|
||||
self._hub,
|
||||
f"{self._subname}.contracts",
|
||||
static=self._contract_dirs,
|
||||
is_contract=True,
|
||||
)
|
||||
else:
|
||||
self._contracts = None
|
||||
|
||||
if self._recursive_contract_dirs:
|
||||
self._recursive_contracts = Sub(
|
||||
self._hub,
|
||||
f"{self._subname}.recursive_contracts",
|
||||
static=self._recursive_contract_dirs,
|
||||
is_contract=True,
|
||||
)
|
||||
else:
|
||||
self._recursive_contracts = getattr(
|
||||
self._root, "_recursive_contracts", None
|
||||
)
|
||||
self._name_root = self._load_name_root()
|
||||
self._scan = pop.scanner.scan(self._dirs)
|
||||
self._loaded = {}
|
||||
self._vmap = {}
|
||||
self._load_errors = {}
|
||||
self._loaded_all = False
|
||||
|
||||
def _load_dyne(self):
|
||||
"""
|
||||
Load up the dynamic dirs for this sub
|
||||
"""
|
||||
if not self._hub._dscan:
|
||||
self._hub._scan_dynamic()
|
||||
for path in self._hub._dynamic.get(self._dyne_name, {}).get("paths", []):
|
||||
self._dirs.append(path)
|
||||
|
||||
def _load_name_root(self):
|
||||
"""
|
||||
Generate the root of the name to be used to apply to the loaded modules
|
||||
"""
|
||||
if self._pypath:
|
||||
return self._pypath[0]
|
||||
elif self._dirs:
|
||||
return secrets.token_hex()
|
||||
|
||||
def __getstate__(self):
|
||||
return dict(
|
||||
_hub=self._hub,
|
||||
_subname=self._subname,
|
||||
_pypath=self._pypath,
|
||||
_static=self._static,
|
||||
_contracts_pypath=self._contracts_pypath,
|
||||
_contracts_static=self._contracts_static,
|
||||
_default_contracts=self._default_contracts,
|
||||
_virtual=self._virtual,
|
||||
_omit_start=self._omit_start,
|
||||
_omit_end=self._omit_end,
|
||||
_omit_func=self._omit_func,
|
||||
_omit_class=self._omit_class,
|
||||
_omit_vars=self._omit_vars,
|
||||
_mod_basename=self._mod_basename,
|
||||
_stop_on_failures=self._stop_on_failures,
|
||||
)
|
||||
|
||||
def __setstate__(self, state: Dict):
|
||||
self.__dict__.update(state)
|
||||
self._prepare()
|
||||
|
||||
def __getattr__(self, item: str):
|
||||
"""
|
||||
If the item should be loaded, load it, else serve it
|
||||
"""
|
||||
if item.startswith("_"):
|
||||
return self.__getattribute__(item)
|
||||
if "." in item:
|
||||
return self._hub.pop.ref.last(f"{self._subname}.{item}")
|
||||
if item in self._loaded:
|
||||
ret = self._loaded[item]
|
||||
# If this previously errored on load, try it again,
|
||||
# it might be ready to load now
|
||||
if isinstance(ret, pop.loader.LoadError):
|
||||
ret = self._find_mod(item)
|
||||
if isinstance(ret, pop.loader.LoadError):
|
||||
# If this is still a LoadError, process it
|
||||
self._process_load_error(ret)
|
||||
return ret
|
||||
elif item in self._subs:
|
||||
return self._subs[item]
|
||||
elif item in self._sub_alias:
|
||||
resolved = self._sub_alias[item]
|
||||
if resolved in self._subs:
|
||||
return self._subs[resolved]
|
||||
|
||||
mod = self._find_mod(item)
|
||||
if mod is None:
|
||||
raise AttributeError(f"'{self._subname}' has no attribute '{item}'")
|
||||
return mod
|
||||
|
||||
def __contains__(self, item: str):
|
||||
try:
|
||||
return hasattr(self, item)
|
||||
except pop.exc.PopLookupError:
|
||||
return False
|
||||
|
||||
def __iter__(self) -> Iterator["Sub"]:
|
||||
self._load_all()
|
||||
|
||||
def iter(loaded):
|
||||
for l in sorted(loaded.keys()):
|
||||
yield loaded[l]
|
||||
|
||||
return iter(self._loaded)
|
||||
|
||||
def __next__(self) -> "Sub":
|
||||
self._load_all()
|
||||
if self._iter_ind == len(self._iter_keys):
|
||||
self._iter_ind = 0
|
||||
raise StopIteration
|
||||
self._iter_ind += 1
|
||||
return self._loaded[self._iter_keys[self._iter_ind - 1]]
|
||||
|
||||
def _sub_init(self):
|
||||
"""
|
||||
Run load init.py for the sub, running '__init__' function if present
|
||||
"""
|
||||
self._find_mod("init", match_only=True)
|
||||
|
||||
def _process_load_error(
|
||||
self, mod: ModuleType, skip_full_stop: bool = False
|
||||
) -> bool:
|
||||
if not isinstance(mod, pop.loader.LoadError):
|
||||
# This is not a LoadError, return now!
|
||||
return False
|
||||
|
||||
if mod.edict["verror"]:
|
||||
error = "{0[msg]}: {0[verror]}".format(mod())
|
||||
if skip_full_stop is False and self._stop_on_failures is True:
|
||||
raise pop.exc.PopError(error)
|
||||
log.info(error)
|
||||
return False
|
||||
error = "{0[msg]}: {0[exception]!r}".format(mod())
|
||||
if mod.traceback:
|
||||
error += "\n" + mod.traceback
|
||||
if skip_full_stop is False and self._stop_on_failures is True:
|
||||
raise pop.exc.PopError(error)
|
||||
if mod.traceback:
|
||||
log.warning(error)
|
||||
else:
|
||||
log.info(error)
|
||||
return True
|
||||
|
||||
def _find_mod(self, item: str, match_only: bool = False) -> Dict:
|
||||
"""
|
||||
Find the module named item
|
||||
:param item: The module to search for (then load) from any scanned interface
|
||||
:param match_only: return the loaded module
|
||||
:return a loaded mod_dict
|
||||
"""
|
||||
for iface in self._scan:
|
||||
for bname in self._scan[iface]:
|
||||
if os.path.basename(bname) == item:
|
||||
self._load_item(iface, bname)
|
||||
if item in self._loaded:
|
||||
return self._loaded[item]
|
||||
if not match_only:
|
||||
for iface in self._scan:
|
||||
for bname in self._scan[iface]:
|
||||
if self._scan[iface][bname].get("loaded"):
|
||||
continue
|
||||
self._load_item(iface, bname)
|
||||
if item in self._loaded:
|
||||
return self._loaded[item]
|
||||
# Let's see if the module being lookup is in the load errors dictionary
|
||||
if item in self._load_errors:
|
||||
# Return the LoadError
|
||||
return self._load_errors[item]
|
||||
|
||||
def _load_item(self, iface: str, bname: str):
|
||||
"""
|
||||
Load the named basename
|
||||
:param iface: A scanned directory type
|
||||
:param bname: The base name of the python path of a module
|
||||
"""
|
||||
if iface not in self._scan:
|
||||
raise pop.exc.PopLoadError(
|
||||
"Bad call to load item, no iface {}".format(iface)
|
||||
)
|
||||
if bname not in self._scan[iface]:
|
||||
raise pop.exc.PopLoadError(
|
||||
"Bad call to load item, no bname {} in iface {}".format(bname, iface)
|
||||
)
|
||||
# The mname is the name to give the module in python's sys.modules
|
||||
# This name must be unique for every loaded module, so we use the full
|
||||
# module path sans the file extention
|
||||
mname = self._scan[iface][bname]["path"].replace(os.sep, ".")
|
||||
mname = mname[mname.index(".") + 1 : mname.rindex(".")].strip(".")
|
||||
mod = pop.loader.load_mod(mname, iface, self._scan[iface][bname]["path"],)
|
||||
if self._process_load_error(mod):
|
||||
self._load_errors[os.path.basename(bname)] = mod
|
||||
return
|
||||
self._prep_mod(mod, iface, bname)
|
||||
|
||||
def _process_vret(self, vret: Dict[str, Any]) -> bool:
|
||||
"""
|
||||
:param vret: The return from a __virtual__ or __sub_virtual__ function
|
||||
:return: True if there was an error, else false
|
||||
"""
|
||||
if "error" in vret:
|
||||
# Virtual Errors should not full stop pop
|
||||
self._process_load_error(vret["error"], skip_full_stop=True)
|
||||
# Store the LoadError under the __virtualname__ if defined
|
||||
self._load_errors[vret["vname"]] = vret["error"]
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
def _prep_mod(self, mod: ModuleType, iface: str, bname: str):
|
||||
"""
|
||||
Prepare the module!
|
||||
:param mod: A python module containing data
|
||||
:param iface: A scanned directory type
|
||||
:param bname: The base name of the python path of a module
|
||||
"""
|
||||
if not self._sub_virtual:
|
||||
return
|
||||
else:
|
||||
vret = pop.loader.load_sub_virtual(self._hub, self._virtual, mod, bname)
|
||||
if self._process_vret(vret):
|
||||
self._sub_virtual = False
|
||||
return
|
||||
vret = pop.loader.load_virtual(self._hub, self._virtual, mod, bname)
|
||||
if self._process_vret(vret):
|
||||
return
|
||||
|
||||
contracts = pop.contract.load_contract(
|
||||
self._contracts, self._default_contracts, mod, vret["name"]
|
||||
)
|
||||
recursive_contracts = set(
|
||||
pop.contract.load_contract(
|
||||
self._recursive_contracts,
|
||||
self._default_recursive_contracts,
|
||||
mod,
|
||||
vret["name"],
|
||||
)
|
||||
)
|
||||
if getattr(self._root, "_recursive_contracts", None):
|
||||
recursive_contracts.update(
|
||||
pop.contract.load_contract(
|
||||
self._root._recursive_contracts,
|
||||
self._root._default_recursive_contracts,
|
||||
mod,
|
||||
vret["name"],
|
||||
)
|
||||
)
|
||||
recursive_contracts = list(recursive_contracts)
|
||||
name = vret["name"]
|
||||
if name.endswith(EXT_SUFFIXES):
|
||||
for ext in EXT_SUFFIXES:
|
||||
if name.endswith(ext):
|
||||
name = name.split(ext)[0]
|
||||
break
|
||||
mod_dict = pop.loader.prep_loaded_mod(
|
||||
self, mod, name, contracts, recursive_contracts
|
||||
)
|
||||
if name != "init":
|
||||
pop.verify.contract(self._hub, contracts + recursive_contracts, mod_dict)
|
||||
self._loaded[name] = mod_dict
|
||||
self._vmap[mod.__file__] = name
|
||||
# Let's mark the module as loaded
|
||||
self._scan[iface][bname]["loaded"] = True
|
||||
if self._process_init:
|
||||
# Now that the module has been added to the sub, call mod_init
|
||||
pop.loader.mod_init(self, mod, name)
|
||||
|
||||
def _load_all(self):
|
||||
"""
|
||||
Load all modules found during the scan.
|
||||
|
||||
.. attention:: This completely disables the lazy loader behavior of pop
|
||||
"""
|
||||
if self._loaded_all is True:
|
||||
return
|
||||
for iface in self._scan:
|
||||
for bname in self._scan[iface]:
|
||||
if self._scan[iface][bname].get("loaded"):
|
||||
continue
|
||||
self._load_item(iface, bname)
|
||||
self._loaded_all = True
|
||||
Reference in New Issue
Block a user