first commit
This commit is contained in:
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,274 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Translate an options data structure into command line args
|
||||
"""
|
||||
|
||||
# Import python libs
|
||||
import sys
|
||||
import inspect
|
||||
import argparse
|
||||
import functools
|
||||
import collections
|
||||
import pop.hub
|
||||
|
||||
__virtualname__ = "args"
|
||||
__contracts__ = [__virtualname__]
|
||||
|
||||
|
||||
class ActionWrapper:
|
||||
"""
|
||||
This class wraps argparse.Action instances in order to mark arguments passed
|
||||
on CLI as explicitly passed
|
||||
"""
|
||||
|
||||
def __init__(self, action):
|
||||
self._action = action
|
||||
functools.update_wrapper(self, action)
|
||||
|
||||
def __call__(self, parser, namespace, values, option_string):
|
||||
# Let's store the call to this option as an explicit CLI call for later
|
||||
# use when overwriting any configuration settings on file with those
|
||||
# from CLI
|
||||
if getattr(parser, "_explicit_cli_args_", None) is None:
|
||||
setattr(parser, "_explicit_cli_args_", set())
|
||||
parser._explicit_cli_args_.add(
|
||||
self._action.dest
|
||||
) # pylint: disable=protected-access
|
||||
# Carry on regular operation
|
||||
return self._action(parser, namespace, values, option_string)
|
||||
|
||||
def __getattribute__(self, name):
|
||||
if name == "_action":
|
||||
return object.__getattribute__(self, name)
|
||||
# Proxy any attribute's search to the _action instance
|
||||
return getattr(self._action, name)
|
||||
|
||||
def __repr__(self):
|
||||
return repr(self._action)
|
||||
|
||||
|
||||
class ActionClassWrapper:
|
||||
"""
|
||||
This class wraps argparse.Action classes in order to mark arguments passed
|
||||
on CLI as explicitly passed
|
||||
"""
|
||||
|
||||
def __init__(self, klass):
|
||||
self._klass = klass
|
||||
|
||||
def __call__(self, *args, **kwargs):
|
||||
return ActionWrapper(self._klass(*args, **kwargs))
|
||||
|
||||
def __repr__(self):
|
||||
return repr(self._klass)
|
||||
|
||||
def __getattribute__(self, name):
|
||||
if name == "_klass":
|
||||
return object.__getattribute__(self, name)
|
||||
# Proxy any attributes search to the _klass instance
|
||||
return getattr(self._klass, name)
|
||||
|
||||
|
||||
class ArgumentParser(argparse.ArgumentParser):
|
||||
def register(self, name, value, obj): # pylint: disable=arguments-differ
|
||||
if name == "action":
|
||||
# Let's wrap it on our action class wrapper so we can latter store
|
||||
# which options were explicitly passed from CLI
|
||||
return super(ArgumentParser, self).register(
|
||||
name, value, ActionClassWrapper(obj)
|
||||
)
|
||||
return super(ArgumentParser, self).register(name, value, obj)
|
||||
|
||||
def parse_known_args(self, args=None, namespace=None):
|
||||
namespace, arg_strings = super().parse_known_args(args, namespace)
|
||||
explicit_cli_args = getattr(self, "_explicit_cli_args_", set())
|
||||
if "_explicit_cli_args_" not in namespace:
|
||||
setattr(namespace, "_explicit_cli_args_", set())
|
||||
namespace._explicit_cli_args_.update(explicit_cli_args)
|
||||
return namespace, arg_strings
|
||||
|
||||
|
||||
def __init__(hub: "pop.hub.Hub"):
|
||||
"""
|
||||
Set up the local memory copy of the parser
|
||||
"""
|
||||
hub.conf._mem["args"] = {}
|
||||
|
||||
|
||||
def _init_parser(hub: "pop.hub.Hub", opts):
|
||||
if "parser" not in hub.conf._mem["args"]:
|
||||
# Instantiate the parser
|
||||
hub.conf._mem["args"]["parser"] = ArgumentParser(**opts.get("_argparser_", {}))
|
||||
|
||||
|
||||
def _keys(opts):
|
||||
"""
|
||||
Return the keys in the right order
|
||||
"""
|
||||
if isinstance(opts, collections.OrderedDict):
|
||||
return sorted(
|
||||
list(opts), key=lambda k: opts[k].get("display_priority", sys.maxsize)
|
||||
)
|
||||
return sorted(opts, key=lambda k: (opts[k].get("display_priority", sys.maxsize), k))
|
||||
|
||||
|
||||
def subs(hub: "pop.hub.Hub", opts):
|
||||
"""
|
||||
Set up sub parsers, if using sub parsers this needs to be called
|
||||
before calling setup.
|
||||
|
||||
opts dict:
|
||||
<sub_title>:
|
||||
[desc]: 'Some subparser'
|
||||
help: 'subparser!'
|
||||
"""
|
||||
_init_parser(hub, opts)
|
||||
hub.conf._mem["args"]["sub"] = hub.conf._mem["args"]["parser"].add_subparsers(
|
||||
dest="_subparser_"
|
||||
)
|
||||
hub.conf._mem["args"]["subs"] = {}
|
||||
for arg in _keys(opts):
|
||||
if arg in ("_argparser_",):
|
||||
continue
|
||||
comps = opts[arg]
|
||||
kwargs = {}
|
||||
if "help" in comps:
|
||||
kwargs["help"] = comps["help"]
|
||||
if "desc" in comps:
|
||||
kwargs["description"] = comps["desc"]
|
||||
hub.conf._mem["args"]["subs"][arg] = hub.conf._mem["args"]["sub"].add_parser(
|
||||
arg, **kwargs
|
||||
)
|
||||
return {"result": True, "return": True}
|
||||
|
||||
|
||||
def setup(hub: "pop.hub.Hub", opts):
|
||||
"""
|
||||
Take in a pre-defined opts dict and translate it to args
|
||||
|
||||
opts dict:
|
||||
<arg>:
|
||||
[group]: foo
|
||||
[default]: bar
|
||||
[action]: store_true
|
||||
[options]: # arg will be turned into --arg
|
||||
- '-A'
|
||||
- '--cheese'
|
||||
[choices]:
|
||||
- foo
|
||||
- bar
|
||||
- baz
|
||||
[nargs]: +
|
||||
[type]: int
|
||||
[dest]: cheese
|
||||
help: Some great help message
|
||||
"""
|
||||
_init_parser(hub, opts)
|
||||
defaults = {}
|
||||
groups = {}
|
||||
ex_groups = {}
|
||||
for arg in _keys(opts):
|
||||
if arg in ("_argparser_",):
|
||||
continue
|
||||
comps = opts[arg]
|
||||
positional = comps.pop("positional", False)
|
||||
if positional:
|
||||
args = [arg]
|
||||
else:
|
||||
long_opts = ["--{}".format(arg.replace("_", "-"))]
|
||||
short_opts = []
|
||||
for o_str in comps.get("options", []):
|
||||
if not o_str.startswith("--") and o_str.startswith("-"):
|
||||
short_opts.append(o_str)
|
||||
continue
|
||||
long_opts.append(o_str)
|
||||
args = short_opts + long_opts
|
||||
kwargs = {}
|
||||
kwargs["action"] = action = comps.get("action", None)
|
||||
|
||||
if action is None:
|
||||
# Non existing option defaults to a StoreAction in argparse
|
||||
action = hub.conf._mem["args"]["parser"]._registry_get(
|
||||
"action", action
|
||||
) # pylint: disable=protected-access
|
||||
|
||||
if isinstance(action, str):
|
||||
signature = inspect.signature(
|
||||
hub.conf._mem["args"]["parser"]._registry_get("action", action).__init__
|
||||
) # pylint: disable=protected-access
|
||||
else:
|
||||
signature = inspect.signature(action.__init__)
|
||||
|
||||
for param in signature.parameters:
|
||||
if param == "self" or param not in comps:
|
||||
continue
|
||||
if param == "dest":
|
||||
kwargs["dest"] = comps.get("dest", arg)
|
||||
continue
|
||||
if param == "help":
|
||||
kwargs["help"] = comps.get("help", "THIS NEEDS SOME DOCUMENTATION!!")
|
||||
continue
|
||||
if param == "default":
|
||||
defaults[comps.get("dest", arg)] = comps[param]
|
||||
kwargs[param] = comps[param]
|
||||
|
||||
if "group" in comps:
|
||||
group = comps["group"]
|
||||
if group not in groups:
|
||||
groups[group] = hub.conf._mem["args"]["parser"].add_argument_group(
|
||||
group
|
||||
)
|
||||
groups[group].add_argument(*args, **kwargs)
|
||||
continue
|
||||
if "ex_group" in comps:
|
||||
group = comps["ex_group"]
|
||||
if group not in ex_groups:
|
||||
ex_groups[group] = hub.conf._mem["args"][
|
||||
"parser"
|
||||
].add_mutually_exclusive_group()
|
||||
ex_groups[group].add_argument(*args, **kwargs)
|
||||
continue
|
||||
if "sub" in comps:
|
||||
subs = comps["sub"]
|
||||
if not isinstance(subs, list):
|
||||
subs = [subs]
|
||||
for sub in subs:
|
||||
sparse = hub.conf._mem["args"]["subs"].get(sub)
|
||||
if not sparse:
|
||||
# Maybe raise exception here? Malformed config?
|
||||
continue
|
||||
sparse.add_argument(*args, **kwargs)
|
||||
continue
|
||||
hub.conf._mem["args"]["parser"].add_argument(*args, **kwargs)
|
||||
return {"result": True, "return": defaults}
|
||||
|
||||
|
||||
def parse(
|
||||
hub: "pop.hub.Hub", args=None, namespace=None, only_parse_known_arguments=False
|
||||
):
|
||||
"""
|
||||
Parse the command line options
|
||||
"""
|
||||
if only_parse_known_arguments:
|
||||
opts, unknown_args = hub.conf._mem["args"]["parser"].parse_known_args(
|
||||
args, namespace
|
||||
)
|
||||
opts_dict = opts.__dict__
|
||||
opts_dict["_unknown_args_"] = unknown_args
|
||||
else:
|
||||
opts = hub.conf._mem["args"]["parser"].parse_args(args, namespace)
|
||||
opts_dict = opts.__dict__
|
||||
return {"result": True, "return": opts_dict}
|
||||
|
||||
|
||||
def render(hub: "pop.hub.Hub", defaults, cli_opts, explicit_cli_args):
|
||||
"""
|
||||
For options specified as such, take the string passed into the cli and
|
||||
render it using the specified render flag
|
||||
"""
|
||||
for key in explicit_cli_args:
|
||||
rend = defaults.get(key, {}).get("render")
|
||||
if rend:
|
||||
ref = f"conf.{rend}.render"
|
||||
cli_opts[key] = hub.pop.ref.last(ref)(cli_opts[key])
|
||||
return cli_opts
|
||||
@@ -0,0 +1,56 @@
|
||||
"""
|
||||
Used to take care of the options that end in `_dir`. The assumption is that
|
||||
`_dir` options need to be treated differently. They need to verified to exist
|
||||
and they need to be rooted based on the user, root option etc.
|
||||
"""
|
||||
# Import python libs
|
||||
import os
|
||||
import pop.hub
|
||||
|
||||
|
||||
def roots(hub: "pop.hub.Hub", default_root, f_opts, root_dir):
|
||||
"""
|
||||
Detect the root dir data and apply it
|
||||
"""
|
||||
os_root = os.path.abspath(os.sep)
|
||||
root = os_root
|
||||
change = False
|
||||
non_priv = False
|
||||
if hasattr(os, "geteuid"):
|
||||
if not os.geteuid() == 0:
|
||||
change = True
|
||||
non_priv = True
|
||||
if root_dir and root_dir != default_root:
|
||||
root = root_dir
|
||||
change = True
|
||||
if not root.endswith(os.sep):
|
||||
root = f"{root}{os.sep}"
|
||||
if change:
|
||||
for imp in f_opts:
|
||||
for key in f_opts[imp]:
|
||||
if key == "root_dir":
|
||||
continue
|
||||
if key.endswith("_dir"):
|
||||
if non_priv:
|
||||
root = os.path.join(os.environ["HOME"], f".{imp}{os.sep}")
|
||||
if imp in f_opts[imp][key]:
|
||||
a_len = len(imp) + 1
|
||||
f_opts[imp][
|
||||
key
|
||||
] = f"{os_root}{f_opts[imp][key][f_opts[imp][key].index(imp)+a_len:]}"
|
||||
f_opts[imp][key] = f_opts[imp][key].replace(os_root, root, 1)
|
||||
|
||||
|
||||
def verify(hub: "pop.hub.Hub", opts):
|
||||
"""
|
||||
Verify that the environment and all named directories in the
|
||||
configuration exist
|
||||
"""
|
||||
for key in opts:
|
||||
if key == "root_dir":
|
||||
continue
|
||||
if key == "config_dir":
|
||||
continue
|
||||
if key.endswith("_dir"):
|
||||
if not os.path.isdir(opts[key]):
|
||||
os.makedirs(opts[key])
|
||||
@@ -0,0 +1,120 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Configuration file core loading functions
|
||||
"""
|
||||
|
||||
# Import python libs
|
||||
import os
|
||||
import glob
|
||||
import fnmatch
|
||||
import pop.hub
|
||||
|
||||
__virtualname__ = "file"
|
||||
__contracts__ = [__virtualname__]
|
||||
|
||||
|
||||
def load_file(hub: "pop.hub.Hub", paths, defaults=None, overrides=None, includes=True):
|
||||
"""
|
||||
Load a single configuration file
|
||||
"""
|
||||
opts = {}
|
||||
if isinstance(defaults, dict):
|
||||
opts.update(defaults)
|
||||
if not isinstance(paths, list):
|
||||
paths = paths.split(",")
|
||||
add = []
|
||||
for fn_ in paths:
|
||||
add.extend(glob.glob(fn_))
|
||||
paths.extend(add)
|
||||
for fn_ in paths:
|
||||
if hub.conf._loader == "yaml":
|
||||
opts.update(hub.conf.yaml.load(fn_))
|
||||
elif hub.conf._loader == "json":
|
||||
opts.update(hub.conf.json.load(fn_))
|
||||
elif hub.conf._loader == "toml":
|
||||
opts.update(hub.conf.toml.load(fn_))
|
||||
if includes:
|
||||
hub.conf.file.proc_include(opts)
|
||||
if isinstance(overrides, dict):
|
||||
opts.update(overrides)
|
||||
return opts
|
||||
|
||||
|
||||
def load_dir(
|
||||
hub,
|
||||
confdir,
|
||||
defaults=None,
|
||||
overrides=None,
|
||||
includes=True,
|
||||
recurse=False,
|
||||
pattern=None,
|
||||
):
|
||||
"""
|
||||
Load takes a directory location to scan for configuration files. These
|
||||
files will be read in. The defaults dict defines what
|
||||
configuration options should exist if not found in the confdir. Overrides
|
||||
are configuration options which should be included regardless of whether
|
||||
those options existed before. If includes is set to True, then the
|
||||
statements 'include' and 'include_dir' found in either the defaults or
|
||||
in configuration files.
|
||||
"""
|
||||
opts = {}
|
||||
if not isinstance(confdir, list):
|
||||
confdir = confdir.split(",")
|
||||
confdirs = []
|
||||
for dirs in confdir:
|
||||
if not isinstance(dirs, (list, tuple)):
|
||||
dirs = [dirs]
|
||||
for dir_ in dirs:
|
||||
confdirs.extend(glob.glob(dir_))
|
||||
if isinstance(defaults, dict):
|
||||
opts.update(defaults)
|
||||
paths = []
|
||||
for dir_ in confdirs:
|
||||
dirpaths = []
|
||||
if os.path.isdir(dir_):
|
||||
if not recurse:
|
||||
for fn_ in os.listdir(dir_):
|
||||
path = os.path.join(dir_, fn_)
|
||||
if os.path.isdir(path):
|
||||
# Don't process directories
|
||||
continue
|
||||
if pattern and not fnmatch.fnmatch(fn_, pattern):
|
||||
continue
|
||||
dirpaths.append(path)
|
||||
else:
|
||||
for root, dirs, files in os.walk(dir_):
|
||||
for fn_ in files:
|
||||
path = os.path.join(root, fn_)
|
||||
if pattern and not fnmatch.fnmatch(fn_, pattern):
|
||||
continue
|
||||
dirpaths.append(path)
|
||||
|
||||
# Sort confdir directory paths like:
|
||||
# /b.txt
|
||||
# /c.txt
|
||||
# /a/x.txt
|
||||
# /b/x.txt
|
||||
paths.extend(sorted(dirpaths, key=lambda p: (p.count(os.path.sep), p)))
|
||||
opts.update(hub.conf.file.load_file(paths, includes))
|
||||
if isinstance(overrides, dict):
|
||||
opts.update(overrides)
|
||||
return opts
|
||||
|
||||
|
||||
def proc_include(hub: "pop.hub.Hub", opts):
|
||||
"""
|
||||
process include and include_dir
|
||||
"""
|
||||
rec = False
|
||||
if opts.get("include_dir"):
|
||||
idir = opts.pop("include_dir")
|
||||
opts.update(hub.conf.file.load_dir(idir))
|
||||
rec = True
|
||||
if opts.get("include"):
|
||||
ifn = opts.pop("include")
|
||||
opts.update(hub.conf.file.load_file(ifn))
|
||||
rec = True
|
||||
if rec:
|
||||
hub.conf.file.proc_include(opts)
|
||||
return opts
|
||||
@@ -0,0 +1,6 @@
|
||||
def __init__(hub):
|
||||
"""
|
||||
Load the subdirs for conf
|
||||
"""
|
||||
hub.__._mem = {}
|
||||
hub.pop.sub.load_subdirs(hub.conf)
|
||||
@@ -0,0 +1,141 @@
|
||||
"""
|
||||
Integrate is used to pull config data from multiple sources and merge it into
|
||||
the hub. Once it is merged then when a sub is loaded the respective config data
|
||||
is loaded into the sub as `OPTS`
|
||||
"""
|
||||
# Take an *args list of modules to import and look for conf.py
|
||||
# Import conf.py if present
|
||||
# After gathering all dicts, modify them to merge CLI options
|
||||
#
|
||||
# Import python libs
|
||||
import importlib
|
||||
import copy
|
||||
import os
|
||||
|
||||
|
||||
def _ex_final(confs, final, override, key_to_ref, ops_to_ref):
|
||||
"""
|
||||
Scan the configuration datasets, create the final config
|
||||
value, and detect collisions
|
||||
"""
|
||||
for arg in confs:
|
||||
for key in confs[arg]:
|
||||
ref = f"{arg}.{key}"
|
||||
if ref in override:
|
||||
s_key = override[ref]["key"]
|
||||
s_opts = override[ref]["options"]
|
||||
else:
|
||||
s_key = key
|
||||
s_opts = confs[arg][key].get("options", [])
|
||||
s_opts.append(f"--{s_key}")
|
||||
final[s_key] = confs[arg][key]
|
||||
if s_opts:
|
||||
final[s_key]["options"] = s_opts
|
||||
if s_key in key_to_ref:
|
||||
key_to_ref[s_key].append(ref)
|
||||
else:
|
||||
key_to_ref[s_key] = [ref]
|
||||
for opt in s_opts:
|
||||
if opt in ops_to_ref:
|
||||
ops_to_ref[opt].append(ref)
|
||||
else:
|
||||
ops_to_ref = [ref]
|
||||
|
||||
|
||||
def load(
|
||||
hub,
|
||||
imports,
|
||||
override=None,
|
||||
cli=None,
|
||||
roots=False,
|
||||
loader="json",
|
||||
logs=True,
|
||||
version=True,
|
||||
):
|
||||
"""
|
||||
This function takes a list of python packages to load and look for
|
||||
respective configs. The configs are then loaded in a non-collision
|
||||
way modifying the cli options dynamically.
|
||||
The args look for the named <package>.conf python module and then
|
||||
looks for dictionaries named after the following convention:
|
||||
|
||||
override = {'<package>.key': 'key': 'new_key', 'options': ['--option1', '--option2']}
|
||||
|
||||
CONFIG: The main configuration for this package - loads to hub.OPT['<import>']
|
||||
CLI_CONFIG: Loaded only if this is the only import or if specified in the cli option
|
||||
SUBS: Used to define the subcommands, only loaded if this is the cli config
|
||||
"""
|
||||
if override is None:
|
||||
override = {}
|
||||
if isinstance(imports, str):
|
||||
if cli is None:
|
||||
cli = imports
|
||||
imports = [imports]
|
||||
primary = imports[0] if cli is None else cli
|
||||
confs = {}
|
||||
final = {}
|
||||
collides = []
|
||||
key_to_ref = {}
|
||||
ops_to_ref = {}
|
||||
subs = {}
|
||||
for imp in imports:
|
||||
try:
|
||||
cmod = importlib.import_module(f"{imp}.conf")
|
||||
except ImportError:
|
||||
continue
|
||||
if hasattr(cmod, "CONFIG"):
|
||||
confs[imp] = copy.deepcopy(cmod.CONFIG)
|
||||
if cli == imp:
|
||||
if hasattr(cmod, "CLI_CONFIG"):
|
||||
confs[imp].update(copy.deepcopy(cmod.CLI_CONFIG))
|
||||
if hasattr(cmod, "SUBS"):
|
||||
subs = copy.deepcopy(cmod.SUBS)
|
||||
if logs:
|
||||
lconf = hub.conf.log.init.conf(primary)
|
||||
lconf.update(confs[primary])
|
||||
confs[primary] = lconf
|
||||
if version:
|
||||
vconf = hub.conf.version.CONFIG
|
||||
vconf.update(confs[primary])
|
||||
confs[primary] = vconf
|
||||
_ex_final(confs, final, override, key_to_ref, ops_to_ref)
|
||||
for opt in ops_to_ref:
|
||||
g_count = 0
|
||||
if len(ops_to_ref[opt]) > 1:
|
||||
collides.append({opt: ops_to_ref[opt]})
|
||||
for key in key_to_ref:
|
||||
col = []
|
||||
for ref in key_to_ref[key]:
|
||||
col.append(ref)
|
||||
if len(col) > 1:
|
||||
collides.append({key: key_to_ref[key]})
|
||||
if collides:
|
||||
raise KeyError(collides)
|
||||
opts = hub.conf.reader.read(final, subs, loader=loader)
|
||||
# This will be put into an immutable data type before it is passed on
|
||||
f_opts = {}
|
||||
for key in opts:
|
||||
if key == "_subparser_":
|
||||
f_opts["_subparser_"] = opts["_subparser_"]
|
||||
continue
|
||||
for ref in key_to_ref[key]:
|
||||
imp = ref[: ref.rindex(".")]
|
||||
local_key = ref[ref.rindex(".") + 1 :]
|
||||
if imp not in f_opts:
|
||||
f_opts[imp] = {}
|
||||
f_opts[imp][local_key] = opts[key]
|
||||
if roots:
|
||||
root_dir = f_opts.get(cli, {}).get("root_dir")
|
||||
hub.conf.dirs.roots(
|
||||
final.get("root_dir", {}).get("default", os.path.abspath(os.sep)),
|
||||
f_opts,
|
||||
root_dir,
|
||||
)
|
||||
for imp in f_opts:
|
||||
hub.conf.dirs.verify(f_opts[imp])
|
||||
hub.OPT = hub.pop.data.imap(f_opts)
|
||||
if logs:
|
||||
log_plugin = hub.OPT[primary].get("log_plugin")
|
||||
getattr(hub, f"conf.log.{log_plugin}.setup")(hub.OPT[primary])
|
||||
if hub.OPT[primary].get("version"):
|
||||
hub.conf.version.run(primary)
|
||||
@@ -0,0 +1,34 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Define the JSON loader interface
|
||||
"""
|
||||
|
||||
# Import python libs
|
||||
import json
|
||||
import pop.hub
|
||||
|
||||
__virtualname__ = "json"
|
||||
__contracts__ = [__virtualname__]
|
||||
|
||||
|
||||
def __virtual__(hub):
|
||||
return True
|
||||
|
||||
|
||||
def load(hub: "pop.hub.Hub", path):
|
||||
"""
|
||||
Use json to read in a file
|
||||
"""
|
||||
try:
|
||||
with open(path, "r") as fp_:
|
||||
ret = json.loads(fp_.read())
|
||||
return ret
|
||||
except FileNotFoundError:
|
||||
return {}
|
||||
|
||||
|
||||
def render(hub: "pop.hub.Hub", val):
|
||||
"""
|
||||
Take the string and render it in json
|
||||
"""
|
||||
return json.loads(val)
|
||||
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,23 @@
|
||||
# Import python libs
|
||||
import logging
|
||||
import pop.hub
|
||||
from typing import Any, Dict
|
||||
|
||||
|
||||
def setup(hub: "pop.hub.Hub", conf: Dict[str, Any]):
|
||||
"""
|
||||
Given the configuration data set up the logger
|
||||
"""
|
||||
level = hub.conf.log.LEVELS.get(conf["log_level"].lower(), logging.INFO)
|
||||
root = logging.getLogger("")
|
||||
root.setLevel(level)
|
||||
cf = logging.Formatter(fmt=conf["log_fmt_console"], datefmt=conf["log_datefmt"])
|
||||
ch = logging.StreamHandler()
|
||||
ch.setLevel(level)
|
||||
ch.setFormatter(cf)
|
||||
root.addHandler(ch)
|
||||
ff = logging.Formatter(fmt=conf["log_fmt_console"], datefmt=conf["log_datefmt"])
|
||||
fh = logging.FileHandler(conf["log_file"])
|
||||
fh.setLevel(level)
|
||||
fh.setFormatter(ff)
|
||||
root.addHandler(fh)
|
||||
@@ -0,0 +1,62 @@
|
||||
"""
|
||||
This sub is used to set up logging for pop projects and injects logging
|
||||
options into conf making it easy to add robust logging
|
||||
"""
|
||||
# Import python libs
|
||||
import logging
|
||||
import pop.hub
|
||||
from typing import Any, Dict
|
||||
|
||||
|
||||
def __init__(hub: "pop.hub.Hub"):
|
||||
"""
|
||||
Set up variables used by the log subsystem
|
||||
"""
|
||||
hub.conf.log.LEVELS = {
|
||||
"debug": logging.DEBUG,
|
||||
"info": logging.INFO,
|
||||
"warning": logging.WARNING,
|
||||
"error": logging.ERROR,
|
||||
"critical": logging.CRITICAL,
|
||||
}
|
||||
|
||||
|
||||
def conf(hub: "pop.hub.Hub", name: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Return the conf dict for logging, this should be merged OVER by the loaded
|
||||
config dict(s)
|
||||
"""
|
||||
# TODO: Make this more robust to handle more logging interfaces
|
||||
ldict = {
|
||||
"log_file": {
|
||||
"default": f"{name}.log",
|
||||
"help": "The location of the log file",
|
||||
"group": "Logging Options",
|
||||
},
|
||||
"log_level": {
|
||||
"default": "warning",
|
||||
"help": "Set the log level, either quiet, info, warning, or error",
|
||||
"group": "Logging Options",
|
||||
},
|
||||
"log_fmt_logfile": {
|
||||
"default": "%(asctime)s,%(msecs)03d [%(name)-17s][%(levelname)-8s] %(message)s",
|
||||
"help": "The format to be given to log file messages",
|
||||
"group": "Logging Options",
|
||||
},
|
||||
"log_fmt_console": {
|
||||
"default": "[%(levelname)-8s] %(message)s",
|
||||
"help": "The log formatting used in the console",
|
||||
"group": "Logging Options",
|
||||
},
|
||||
"log_datefmt": {
|
||||
"default": "%H:%M:%S",
|
||||
"help": "The date format to display in the logs",
|
||||
"group": "Logging Options",
|
||||
},
|
||||
"log_plugin": {
|
||||
"default": "basic",
|
||||
"help": "The logging plugin to use",
|
||||
"group": "Logging Options",
|
||||
},
|
||||
}
|
||||
return ldict
|
||||
@@ -0,0 +1,37 @@
|
||||
"""
|
||||
The os module is used to gather configuration options from the OS facility
|
||||
to send configuration options into applications. In the case of Unix like
|
||||
systems this translates to the environment variables. On Windows systems
|
||||
this translates to the registry.
|
||||
"""
|
||||
# Import python libs
|
||||
import os
|
||||
import pop.hub
|
||||
|
||||
__virtualname__ = "os"
|
||||
|
||||
|
||||
def __virtual__(hub):
|
||||
"""
|
||||
Don't load on Windows, this is for *nix style platforms
|
||||
"""
|
||||
# TODO: detect if windows
|
||||
return True
|
||||
|
||||
|
||||
def gather(hub: "pop.hub.Hub", defaults):
|
||||
"""
|
||||
Iterate over the default config data and look for os: True/str options. When set
|
||||
gather the option from environment variables is present
|
||||
"""
|
||||
ret = {}
|
||||
for key in defaults:
|
||||
if not "os" in defaults[key]:
|
||||
continue
|
||||
os_var = defaults[key]["os"]
|
||||
if os_var is True:
|
||||
os_var = key
|
||||
os_var = os_var.upper()
|
||||
if os_var in os.environ:
|
||||
ret[key] = os.environ[os_var]
|
||||
return ret
|
||||
@@ -0,0 +1,114 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
The reader module is used to read the config data. This will read in cli
|
||||
arguments and merge them with config fie arguments.
|
||||
"""
|
||||
# Import python libs
|
||||
import warnings
|
||||
|
||||
# Priority order: cli, config, cli_defaults
|
||||
|
||||
__virtualname__ = "reader"
|
||||
__contracts__ = [__virtualname__]
|
||||
|
||||
|
||||
def _merge_dicts(opts, updates, os_opts, explicit_cli_args):
|
||||
"""
|
||||
recursively merge updates into opts
|
||||
"""
|
||||
for key, val in os_opts.items():
|
||||
if not val:
|
||||
# Don't use empty os vals
|
||||
continue
|
||||
if key in opts:
|
||||
opts[key] = val
|
||||
for key, val in updates.items():
|
||||
if isinstance(val, dict) and isinstance(opts.get(key), dict):
|
||||
_merge_dicts(opts.get(key, {}), val, os_opts, explicit_cli_args)
|
||||
elif val is not None:
|
||||
if key not in opts:
|
||||
# The key is not in opts(from config file), let's add it
|
||||
opts[key] = val
|
||||
continue
|
||||
|
||||
# We already have a value for the key in opts
|
||||
if opts[key] == val:
|
||||
# The value is the same, carry on
|
||||
continue
|
||||
|
||||
if key in explicit_cli_args:
|
||||
# We have a value for the key in opts(from config file) but
|
||||
# this option was explicitly passed on the CLI, ie, it's not
|
||||
# a default value.
|
||||
# Overwrite what's in opts
|
||||
opts[key] = val
|
||||
continue
|
||||
return opts
|
||||
|
||||
|
||||
def read(
|
||||
hub,
|
||||
defaults,
|
||||
subs=None,
|
||||
loader="json",
|
||||
process_cli=True,
|
||||
process_cli_known_args_only=False,
|
||||
args=None,
|
||||
namespace=None,
|
||||
):
|
||||
"""
|
||||
Pass in the default options dict to use
|
||||
:param opts:
|
||||
:param process_cli: Process the passed args or sys.argv
|
||||
:param process_cli_known_args_only: Tells the ArgumentParser to only process known arguments
|
||||
:param args: Arguments to pass to ArgumentParser
|
||||
:param namespace: argparse.Namespace to pass to ArgumentParser
|
||||
:return: options
|
||||
"""
|
||||
msg = "Pop-config is the new means to load configs in pop, reader.read will be removed in pop 13"
|
||||
warnings.warn(msg, DeprecationWarning, stacklevel=2)
|
||||
hub.conf._loader = loader
|
||||
if subs:
|
||||
hub.conf.args.subs(subs)
|
||||
opts = hub.conf.args.setup(defaults)["return"]
|
||||
os_opts = hub.conf.os.gather(defaults)
|
||||
if process_cli is True:
|
||||
cli_opts = hub.conf.args.parse(args, namespace, process_cli_known_args_only)[
|
||||
"return"
|
||||
]
|
||||
else:
|
||||
cli_opts = {}
|
||||
explicit_cli_args = cli_opts.pop("_explicit_cli_args_", set())
|
||||
cli_opts = hub.conf.args.render(defaults, cli_opts, explicit_cli_args)
|
||||
kwargs = {}
|
||||
# Due to the order of priorities and the representation of defaults in the
|
||||
# Argparser we need to manually check if the config option values are from
|
||||
# the cli or from defaults
|
||||
f_func = False
|
||||
if "config_dir" in cli_opts:
|
||||
if cli_opts["config_dir"]:
|
||||
kwargs["confdir"] = cli_opts["config_dir"]
|
||||
else:
|
||||
kwargs["confdir"] = opts["config_dir"]
|
||||
if "config_recurse" in cli_opts:
|
||||
if cli_opts["config_recurse"]:
|
||||
kwargs["recurse"] = cli_opts["config_recurse"]
|
||||
else:
|
||||
kwargs["recurse"] = opts["config_recurse"]
|
||||
# If the config_dir configuration dictionary provides a configuration
|
||||
# file pattern to read, pass it along
|
||||
kwargs["pattern"] = defaults["config_dir"].get("pattern")
|
||||
f_func = hub.conf.file.load_dir
|
||||
elif "config" in cli_opts:
|
||||
if cli_opts["config"]:
|
||||
kwargs["paths"] = cli_opts["config"]
|
||||
else:
|
||||
kwargs["paths"] = opts["config"]
|
||||
f_func = hub.conf.file.load_file
|
||||
# Render args before config parsing
|
||||
if f_func:
|
||||
f_opts = f_func(**kwargs)
|
||||
opts.update(f_opts)
|
||||
return _merge_dicts(opts, cli_opts, os_opts, explicit_cli_args)
|
||||
else:
|
||||
return _merge_dicts(opts, cli_opts, os_opts, explicit_cli_args)
|
||||
@@ -0,0 +1,41 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Define the yaml loader interface
|
||||
"""
|
||||
import pop.hub
|
||||
|
||||
# Import third party libs
|
||||
try:
|
||||
import toml
|
||||
|
||||
HAS_TOML = True
|
||||
except ImportError:
|
||||
HAS_TOML = False
|
||||
|
||||
__virtualname__ = "toml"
|
||||
# __contracts__ = [__virtualname__]
|
||||
|
||||
|
||||
def __virtual__(hub: "pop.hub.Hub"):
|
||||
if HAS_TOML:
|
||||
return True
|
||||
return (False, "TOML could not be loaded")
|
||||
|
||||
|
||||
def load(hub: "pop.hub.Hub", path):
|
||||
"""
|
||||
use toml to read in a file
|
||||
"""
|
||||
try:
|
||||
with open(path, "rb") as fp_:
|
||||
return toml.load(fp_.read())
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
return {}
|
||||
|
||||
|
||||
def render(hub: "pop.hub.Hub", val):
|
||||
"""
|
||||
Take the string and render it in json
|
||||
"""
|
||||
return toml.loads(val)
|
||||
@@ -0,0 +1,25 @@
|
||||
"""
|
||||
Support embedding version number lookup into cli
|
||||
"""
|
||||
# IMport python libs
|
||||
import importlib
|
||||
import pop.hub
|
||||
import sys
|
||||
|
||||
|
||||
CONFIG = {
|
||||
"version": {
|
||||
"default": False,
|
||||
"action": "store_true",
|
||||
"help": "Display version information",
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def run(hub: "pop.hub.Hub", primary):
|
||||
"""
|
||||
Check the version number and then exit
|
||||
"""
|
||||
mod = importlib.import_module(f"{primary}.version")
|
||||
print(f"{primary} {mod.version}")
|
||||
sys.exit(0)
|
||||
@@ -0,0 +1,42 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Define the yaml loader interface
|
||||
"""
|
||||
|
||||
# Import third party libs
|
||||
import pop.hub
|
||||
|
||||
try:
|
||||
import yaml
|
||||
|
||||
HAS_YAML = True
|
||||
except ImportError:
|
||||
HAS_YAML = False
|
||||
|
||||
__virtualname__ = "yaml"
|
||||
__contracts__ = [__virtualname__]
|
||||
|
||||
|
||||
def __virtual__(hub):
|
||||
if HAS_YAML:
|
||||
return True
|
||||
return (False, "PyYaml could not be loaded")
|
||||
|
||||
|
||||
def load(hub: "pop.hub.Hub", path):
|
||||
"""
|
||||
use yaml to read in a file
|
||||
"""
|
||||
try:
|
||||
with open(path, "rb") as fp_:
|
||||
return yaml.safe_load(fp_.read())
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
return {}
|
||||
|
||||
|
||||
def render(hub: "pop.hub.Hub", val):
|
||||
"""
|
||||
Take the string and render it in json
|
||||
"""
|
||||
return yaml.safe_load(val)
|
||||
Reference in New Issue
Block a user