Coverage for azmlclient/clients_config.py : 86%
Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# Authors: Sylvain MARIE <sylvain.marie@se.com>
2# + All contributors to <https://github.com/smarie/python-azureml-client>
3#
4# License: 3-clause BSD, <https://github.com/smarie/python-azureml-client/blob/master/LICENSE>
5import sys
6from collections import OrderedDict
7from distutils.util import strtobool
8from warnings import warn
10from jinja2 import Environment, StrictUndefined
12try: # python 3+
13 from configparser import ConfigParser
14except ImportError:
15 from ConfigParser import ConfigParser
17try: # python 3.5+
18 from typing import Dict, List, Callable, Union, Iterable, Optional, Any
19 from logging import Logger
20except ImportError:
21 pass
23from autoclass import autodict
24from yamlable import YamlAble, yaml_info
26from .requests_utils import set_http_proxy
29PY2 = sys.version_info < (3, 0)
32YAML_NS = 'org.pypi.azmlclient'
33"""The namespace used for yaml conversion"""
36@autodict
37class GlobalConfig:
38 """
39 Represents a global component client configuration, that is, configuration that is transverse to all web services.
40 """
41 def __init__(self,
42 http_proxy=None, # type: str
43 https_proxy=None, # type: str
44 ssl_verify=True, # type: Union[str, bool]
45 ):
46 """
47 Global information used for all the calls.
49 :param http_proxy: an optional string representing the http proxy to use. For example "http://localhost:8888"
50 :param https_proxy: an optional string representing the https proxy to use. For example "http://localhost:8888"
51 :param ssl_verify: a boolean or string representing a boolean, indicating if we should validate the SSL
52 certificate. This is True by default (highly recommended in production)
53 """
54 self.http_proxy = http_proxy
55 self.https_proxy = https_proxy
56 self.ssl_verify = ssl_verify
58 def configure_session(self, session):
59 """
60 Helper to get a `requests` (http client) session object, based on the local configuration.
62 If the client is configured for use with a proxy the session will be created accordingly.
63 Note that if this client has no particular configuration for the http proxy this function will return None.
65 :param session:
66 :return:
67 """
68 use_http_for_https = self.http_proxy and not self.https_proxy
69 set_http_proxy(session, http_url=self.http_proxy, https_url=self.https_proxy,
70 use_http_proxy_for_https_requests=use_http_for_https)
72 if self.ssl_verify is not None: 72 ↛ exitline 72 didn't return from function 'configure_session', because the condition on line 72 was never false
73 try:
74 # try to parse a boolean
75 session.verify = bool(strtobool(self.ssl_verify))
76 except:
77 # otherwise this is a path
78 session.verify = self.ssl_verify
81@autodict
82class ServiceConfig:
83 """
84 Represents the configuration to use to interact with an azureml service.
86 A service has a main endpoint defined by a base url and an api key.
87 An optional blob storage configuration can be specified to interact with the service in batch mode (BES)
89 Finally, an optional alternate endpoint can be specified for "some inputs by reference" calls. In that case the
90 endpoint should correspond to a service able to understand the input references and to retrieve them (this is not a
91 standard AzureML mechanism).
92 """
93 def __init__(self,
94 base_url, # type: str
95 api_key, # type: str
96 by_ref_base_url=None, # type: str
97 by_ref_api_key=None, # type: str
98 blob_account=None, # type: str
99 blob_api_key=None, # type: str
100 blob_container=None, # type: str
101 blob_path_prefix=None # type: str
102 ):
103 """
104 Constructor with
106 * an url and api key for normal (anonymized request-response + batch),
107 * an optional url and api key for non-anonymized request-response with input by reference,
108 * an optional account, api key and container name for batch
110 :param base_url:
111 :param api_key:
112 :param by_ref_base_url: an alternate URL to use in 'input by reference' mode. If not provided, the base URL
113 will be used.
114 :param by_ref_api_key: an alternate api key to use in 'input by reference' mode. If not provided, the base
115 api key will be used.
116 :param blob_account: an optional blob account that should be used in batch mode. A non-None value has
117 to be provided
118 :param blob_api_key:
119 :param blob_container:
120 :param blob_path_prefix: an optional prefix path for the blobs to be stored
121 """
122 self.base_url = base_url
123 self.api_key = api_key
125 self._by_ref_base_url = by_ref_base_url
126 self._by_ref_api_key = by_ref_api_key
128 self.blob_account = blob_account
129 self.blob_api_key = blob_api_key
130 self.blob_container = blob_container
131 self.blob_path_prefix = blob_path_prefix
133 @property
134 def by_ref_base_url(self):
135 if self._by_ref_base_url is None:
136 return self.base_url
137 else:
138 return self._by_ref_base_url
140 @property
141 def by_ref_api_key(self):
142 if self._by_ref_api_key is None:
143 return self.api_key
144 else:
145 return self._by_ref_api_key
148@yaml_info(yaml_tag_ns=YAML_NS)
149@autodict
150class ClientConfig(YamlAble):
151 """
152 An AzureML client configuration. It is made of two parts:
154 * A 'global' configuration (a `GlobalConfig`)
155 * services configurations (one `ServiceConfig` for each. Each is registered under a name that will be used
156 to bind the configuration with the appropriate method in `AzureMLClient`. See `@azureml_service` for details.
157 """
158 def __init__(self,
159 global_config=None, # type: GlobalConfig
160 **services_configs # type: ServiceConfig
161 ):
162 """
164 :param global_config: the global configuration, a GlobalConfig
165 :param services_configs: a dictionary of {service_name: ServiceConfig}
166 """
167 if global_config is None:
168 global_config = GlobalConfig()
169 self.global_config = global_config
171 self.services_configs = services_configs
173 def assert_valid_for_services(self,
174 service_names # type: Iterable[str]
175 ):
176 """
177 Asserts that the configuration corresponds to the list of services provided
178 :param service_names:
179 :return:
180 """
181 unknown_services = set(self.services_configs.keys()) - set(service_names)
182 if len(unknown_services) > 0: 182 ↛ 183line 182 didn't jump to line 183, because the condition on line 182 was never true
183 raise ValueError("Configuration is not able to handle services: '" + str(unknown_services)
184 + "'. The list of services supported by this client is '" + str(service_names)
185 + "'")
187 # ---- yamlable interface ----
189 def __to_yaml_dict__(self):
190 # type: (...) -> Dict[str, Any]
191 """ This optional method is called when you call yaml.dump(). See `yamlable` for details."""
192 # notes:
193 # - we do not make `GlobalConfig` and `ServiceConfig` yamlable objects because their custom yaml names would
194 # have to appear in the configuration, which seems tideous
195 # - we use `dict()` not `var()` so that we benefit from their `@autodict` capability to hide private fields
196 return {'global': dict(self.global_config),
197 'services': {service_name: dict(service) for service_name, service in self.services_configs.items()}}
199 @classmethod
200 def __from_yaml_dict__(cls, dct, yaml_tag):
201 # type: (...) -> ClientConfig
202 """ This optional method is called when you call yaml.load(). See `yamlable` for details."""
203 global_cfg = GlobalConfig(**dct['global'])
204 services_cfg = {service_name: ServiceConfig(**service_cfg_dct)
205 for service_name, service_cfg_dct in dct['services'].items()}
206 return ClientConfig(global_cfg, **services_cfg)
208 @classmethod
209 def load_yaml(cls, # type: Type[Y]
210 file_path_or_stream, # type: Union[str, IOBase, StringIO]
211 safe=True, # type:
212 **var_values # type: Any
213 ): # type: (...) -> Y
214 """ applies the template before loading """
215 contents = read_file_and_apply_template(file_path_or_stream, **var_values)
216 return YamlAble.loads_yaml(contents, safe=safe)
218 @classmethod
219 def loads_yaml(cls, # type: Type[Y]
220 yaml_str, # type: str
221 safe=True, # type: bool
222 **var_values # type: Any
223 ): # type: (...) -> Y
224 """ applies the template before loading """
225 contents = apply_template(yaml_str, **var_values)
226 return YamlAble.loads_yaml(contents, safe=safe)
228 # ---- configparser interface ----
230 @staticmethod
231 def load_config(cfg_file_path, # type: str
232 **var_values # type: Any
233 ):
234 # type: (...) -> ClientConfig
235 """
236 Utility method to create a `ClientConfig` from a configuration file (.ini or .cfg, see `ConfigParser`).
237 That configuration file should have a 'global' section, and one section per service named with the service name.
239 :param cfg_file_path: the path to the config file in `ConfigParser` supported format
240 :param var_values: variables to replace in the configuration file. For example `api_key="abcd"` will inject
241 `"abcd"` everywhere where `{{api_key}}` will be found in the file.
242 :return:
243 """
244 # read and apply template
245 contents = read_file_and_apply_template(cfg_file_path, **var_values)
247 # load the config
248 config = ConfigParser()
249 config.read_string(contents, source=cfg_file_path)
251 global_cfg = GlobalConfig()
252 services_cfgs = dict()
254 if PY2: 254 ↛ 255line 254 didn't jump to line 255, because the condition on line 254 was never true
255 _config = config
256 config = OrderedDict()
257 for section_name in _config.sections():
258 config[section_name] = OrderedDict(_config.items(section_name))
259 config['DEFAULT'] = _config.defaults()
261 for section_name, section_contents in config.items():
262 if section_name == 'global':
263 global_cfg = GlobalConfig(**section_contents)
264 elif section_name == 'DEFAULT':
265 if len(section_contents) > 0: 265 ↛ 266line 265 didn't jump to line 266, because the condition on line 265 was never true
266 warn('Configuration contains a DEFAULT section, that will be ignored')
267 else:
268 services_cfgs[section_name] = ServiceConfig(**section_contents)
270 return ClientConfig(global_cfg, **services_cfgs)
273def read_file_and_apply_template(file_path_or_stream, # type: Union[str, IOBase, StringIO]
274 **var_values
275 ):
276 # type: (...) -> str
277 """
279 :param file_path_or_stream:
280 :param var_values:
281 :return:
282 """
283 # first read the file or stream
284 if isinstance(file_path_or_stream, str): 284 ↛ 288line 284 didn't jump to line 288, because the condition on line 284 was never false
285 with open(file_path_or_stream, mode='rt') as f:
286 contents = f.read()
287 else:
288 with file_path_or_stream as f:
289 contents = f.read()
291 return apply_template(contents, **var_values)
294# the jinja2 environment that will be used
295env = Environment(undefined=StrictUndefined)
298class ConfigTemplateSyntaxError(Exception):
299 def __init__(self, contents, idx, original_path):
300 self.extract = contents[idx-30:idx+32]
301 self.original_path = original_path
303 def __str__(self):
304 if self.original_path is not None:
305 tmpstr = "File: %s. " % self.original_path
306 else:
307 tmpstr = ""
308 return "Syntax error in template: a double curly brace remains after template processing. %s" \
309 "Extract: %s" % (tmpstr, self.extract)
312def apply_template(contents, # type: str
313 original_path=None, # type: str
314 **var_values
315 ):
316 # type: (...) -> str
318 # apply the template using Jinja2
319 template = env.from_string(contents)
320 contents = template.render(**var_values)
322 # this check can not be done by Jinja2, do it ourselves
323 if '{{' in contents or '}}' in contents:
324 try:
325 idx = contents.index("{{")
326 except ValueError:
327 idx = contents.index("}}")
328 raise ConfigTemplateSyntaxError(contents, idx, original_path)
330 return contents