azmlclient/clients_config.py source
1 # Authors: Sylvain MARIE <sylvain.marie@se.com>
2 # + All contributors to <https://github.com/smarie/python-azureml-client>
3 #
4 # License: 3-clause BSD, <https://github.com/smarie/python-azureml-client/blob/master/LICENSE>
5 import sys
6 from collections import OrderedDict
7 from distutils.util import strtobool
8 from warnings import warn
9
10 from jinja2 import Environment, StrictUndefined
11
12 try: # python 3+
13 from configparser import ConfigParser
14 except ImportError:
15 from ConfigParser import ConfigParser
16
17 try: # python 3.5+
- F401 'typing.List' imported but unused
- F401 'typing.Callable' imported but unused
- F401 'typing.Optional' imported but unused
- F401 'logging.Logger' imported but unused
20 except ImportError:
21 pass
22
23 from autoclass import autodict
24 from yamlable import YamlAble, yaml_info
25
26 from .requests_utils import set_http_proxy
27
28
29 PY2 = sys.version_info < (3, 0)
30
31
32 YAML_NS = 'org.pypi.azmlclient'
33 """The namespace used for yaml conversion"""
34
35
36 @autodict
37 class GlobalConfig:
38 """
39 Represents a global component client configuration, that is, configuration that is transverse to all web services.
40 """
41 def __init__(self,
42 http_proxy=None, # type: str
43 https_proxy=None, # type: str
44 ssl_verify=True, # type: Union[str, bool]
45 ):
46 """
47 Global information used for all the calls.
48
49 :param http_proxy: an optional string representing the http proxy to use. For example "http://localhost:8888"
50 :param https_proxy: an optional string representing the https proxy to use. For example "http://localhost:8888"
51 :param ssl_verify: a boolean or string representing a boolean, indicating if we should validate the SSL
52 certificate. This is True by default (highly recommended in production)
53 """
54 self.http_proxy = http_proxy
55 self.https_proxy = https_proxy
56 self.ssl_verify = ssl_verify
57
58 def configure_session(self, session):
59 """
60 Helper to get a `requests` (http client) session object, based on the local configuration.
61
62 If the client is configured for use with a proxy the session will be created accordingly.
63 Note that if this client has no particular configuration for the http proxy this function will return None.
64
65 :param session:
66 :return:
67 """
68 use_http_for_https = self.http_proxy and not self.https_proxy
69 set_http_proxy(session, http_url=self.http_proxy, https_url=self.https_proxy,
70 use_http_proxy_for_https_requests=use_http_for_https)
71
72 if self.ssl_verify is not None:
73 try:
74 # try to parse a boolean
75 session.verify = bool(strtobool(self.ssl_verify))
- E722 Do not use bare 'except'
- B001 Do not use bare `except:`, it also catches unexpected events like memory errors, interrupts, system exit, and so on. Prefer `except Exception:`. If you're sure what you're doing, be explicit and write `except BaseException:`.
77 # otherwise this is a path
78 session.verify = self.ssl_verify
79
80
81 @autodict
82 class ServiceConfig:
83 """
84 Represents the configuration to use to interact with an azureml service.
85
86 A service has a main endpoint defined by a base url and an api key.
87 An optional blob storage configuration can be specified to interact with the service in batch mode (BES)
88
89 Finally, an optional alternate endpoint can be specified for "some inputs by reference" calls. In that case the
90 endpoint should correspond to a service able to understand the input references and to retrieve them (this is not a
91 standard AzureML mechanism).
92 """
93 def __init__(self,
94 base_url, # type: str
95 api_key, # type: str
96 by_ref_base_url=None, # type: str
97 by_ref_api_key=None, # type: str
98 blob_account=None, # type: str
99 blob_api_key=None, # type: str
100 blob_container=None, # type: str
101 blob_path_prefix=None # type: str
102 ):
103 """
104 Constructor with
105
106 * an url and api key for normal (anonymized request-response + batch),
107 * an optional url and api key for non-anonymized request-response with input by reference,
108 * an optional account, api key and container name for batch
109
110 :param base_url:
111 :param api_key:
112 :param by_ref_base_url: an alternate URL to use in 'input by reference' mode. If not provided, the base URL
113 will be used.
114 :param by_ref_api_key: an alternate api key to use in 'input by reference' mode. If not provided, the base
115 api key will be used.
116 :param blob_account: an optional blob account that should be used in batch mode. A non-None value has
117 to be provided
118 :param blob_api_key:
119 :param blob_container:
120 :param blob_path_prefix: an optional prefix path for the blobs to be stored
121 """
122 self.base_url = base_url
123 self.api_key = api_key
124
125 self._by_ref_base_url = by_ref_base_url
126 self._by_ref_api_key = by_ref_api_key
127
128 self.blob_account = blob_account
129 self.blob_api_key = blob_api_key
130 self.blob_container = blob_container
131 self.blob_path_prefix = blob_path_prefix
132
133 @property
134 def by_ref_base_url(self):
135 if self._by_ref_base_url is None:
136 return self.base_url
137 else:
138 return self._by_ref_base_url
139
140 @property
141 def by_ref_api_key(self):
142 if self._by_ref_api_key is None:
143 return self.api_key
144 else:
145 return self._by_ref_api_key
146
147
148 @yaml_info(yaml_tag_ns=YAML_NS)
149 @autodict
150 class ClientConfig(YamlAble):
151 """
152 An AzureML client configuration. It is made of two parts:
153
154 * A 'global' configuration (a `GlobalConfig`)
155 * services configurations (one `ServiceConfig` for each. Each is registered under a name that will be used
156 to bind the configuration with the appropriate method in `AzureMLClient`. See `@azureml_service` for details.
157 """
158 def __init__(self,
159 global_config=None, # type: GlobalConfig
160 **services_configs # type: ServiceConfig
161 ):
162 """
163
164 :param global_config: the global configuration, a GlobalConfig
165 :param services_configs: a dictionary of {service_name: ServiceConfig}
166 """
167 if global_config is None:
168 global_config = GlobalConfig()
169 self.global_config = global_config
170
171 self.services_configs = services_configs
172
173 def assert_valid_for_services(self,
174 service_names # type: Iterable[str]
175 ):
176 """
177 Asserts that the configuration corresponds to the list of services provided
178 :param service_names:
179 :return:
180 """
181 unknown_services = set(self.services_configs.keys()) - set(service_names)
182 if len(unknown_services) > 0:
183 raise ValueError("Configuration is not able to handle services: '" + str(unknown_services)
184 + "'. The list of services supported by this client is '" + str(service_names)
185 + "'")
186
187 # ---- yamlable interface ----
188
189 def __to_yaml_dict__(self):
190 # type: (...) -> Dict[str, Any]
191 """ This optional method is called when you call yaml.dump(). See `yamlable` for details."""
192 # notes:
193 # - we do not make `GlobalConfig` and `ServiceConfig` yamlable objects because their custom yaml names would
194 # have to appear in the configuration, which seems tideous
195 # - we use `dict()` not `var()` so that we benefit from their `@autodict` capability to hide private fields
196 return {'global': dict(self.global_config),
197 'services': {service_name: dict(service) for service_name, service in self.services_configs.items()}}
198
199 @classmethod
200 def __from_yaml_dict__(cls, dct, yaml_tag):
201 # type: (...) -> ClientConfig
202 """ This optional method is called when you call yaml.load(). See `yamlable` for details."""
203 global_cfg = GlobalConfig(**dct['global'])
204 services_cfg = {service_name: ServiceConfig(**service_cfg_dct)
205 for service_name, service_cfg_dct in dct['services'].items()}
206 return ClientConfig(global_cfg, **services_cfg)
207
208 @classmethod
- F821 Undefined name 'Type'
- F821 Undefined name 'Y'
- F821 Undefined name 'IOBase'
- F821 Undefined name 'StringIO'
- F723 Syntax error in type comment ''
212 **var_values # type: Any
- F821 Undefined name 'Y'
214 """ applies the template before loading """
215 contents = read_file_and_apply_template(file_path_or_stream, **var_values)
216 return YamlAble.loads_yaml(contents, safe=safe)
217
218 @classmethod
- F821 Undefined name 'Type'
- F821 Undefined name 'Y'
220 yaml_str, # type: str
221 safe=True, # type: bool
222 **var_values # type: Any
- F821 Undefined name 'Y'
224 """ applies the template before loading """
225 contents = apply_template(yaml_str, **var_values)
226 return YamlAble.loads_yaml(contents, safe=safe)
227
228 # ---- configparser interface ----
229
230 @staticmethod
231 def load_config(cfg_file_path, # type: str
232 **var_values # type: Any
233 ):
234 # type: (...) -> ClientConfig
235 """
236 Utility method to create a `ClientConfig` from a configuration file (.ini or .cfg, see `ConfigParser`).
237 That configuration file should have a 'global' section, and one section per service named with the service name.
238
239 :param cfg_file_path: the path to the config file in `ConfigParser` supported format
240 :param var_values: variables to replace in the configuration file. For example `api_key="abcd"` will inject
241 `"abcd"` everywhere where `{{api_key}}` will be found in the file.
242 :return:
243 """
244 # read and apply template
245 contents = read_file_and_apply_template(cfg_file_path, **var_values)
246
247 # load the config
248 config = ConfigParser()
249 config.read_string(contents, source=cfg_file_path)
250
251 global_cfg = GlobalConfig()
252 services_cfgs = dict()
253
254 if PY2:
255 _config = config
256 config = OrderedDict()
257 for section_name in _config.sections():
258 config[section_name] = OrderedDict(_config.items(section_name))
259 config['DEFAULT'] = _config.defaults()
260
261 for section_name, section_contents in config.items():
262 if section_name == 'global':
263 global_cfg = GlobalConfig(**section_contents)
264 elif section_name == 'DEFAULT':
265 if len(section_contents) > 0:
266 warn('Configuration contains a DEFAULT section, that will be ignored')
267 else:
268 services_cfgs[section_name] = ServiceConfig(**section_contents)
269
270 return ClientConfig(global_cfg, **services_cfgs)
271
272
- F821 Undefined name 'IOBase'
- F821 Undefined name 'StringIO'
274 **var_values
275 ):
276 # type: (...) -> str
277 """
- W293 Blank line contains whitespace
279 :param file_path_or_stream:
- W291 Trailing whitespace
- W291 Trailing whitespace
282 """
283 # first read the file or stream
284 if isinstance(file_path_or_stream, str):
285 with open(file_path_or_stream, mode='rt') as f:
286 contents = f.read()
287 else:
288 with file_path_or_stream as f:
289 contents = f.read()
290
291 return apply_template(contents, **var_values)
292
293
294 # the jinja2 environment that will be used
- S701 By default, jinja2 sets autoescape to False. Consider using autoescape=True or use the select_autoescape function to mitigate XSS vulnerabilities.
296
297
298 class ConfigTemplateSyntaxError(Exception):
299 def __init__(self, contents, idx, original_path):
300 self.extract = contents[idx-30:idx+32]
301 self.original_path = original_path
302
303 def __str__(self):
304 if self.original_path is not None:
305 tmpstr = "File: %s. " % self.original_path
306 else:
307 tmpstr = ""
308 return "Syntax error in template: a double curly brace remains after template processing. %s" \
309 "Extract: %s" % (tmpstr, self.extract)
310
311
312 def apply_template(contents, # type: str
313 original_path=None, # type: str
314 **var_values
315 ):
316 # type: (...) -> str
317
318 # apply the template using Jinja2
319 template = env.from_string(contents)
320 contents = template.render(**var_values)
321
322 # this check can not be done by Jinja2, do it ourselves
323 if '{{' in contents or '}}' in contents:
324 try:
325 idx = contents.index("{{")
326 except ValueError:
327 idx = contents.index("}}")
328 raise ConfigTemplateSyntaxError(contents, idx, original_path)
329
330 return contents