Coverage for azmlclient/clients.py : 83%
Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# Authors: Sylvain MARIE <sylvain.marie@se.com>
2# + All contributors to <https://github.com/smarie/python-azureml-client>
3#
4# License: 3-clause BSD, <https://github.com/smarie/python-azureml-client/blob/master/LICENSE>
5import sys
6from contextlib import contextmanager
7from inspect import getmembers, isroutine
8from logging import getLogger, StreamHandler, INFO
9from warnings import warn
11try: # python 3+
12 from configparser import ConfigParser
13except ImportError:
14 from ConfigParser import ConfigParser
16try: # python 3.5+
17 from typing import Dict, List, Callable, Union, Optional
18 from logging import Logger
19except ImportError:
20 pass
22from decopatch import function_decorator, DECORATED
23from makefun import wraps
24from requests import Session
26import pandas as pd
28from azmlclient.clients_callmodes import CallMode, Batch, RequestResponse, LocalCallMode
29from azmlclient.clients_config import ClientConfig
30from azmlclient.utils_requests import debug_requests
33# default logger that may be used by clients
34default_logger = getLogger('azmlclient')
35ch = StreamHandler(sys.stdout)
36default_logger.addHandler(ch)
37default_logger.setLevel(INFO)
40AZML_SERVICE_ID = '__azml_service__'
43class LocalCallModeNotAllowed(Exception):
44 """
45 Exception raised when users call a method corresponding to a
46 """
47 __slots__ = 'f',
49 def __init__(self, f):
50 self.f = f
51 super(LocalCallModeNotAllowed, self).__init__()
53 def __str__(self):
54 return repr(self)
56 def __repr__(self):
57 azml_service_name = get_azureml_service_name(self.f)
58 return "function '%s' (service '%s') is remote-only and can not be executed in local mode " \
59 "(`allow_local=False`). Please change call mode to request-response or batch before using it." \
60 % (self.f.__name__, azml_service_name)
63@function_decorator
64def azureml_service(service_name=None, # type: str
65 remote_only=False, # type: bool
66 f=DECORATED,
67 ):
68 """
69 A decorator for methods in your `AzureMLClient` subclasses, that you should use to indicate that a given method
70 corresponds to an AzureML service. That way, the `AzureMLClient` base class will be able to link this method
71 with local implementation and with the service configuration (url, api key).
73 This decorator performs two things:
74 - It wraps the decorated method into a method able to route "local"-mode calls to `self.call_local_service`
75 - It adds the `AZML_SERVICE_ID` attribute with the `service_name` so that the method is known as being
76 AzureML-related, and therefore the appropriate service configuration can be looked up.
78 :param service_name: the optional service name appearing in the `AzureMLClient` configuration (`ClientConfig`). By
79 default this is `None` and means that the method name should be used as the service name.
80 :param remote_only: a boolean (default False) indicating if a service should be considered remote-only. If True, an
81 appropriate exception will be raised if the service is used in local mode.
82 """
83 @wraps(f)
84 def f_wrapper(self, # type: AzureMLClient
85 *args,
86 **kwargs):
87 """
89 :param self:
90 :param args:
91 :param kwargs:
92 :return:
93 """
94 if self.is_local_mode():
95 if not remote_only:
96 # execute the same method on local implementor rather than client.
97 return self.call_local_service(f.__name__, *args, **kwargs)
98 else:
99 raise LocalCallModeNotAllowed(f_wrapper)
100 else:
101 # execute as usual
102 return f(self, *args, **kwargs)
104 # tag the method as being related to an AzureML service with given id
105 setattr(f_wrapper, AZML_SERVICE_ID, service_name)
106 return f_wrapper
109def get_azureml_service_name(f):
110 """
111 Returns the AzureML service name associated with method `f`.
112 :param f:
113 :return:
114 """
115 try:
116 # If this is the bound (=instance) method, get the unbound (=class) one
117 if hasattr(f, '__func__'):
118 f = f.__func__
119 azml_name = getattr(f, AZML_SERVICE_ID)
120 except AttributeError:
121 raise ValueError("Method '%s' can not be bound to an AzureML service, please decorate it with "
122 "@azureml_service." % f.__name__)
123 else:
124 return azml_name if azml_name is not None else f.__name__
127class AzureMLClient:
128 """
129 Base class for AzureML clients.
131 A client is configured with a mandatory `ClientConfig` object describing global and per-service options (endpoint
132 urls, api keys).
134 It provides a way to create them from a configuration containing endpoint definitions,
135 and to declare a local implementation
136 """
138 def __init__(self,
139 client_config, # type: ClientConfig
140 logger=default_logger, # type: Logger
141 default_call_mode=None, # type: CallMode
142 requests_session=None, # type: Session
143 auto_close_session=None # type: bool
144 ):
145 """
146 Creates an `AzureMLClient` with an initial `ClientConfig` containing the global and per-service configurations.
148 Constructor with a global configuration and service endpoint configurations. The service endpoint
149 configurations should be provided in a dictionary with keys being the service names. Only names declared in the
150 'services' meta attribute of the class will be accepted, otherwise and error will be raised. Note that you may
151 provide configurations for some services only.
153 A `requests.Session` object is automatically created when the client is created, and possibly configured with
154 the proxy information obtained from the `ClientConfig`. The `Session` is automatically closed when the client
155 instance is garbaged out. A custom `Session` can be passed to the constructor instead. It won't be closed nor
156 configured by default, the user should do it (using `session.close()` and `<config>.configure_session(session)`
157 respectively).
159 :param client_config: a configuration for this component client. It should be valid = contain sections for
160 each service in this client. The configuration can contain proxy information, in which case it will
161 be used to configure the underlying requests Session that is created.
162 :param logger:
163 :param default_call_mode: (advanced) if a non-None `CallMode` instance is provided, it will be used as the
164 default call mode for this client. Otherwise by default a request-response call mode will be set as the
165 default call mode (`RequestResponse()`)
166 :param requests_session: (advanced) an optional `Session` object to use (from `requests` lib). If `None` is
167 provided, a new `Session` will be used, possibly configured with the proxy information in the `ClientConfig`
168 and deleted when this object is garbaged out. If a custom object is provided, you should close it yourself
169 or switch `auto_close_session` to `True` explicitly. You should also configure it yourself, for example
170 with `<config>.configure_session(session)`.
171 :param auto_close_session: an optional boolean indicating if `self.session` should be closed when this object
172 is garbaged out. By default this is `None` and means "`True` if no custom `requests_session` is passed, else
173 `False`"). Turning this to `False` can leave hanging Sockets unclosed.
174 """
175 # save the attributes
176 self.client_config = client_config
177 self.logger = logger
178 if default_call_mode is None: 178 ↛ 181line 178 didn't jump to line 181, because the condition on line 178 was never false
179 # by default make this a request response
180 default_call_mode = RequestResponse()
181 self._current_call_mode = default_call_mode
183 # init the local impl property
184 self._local_impl = None
186 if requests_session is None: 186 ↛ 192line 186 didn't jump to line 192, because the condition on line 186 was never false
187 # create and configure a session
188 self.session = Session()
189 self.global_cfg.configure_session(self.session)
190 else:
191 # custom provided : do not configure it
192 self.session = requests_session
194 # auto-close behaviour
195 if auto_close_session is None: 195 ↛ 198line 195 didn't jump to line 198, because the condition on line 195 was never false
196 # default: only auto-close if this session was created by us.
197 auto_close_session = requests_session is None
198 self.auto_close_session = auto_close_session
200 def __del__(self):
201 """
202 This is called when the garbage collector deletes this object.
203 Let's use this opportunity to close the requests Session to avoid
204 leaving hanging Sockets, see https://github.com/smarie/python-odsclient/issues/27
205 """
206 if self.auto_close_session and self.session is not None: 206 ↛ exitline 206 didn't return from function '__del__', because the condition on line 206 was never false
207 try:
208 # close the underlying `requests.Session`
209 self.session.close()
210 except Exception as e:
211 warn("Error while closing session: %r" % e)
213 # --------- remote service calls implementation
215 @property
216 def service_methods(self):
217 """
218 returns a dictionary of all service methods referenced by AzureML service name.
219 These are all methods in the class that have been decorated with `@azureml_service`
220 :return:
221 """
222 return {get_azureml_service_name(v[1]): v[1]
223 for v in getmembers(self.__class__, predicate=lambda x: isroutine(x) and hasattr(x, AZML_SERVICE_ID))}
225 @property
226 def service_names(self):
227 """
228 Returns the list of all service names - basically the names of the `service_methods`
229 :return:
230 """
231 return self.service_methods.keys()
233 # --------- local implementor
235 def __init_local_impl__(self):
236 """
237 Implementors should create a local implementation and return it
238 :return:
239 """
240 raise NotImplementedError("Local execution is not available for this client. Please override "
241 "`__init_local_impl__` or set a non-none `self._local_impl` if you wish local calls "
242 "to be made available")
244 @property
245 def local_impl(self):
246 if self._local_impl is None: 246 ↛ 248line 246 didn't jump to line 248, because the condition on line 246 was never false
247 self._local_impl = self.__init_local_impl__()
248 return self._local_impl
250 def call_local_service(self,
251 function_name, # type: str
252 *args, **kwargs):
253 """
254 This method is called automatically when a service method (i.e. decorated with `@azureml_service`)
255 is called and this instance is in "local" mode. It delegates to local.
257 :param function_name:
258 :param args:
259 :param kwargs:
260 :return:
261 """
262 local_provider = self.local_impl
263 local_method = getattr(local_provider, function_name)
264 return local_method(*args, **kwargs)
266 # --------- configuration
268 @property
269 def client_config(self):
270 return self._client_config
272 @client_config.setter
273 def client_config(self,
274 client_config # type: ClientConfig
275 ):
276 # validate configuration before accepting it
277 client_config.assert_valid_for_services(self.service_names)
278 self._client_config = client_config
280 # ------ convenience methods
282 @property
283 def global_cfg(self):
284 return self.client_config.global_config
286 @property
287 def services_cfg_dct(self):
288 return self.client_config.services_configs
290 # ------ call modes
291 @property
292 def current_call_mode(self):
293 if self._current_call_mode is None: 293 ↛ 294line 293 didn't jump to line 294, because the condition on line 293 was never true
294 raise ValueError("Current call mode is None. Please set a call mode (local, rr, batch...) by using the "
295 "appropriate context manager")
296 return self._current_call_mode
298 @current_call_mode.setter
299 def current_call_mode(self, current_call_mode):
300 self._current_call_mode = current_call_mode
302 def is_local_mode(self):
303 """
305 :return:
306 """
307 return isinstance(self.current_call_mode, LocalCallMode)
309 # --- context managers to switch call mode
311 def local_calls(self):
312 """
313 Alias for the `call_mode` context manager to temporarily switch this client to 'local' mode
315 >>> with client.local_calls():
316 >>> client.my_service(foo)
317 """
318 return self.call_mode(LocalCallMode())
320 def rr_calls(self,
321 use_swagger_format=False # type: bool
322 ):
323 """
324 Alias for the `call_mode` context manager to temporarily switch this client to 'request response' mode
326 >>> with client.rr_calls():
327 >>> client.my_service(foo)
328 """
329 return self.call_mode(RequestResponse(use_swagger_format=use_swagger_format))
331 def batch_calls(self,
332 polling_period_seconds=5, # type: int
333 ):
334 """
335 Alias for the `call_mode` context manager to temporarily switch this client to 'batch' mode
337 >>> with client.batch_calls(polling_period_seconds=5):
338 >>> client.my_service(foo)
339 """
340 return self.call_mode(Batch(polling_period_seconds=polling_period_seconds))
342 @contextmanager
343 def call_mode(self,
344 mode # type: CallMode
345 ):
346 """
347 Context manager to temporarily switch this client to `mode` CallMode
349 >>> with client.call_mode(Batch(polling_period_seconds=20)):
350 >>> client.my_service(foo)
352 :param mode: the `CallMode` to switch to
353 :return:
354 """
355 previous_mode = self.current_call_mode
356 self.current_call_mode = mode
357 yield
358 self.current_call_mode = previous_mode
360 def debug_requests(self):
361 """
362 Context manager to temporarily enable debug mode on requests.
364 :return:
365 """
366 return debug_requests()
368 # ------
370 def call_azureml(self,
371 service_id, # type: Union[str, Callable]
372 ws_inputs, # type: Dict[str, pd.DataFrame]
373 ws_output_names, # type: Optional[List[str]]
374 ws_params=None, # type: Dict[str, str]
375 ):
376 """
377 Calls the service identified with id service_id in the services configuration.
379 Inputs
381 :param service_id: a string identifier or a method representing the service
382 :param ws_inputs: a (name, DataFrame) dictionary of web service inputs
383 :param ws_output_names: a list of web service outputs, or `None` to allow all outputs to be received
384 :param ws_params: a (param_name, value) dictionary of web service parameters
385 :return:
386 """
387 # -- one can provide a method as the service id
388 if callable(service_id): 388 ↛ 392line 388 didn't jump to line 392, because the condition on line 388 was never false
389 service_id = get_azureml_service_name(service_id)
391 # -- Retrieve service configuration
392 if service_id not in self.client_config.services_configs.keys(): 392 ↛ 393line 392 didn't jump to line 393, because the condition on line 392 was never true
393 raise ValueError('Unknown service_id: \'' + service_id + '\'')
394 else:
395 service_config = self.client_config.services_configs[service_id]
397 # -- Perform call according to options
398 return self.current_call_mode.call_azureml(service_id,
399 service_config=service_config, ws_inputs=ws_inputs,
400 ws_output_names=ws_output_names, ws_params=ws_params,
401 session=self.session)
404def unpack_single_value_from_df(name, # type: str
405 df, # type: pd.DataFrame
406 allow_empty=True # type: bool
407 ):
408 """
409 Utility method to unpack a single value from a DataFrame.
410 If allow_empty is True (default), an empty DataFrame will be accepted and None will be returned.
412 :param name: the name of the DataFrame, for validation purposes
413 :param df:
414 :param allow_empty:
415 :return:
416 """
417 values = df.values.ravel()
418 if len(values) == 1:
419 return values[0]
420 elif len(values) == 0 and allow_empty:
421 return None
422 else:
423 raise ValueError("DataFrame '%s' is supposed to contain a single value but does not: \n%s" % (name, df))