azmlclient/clients.py source
1 # Authors: Sylvain MARIE <sylvain.marie@se.com>
2 # + All contributors to <https://github.com/smarie/python-azureml-client>
3 #
4 # License: 3-clause BSD, <https://github.com/smarie/python-azureml-client/blob/master/LICENSE>
5 import sys
6 from contextlib import contextmanager
7 from inspect import getmembers, isroutine
8 from logging import getLogger, StreamHandler, INFO
9 from warnings import warn
10
11 try: # python 3+
12 from configparser import ConfigParser
13 except ImportError:
- F401 'ConfigParser.ConfigParser' imported but unused
15
16 try: # python 3.5+
17 from typing import Dict, List, Callable, Union, Optional
18 from logging import Logger
19 except ImportError:
20 pass
21
22 from decopatch import function_decorator, DECORATED
23 from makefun import wraps
24 from requests import Session
25
26 import pandas as pd
27
28 from azmlclient.clients_callmodes import CallMode, Batch, RequestResponse, LocalCallMode
29 from azmlclient.clients_config import ClientConfig
30 from azmlclient.utils_requests import debug_requests
31
32
33 # default logger that may be used by clients
34 default_logger = getLogger('azmlclient')
35 ch = StreamHandler(sys.stdout)
36 default_logger.addHandler(ch)
37 default_logger.setLevel(INFO)
38
39
40 AZML_SERVICE_ID = '__azml_service__'
41
42
43 class LocalCallModeNotAllowed(Exception):
44 """
45 Exception raised when users call a method corresponding to a
46 """
47 __slots__ = 'f',
48
49 def __init__(self, f):
50 self.f = f
51 super(LocalCallModeNotAllowed, self).__init__()
52
53 def __str__(self):
54 return repr(self)
55
56 def __repr__(self):
57 azml_service_name = get_azureml_service_name(self.f)
58 return "function '%s' (service '%s') is remote-only and can not be executed in local mode " \
59 "(`allow_local=False`). Please change call mode to request-response or batch before using it." \
60 % (self.f.__name__, azml_service_name)
61
62
63 @function_decorator
64 def azureml_service(service_name=None, # type: str
65 remote_only=False, # type: bool
66 f=DECORATED,
67 ):
68 """
69 A decorator for methods in your `AzureMLClient` subclasses, that you should use to indicate that a given method
70 corresponds to an AzureML service. That way, the `AzureMLClient` base class will be able to link this method
71 with local implementation and with the service configuration (url, api key).
72
73 This decorator performs two things:
74 - It wraps the decorated method into a method able to route "local"-mode calls to `self.call_local_service`
75 - It adds the `AZML_SERVICE_ID` attribute with the `service_name` so that the method is known as being
76 AzureML-related, and therefore the appropriate service configuration can be looked up.
77
78 :param service_name: the optional service name appearing in the `AzureMLClient` configuration (`ClientConfig`). By
79 default this is `None` and means that the method name should be used as the service name.
80 :param remote_only: a boolean (default False) indicating if a service should be considered remote-only. If True, an
81 appropriate exception will be raised if the service is used in local mode.
82 """
83 @wraps(f)
84 def f_wrapper(self, # type: AzureMLClient
85 *args,
86 **kwargs):
87 """
88
89 :param self:
90 :param args:
91 :param kwargs:
92 :return:
93 """
94 if self.is_local_mode():
95 if not remote_only:
96 # execute the same method on local implementor rather than client.
97 return self.call_local_service(f.__name__, *args, **kwargs)
98 else:
99 raise LocalCallModeNotAllowed(f_wrapper)
100 else:
101 # execute as usual
102 return f(self, *args, **kwargs)
103
104 # tag the method as being related to an AzureML service with given id
105 setattr(f_wrapper, AZML_SERVICE_ID, service_name)
106 return f_wrapper
107
108
109 def get_azureml_service_name(f):
110 """
111 Returns the AzureML service name associated with method `f`.
112 :param f:
113 :return:
114 """
115 try:
116 # If this is the bound (=instance) method, get the unbound (=class) one
117 if hasattr(f, '__func__'):
118 f = f.__func__
119 azml_name = getattr(f, AZML_SERVICE_ID)
120 except AttributeError:
121 raise ValueError("Method '%s' can not be bound to an AzureML service, please decorate it with "
122 "@azureml_service." % f.__name__)
123 else:
124 return azml_name if azml_name is not None else f.__name__
125
126
127 class AzureMLClient:
128 """
129 Base class for AzureML clients.
130
131 A client is configured with a mandatory `ClientConfig` object describing global and per-service options (endpoint
132 urls, api keys).
133
134 It provides a way to create them from a configuration containing endpoint definitions,
135 and to declare a local implementation
136 """
137
138 def __init__(self,
139 client_config, # type: ClientConfig
140 logger=default_logger, # type: Logger
141 default_call_mode=None, # type: CallMode
142 requests_session=None, # type: Session
143 auto_close_session=None # type: bool
144 ):
145 """
146 Creates an `AzureMLClient` with an initial `ClientConfig` containing the global and per-service configurations.
147
148 Constructor with a global configuration and service endpoint configurations. The service endpoint
149 configurations should be provided in a dictionary with keys being the service names. Only names declared in the
150 'services' meta attribute of the class will be accepted, otherwise and error will be raised. Note that you may
151 provide configurations for some services only.
152
153 A `requests.Session` object is automatically created when the client is created, and possibly configured with
154 the proxy information obtained from the `ClientConfig`. The `Session` is automatically closed when the client
155 instance is garbaged out. A custom `Session` can be passed to the constructor instead. It won't be closed nor
156 configured by default, the user should do it (using `session.close()` and `<config>.configure_session(session)`
157 respectively).
158
159 :param client_config: a configuration for this component client. It should be valid = contain sections for
160 each service in this client. The configuration can contain proxy information, in which case it will
161 be used to configure the underlying requests Session that is created.
162 :param logger:
163 :param default_call_mode: (advanced) if a non-None `CallMode` instance is provided, it will be used as the
164 default call mode for this client. Otherwise by default a request-response call mode will be set as the
165 default call mode (`RequestResponse()`)
166 :param requests_session: (advanced) an optional `Session` object to use (from `requests` lib). If `None` is
167 provided, a new `Session` will be used, possibly configured with the proxy information in the `ClientConfig`
168 and deleted when this object is garbaged out. If a custom object is provided, you should close it yourself
169 or switch `auto_close_session` to `True` explicitly. You should also configure it yourself, for example
170 with `<config>.configure_session(session)`.
171 :param auto_close_session: an optional boolean indicating if `self.session` should be closed when this object
172 is garbaged out. By default this is `None` and means "`True` if no custom `requests_session` is passed, else
173 `False`"). Turning this to `False` can leave hanging Sockets unclosed.
174 """
175 # save the attributes
176 self.client_config = client_config
177 self.logger = logger
178 if default_call_mode is None:
179 # by default make this a request response
180 default_call_mode = RequestResponse()
181 self._current_call_mode = default_call_mode
182
183 # init the local impl property
184 self._local_impl = None
185
186 if requests_session is None:
187 # create and configure a session
188 self.session = Session()
189 self.global_cfg.configure_session(self.session)
190 else:
191 # custom provided : do not configure it
192 self.session = requests_session
193
194 # auto-close behaviour
195 if auto_close_session is None:
196 # default: only auto-close if this session was created by us.
197 auto_close_session = requests_session is None
198 self.auto_close_session = auto_close_session
199
200 def __del__(self):
201 """
202 This is called when the garbage collector deletes this object.
203 Let's use this opportunity to close the requests Session to avoid
204 leaving hanging Sockets, see https://github.com/smarie/python-odsclient/issues/27
205 """
206 if self.auto_close_session and self.session is not None:
207 try:
208 # close the underlying `requests.Session`
209 self.session.close()
210 except Exception as e:
211 warn("Error while closing session: %r" % e)
212
213 # --------- remote service calls implementation
214
215 @property
216 def service_methods(self):
217 """
218 returns a dictionary of all service methods referenced by AzureML service name.
219 These are all methods in the class that have been decorated with `@azureml_service`
220 :return:
221 """
222 return {get_azureml_service_name(v[1]): v[1]
223 for v in getmembers(self.__class__, predicate=lambda x: isroutine(x) and hasattr(x, AZML_SERVICE_ID))}
224
225 @property
226 def service_names(self):
227 """
228 Returns the list of all service names - basically the names of the `service_methods`
229 :return:
230 """
231 return self.service_methods.keys()
232
233 # --------- local implementor
234
235 def __init_local_impl__(self):
236 """
237 Implementors should create a local implementation and return it
238 :return:
239 """
240 raise NotImplementedError("Local execution is not available for this client. Please override "
241 "`__init_local_impl__` or set a non-none `self._local_impl` if you wish local calls "
242 "to be made available")
243
244 @property
245 def local_impl(self):
246 if self._local_impl is None:
247 self._local_impl = self.__init_local_impl__()
248 return self._local_impl
249
250 def call_local_service(self,
251 function_name, # type: str
252 *args, **kwargs):
253 """
254 This method is called automatically when a service method (i.e. decorated with `@azureml_service`)
255 is called and this instance is in "local" mode. It delegates to local.
256
257 :param function_name:
258 :param args:
259 :param kwargs:
260 :return:
261 """
262 local_provider = self.local_impl
263 local_method = getattr(local_provider, function_name)
264 return local_method(*args, **kwargs)
265
266 # --------- configuration
267
268 @property
269 def client_config(self):
270 return self._client_config
271
272 @client_config.setter
273 def client_config(self,
274 client_config # type: ClientConfig
275 ):
276 # validate configuration before accepting it
277 client_config.assert_valid_for_services(self.service_names)
278 self._client_config = client_config
279
280 # ------ convenience methods
281
282 @property
283 def global_cfg(self):
284 return self.client_config.global_config
285
286 @property
287 def services_cfg_dct(self):
288 return self.client_config.services_configs
289
290 # ------ call modes
291 @property
292 def current_call_mode(self):
293 if self._current_call_mode is None:
294 raise ValueError("Current call mode is None. Please set a call mode (local, rr, batch...) by using the "
295 "appropriate context manager")
296 return self._current_call_mode
297
298 @current_call_mode.setter
299 def current_call_mode(self, current_call_mode):
300 self._current_call_mode = current_call_mode
301
302 def is_local_mode(self):
303 """
304
305 :return:
306 """
307 return isinstance(self.current_call_mode, LocalCallMode)
308
309 # --- context managers to switch call mode
310
311 def local_calls(self):
312 """
313 Alias for the `call_mode` context manager to temporarily switch this client to 'local' mode
314
315 >>> with client.local_calls():
316 >>> client.my_service(foo)
317 """
318 return self.call_mode(LocalCallMode())
319
320 def rr_calls(self,
321 use_swagger_format=False # type: bool
322 ):
323 """
324 Alias for the `call_mode` context manager to temporarily switch this client to 'request response' mode
325
326 >>> with client.rr_calls():
327 >>> client.my_service(foo)
328 """
329 return self.call_mode(RequestResponse(use_swagger_format=use_swagger_format))
330
331 def batch_calls(self,
332 polling_period_seconds=5, # type: int
333 ):
334 """
335 Alias for the `call_mode` context manager to temporarily switch this client to 'batch' mode
336
337 >>> with client.batch_calls(polling_period_seconds=5):
338 >>> client.my_service(foo)
339 """
340 return self.call_mode(Batch(polling_period_seconds=polling_period_seconds))
341
342 @contextmanager
343 def call_mode(self,
344 mode # type: CallMode
345 ):
346 """
347 Context manager to temporarily switch this client to `mode` CallMode
348
349 >>> with client.call_mode(Batch(polling_period_seconds=20)):
350 >>> client.my_service(foo)
351
352 :param mode: the `CallMode` to switch to
353 :return:
354 """
355 previous_mode = self.current_call_mode
356 self.current_call_mode = mode
357 yield
358 self.current_call_mode = previous_mode
359
360 def debug_requests(self):
361 """
362 Context manager to temporarily enable debug mode on requests.
363
364 :return:
365 """
366 return debug_requests()
367
368 # ------
369
370 def call_azureml(self,
371 service_id, # type: Union[str, Callable]
372 ws_inputs, # type: Dict[str, pd.DataFrame]
373 ws_output_names, # type: Optional[List[str]]
374 ws_params=None, # type: Dict[str, str]
375 ):
376 """
377 Calls the service identified with id service_id in the services configuration.
378
379 Inputs
380
381 :param service_id: a string identifier or a method representing the service
382 :param ws_inputs: a (name, DataFrame) dictionary of web service inputs
383 :param ws_output_names: a list of web service outputs, or `None` to allow all outputs to be received
384 :param ws_params: a (param_name, value) dictionary of web service parameters
385 :return:
386 """
387 # -- one can provide a method as the service id
388 if callable(service_id):
389 service_id = get_azureml_service_name(service_id)
390
391 # -- Retrieve service configuration
392 if service_id not in self.client_config.services_configs.keys():
393 raise ValueError('Unknown service_id: \'' + service_id + '\'')
394 else:
395 service_config = self.client_config.services_configs[service_id]
396
397 # -- Perform call according to options
398 return self.current_call_mode.call_azureml(service_id,
399 service_config=service_config, ws_inputs=ws_inputs,
400 ws_output_names=ws_output_names, ws_params=ws_params,
401 session=self.session)
402
403
404 def unpack_single_value_from_df(name, # type: str
405 df, # type: pd.DataFrame
406 allow_empty=True # type: bool
407 ):
408 """
409 Utility method to unpack a single value from a DataFrame.
410 If allow_empty is True (default), an empty DataFrame will be accepted and None will be returned.
411
412 :param name: the name of the DataFrame, for validation purposes
413 :param df:
414 :param allow_empty:
415 :return:
416 """
417 values = df.values.ravel()
418 if len(values) == 1:
419 return values[0]
420 elif len(values) == 0 and allow_empty:
421 return None
422 else:
423 raise ValueError("DataFrame '%s' is supposed to contain a single value but does not: \n%s" % (name, df))