1
2
3
4
5
6
7
8
9 """Base class for data measures: algorithms that quantify properties of
10 datasets.
11
12 Besides the `DatasetMeasure` base class this module also provides the
13 (abstract) `FeaturewiseDatasetMeasure` class. The difference between a general
14 measure and the output of the `FeaturewiseDatasetMeasure` is that the latter
15 returns a 1d map (one value per feature in the dataset). In contrast there are
16 no restrictions on the returned value of `DatasetMeasure` except for that it
17 has to be in some iterable container.
18
19 """
20
21 __docformat__ = 'restructuredtext'
22
23 import numpy as N
24 import mvpa.support.copy as copy
25
26 from mvpa.misc.state import StateVariable, ClassWithCollections
27 from mvpa.misc.args import group_kwargs
28 from mvpa.misc.transformers import FirstAxisMean, SecondAxisSumOfAbs
29 from mvpa.base.dochelpers import enhancedDocString
30 from mvpa.base import externals
31 from mvpa.clfs.stats import autoNullDist
32
33 if __debug__:
34 from mvpa.base import debug
38 """A measure computed from a `Dataset`
39
40 All dataset measures support arbitrary transformation of the measure
41 after it has been computed. Transformation are done by processing the
42 measure with a functor that is specified via the `transformer` keyword
43 argument of the constructor. Upon request, the raw measure (before
44 transformations are applied) is stored in the `raw_result` state variable.
45
46 Additionally all dataset measures support the estimation of the
47 probabilit(y,ies) of a measure under some distribution. Typically this will
48 be the NULL distribution (no signal), that can be estimated with
49 permutation tests. If a distribution estimator instance is passed to the
50 `null_dist` keyword argument of the constructor the respective
51 probabilities are automatically computed and stored in the `null_prob`
52 state variable.
53
54 .. note::
55 For developers: All subclasses shall get all necessary parameters via
56 their constructor, so it is possible to get the same type of measure for
57 multiple datasets by passing them to the __call__() method successively.
58 """
59
60 raw_result = StateVariable(enabled=False,
61 doc="Computed results before applying any " +
62 "transformation algorithm")
63 null_prob = StateVariable(enabled=True)
64 """Stores the probability of a measure under the NULL hypothesis"""
65 null_t = StateVariable(enabled=False)
66 """Stores the t-score corresponding to null_prob under assumption
67 of Normal distribution"""
68
69 - def __init__(self, transformer=None, null_dist=None, **kwargs):
70 """Does nothing special.
71
72 :Parameter:
73 transformer: Functor
74 This functor is called in `__call__()` to perform a final
75 processing step on the to be returned dataset measure. If None,
76 nothing is called
77 null_dist: instance of distribution estimator
78 The estimated distribution is used to assign a probability for a
79 certain value of the computed measure.
80 """
81 ClassWithCollections.__init__(self, **kwargs)
82
83 self.__transformer = transformer
84 """Functor to be called in return statement of all subclass __call__()
85 methods."""
86 null_dist_ = autoNullDist(null_dist)
87 if __debug__:
88 debug('SA', 'Assigning null_dist %s whenever original given was %s'
89 % (null_dist_, null_dist))
90 self.__null_dist = null_dist_
91
92
93 __doc__ = enhancedDocString('DatasetMeasure', locals(), ClassWithCollections)
94
95
97 """Compute measure on a given `Dataset`.
98
99 Each implementation has to handle a single arguments: the source
100 dataset.
101
102 Returns the computed measure in some iterable (list-like)
103 container applying transformer if such is defined
104 """
105 result = self._call(dataset)
106 result = self._postcall(dataset, result)
107 return result
108
109
110 - def _call(self, dataset):
111 """Actually compute measure on a given `Dataset`.
112
113 Each implementation has to handle a single arguments: the source
114 dataset.
115
116 Returns the computed measure in some iterable (list-like) container.
117 """
118 raise NotImplemented
119
120
121 - def _postcall(self, dataset, result):
122 """Some postprocessing on the result
123 """
124 self.raw_result = result
125 if not self.__transformer is None:
126 if __debug__:
127 debug("SA_", "Applying transformer %s" % self.__transformer)
128 result = self.__transformer(result)
129
130
131 if not self.__null_dist is None:
132 if __debug__:
133 debug("SA_", "Estimating NULL distribution using %s"
134 % self.__null_dist)
135
136
137
138
139 measure = copy.copy(self)
140 measure.__null_dist = None
141 self.__null_dist.fit(measure, dataset)
142
143 if self.states.isEnabled('null_t'):
144
145
146 null_prob, null_right_tail = \
147 self.__null_dist.p(result, return_tails=True)
148 self.null_prob = null_prob
149
150 externals.exists('scipy', raiseException=True)
151 from scipy.stats import norm
152
153
154
155 tail = self.null_dist.tail
156 if tail == 'left':
157 acdf = N.abs(null_prob)
158 elif tail == 'right':
159 acdf = 1.0 - N.abs(null_prob)
160 elif tail in ['any', 'both']:
161 acdf = 1.0 - N.clip(N.abs(null_prob), 0, 0.5)
162 else:
163 raise RuntimeError, 'Unhandled tail %s' % tail
164
165
166
167
168
169
170
171 clip = 1e-16
172 null_t = norm.ppf(N.clip(acdf, clip, 1.0 - clip))
173 null_t[~null_right_tail] *= -1.0
174 self.null_t = null_t
175 else:
176
177
178 self.null_prob = self.__null_dist.p(result)
179
180 return result
181
182
184 """String representation of DatasetMeasure
185
186 Includes only arguments which differ from default ones
187 """
188 prefixes = prefixes[:]
189 if self.__transformer is not None:
190 prefixes.append("transformer=%s" % self.__transformer)
191 if self.__null_dist is not None:
192 prefixes.append("null_dist=%s" % self.__null_dist)
193 return super(DatasetMeasure, self).__repr__(prefixes=prefixes)
194
195
196 @property
198 """Return Null Distribution estimator"""
199 return self.__null_dist
200
201 @property
205
208 """A per-feature-measure computed from a `Dataset` (base class).
209
210 Should behave like a DatasetMeasure.
211 """
212
213 base_sensitivities = StateVariable(enabled=False,
214 doc="Stores basic sensitivities if the sensitivity " +
215 "relies on combining multiple ones")
216
217
218
219
220
221
222
223
224
225
226
227
228
230 """Initialize
231
232 :Parameters:
233 combiner : Functor
234 The combiner is only applied if the computed featurewise dataset
235 measure is more than one-dimensional. This is different from a
236 `transformer`, which is always applied. By default, the sum of
237 absolute values along the second axis is computed.
238 """
239 DatasetMeasure.__init__(self, **kwargs)
240
241 self.__combiner = combiner
242
250
251
252 - def _call(self, dataset):
253 """Computes a per-feature-measure on a given `Dataset`.
254
255 Behaves like a `DatasetMeasure`, but computes and returns a 1d ndarray
256 with one value per feature.
257 """
258 raise NotImplementedError
259
260
261 - def _postcall(self, dataset, result):
262 """Adjusts per-feature-measure for computed `result`
263
264
265 TODO: overlaps in what it does heavily with
266 CombinedSensitivityAnalyzer, thus this one might make use of
267 CombinedSensitivityAnalyzer yoh thinks, and here
268 base_sensitivities doesn't sound appropriate.
269 MH: There is indeed some overlap, but also significant differences.
270 This one operates on a single sensana and combines over second
271 axis, CombinedFeaturewiseDatasetMeasure uses first axis.
272 Additionally, 'Sensitivity' base class is
273 FeaturewiseDatasetMeasures which would have to be changed to
274 CombinedFeaturewiseDatasetMeasure to deal with stuff like
275 SMLRWeights that return multiple sensitivity values by default.
276 Not sure if unification of both (and/or removal of functionality
277 here does not lead to an overall more complicated situation,
278 without any real gain -- after all this one works ;-)
279 """
280 result_sq = result.squeeze()
281 if len(result_sq.shape)>1:
282 n_base = result.shape[1]
283 """Number of base sensitivities"""
284 if self.states.isEnabled('base_sensitivities'):
285 b_sensitivities = []
286 if not self.states.isKnown('biases'):
287 biases = None
288 else:
289 biases = self.biases
290 if len(self.biases) != n_base:
291 raise ValueError, \
292 "Number of biases %d is " % len(self.biases) \
293 + "different from number of base sensitivities" \
294 + "%d" % n_base
295 for i in xrange(n_base):
296 if not biases is None:
297 bias = biases[i]
298 else:
299 bias = None
300 b_sensitivities = StaticDatasetMeasure(
301 measure = result[:,i],
302 bias = bias)
303 self.base_sensitivities = b_sensitivities
304
305
306
307 if self.__combiner is not None:
308 result = self.__combiner(result)
309 else:
310
311
312
313 result = result_sq
314
315
316 result = DatasetMeasure._postcall(self, dataset, result)
317
318 return result
319
320 @property
322 """Return combiner"""
323 return self.__combiner
324
328 """A static (assigned) sensitivity measure.
329
330 Since implementation is generic it might be per feature or
331 per whole dataset
332 """
333
334 - def __init__(self, measure=None, bias=None, *args, **kwargs):
335 """Initialize.
336
337 :Parameters:
338 measure
339 actual sensitivity to be returned
340 bias
341 optionally available bias
342 """
343 DatasetMeasure.__init__(self, *args, **kwargs)
344 if measure is None:
345 raise ValueError, "Sensitivity measure has to be provided"
346 self.__measure = measure
347 self.__bias = bias
348
349 - def _call(self, dataset):
350 """Returns assigned sensitivity
351 """
352 return self.__measure
353
354
355 bias = property(fget=lambda self:self.__bias)
356
357
358
359
360
361
362 -class Sensitivity(FeaturewiseDatasetMeasure):
363
364 _LEGAL_CLFS = []
365 """If Sensitivity is classifier specific, classes of classifiers
366 should be listed in the list
367 """
368
369 - def __init__(self, clf, force_training=True, **kwargs):
370 """Initialize the analyzer with the classifier it shall use.
371
372 :Parameters:
373 clf : :class:`Classifier`
374 classifier to use.
375 force_training : Bool
376 if classifier was already trained -- do not retrain
377 """
378
379 """Does nothing special."""
380 FeaturewiseDatasetMeasure.__init__(self, **kwargs)
381
382 _LEGAL_CLFS = self._LEGAL_CLFS
383 if len(_LEGAL_CLFS) > 0:
384 found = False
385 for clf_class in _LEGAL_CLFS:
386 if isinstance(clf, clf_class):
387 found = True
388 break
389 if not found:
390 raise ValueError, \
391 "Classifier %s has to be of allowed class (%s), but is %s" \
392 % (clf, _LEGAL_CLFS, `type(clf)`)
393
394 self.__clf = clf
395 """Classifier used to computed sensitivity"""
396
397 self._force_training = force_training
398 """Either to force it to train"""
399
401 if prefixes is None:
402 prefixes = []
403 prefixes.append("clf=%s" % repr(self.clf))
404 if not self._force_training:
405 prefixes.append("force_training=%s" % self._force_training)
406 return super(Sensitivity, self).__repr__(prefixes=prefixes)
407
408
410 """Train classifier on `dataset` and then compute actual sensitivity.
411
412 If the classifier is already trained it is possible to extract the
413 sensitivities without passing a dataset.
414 """
415
416 clf = self.__clf
417 if not clf.trained or self._force_training:
418 if dataset is None:
419 raise ValueError, \
420 "Training classifier to compute sensitivities requires " \
421 "a dataset."
422 if __debug__:
423 debug("SA", "Training classifier %s %s" %
424 (`clf`,
425 {False: "since it wasn't yet trained",
426 True: "although it was trained previousely"}
427 [clf.trained]))
428 clf.train(dataset)
429
430 return FeaturewiseDatasetMeasure.__call__(self, dataset)
431
432
435
436
437 @property
439 """Return feature_ids used by the underlying classifier
440 """
441 return self.__clf._getFeatureIds()
442
443
444 clf = property(fget=lambda self:self.__clf,
445 fset=_setClassifier)
446
450 """Set sensitivity analyzers to be merged into a single output"""
451
452 sensitivities = StateVariable(enabled=False,
453 doc="Sensitivities produced by each analyzer")
454
455
456
457
458 - def __init__(self, analyzers=None,
459 combiner=None,
460 **kwargs):
461 """Initialize CombinedFeaturewiseDatasetMeasure
462
463 :Parameters:
464 analyzers : list or None
465 List of analyzers to be used. There is no logic to populate
466 such a list in __call__, so it must be either provided to
467 the constructor or assigned to .analyzers prior calling
468 """
469 if analyzers is None:
470 analyzers = []
471
472 FeaturewiseDatasetMeasure.__init__(self, **kwargs)
473 self.__analyzers = analyzers
474 """List of analyzers to use"""
475
476 self.__combiner = combiner
477 """Which functor to use to combine all sensitivities"""
478
479
480 - def _call(self, dataset):
501
502
504 """Set the analyzers
505 """
506 self.__analyzers = analyzers
507 """Analyzers to use"""
508
509 analyzers = property(fget=lambda x:x.__analyzers,
510 fset=_setAnalyzers,
511 doc="Used analyzers")
512
519 """Compute measures across splits for a specific analyzer"""
520
521
522
523
524 sensitivities = StateVariable(enabled=False,
525 doc="Sensitivities produced for each split")
526
527 splits = StateVariable(enabled=False, doc=
528 """Store the actual splits of the data. Can be memory expensive""")
529
530 - def __init__(self, splitter, analyzer,
531 insplit_index=0, combiner=None, **kwargs):
532 """Initialize SplitFeaturewiseDatasetMeasure
533
534 :Parameters:
535 splitter : Splitter
536 Splitter to use to split the dataset
537 analyzer : DatasetMeasure
538 Measure to be used. Could be analyzer as well (XXX)
539 insplit_index : int
540 splitter generates tuples of dataset on each iteration
541 (usually 0th for training, 1st for testing).
542 On what split index in that tuple to operate.
543 """
544
545
546
547
548
549
550
551
552 FeaturewiseDatasetMeasure.__init__(self, combiner=None, **kwargs)
553
554 self.__analyzer = analyzer
555 """Analyzer to use per split"""
556
557 self.__combiner = combiner
558 """Which functor to use to combine all sensitivities"""
559
560 self.__splitter = splitter
561 """Splitter to be used on the dataset"""
562
563 self.__insplit_index = insplit_index
564
565 - def _call(self, dataset):
596
599 """Set sensitivity analyzers to be merged into a single output"""
600
601
602
603 @group_kwargs(prefixes=['slave_'], assign=True)
604 - def __init__(self,
605 clf,
606 analyzer=None,
607 combined_analyzer=None,
608 slave_kwargs={},
609 **kwargs):
610 """Initialize Sensitivity Analyzer for `BoostedClassifier`
611
612 :Parameters:
613 clf : `BoostedClassifier`
614 Classifier to be used
615 analyzer : analyzer
616 Is used to populate combined_analyzer
617 slave_*
618 Arguments to pass to created analyzer if analyzer is None
619 """
620 Sensitivity.__init__(self, clf, **kwargs)
621 if combined_analyzer is None:
622
623 kwargs.pop('force_training', None)
624 combined_analyzer = CombinedFeaturewiseDatasetMeasure(**kwargs)
625 self.__combined_analyzer = combined_analyzer
626 """Combined analyzer to use"""
627
628 if analyzer is not None and len(self._slave_kwargs):
629 raise ValueError, \
630 "Provide either analyzer of slave_* arguments, not both"
631 self.__analyzer = analyzer
632 """Analyzer to use for basic classifiers within boosted classifier"""
633
634
635 - def _call(self, dataset):
666
667 combined_analyzer = property(fget=lambda x:x.__combined_analyzer)
668
671 """Set sensitivity analyzer output just to pass through"""
672
673 clf_sensitivities = StateVariable(enabled=False,
674 doc="Stores sensitivities of the proxied classifier")
675
676
677 @group_kwargs(prefixes=['slave_'], assign=True)
678 - def __init__(self,
679 clf,
680 analyzer=None,
681 **kwargs):
682 """Initialize Sensitivity Analyzer for `BoostedClassifier`
683 """
684 Sensitivity.__init__(self, clf, **kwargs)
685
686 if analyzer is not None and len(self._slave_kwargs):
687 raise ValueError, \
688 "Provide either analyzer of slave_* arguments, not both"
689
690 self.__analyzer = analyzer
691 """Analyzer to use for basic classifiers within boosted classifier"""
692
693
694 - def _call(self, dataset):
724
725 analyzer = property(fget=lambda x:x.__analyzer)
726
729 """Set sensitivity analyzer output be reverse mapped using mapper of the
730 slave classifier"""
731
732 - def _call(self, dataset):
740
743 """Set sensitivity analyzer output be reverse mapped using mapper of the
744 slave classifier"""
745
746 - def _call(self, dataset):
754