mvpa.clfs.meta

1 # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- 2 # vi: set ft=python sts=4 ts=4 sw=4 et: 3 ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## 4 # 5 # See COPYING file distributed along with the PyMVPA package for the 6 # copyright and license terms. 7 # 8 ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## 9 """Classes for meta classifiers -- classifiers which use other classifiers 10 11 Meta Classifiers can be grouped according to their function as 12 13 :group BoostedClassifiers: CombinedClassifier MulticlassClassifier 14 SplitClassifier 15 :group ProxyClassifiers: ProxyClassifier BinaryClassifier MappedClassifier 16 FeatureSelectionClassifier 17 :group PredictionsCombiners for CombinedClassifier: PredictionsCombiner 18 MaximalVote MeanPrediction 19 20 """ 21 22 __docformat__ = 'restructuredtext' 23 24 import operator 25 import numpy as N 26 27 from sets import Set 28 29 from mvpa.misc.args import group_kwargs 30 from mvpa.mappers.mask import MaskMapper 31 from mvpa.datasets.splitters import NFoldSplitter 32 from mvpa.misc.state import StateVariable, ClassWithCollections, Harvestable 33 34 from mvpa.clfs.base import Classifier 35 from mvpa.misc.transformers import FirstAxisMean 36 37 from mvpa.measures.base import \ 38 BoostedClassifierSensitivityAnalyzer, ProxyClassifierSensitivityAnalyzer, \ 39 MappedClassifierSensitivityAnalyzer, \ 40 FeatureSelectionClassifierSensitivityAnalyzer 41 42 from mvpa.base import warning 43 44 if __debug__: 45 from mvpa.base import debug

46 47 48 -class BoostedClassifier(Classifier, Harvestable):

49 """Classifier containing the farm of other classifiers. 50 51 Should rarely be used directly. Use one of its childs instead 52 """ 53 54 # should not be needed if we have prediction_values upstairs 55 # raw_predictions should be handled as Harvestable??? 56 raw_predictions = StateVariable(enabled=False, 57 doc="Predictions obtained from each classifier") 58 59 raw_values = StateVariable(enabled=False, 60 doc="Values obtained from each classifier") 61 62

63 - def __init__(self, clfs=None, propagate_states=True, 64 harvest_attribs=None, copy_attribs='copy', 65 **kwargs):

66 """Initialize the instance. 67 68 :Parameters: 69 clfs : list 70 list of classifier instances to use (slave classifiers) 71 propagate_states : bool 72 either to propagate enabled states into slave classifiers. 73 It is in effect only when slaves get assigned - so if state 74 is enabled not during construction, it would not necessarily 75 propagate into slaves 76 kwargs : dict 77 dict of keyworded arguments which might get used 78 by State or Classifier 79 """ 80 if clfs == None: 81 clfs = [] 82 83 Classifier.__init__(self, **kwargs) 84 Harvestable.__init__(self, harvest_attribs, copy_attribs) 85 86 self.__clfs = None 87 """Pylint friendly definition of __clfs""" 88 89 self.__propagate_states = propagate_states 90 """Enable current enabled states in slave classifiers""" 91 92 self._setClassifiers(clfs) 93 """Store the list of classifiers"""

94 95

96 - def __repr__(self, prefixes=[]):

97 if self.__clfs is None or len(self.__clfs)==0: 98 #prefix_ = "clfs=%s" % repr(self.__clfs) 99 prefix_ = [] 100 else: 101 prefix_ = ["clfs=[%s,...]" % repr(self.__clfs[0])] 102 return super(BoostedClassifier, self).__repr__(prefix_ + prefixes)

103 104

105 - def _train(self, dataset):

106 """Train `BoostedClassifier` 107 """ 108 for clf in self.__clfs: 109 clf.train(dataset)

110 111

112 - def _posttrain(self, dataset):

113 """Custom posttrain of `BoostedClassifier` 114 115 Harvest over the trained classifiers if it was asked to so 116 """ 117 Classifier._posttrain(self, dataset) 118 if self.states.isEnabled('harvested'): 119 for clf in self.__clfs: 120 self._harvest(locals()) 121 if self.params.retrainable: 122 self.__changedData_isset = False

123 124

125 - def _getFeatureIds(self):

126 """Custom _getFeatureIds for `BoostedClassifier` 127 """ 128 # return union of all used features by slave classifiers 129 feature_ids = Set([]) 130 for clf in self.__clfs: 131 feature_ids = feature_ids.union(Set(clf.feature_ids)) 132 return list(feature_ids)

133 134

135 - def _predict(self, data):

136 """Predict using `BoostedClassifier` 137 """ 138 raw_predictions = [ clf.predict(data) for clf in self.__clfs ] 139 self.raw_predictions = raw_predictions 140 assert(len(self.__clfs)>0) 141 if self.states.isEnabled("values"): 142 if N.array([x.states.isEnabled("values") 143 for x in self.__clfs]).all(): 144 values = [ clf.values for clf in self.__clfs ] 145 self.raw_values = values 146 else: 147 warning("One or more classifiers in %s has no 'values' state" % 148 self + "enabled, thus BoostedClassifier can't have" + 149 " 'raw_values' state variable defined") 150 151 return raw_predictions

152 153

154 - def _setClassifiers(self, clfs):

155 """Set the classifiers used by the boosted classifier 156 157 We have to allow to set list of classifiers after the object 158 was actually created. It will be used by 159 MulticlassClassifier 160 """ 161 self.__clfs = clfs 162 """Classifiers to use""" 163 164 for flag in ['regression']: 165 values = N.array([clf.params[flag].value for clf in self.__clfs]) 166 value = values.any() 167 if __debug__: 168 debug("CLFBST", "Setting %(flag)s=%(value)s for classifiers " 169 "%(clfs)s with %(values)s", 170 msgargs={'flag' : flag, 'value' : value, 171 'clfs' : self.__clfs, 172 'values' : values}) 173 # set flag if it needs to be trained before predicting 174 self.params[flag].value = value 175 176 # enable corresponding states in the slave-classifiers 177 if self.__propagate_states: 178 for clf in self.__clfs: 179 clf.states.enable(self.states.enabled, missingok=True) 180 181 # adhere to their capabilities + 'multiclass' 182 # XXX do intersection across all classifiers! 183 self._clf_internals = [ 'binary', 'multiclass', 'meta' ] 184 if len(clfs)>0: 185 self._clf_internals += self.__clfs[0]._clf_internals

186

187 - def untrain(self):

188 """Untrain `BoostedClassifier` 189 190 Has to untrain any known classifier 191 """ 192 if not self.trained: 193 return 194 for clf in self.clfs: 195 clf.untrain() 196 super(BoostedClassifier, self).untrain()

197

198 - def getSensitivityAnalyzer(self, **kwargs):

199 """Return an appropriate SensitivityAnalyzer""" 200 return BoostedClassifierSensitivityAnalyzer( 201 self, 202 **kwargs)

203 204 205 clfs = property(fget=lambda x:x.__clfs, 206 fset=_setClassifiers, 207 doc="Used classifiers")

208

209 210 211 -class ProxyClassifier(Classifier):

212 """Classifier which decorates another classifier 213 214 Possible uses: 215 216 - modify data somehow prior training/testing: 217 * normalization 218 * feature selection 219 * modification 220 221 - optimized classifier? 222 223 """ 224

225 - def __init__(self, clf, **kwargs):

226 """Initialize the instance 227 228 :Parameters: 229 clf : Classifier 230 classifier based on which mask classifiers is created 231 """ 232 233 Classifier.__init__(self, regression=clf.regression, **kwargs) 234 235 self.__clf = clf 236 """Store the classifier to use.""" 237 238 # adhere to slave classifier capabilities 239 # TODO: unittest 240 self._clf_internals = self._clf_internals[:] + ['meta'] 241 if clf is not None: 242 self._clf_internals += clf._clf_internals

243 244

245 - def __repr__(self, prefixes=[]):

246 return super(ProxyClassifier, self).__repr__( 247 ["clf=%s" % repr(self.__clf)] + prefixes)

248

249 - def summary(self):

250 s = super(ProxyClassifier, self).summary() 251 if self.trained: 252 s += "\n Slave classifier summary:" + \ 253 '\n + %s' % \ 254 (self.__clf.summary().replace('\n', '\n |')) 255 return s

256 257 258

259 - def _train(self, dataset):

260 """Train `ProxyClassifier` 261 """ 262 # base class does nothing much -- just proxies requests to underlying 263 # classifier 264 self.__clf.train(dataset)

265 266 # for the ease of access 267 # TODO: if to copy we should exclude some states which are defined in 268 # base Classifier (such as training_time, predicting_time) 269 # YOH: for now _copy_states_ would copy only set states variables. If 270 # anything needs to be overriden in the parent's class, it is 271 # welcome to do so 272 #self.states._copy_states_(self.__clf, deep=False) 273 274

275 - def _predict(self, data):

276 """Predict using `ProxyClassifier` 277 """ 278 clf = self.__clf 279 if self.states.isEnabled('values'): 280 clf.states.enable(['values']) 281 282 result = clf.predict(data) 283 # for the ease of access 284 self.states._copy_states_(self.__clf, ['values'], deep=False) 285 return result

286 287

288 - def untrain(self):

289 """Untrain ProxyClassifier 290 """ 291 if not self.__clf is None: 292 self.__clf.untrain() 293 super(ProxyClassifier, self).untrain()

294 295 296 @group_kwargs(prefixes=['slave_'], passthrough=True)

297 - def getSensitivityAnalyzer(self, slave_kwargs, **kwargs):

298 """Return an appropriate SensitivityAnalyzer""" 299 return ProxyClassifierSensitivityAnalyzer( 300 self, 301 analyzer=self.__clf.getSensitivityAnalyzer(**slave_kwargs), 302 **kwargs)

303 304 305 clf = property(lambda x:x.__clf, doc="Used `Classifier`")

306

307 308 309 # 310 # Various combiners for CombinedClassifier 311 # 312 313 -class PredictionsCombiner(ClassWithCollections):

314 """Base class for combining decisions of multiple classifiers""" 315

316 - def train(self, clfs, dataset):

317 """PredictionsCombiner might need to be trained 318 319 :Parameters: 320 clfs : list of Classifier 321 List of classifiers to combine. Has to be classifiers (not 322 pure predictions), since combiner might use some other 323 state variables (value's) instead of pure prediction's 324 dataset : Dataset 325 training data in this case 326 """ 327 pass

328 329

330 - def __call__(self, clfs, dataset):

331 """Call function 332 333 :Parameters: 334 clfs : list of Classifier 335 List of classifiers to combine. Has to be classifiers (not 336 pure predictions), since combiner might use some other 337 state variables (value's) instead of pure prediction's 338 """ 339 raise NotImplementedError

340

341 342 343 -class MaximalVote(PredictionsCombiner):

344 """Provides a decision using maximal vote rule""" 345 346 predictions = StateVariable(enabled=True, 347 doc="Voted predictions") 348 all_label_counts = StateVariable(enabled=False, 349 doc="Counts across classifiers for each label/sample") 350

351 - def __init__(self):

352 """XXX Might get a parameter to use raw decision values if 353 voting is not unambigous (ie two classes have equal number of 354 votes 355 """ 356 PredictionsCombiner.__init__(self)

357 358

359 - def __call__(self, clfs, dataset):

360 """Actuall callable - perform voting 361 362 Extended functionality which might not be needed actually: 363 Since `BinaryClassifier` might return a list of possible 364 predictions (not just a single one), we should consider all of those 365 366 MaximalVote doesn't care about dataset itself 367 """ 368 if len(clfs)==0: 369 return [] # to don't even bother 370 371 all_label_counts = None 372 for clf in clfs: 373 # Lets check first if necessary state variable is enabled 374 if not clf.states.isEnabled("predictions"): 375 raise ValueError, "MaximalVote needs classifiers (such as " + \ 376 "%s) with state 'predictions' enabled" % clf 377 predictions = clf.predictions 378 if all_label_counts is None: 379 all_label_counts = [ {} for i in xrange(len(predictions)) ] 380 381 # for every sample 382 for i in xrange(len(predictions)): 383 prediction = predictions[i] 384 if not operator.isSequenceType(prediction): 385 prediction = (prediction,) 386 for label in prediction: # for every label 387 # XXX we might have multiple labels assigned 388 # but might not -- don't remember now 389 if not all_label_counts[i].has_key(label): 390 all_label_counts[i][label] = 0 391 all_label_counts[i][label] += 1 392 393 predictions = [] 394 # select maximal vote now for each sample 395 for i in xrange(len(all_label_counts)): 396 label_counts = all_label_counts[i] 397 # lets do explicit search for max so we know 398 # if it is unique 399 maxk = [] # labels of elements with max vote 400 maxv = -1 401 for k, v in label_counts.iteritems(): 402 if v > maxv: 403 maxk = [k] 404 maxv = v 405 elif v == maxv: 406 maxk.append(k) 407 408 assert len(maxk) >= 1, \ 409 "We should have obtained at least a single key of max label" 410 411 if len(maxk) > 1: 412 warning("We got multiple labels %s which have the " % maxk + 413 "same maximal vote %d. XXX disambiguate" % maxv) 414 predictions.append(maxk[0]) 415 416 self.all_label_counts = all_label_counts 417 self.predictions = predictions 418 return predictions

419

420 421 422 -class MeanPrediction(PredictionsCombiner):

423 """Provides a decision by taking mean of the results 424 """ 425 426 predictions = StateVariable(enabled=True, 427 doc="Mean predictions") 428

429 - def __call__(self, clfs, dataset):

430 """Actuall callable - perform meaning 431 432 """ 433 if len(clfs)==0: 434 return [] # to don't even bother 435 436 all_predictions = [] 437 for clf in clfs: 438 # Lets check first if necessary state variable is enabled 439 if not clf.states.isEnabled("predictions"): 440 raise ValueError, "MeanPrediction needs classifiers (such " \ 441 " as %s) with state 'predictions' enabled" % clf 442 all_predictions.append(clf.predictions) 443 444 # compute mean 445 predictions = N.mean(N.asarray(all_predictions), axis=0) 446 self.predictions = predictions 447 return predictions

448

449 450 -class ClassifierCombiner(PredictionsCombiner):

451 """Provides a decision using training a classifier on predictions/values 452 453 TODO: implement 454 """ 455 456 predictions = StateVariable(enabled=True, 457 doc="Trained predictions") 458 459

460 - def __init__(self, clf, variables=None):

461 """Initialize `ClassifierCombiner` 462 463 :Parameters: 464 clf : Classifier 465 Classifier to train on the predictions 466 variables : list of basestring 467 List of state variables stored in 'combined' classifiers, which 468 to use as features for training this classifier 469 """ 470 PredictionsCombiner.__init__(self) 471 472 self.__clf = clf 473 """Classifier to train on `variables` states of provided classifiers""" 474 475 if variables == None: 476 variables = ['predictions'] 477 self.__variables = variables 478 """What state variables of the classifiers to use"""

479 480

481 - def untrain(self):

482 """It might be needed to untrain used classifier""" 483 if self.__clf: 484 self.__clf.untrain()

485

486 - def __call__(self, clfs, dataset):

487 """ 488 """ 489 if len(clfs)==0: 490 return [] # to don't even bother 491 492 raise NotImplementedError

493

494 495 496 -class CombinedClassifier(BoostedClassifier):

497 """`BoostedClassifier` which combines predictions using some 498 `PredictionsCombiner` functor. 499 """ 500

501 - def __init__(self, clfs=None, combiner=None, **kwargs):

502 """Initialize the instance. 503 504 :Parameters: 505 clfs : list of Classifier 506 list of classifier instances to use 507 combiner : PredictionsCombiner 508 callable which takes care about combining multiple 509 results into a single one (e.g. maximal vote for 510 classification, MeanPrediction for regression)) 511 kwargs : dict 512 dict of keyworded arguments which might get used 513 by State or Classifier 514 515 NB: `combiner` might need to operate not on 'predictions' descrete 516 labels but rather on raw 'class' values classifiers 517 estimate (which is pretty much what is stored under 518 `values` 519 """ 520 if clfs == None: 521 clfs = [] 522 523 BoostedClassifier.__init__(self, clfs, **kwargs) 524 525 # assign default combiner 526 if combiner is None: 527 combiner = (MaximalVote, MeanPrediction)[int(self.regression)]() 528 self.__combiner = combiner 529 """Functor destined to combine results of multiple classifiers"""

530 531

532 - def __repr__(self, prefixes=[]):

533 """Literal representation of `CombinedClassifier`. 534 """ 535 return super(CombinedClassifier, self).__repr__( 536 ["combiner=%s" % repr(self.__combiner)] + prefixes)

537 538

539 - def summary(self):

540 """Provide summary for the `CombinedClassifier`. 541 """ 542 s = super(CombinedClassifier, self).summary() 543 if self.trained: 544 s += "\n Slave classifiers summaries:" 545 for i, clf in enumerate(self.clfs): 546 s += '\n + %d clf: %s' % \ 547 (i, clf.summary().replace('\n', '\n |')) 548 return s

549 550

551 - def untrain(self):

552 """Untrain `CombinedClassifier` 553 """ 554 try: 555 self.__combiner.untrain() 556 except: 557 pass 558 super(CombinedClassifier, self).untrain()

559

560 - def _train(self, dataset):

561 """Train `CombinedClassifier` 562 """ 563 BoostedClassifier._train(self, dataset) 564 # combiner might need to train as well 565 self.__combiner.train(self.clfs, dataset)

566 567

568 - def _predict(self, data):

569 """Predict using `CombinedClassifier` 570 """ 571 BoostedClassifier._predict(self, data) 572 # combiner will make use of state variables instead of only predictions 573 # returned from _predict 574 predictions = self.__combiner(self.clfs, data) 575 self.predictions = predictions 576 577 if self.states.isEnabled("values"): 578 if self.__combiner.states.isActive("values"): 579 # XXX or may be we could leave simply up to accessing .combiner? 580 self.values = self.__combiner.values 581 else: 582 if __debug__: 583 warning("Boosted classifier %s has 'values' state" % self + 584 " enabled, but combiner has it active, thus no" + 585 " values could be provided directly, access .clfs") 586 return predictions

587 588 589 combiner = property(fget=lambda x:x.__combiner, 590 doc="Used combiner to derive a single result")

591

592 593 594 -class BinaryClassifier(ProxyClassifier):

595 """`ProxyClassifier` which maps set of two labels into +1 and -1 596 """ 597

598 - def __init__(self, clf, poslabels, neglabels, **kwargs):

599 """ 600 :Parameters: 601 clf : Classifier 602 classifier to use 603 poslabels : list 604 list of labels which are treated as +1 category 605 neglabels : list 606 list of labels which are treated as -1 category 607 """ 608 609 ProxyClassifier.__init__(self, clf, **kwargs) 610 611 self._regressionIsBogus() 612 613 # Handle labels 614 sposlabels = Set(poslabels) # so to remove duplicates 615 sneglabels = Set(neglabels) # so to remove duplicates 616 617 # check if there is no overlap 618 overlap = sposlabels.intersection(sneglabels) 619 if len(overlap)>0: 620 raise ValueError("Sets of positive and negative labels for " + 621 "BinaryClassifier must not overlap. Got overlap " % 622 overlap) 623 624 self.__poslabels = list(sposlabels) 625 self.__neglabels = list(sneglabels) 626 627 # define what values will be returned by predict: if there is 628 # a single label - return just it alone, otherwise - whole 629 # list 630 # Such approach might come useful if we use some classifiers 631 # over different subsets of data with some voting later on 632 # (1-vs-therest?) 633 634 if len(self.__poslabels) > 1: 635 self.__predictpos = self.__poslabels 636 else: 637 self.__predictpos = self.__poslabels[0] 638 639 if len(self.__neglabels) > 1: 640 self.__predictneg = self.__neglabels 641 else: 642 self.__predictneg = self.__neglabels[0]

643 644

645 - def __repr__(self, prefixes=[]):

646 prefix = "poslabels=%s, neglabels=%s" % ( 647 repr(self.__poslabels), repr(self.__neglabels)) 648 return super(BinaryClassifier, self).__repr__([prefix] + prefixes)

649 650

651 - def _train(self, dataset):

652 """Train `BinaryClassifier` 653 """ 654 idlabels = [(x, +1) for x in dataset.idsbylabels(self.__poslabels)] + \ 655 [(x, -1) for x in dataset.idsbylabels(self.__neglabels)] 656 # XXX we have to sort ids since at the moment Dataset.selectSamples 657 # doesn't take care about order 658 idlabels.sort() 659 # select the samples 660 orig_labels = None 661 662 # If we need all samples, why simply not perform on original 663 # data, an just store/restore labels. But it really should be done 664 # within Dataset.selectSamples 665 if len(idlabels) == dataset.nsamples \ 666 and [x[0] for x in idlabels] == range(dataset.nsamples): 667 # the last condition is not even necessary... just overly 668 # cautious 669 datasetselected = dataset # no selection is needed 670 orig_labels = dataset.labels # but we would need to restore labels 671 if __debug__: 672 debug('CLFBIN', 673 "Assigned all %d samples for binary " % 674 (dataset.nsamples) + 675 " classification among labels %s/+1 and %s/-1" % 676 (self.__poslabels, self.__neglabels)) 677 else: 678 datasetselected = dataset.selectSamples([ x[0] for x in idlabels ]) 679 if __debug__: 680 debug('CLFBIN', 681 "Selected %d samples out of %d samples for binary " % 682 (len(idlabels), dataset.nsamples) + 683 " classification among labels %s/+1 and %s/-1" % 684 (self.__poslabels, self.__neglabels) + 685 ". Selected %s" % datasetselected) 686 687 # adjust the labels 688 datasetselected.labels = [ x[1] for x in idlabels ] 689 690 # now we got a dataset with only 2 labels 691 if __debug__: 692 assert((datasetselected.uniquelabels == [-1, 1]).all()) 693 694 self.clf.train(datasetselected) 695 696 if not orig_labels is None: 697 dataset.labels = orig_labels

698

699 - def _predict(self, data):

700 """Predict the labels for a given `data` 701 702 Predicts using binary classifier and spits out list (for each sample) 703 where with either poslabels or neglabels as the "label" for the sample. 704 If there was just a single label within pos or neg labels then it would 705 return not a list but just that single label. 706 """ 707 binary_predictions = ProxyClassifier._predict(self, data) 708 self.values = binary_predictions 709 predictions = [ {-1: self.__predictneg, 710 +1: self.__predictpos}[x] for x in binary_predictions] 711 self.predictions = predictions 712 return predictions

713

714 715 716 -class MulticlassClassifier(CombinedClassifier):

717 """`CombinedClassifier` to perform multiclass using a list of 718 `BinaryClassifier`. 719 720 such as 1-vs-1 (ie in pairs like libsvm doesn) or 1-vs-all (which 721 is yet to think about) 722 """ 723

724 - def __init__(self, clf, bclf_type="1-vs-1", **kwargs):

725 """Initialize the instance 726 727 :Parameters: 728 clf : Classifier 729 classifier based on which multiple classifiers are created 730 for multiclass 731 bclf_type 732 "1-vs-1" or "1-vs-all", determines the way to generate binary 733 classifiers 734 """ 735 CombinedClassifier.__init__(self, **kwargs) 736 self._regressionIsBogus() 737 if not clf is None: 738 clf._regressionIsBogus() 739 740 self.__clf = clf 741 """Store sample instance of basic classifier""" 742 743 # Some checks on known ways to do multiclass 744 if bclf_type == "1-vs-1": 745 pass 746 elif bclf_type == "1-vs-all": # TODO 747 raise NotImplementedError 748 else: 749 raise ValueError, \ 750 "Unknown type of classifier %s for " % bclf_type + \ 751 "BoostedMulticlassClassifier" 752 self.__bclf_type = bclf_type

753 754 # XXX fix it up a bit... it seems that MulticlassClassifier should 755 # be actually ProxyClassifier and use BoostedClassifier internally

756 - def __repr__(self, prefixes=[]):

757 prefix = "bclf_type=%s, clf=%s" % (repr(self.__bclf_type), 758 repr(self.__clf)) 759 return super(MulticlassClassifier, self).__repr__([prefix] + prefixes)

760 761

762 - def _train(self, dataset):

763 """Train classifier 764 """ 765 # construct binary classifiers 766 ulabels = dataset.uniquelabels 767 if self.__bclf_type == "1-vs-1": 768 # generate pairs and corresponding classifiers 769 biclfs = [] 770 for i in xrange(len(ulabels)): 771 for j in xrange(i+1, len(ulabels)): 772 clf = self.__clf.clone() 773 biclfs.append( 774 BinaryClassifier( 775 clf, 776 poslabels=[ulabels[i]], neglabels=[ulabels[j]])) 777 if __debug__: 778 debug("CLFMC", "Created %d binary classifiers for %d labels" % 779 (len(biclfs), len(ulabels))) 780 781 self.clfs = biclfs 782 783 elif self.__bclf_type == "1-vs-all": 784 raise NotImplementedError 785 786 # perform actual training 787 CombinedClassifier._train(self, dataset)

788

789 790 791 -class SplitClassifier(CombinedClassifier):

792 """`BoostedClassifier` to work on splits of the data 793 794 """ 795 796 """ 797 TODO: SplitClassifier and MulticlassClassifier have too much in 798 common -- need to refactor: just need a splitter which would 799 split dataset in pairs of class labels. MulticlassClassifier 800 does just a tiny bit more which might be not necessary at 801 all: map sets of labels into 2 categories... 802 """ 803 804 # TODO: unify with CrossValidatedTransferError which now uses 805 # harvest_attribs to expose gathered attributes 806 confusion = StateVariable(enabled=False, 807 doc="Resultant confusion whenever classifier trained " + 808 "on 1 part and tested on 2nd part of each split") 809 810 splits = StateVariable(enabled=False, doc= 811 """Store the actual splits of the data. Can be memory expensive""") 812 813 # ??? couldn't be training_confusion since it has other meaning 814 # here, BUT it is named so within CrossValidatedTransferError 815 # -- unify 816 # decided to go with overriding semantics tiny bit. For split 817 # classifier training_confusion would correspond to summary 818 # over training errors across all splits. Later on if need comes 819 # we might want to implement global_training_confusion which would 820 # correspond to overall confusion on full training dataset as it is 821 # done in base Classifier 822 #global_training_confusion = StateVariable(enabled=False, 823 # doc="Summary over training confusions acquired at each split") 824

825 - def __init__(self, clf, splitter=NFoldSplitter(cvtype=1), **kwargs):

826 """Initialize the instance 827 828 :Parameters: 829 clf : Classifier 830 classifier based on which multiple classifiers are created 831 for multiclass 832 splitter : Splitter 833 `Splitter` to use to split the dataset prior training 834 """ 835 836 CombinedClassifier.__init__(self, regression=clf.regression, **kwargs) 837 self.__clf = clf 838 """Store sample instance of basic classifier""" 839 840 if isinstance(splitter, type): 841 raise ValueError, \ 842 "Please provide an instance of a splitter, not a type." \ 843 " Got %s" % splitter 844 845 self.__splitter = splitter

846 847

848 - def _train(self, dataset):

849 """Train `SplitClassifier` 850 """ 851 # generate pairs and corresponding classifiers 852 bclfs = [] 853 854 # local binding 855 states = self.states 856 857 clf_template = self.__clf 858 if states.isEnabled('confusion'): 859 states.confusion = clf_template._summaryClass() 860 if states.isEnabled('training_confusion'): 861 clf_template.states.enable(['training_confusion']) 862 states.training_confusion = clf_template._summaryClass() 863 864 clf_hastestdataset = hasattr(clf_template, 'testdataset') 865 866 # for proper and easier debugging - first define classifiers and then 867 # train them 868 for split in self.__splitter.splitcfg(dataset): 869 if __debug__: 870 debug("CLFSPL", 871 "Deepcopying %(clf)s for %(sclf)s", 872 msgargs={'clf':clf_template, 873 'sclf':self}) 874 clf = clf_template.clone() 875 bclfs.append(clf) 876 self.clfs = bclfs 877 878 self.splits = [] 879 880 for i, split in enumerate(self.__splitter(dataset)): 881 if __debug__: 882 debug("CLFSPL", "Training classifier for split %d" % (i)) 883 884 if states.isEnabled("splits"): 885 self.splits.append(split) 886 887 clf = self.clfs[i] 888 889 # assign testing dataset if given classifier can digest it 890 if clf_hastestdataset: 891 clf.testdataset = split[1] 892 893 clf.train(split[0]) 894 895 # unbind the testdataset from the classifier 896 if clf_hastestdataset: 897 clf.testdataset = None 898 899 if states.isEnabled("confusion"): 900 predictions = clf.predict(split[1].samples) 901 self.confusion.add(split[1].labels, predictions, 902 clf.states.get('values', None)) 903 if states.isEnabled("training_confusion"): 904 states.training_confusion += \ 905 clf.states.training_confusion 906 # hackish way -- so it should work only for ConfusionMatrix??? 907 try: 908 if states.isEnabled("confusion"): 909 states.confusion.labels_map = dataset.labels_map 910 if states.isEnabled("training_confusion"): 911 states.training_confusion.labels_map = dataset.labels_map 912 except: 913 pass

914 915 916 @group_kwargs(prefixes=['slave_'], passthrough=True)

917 - def getSensitivityAnalyzer(self, slave_kwargs, **kwargs):

918 """Return an appropriate SensitivityAnalyzer for `SplitClassifier` 919 920 :Parameters: 921 combiner 922 If not provided, FirstAxisMean is assumed 923 """ 924 kwargs.setdefault('combiner', FirstAxisMean) 925 return BoostedClassifierSensitivityAnalyzer( 926 self, 927 analyzer=self.__clf.getSensitivityAnalyzer(**slave_kwargs), 928 **kwargs)

929 930 splitter = property(fget=lambda x:x.__splitter, 931 doc="Splitter user by SplitClassifier")

932

933 934 -class MappedClassifier(ProxyClassifier):

935 """`ProxyClassifier` which uses some mapper prior training/testing. 936 937 `MaskMapper` can be used just a subset of features to 938 train/classify. 939 Having such classifier we can easily create a set of classifiers 940 for BoostedClassifier, where each classifier operates on some set 941 of features, e.g. set of best spheres from SearchLight, set of 942 ROIs selected elsewhere. It would be different from simply 943 applying whole mask over the dataset, since here initial decision 944 is made by each classifier and then later on they vote for the 945 final decision across the set of classifiers. 946 """ 947

948 - def __init__(self, clf, mapper, **kwargs):

949 """Initialize the instance 950 951 :Parameters: 952 clf : Classifier 953 classifier based on which mask classifiers is created 954 mapper 955 whatever `Mapper` comes handy 956 """ 957 ProxyClassifier.__init__(self, clf, **kwargs) 958 959 self.__mapper = mapper 960 """mapper to help us our with prepping data to 961 training/classification"""

962 963

964 - def _train(self, dataset):

965 """Train `MappedClassifier` 966 """ 967 # first train the mapper 968 # XXX: should training be done using whole dataset or just samples 969 # YYY: in some cases labels might be needed, thus better full dataset 970 self.__mapper.train(dataset) 971 972 # for train() we have to provide dataset -- not just samples to train! 973 wdataset = dataset.applyMapper(featuresmapper = self.__mapper) 974 ProxyClassifier._train(self, wdataset)

975 976

977 - def _predict(self, data):

978 """Predict using `MappedClassifier` 979 """ 980 return ProxyClassifier._predict(self, self.__mapper.forward(data))

981 982 983 @group_kwargs(prefixes=['slave_'], passthrough=True)

984 - def getSensitivityAnalyzer(self, slave_kwargs, **kwargs):

985 """Return an appropriate SensitivityAnalyzer""" 986 return MappedClassifierSensitivityAnalyzer( 987 self, 988 analyzer=self.clf.getSensitivityAnalyzer(**slave_kwargs), 989 **kwargs)

990 991 992 mapper = property(lambda x:x.__mapper, doc="Used mapper")

993

994 995 996 -class FeatureSelectionClassifier(ProxyClassifier):

997 """`ProxyClassifier` which uses some `FeatureSelection` prior training. 998 999 `FeatureSelection` is used first to select features for the classifier to 1000 use for prediction. Internally it would rely on MappedClassifier which 1001 would use created MaskMapper. 1002 1003 TODO: think about removing overhead of retraining the same classifier if 1004 feature selection was carried out with the same classifier already. It 1005 has been addressed by adding .trained property to classifier, but now 1006 we should expclitely use isTrained here if we want... need to think more 1007 """ 1008 1009 _clf_internals = [ 'does_feature_selection', 'meta' ] 1010

1011 - def __init__(self, clf, feature_selection, testdataset=None, **kwargs):

1012 """Initialize the instance 1013 1014 :Parameters: 1015 clf : Classifier 1016 classifier based on which mask classifiers is created 1017 feature_selection : FeatureSelection 1018 whatever `FeatureSelection` comes handy 1019 testdataset : Dataset 1020 optional dataset which would be given on call to feature_selection 1021 """ 1022 ProxyClassifier.__init__(self, clf, **kwargs) 1023 1024 self.__maskclf = None 1025 """Should become `MappedClassifier`(mapper=`MaskMapper`) later on.""" 1026 1027 self.__feature_selection = feature_selection 1028 """`FeatureSelection` to select the features prior training""" 1029 1030 self.__testdataset = testdataset 1031 """`FeatureSelection` might like to use testdataset"""

1032 1033

1034 - def untrain(self):

1035 """Untrain `FeatureSelectionClassifier` 1036 1037 Has to untrain any known classifier 1038 """ 1039 if not self.trained: 1040 return 1041 if not self.__maskclf is None: 1042 self.__maskclf.untrain() 1043 super(FeatureSelectionClassifier, self).untrain()

1044 1045

1046 - def _train(self, dataset):

1047 """Train `FeatureSelectionClassifier` 1048 """ 1049 # temporarily enable selected_ids 1050 self.__feature_selection.states._changeTemporarily( 1051 enable_states=["selected_ids"]) 1052 1053 if __debug__: 1054 debug("CLFFS", "Performing feature selection using %s" % 1055 self.__feature_selection + " on %s" % dataset) 1056 1057 (wdataset, tdataset) = self.__feature_selection(dataset, 1058 self.__testdataset) 1059 if __debug__: 1060 add_ = "" 1061 if "CLFFS_" in debug.active: 1062 add_ = " Selected features: %s" % \ 1063 self.__feature_selection.selected_ids 1064 debug("CLFFS", "%(fs)s selected %(nfeat)d out of " + 1065 "%(dsnfeat)d features.%(app)s", 1066 msgargs={'fs':self.__feature_selection, 1067 'nfeat':wdataset.nfeatures, 1068 'dsnfeat':dataset.nfeatures, 1069 'app':add_}) 1070 1071 # create a mask to devise a mapper 1072 # TODO -- think about making selected_ids a MaskMapper 1073 mappermask = N.zeros(dataset.nfeatures) 1074 mappermask[self.__feature_selection.selected_ids] = 1 1075 mapper = MaskMapper(mappermask) 1076 1077 self.__feature_selection.states._resetEnabledTemporarily() 1078 1079 # create and assign `MappedClassifier` 1080 self.__maskclf = MappedClassifier(self.clf, mapper) 1081 # we could have called self.__clf.train(dataset), but it would 1082 # cause unnecessary masking 1083 self.__maskclf.clf.train(wdataset)

1084 1085 # for the ease of access 1086 # TODO see for ProxyClassifier 1087 #self.states._copy_states_(self.__maskclf, deep=False) 1088

1089 - def _getFeatureIds(self):

1090 """Return used feature ids for `FeatureSelectionClassifier` 1091 1092 """ 1093 return self.__feature_selection.selected_ids

1094

1095 - def _predict(self, data):

1096 """Predict using `FeatureSelectionClassifier` 1097 """ 1098 clf = self.__maskclf 1099 if self.states.isEnabled('values'): 1100 clf.states.enable(['values']) 1101 1102 result = clf._predict(data) 1103 # for the ease of access 1104 self.states._copy_states_(clf, ['values'], deep=False) 1105 return result

1106

1107 - def setTestDataset(self, testdataset):

1108 """Set testing dataset to be used for feature selection 1109 """ 1110 self.__testdataset = testdataset

1111 1112 maskclf = property(lambda x:x.__maskclf, doc="Used `MappedClassifier`") 1113 feature_selection = property(lambda x:x.__feature_selection, 1114 doc="Used `FeatureSelection`") 1115 1116 @group_kwargs(prefixes=['slave_'], passthrough=True)

1117 - def getSensitivityAnalyzer(self, slave_kwargs, **kwargs):

1118 """Return an appropriate SensitivityAnalyzer 1119 1120 had to clone from mapped classifier??? 1121 """ 1122 return FeatureSelectionClassifierSensitivityAnalyzer( 1123 self, 1124 analyzer=self.clf.getSensitivityAnalyzer(**slave_kwargs), 1125 **kwargs)

1126 1127 1128 1129 testdataset = property(fget=lambda x:x.__testdataset, 1130 fset=setTestDataset)

1131

Source Code for Module mvpa.clfs.meta