mvpa.datasets.base

1 # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- 2 # vi: set ft=python sts=4 ts=4 sw=4 et: 3 ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## 4 # 5 # See COPYING file distributed along with the PyMVPA package for the 6 # copyright and license terms. 7 # 8 ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## 9 """Dataset container""" 10 11 __docformat__ = 'restructuredtext' 12 13 import operator 14 import random 15 import mvpa.support.copy as copy 16 import numpy as N 17 18 from sets import Set 19 20 # Sooner or later Dataset would become ClassWithCollections as well, but for 21 # now just an object -- thus commenting out tentative changes 22 # 23 #XXX from mvpa.misc.state import ClassWithCollections, SampleAttribute 24 25 from mvpa.misc.exceptions import DatasetError 26 from mvpa.misc.support import idhash as idhash_ 27 from mvpa.base.dochelpers import enhancedDocString, table2string 28 29 if __debug__: 30 from mvpa.base import debug, warning

31 32 - def _validate_indexes_uniq_sorted(seq, fname, item):

33 """Helper function to validate that seq contains unique sorted values 34 """ 35 if operator.isSequenceType(seq): 36 seq_unique = N.unique(seq) 37 if len(seq) != len(seq_unique): 38 warning("%s() operates only with indexes for %s without" 39 " repetitions. Repetitions were removed." 40 % (fname, item)) 41 if N.any(N.sort(seq) != seq_unique): 42 warning("%s() does not guarantee the original order" 43 " of selected %ss. Use selectSamples() and " 44 " selectFeatures(sort=False) instead" % (fname, item))

45

46 47 #XXX class Dataset(ClassWithCollections): 48 -class Dataset(object):

49 """*The* Dataset. 50 51 This class provides a container to store all necessary data to 52 perform MVPA analyses. These are the data samples, as well as the 53 labels associated with the samples. Additionally, samples can be 54 grouped into chunks. 55 56 :Groups: 57 - `Creators`: `__init__`, `selectFeatures`, `selectSamples`, 58 `applyMapper` 59 - `Mutators`: `permuteLabels` 60 61 Important: labels assumed to be immutable, i.e. noone should modify 62 them externally by accessing indexed items, ie something like 63 ``dataset.labels[1] += "_bad"`` should not be used. If a label has 64 to be modified, full copy of labels should be obtained, operated on, 65 and assigned back to the dataset, otherwise dataset.uniquelabels 66 would not work. The same applies to any other attribute which has 67 corresponding unique* access property. 68 69 """ 70 # XXX Notes about migration to use Collections to store data and 71 # attributes for samples, features, and dataset itself: 72 73 # changes: 74 # _data -> s_attr collection (samples attributes) 75 # _dsattr -> ds_attr collection 76 # f_attr collection (features attributes) 77 78 # static definition to track which unique attributes 79 # have to be reset/recomputed whenever anything relevant 80 # changes 81 82 # unique{labels,chunks} become a part of dsattr 83 _uniqueattributes = [] 84 """Unique attributes associated with the data""" 85 86 _registeredattributes = [] 87 """Registered attributes (stored in _data)""" 88 89 _requiredattributes = ['samples', 'labels'] 90 """Attributes which have to be provided to __init__, or otherwise 91 no default values would be assumed and construction of the 92 instance would fail""" 93 94 #XXX _ATTRIBUTE_COLLECTIONS = [ 's_attr', 'f_attr', 'ds_attr' ] 95 #XXX """Assure those 3 collections to be present in all datasets""" 96 #XXX 97 #XXX samples__ = SampleAttribute(doc="Samples data. 0th index is time", hasunique=False) # XXX 98 #XXX labels__ = SampleAttribute(doc="Labels for the samples", hasunique=True) 99 #XXX chunks__ = SampleAttribute(doc="Chunk identities for the samples", hasunique=True) 100 #XXX # samples ids (already unique by definition) 101 #XXX origids__ = SampleAttribute(doc="Chunk identities for the samples", hasunique=False) 102

103 - def __init__(self, 104 # for copy constructor 105 data=None, 106 dsattr=None, 107 # automatic dtype conversion 108 dtype=None, 109 # new instances 110 samples=None, 111 labels=None, 112 labels_map=None, 113 chunks=None, 114 origids=None, 115 # flags 116 check_data=True, 117 copy_samples=False, 118 copy_data=True, 119 copy_dsattr=True):

120 """Initialize dataset instance 121 122 There are basically two different way to create a dataset: 123 124 1. Create a new dataset from samples and sample attributes. In 125 this mode a two-dimensional `ndarray` has to be passed to the 126 `samples` keyword argument and the corresponding samples 127 attributes are provided via the `labels` and `chunks` 128 arguments. 129 130 2. Copy contructor mode 131 The second way is used internally to perform quick coyping 132 of datasets, e.g. when performing feature selection. In this 133 mode and the two dictionaries (`data` and `dsattr`) are 134 required. For performance reasons this mode bypasses most of 135 the sanity check performed by the previous mode, as for 136 internal operations data integrity is assumed. 137 138 139 :Parameters: 140 data : dict 141 Dictionary with an arbitrary number of entries. The value for 142 each key in the dict has to be an ndarray with the 143 same length as the number of rows in the samples array. 144 A special entry in this dictionary is 'samples', a 2d array 145 (samples x features). A shallow copy is stored in the object. 146 dsattr : dict 147 Dictionary of dataset attributes. An arbitrary number of 148 arbitrarily named and typed objects can be stored here. A 149 shallow copy of the dictionary is stored in the object. 150 dtype: type | None 151 If None -- do not change data type if samples 152 is an ndarray. Otherwise convert samples to dtype. 153 154 155 :Keywords: 156 samples : ndarray 157 2d array (samples x features) 158 labels 159 An array or scalar value defining labels for each samples 160 labels_map : None or bool or dict 161 Map from labels into literal names. If is None or True, 162 the mapping is computed, from labels which must be literal. 163 If is False, no mapping is computed. If dict -- mapping is 164 verified and taken, labels get remapped. Dict must map 165 literal -> number 166 chunks 167 An array or scalar value defining chunks for each sample 168 169 Each of the Keywords arguments overwrites what is/might be 170 already in the `data` container. 171 172 """ 173 174 #XXX ClassWithCollections.__init__(self) 175 176 # see if data and dsattr are none, if so, make them empty dicts 177 if data is None: 178 data = {} 179 if dsattr is None: 180 dsattr = {} 181 182 # initialize containers; default values are empty dicts 183 # always make a shallow copy of what comes in, otherwise total chaos 184 # is likely to happen soon 185 if copy_data: 186 # deep copy (cannot use copy.deepcopy, because samples is an 187 # exception 188 # but shallow copy first to get a shared version of the data in 189 # any case 190 lcl_data = data.copy() 191 for k, v in data.iteritems(): 192 # skip copying samples if requested 193 if k == 'samples' and not copy_samples: 194 continue 195 lcl_data[k] = v.copy() 196 else: 197 # shallow copy 198 # XXX? yoh: it might be better speed wise just assign dictionary 199 # without any shallow .copy 200 lcl_data = data.copy() 201 202 if copy_dsattr and len(dsattr)>0: 203 # deep copy 204 if __debug__: 205 debug('DS', "Deep copying dsattr %s" % `dsattr`) 206 lcl_dsattr = copy.deepcopy(dsattr) 207 208 else: 209 # shallow copy 210 lcl_dsattr = copy.copy(dsattr) 211 212 # has to be not private since otherwise derived methods 213 # would have problem accessing it and _registerAttribute 214 # would fail on lambda getters 215 self._data = lcl_data 216 """What makes a dataset.""" 217 218 self._dsattr = lcl_dsattr 219 """Dataset attriibutes.""" 220 221 # store samples (and possibly transform/reshape/retype them) 222 if not samples == None: 223 if __debug__: 224 if lcl_data.has_key('samples'): 225 debug('DS', 226 "`Data` dict has `samples` (%s) but there is also" \ 227 " __init__ parameter `samples` which overrides " \ 228 " stored in `data`" % (`lcl_data['samples'].shape`)) 229 lcl_data['samples'] = self._shapeSamples(samples, dtype, 230 copy_samples) 231 232 # TODO? we might want to have the same logic for chunks and labels 233 # ie if no labels present -- assign arange 234 # MH: don't think this is necessary -- or is there a use case? 235 # labels 236 if not labels == None: 237 if __debug__: 238 if lcl_data.has_key('labels'): 239 debug('DS', 240 "`Data` dict has `labels` (%s) but there is also" + 241 " __init__ parameter `labels` which overrides " + 242 " stored in `data`" % (`lcl_data['labels']`)) 243 if lcl_data.has_key('samples'): 244 lcl_data['labels'] = \ 245 self._expandSampleAttribute(labels, 'labels') 246 247 # check if we got all required attributes 248 for attr in self._requiredattributes: 249 if not lcl_data.has_key(attr): 250 raise DatasetError, \ 251 "Attribute %s is required to initialize dataset" % \ 252 attr 253 254 nsamples = self.nsamples 255 256 # chunks 257 if not chunks == None: 258 lcl_data['chunks'] = \ 259 self._expandSampleAttribute(chunks, 'chunks') 260 elif not lcl_data.has_key('chunks'): 261 # if no chunk information is given assume that every pattern 262 # is its own chunk 263 lcl_data['chunks'] = N.arange(nsamples) 264 265 # samples origids 266 if not origids is None: 267 # simply assign if provided 268 lcl_data['origids'] = origids 269 elif not lcl_data.has_key('origids'): 270 # otherwise contruct unqiue ones 271 lcl_data['origids'] = N.arange(len(lcl_data['labels'])) 272 else: 273 # assume origids have been specified already (copy constructor 274 # mode) leave them as they are, e.g. to make origids survive 275 # selectSamples() 276 pass 277 278 # Initialize attributes which are registered but were not setup 279 for attr in self._registeredattributes: 280 if not lcl_data.has_key(attr): 281 if __debug__: 282 debug("DS", "Initializing attribute %s" % attr) 283 lcl_data[attr] = N.zeros(nsamples) 284 285 # labels_map 286 labels_ = N.asarray(lcl_data['labels']) 287 labels_map_known = lcl_dsattr.has_key('labels_map') 288 if labels_map is True: 289 # need to composte labels_map 290 if labels_.dtype.char == 'S' or not labels_map_known: 291 # Create mapping 292 ulabels = list(Set(labels_)) 293 ulabels.sort() 294 labels_map = dict([ (x[1], x[0]) for x in enumerate(ulabels) ]) 295 if __debug__: 296 debug('DS', 'Mapping for the labels computed to be %s' 297 % labels_map) 298 else: 299 if __debug__: 300 debug('DS', 'Mapping of labels was requested but labels ' 301 'are not strings. Skipped') 302 labels_map = None 303 pass 304 elif labels_map is False: 305 labels_map = None 306 307 if isinstance(labels_map, dict): 308 if labels_map_known: 309 if __debug__: 310 debug('DS', 311 "`dsattr` dict has `labels_map` (%s) but there is also" \ 312 " __init__ parameter `labels_map` (%s) which overrides " \ 313 " stored in `dsattr`" % (lcl_dsattr['labels_map'], labels_map)) 314 315 lcl_dsattr['labels_map'] = labels_map 316 # map labels if needed (if strings or was explicitely requested) 317 if labels_.dtype.char == 'S' or not labels_map_known: 318 if __debug__: 319 debug('DS_', "Remapping labels using mapping %s" % labels_map) 320 # need to remap 321 # !!! N.array is important here 322 try: 323 lcl_data['labels'] = N.array( 324 [labels_map[x] for x in lcl_data['labels']]) 325 except KeyError, e: 326 raise ValueError, "Provided labels_map %s is insufficient " \ 327 "to map all the labels. Mapping for label %s is " \ 328 "missing" % (labels_map, e) 329 330 elif not lcl_dsattr.has_key('labels_map'): 331 lcl_dsattr['labels_map'] = labels_map 332 elif __debug__: 333 debug('DS_', 'Not overriding labels_map in dsattr since it has one') 334 335 if check_data: 336 self._checkData() 337 338 # lazy computation of unique members 339 #self._resetallunique('_dsattr', self._dsattr) 340 341 # Michael: we cannot do this conditional here. When selectSamples() 342 # removes a whole data chunk the uniquechunks values will be invalid. 343 # Same applies to labels of course. 344 if not labels is None or not chunks is None: 345 # for a speed up to don't go through all uniqueattributes 346 # when no need 347 lcl_dsattr['__uniquereseted'] = False 348 self._resetallunique(force=True)

349 350 351 __doc__ = enhancedDocString('Dataset', locals()) 352 353 354 @property

355 - def idhash(self):

356 """To verify if dataset is in the same state as when smth else was done 357 358 Like if classifier was trained on the same dataset as in question""" 359 360 _data = self._data 361 res = idhash_(_data) 362 363 # we cannot count on the order the values in the dict will show up 364 # with `self._data.value()` and since idhash will be order-dependent 365 # we have to make it deterministic 366 keys = _data.keys() 367 keys.sort() 368 for k in keys: 369 res += idhash_(_data[k]) 370 return res

371 372

373 - def _resetallunique(self, force=False):

374 """Set to None all unique* attributes of corresponding dictionary 375 """ 376 _dsattr = self._dsattr 377 378 if not force and _dsattr['__uniquereseted']: 379 return 380 381 _uniqueattributes = self._uniqueattributes 382 383 if __debug__ and "DS_" in debug.active: 384 debug("DS_", "Reseting all attributes %s for dataset %s" 385 % (_uniqueattributes, 386 self.summary(uniq=False, idhash=False, 387 stats=False, lstats=False))) 388 389 # I guess we better checked if dictname is known but... 390 for k in _uniqueattributes: 391 _dsattr[k] = None 392 _dsattr['__uniquereseted'] = True

393 394

395 - def _getuniqueattr(self, attrib, dict_):

396 """Provide common facility to return unique attributes 397 398 XXX `dict_` can be simply replaced now with self._dsattr 399 """ 400 401 # local bindings 402 _dsattr = self._dsattr 403 404 if not _dsattr.has_key(attrib) or _dsattr[attrib] is None: 405 if __debug__ and 'DS_' in debug.active: 406 debug("DS_", "Recomputing unique set for attrib %s within %s" % 407 (attrib, self.summary(uniq=False, 408 stats=False, lstats=False))) 409 # uff... might come up with better strategy to keep relevant 410 # attribute name 411 _dsattr[attrib] = N.unique( N.asanyarray(dict_[attrib[6:]]) ) 412 assert(not _dsattr[attrib] is None) 413 _dsattr['__uniquereseted'] = False 414 415 return _dsattr[attrib]

416 417

418 - def _setdataattr(self, attrib, value):

419 """Provide common facility to set attributes 420 421 """ 422 if len(value) != self.nsamples: 423 raise ValueError, \ 424 "Provided %s have %d entries while there is %d samples" % \ 425 (attrib, len(value), self.nsamples) 426 self._data[attrib] = N.asarray(value) 427 uniqueattr = "unique" + attrib 428 429 _dsattr = self._dsattr 430 if _dsattr.has_key(uniqueattr): 431 _dsattr[uniqueattr] = None

432 433

434 - def _getNSamplesPerAttr( self, attrib='labels' ):

435 """Returns the number of samples per unique label. 436 """ 437 # local bindings 438 _data = self._data 439 440 # XXX hardcoded dict_=self._data.... might be in self._dsattr 441 uniqueattr = self._getuniqueattr(attrib="unique" + attrib, 442 dict_=_data) 443 444 # use dictionary to cope with arbitrary labels 445 result = dict(zip(uniqueattr, [ 0 ] * len(uniqueattr))) 446 for l in _data[attrib]: 447 result[l] += 1 448 449 # XXX only return values to mimic the old interface but we might want 450 # to return the full dict instead 451 # return result 452 return result

453 454

455 - def _getSampleIdsByAttr(self, values, attrib="labels", 456 sort=True):

457 """Return indecies of samples given a list of attributes 458 """ 459 460 if not operator.isSequenceType(values) \ 461 or isinstance(values, basestring): 462 values = [ values ] 463 464 # TODO: compare to plain for loop through the labels 465 # on a real data example 466 sel = N.array([], dtype=N.int16) 467 _data = self._data 468 for value in values: 469 sel = N.concatenate(( 470 sel, N.where(_data[attrib]==value)[0])) 471 472 if sort: 473 # place samples in the right order 474 sel.sort() 475 476 return sel

477 478

479 - def idsonboundaries(self, prior=0, post=0, 480 attributes_to_track=['labels', 'chunks'], 481 affected_labels=None, 482 revert=False):

483 """Find samples which are on the boundaries of the blocks 484 485 Such samples might need to be removed. By default (with 486 prior=0, post=0) ids of the first samples in a 'block' are 487 reported 488 489 :Parameters: 490 prior : int 491 how many samples prior to transition sample to include 492 post : int 493 how many samples post the transition sample to include 494 attributes_to_track : list of basestring 495 which attributes to track to decide on the boundary condition 496 affected_labels : list of basestring 497 for which labels to perform selection. If None - for all 498 revert : bool 499 either to revert the meaning and provide ids of samples which are found 500 to not to be boundary samples 501 """ 502 # local bindings 503 _data = self._data 504 labels = self.labels 505 nsamples = self.nsamples 506 507 lastseen = [None for attr in attributes_to_track] 508 transitions = [] 509 510 for i in xrange(nsamples): 511 current = [_data[attr][i] for attr in attributes_to_track] 512 if lastseen != current: 513 # transition point 514 new_transitions = range(max(0, i-prior), 515 min(nsamples-1, i+post)+1) 516 if affected_labels is not None: 517 new_transitions = filter(lambda i: labels[i] in affected_labels, 518 new_transitions) 519 transitions += new_transitions 520 lastseen = current 521 522 transitions = Set(transitions) 523 if revert: 524 transitions = Set(range(nsamples)).difference(transitions) 525 526 # postprocess 527 transitions = N.array(list(transitions)) 528 transitions.sort() 529 return list(transitions)

530 531

532 - def _shapeSamples(self, samples, dtype, copy):

533 """Adapt different kinds of samples 534 535 Handle all possible input value for 'samples' and tranform 536 them into a 2d (samples x feature) representation. 537 """ 538 # put samples array into correct shape 539 # 1d arrays or simple sequences are assumed to be a single pattern 540 if (not isinstance(samples, N.ndarray)): 541 # it is safe to provide dtype which defaults to None, 542 # when N would choose appropriate dtype automagically 543 samples = N.array(samples, ndmin=2, dtype=dtype, copy=copy) 544 else: 545 if samples.ndim < 2 \ 546 or (not dtype is None and dtype != samples.dtype): 547 if dtype is None: 548 dtype = samples.dtype 549 samples = N.array(samples, ndmin=2, dtype=dtype, copy=copy) 550 elif copy: 551 samples = samples.copy() 552 553 # only samples x features matrices are supported 554 if len(samples.shape) > 2: 555 raise DatasetError, "Only (samples x features) -> 2d sample " \ 556 + "are supported (got %s shape of samples)." \ 557 % (`samples.shape`) \ 558 +" Consider MappedDataset if applicable." 559 560 return samples

561 562

563 - def _checkData(self):

564 """Checks `_data` members to have the same # of samples. 565 """ 566 # 567 # XXX: Maybe just run this under __debug__ and remove the `check_data` 568 # from the constructor, which is too complicated anyway? 569 # 570 571 # local bindings 572 nsamples = self.nsamples 573 _data = self._data 574 575 for k, v in _data.iteritems(): 576 if not len(v) == nsamples: 577 raise DatasetError, \ 578 "Length of sample attribute '%s' [%i] does not " \ 579 "match the number of samples in the dataset [%i]." \ 580 % (k, len(v), nsamples) 581 582 # check for unique origids 583 uniques = N.unique(_data['origids']) 584 uniques.sort() 585 # need to copy to prevent sorting the original array 586 sorted_ids = _data['origids'].copy() 587 sorted_ids.sort() 588 589 if not (uniques == sorted_ids).all(): 590 raise DatasetError, "Samples IDs are not unique."

591 592

593 - def _expandSampleAttribute(self, attr, attr_name):

594 """If a sample attribute is given as a scalar expand/repeat it to a 595 length matching the number of samples in the dataset. 596 """ 597 try: 598 # if we are initializing with a single string -- we should 599 # treat it as a single label 600 if isinstance(attr, basestring): 601 raise TypeError 602 if len(attr) != self.nsamples: 603 raise DatasetError, \ 604 "Length of sample attribute '%s' [%d]" \ 605 % (attr_name, len(attr)) \ 606 + " has to match the number of samples" \ 607 + " [%d]." % self.nsamples 608 # store the sequence as array 609 return N.array(attr) 610 611 except TypeError: 612 # make sequence of identical value matching the number of 613 # samples 614 return N.repeat(attr, self.nsamples)

615 616 617 @classmethod

618 - def _registerAttribute(cls, key, dictname="_data", abbr=None, hasunique=False):

619 """Register an attribute for any Dataset class. 620 621 Creates property assigning getters/setters depending on the 622 availability of corresponding _get, _set functions. 623 """ 624 classdict = cls.__dict__ 625 if not classdict.has_key(key): 626 if __debug__: 627 debug("DS", "Registering new attribute %s" % key) 628 # define get function and use corresponding 629 # _getATTR if such defined 630 getter = '_get%s' % key 631 if classdict.has_key(getter): 632 getter = '%s.%s' % (cls.__name__, getter) 633 else: 634 getter = "lambda x: x.%s['%s']" % (dictname, key) 635 636 # define set function and use corresponding 637 # _setATTR if such defined 638 setter = '_set%s' % key 639 if classdict.has_key(setter): 640 setter = '%s.%s' % (cls.__name__, setter) 641 elif dictname=="_data": 642 setter = "lambda self,x: self._setdataattr" + \ 643 "(attrib='%s', value=x)" % (key) 644 else: 645 setter = None 646 647 if __debug__: 648 debug("DS", "Registering new property %s.%s" % 649 (cls.__name__, key)) 650 exec "%s.%s = property(fget=%s, fset=%s)" % \ 651 (cls.__name__, key, getter, setter) 652 653 if abbr is not None: 654 exec "%s.%s = property(fget=%s, fset=%s)" % \ 655 (cls.__name__, abbr, getter, setter) 656 657 if hasunique: 658 uniquekey = "unique%s" % key 659 getter = '_get%s' % uniquekey 660 if classdict.has_key(getter): 661 getter = '%s.%s' % (cls.__name__, getter) 662 else: 663 getter = "lambda x: x._getuniqueattr" + \ 664 "(attrib='%s', dict_=x.%s)" % (uniquekey, dictname) 665 666 if __debug__: 667 debug("DS", "Registering new property %s.%s" % 668 (cls.__name__, uniquekey)) 669 670 exec "%s.%s = property(fget=%s)" % \ 671 (cls.__name__, uniquekey, getter) 672 if abbr is not None: 673 exec "%s.U%s = property(fget=%s)" % \ 674 (cls.__name__, abbr, getter) 675 676 # create samplesper<ATTR> properties 677 sampleskey = "samplesper%s" % key[:-1] # remove ending 's' XXX 678 if __debug__: 679 debug("DS", "Registering new property %s.%s" % 680 (cls.__name__, sampleskey)) 681 682 exec "%s.%s = property(fget=%s)" % \ 683 (cls.__name__, sampleskey, 684 "lambda x: x._getNSamplesPerAttr(attrib='%s')" % key) 685 686 cls._uniqueattributes.append(uniquekey) 687 688 # create idsby<ATTR> properties 689 sampleskey = "idsby%s" % key # remove ending 's' XXX 690 if __debug__: 691 debug("DS", "Registering new property %s.%s" % 692 (cls.__name__, sampleskey)) 693 694 exec "%s.%s = %s" % (cls.__name__, sampleskey, 695 "lambda self, x: " + 696 "self._getSampleIdsByAttr(x,attrib='%s')" % key) 697 698 cls._uniqueattributes.append(uniquekey) 699 700 cls._registeredattributes.append(key) 701 elif __debug__: 702 warning('Trying to reregister attribute `%s`. For now ' % key + 703 'such capability is not present')

704 705

706 - def __str__(self):

707 """String summary over the object 708 """ 709 return self.summary(uniq=True, 710 idhash=__debug__ and ('DS_ID' in debug.active), 711 stats=__debug__ and ('DS_STATS' in debug.active), 712 lstats=__debug__ and ('DS_STATS' in debug.active), 713 )

714 715

716 - def __repr__(self):

717 return "<%s>" % str(self)

718 719

720 - def summary(self, uniq=True, stats=True, idhash=False, lstats=True, 721 maxc=30, maxl=20):

722 """String summary over the object 723 724 :Parameters: 725 uniq : bool 726 Include summary over data attributes which have unique 727 idhash : bool 728 Include idhash value for dataset and samples 729 stats : bool 730 Include some basic statistics (mean, std, var) over dataset samples 731 lstats : bool 732 Include statistics on chunks/labels 733 maxc : int 734 Maximal number of chunks when provide details on labels/chunks 735 maxl : int 736 Maximal number of labels when provide details on labels/chunks 737 """ 738 # local bindings 739 samples = self.samples 740 _data = self._data 741 _dsattr = self._dsattr 742 743 if idhash: 744 idhash_ds = "{%s}" % self.idhash 745 idhash_samples = "{%s}" % idhash_(samples) 746 else: 747 idhash_ds = "" 748 idhash_samples = "" 749 750 s = """Dataset %s/ %s %d%s x %d""" % \ 751 (idhash_ds, samples.dtype, 752 self.nsamples, idhash_samples, self.nfeatures) 753 754 ssep = (' ', '\n')[lstats] 755 if uniq: 756 s += "%suniq:" % ssep 757 for uattr in _dsattr.keys(): 758 if not uattr.startswith("unique"): 759 continue 760 attr = uattr[6:] 761 try: 762 value = self._getuniqueattr(attrib=uattr, 763 dict_=_data) 764 s += " %d %s" % (len(value), attr) 765 except: 766 pass 767 768 if isinstance(self.labels_map, dict): 769 s += ' labels_mapped' 770 771 if stats: 772 # TODO -- avg per chunk? 773 # XXX We might like to use scipy.stats.describe to get 774 # quick summary statistics (mean/range/skewness/kurtosis) 775 s += "%sstats: mean=%g std=%g var=%g min=%g max=%g\n" % \ 776 (ssep, N.mean(samples), N.std(samples), 777 N.var(samples), N.min(samples), N.max(samples)) 778 779 if lstats: 780 s += self.summary_labels(maxc=maxc, maxl=maxl) 781 782 return s

783 784

785 - def summary_labels(self, maxc=30, maxl=20):

786 """Provide summary statistics over the labels and chunks 787 788 :Parameters: 789 maxc : int 790 Maximal number of chunks when provide details 791 maxl : int 792 Maximal number of labels when provide details 793 """ 794 # We better avoid bound function since if people only 795 # imported Dataset without miscfx it would fail 796 from mvpa.datasets.miscfx import getSamplesPerChunkLabel 797 spcl = getSamplesPerChunkLabel(self) 798 # XXX couldn't they be unordered? 799 ul = self.uniquelabels.tolist() 800 uc = self.uniquechunks.tolist() 801 s = "" 802 if len(ul) < maxl and len(uc) < maxc: 803 s += "\nCounts of labels in each chunk:" 804 # only in a resonable case do printing 805 table = [[' chunks\labels'] + ul] 806 table += [[''] + ['---'] * len(ul)] 807 for c, counts in zip(uc, spcl): 808 table.append([ str(c) ] + counts.tolist()) 809 s += '\n' + table2string(table) 810 else: 811 s += "No details due to large number of labels or chunks. " \ 812 "Increase maxc and maxl if desired" 813 814 labels_map = self.labels_map 815 if isinstance(labels_map, dict): 816 s += "\nOriginal labels were mapped using following mapping:" 817 s += '\n\t'+'\n\t'.join([':\t'.join(map(str, x)) 818 for x in labels_map.items()]) + '\n' 819 820 def cl_stats(axis, u, name1, name2): 821 """ Compute statistics per label 822 """ 823 stats = {'min': N.min(spcl, axis=axis), 824 'max': N.max(spcl, axis=axis), 825 'mean': N.mean(spcl, axis=axis), 826 'std': N.std(spcl, axis=axis), 827 '#%ss' % name2: N.sum(spcl>0, axis=axis)} 828 entries = [' ' + name1, 'mean', 'std', 'min', 'max', '#%ss' % name2] 829 table = [ entries ] 830 for i, l in enumerate(u): 831 d = {' ' + name1 : l} 832 d.update(dict([ (k, stats[k][i]) for k in stats.keys()])) 833 table.append( [ ('%.3g', '%s')[isinstance(d[e], basestring)] 834 % d[e] for e in entries] ) 835 return '\nSummary per %s across %ss\n' % (name1, name2) \ 836 + table2string(table)

837 838 if len(ul) < maxl: 839 s += cl_stats(0, ul, 'label', 'chunk') 840 if len(uc) < maxc: 841 s += cl_stats(1, uc, 'chunk', 'label') 842 return s

843 844

845 - def __iadd__(self, other):

846 """Merge the samples of one Dataset object to another (in-place). 847 848 No dataset attributes, besides labels_map, will be merged! 849 Additionally, a new set of unique `origids` will be generated. 850 """ 851 # local bindings 852 _data = self._data 853 other_data = other._data 854 855 if not self.nfeatures == other.nfeatures: 856 raise DatasetError, "Cannot add Dataset, because the number of " \ 857 "feature do not match." 858 859 # take care about labels_map and labels 860 slm = self.labels_map 861 olm = other.labels_map 862 if N.logical_xor(slm is None, olm is None): 863 raise ValueError, "Cannot add datasets where only one of them " \ 864 "has labels map assigned. If needed -- implement it" 865 866 # concatenate all sample attributes 867 for k,v in _data.iteritems(): 868 if k == 'origids': 869 # special case samples origids: for now just regenerate unique 870 # ones could also check if concatenation is unique, but it 871 # would be costly performance-wise 872 _data[k] = N.arange(len(v) + len(other_data[k])) 873 874 elif k == 'labels' and slm is not None: 875 # special care about labels if mapping was in effect, 876 # we need to append 2nd map to the first one and 877 # relabel 2nd dataset 878 nlm = slm.copy() 879 # figure out maximal numerical label used now 880 nextid = N.sort(nlm.values())[-1] + 1 881 olabels = other.labels 882 olabels_remap = {} 883 for ol, olnum in olm.iteritems(): 884 if not nlm.has_key(ol): 885 # check if we can preserve old numberic label 886 # if not -- assign some new one not yet present 887 # in any dataset 888 if olnum in nlm.values(): 889 nextid = N.sort(nlm.values() + olm.values())[-1] + 1 890 else: 891 nextid = olnum 892 olabels_remap[olnum] = nextid 893 nlm[ol] = nextid 894 nextid += 1 895 else: 896 olabels_remap[olnum] = nlm[ol] 897 olabels = [olabels_remap[x] for x in olabels] 898 # finally compose new labels 899 _data['labels'] = N.concatenate((v, olabels), axis=0) 900 # and reassign new mapping 901 self._dsattr['labels_map'] = nlm 902 903 if __debug__: 904 # check if we are not dealing with colliding 905 # mapping, since it is problematic and might lead 906 # to various complications 907 if (len(Set(slm.keys())) != len(Set(slm.values()))) or \ 908 (len(Set(olm.keys())) != len(Set(olm.values()))): 909 warning("Adding datasets where multiple labels " 910 "mapped to the same ID is not recommended. " 911 "Please check the outcome. Original mappings " 912 "were %s and %s. Resultant is %s" 913 % (slm, olm, nlm)) 914 915 else: 916 _data[k] = N.concatenate((v, other_data[k]), axis=0) 917 918 # might be more sophisticated but for now just reset -- it is safer ;) 919 self._resetallunique() 920 921 return self

922 923

924 - def __add__( self, other ):

925 """Merge the samples two Dataset objects. 926 927 All data of both datasets is copied, concatenated and a new Dataset is 928 returned. 929 930 NOTE: This can be a costly operation (both memory and time). If 931 performance is important consider the '+=' operator. 932 """ 933 # create a new object of the same type it is now and NOT only Dataset 934 out = super(Dataset, self).__new__(self.__class__) 935 936 # now init it: to make it work all Dataset contructors have to accept 937 # Class(data=Dict, dsattr=Dict) 938 out.__init__(data=self._data, 939 dsattr=self._dsattr, 940 copy_samples=True, 941 copy_data=True, 942 copy_dsattr=True) 943 944 out += other 945 946 return out

947 948

949 - def copy(self):

950 """Create a copy (clone) of the dataset, by fully copying current one 951 952 """ 953 # create a new object of the same type it is now and NOT only Dataset 954 out = super(Dataset, self).__new__(self.__class__) 955 956 # now init it: to make it work all Dataset contructors have to accept 957 # Class(data=Dict, dsattr=Dict) 958 out.__init__(data=self._data, 959 dsattr=self._dsattr, 960 copy_samples=True, 961 copy_data=True, 962 copy_dsattr=True) 963 964 return out

965 966

967 - def selectFeatures(self, ids=None, sort=True, groups=None):

968 """Select a number of features from the current set. 969 970 :Parameters: 971 ids 972 iterable container to select ids 973 sort : bool 974 if to sort Ids. Order matters and `selectFeatures` assumes 975 incremental order. If not such, in non-optimized code 976 selectFeatures would verify the order and sort 977 978 Returns a new Dataset object with a view of the original 979 samples array (no copying is performed). 980 981 WARNING: The order of ids determines the order of features in 982 the returned dataset. This might be useful sometimes, but can 983 also cause major headaches! Order would is verified when 984 running in non-optimized code (if __debug__) 985 """ 986 ids = copy.deepcopy(ids) 987 988 if ids is None and groups is None: 989 raise ValueError, "No feature selection specified." 990 991 # start with empty list if no ids where specified (so just groups) 992 if ids is None: 993 ids = [] 994 995 if not groups is None: 996 if not self._dsattr.has_key('featuregroups'): 997 raise RuntimeError, \ 998 "Dataset has no feature grouping information." 999 1000 for g in groups: 1001 ids += (self._dsattr['featuregroups'] == g).nonzero()[0].tolist() 1002 1003 # XXX set sort default to True, now sorting has to be explicitely 1004 # disabled and warning is not necessary anymore 1005 if sort: 1006 ids.sort() 1007 elif __debug__ and 'CHECK_DS_SORTED' in debug.active: 1008 from mvpa.misc.support import isSorted 1009 if not isSorted(ids): 1010 warning("IDs for selectFeatures must be provided " + 1011 "in sorted order, otherwise major headache might occur") 1012 1013 # shallow-copy all stuff from current data dict 1014 new_data = self._data.copy() 1015 1016 # assign the selected features -- data is still shared with 1017 # current dataset 1018 new_data['samples'] = self._data['samples'][:, ids] 1019 1020 # apply selection to feature groups as well 1021 if self._dsattr.has_key('featuregroups'): 1022 new_dsattr = self._dsattr.copy() 1023 new_dsattr['featuregroups'] = self._dsattr['featuregroups'][ids] 1024 else: 1025 new_dsattr = self._dsattr 1026 1027 # create a new object of the same type it is now and NOT onyl Dataset 1028 dataset = super(Dataset, self).__new__(self.__class__) 1029 1030 # now init it: to make it work all Dataset contructors have to accept 1031 # Class(data=Dict, dsattr=Dict) 1032 dataset.__init__(data=new_data, 1033 dsattr=new_dsattr, 1034 check_data=False, 1035 copy_samples=False, 1036 copy_data=False, 1037 copy_dsattr=False 1038 ) 1039 1040 return dataset

1041 1042

1043 - def applyMapper(self, featuresmapper=None, samplesmapper=None, 1044 train=True):

1045 """Obtain new dataset by applying mappers over features and/or samples. 1046 1047 While featuresmappers leave the sample attributes information 1048 unchanged, as the number of samples in the dataset is invariant, 1049 samplesmappers are also applied to the samples attributes themselves! 1050 1051 Applying a featuresmapper will destroy any feature grouping information. 1052 1053 :Parameters: 1054 featuresmapper : Mapper 1055 `Mapper` to somehow transform each sample's features 1056 samplesmapper : Mapper 1057 `Mapper` to transform each feature across samples 1058 train : bool 1059 Flag whether to train the mapper with this dataset before applying 1060 it. 1061 1062 TODO: selectFeatures is pretty much 1063 applyMapper(featuresmapper=MaskMapper(...)) 1064 """ 1065 1066 # shallow-copy all stuff from current data dict 1067 new_data = self._data.copy() 1068 1069 # apply mappers 1070 1071 if samplesmapper: 1072 if __debug__: 1073 debug("DS", "Training samplesmapper %s" % `samplesmapper`) 1074 samplesmapper.train(self) 1075 1076 if __debug__: 1077 debug("DS", "Applying samplesmapper %s" % `samplesmapper` + 1078 " to samples of dataset `%s`" % `self`) 1079 1080 # get rid of existing 'origids' as they are not valid anymore and 1081 # applying a mapper to them is not really meaningful 1082 if new_data.has_key('origids'): 1083 del(new_data['origids']) 1084 1085 # apply mapper to all sample-wise data in dataset 1086 for k in new_data.keys(): 1087 new_data[k] = samplesmapper.forward(self._data[k]) 1088 1089 # feature mapping might affect dataset attributes 1090 # XXX: might be obsolete when proper feature attributes are implemented 1091 new_dsattr = self._dsattr 1092 1093 if featuresmapper: 1094 if __debug__: 1095 debug("DS", "Training featuresmapper %s" % `featuresmapper`) 1096 featuresmapper.train(self) 1097 1098 if __debug__: 1099 debug("DS", "Applying featuresmapper %s" % `featuresmapper` + 1100 " to samples of dataset `%s`" % `self`) 1101 new_data['samples'] = featuresmapper.forward(self._data['samples']) 1102 1103 # remove feature grouping, who knows what the mapper did to the 1104 # features 1105 if self._dsattr.has_key('featuregroups'): 1106 new_dsattr = self._dsattr.copy() 1107 del(new_dsattr['featuregroups']) 1108 else: 1109 new_dsattr = self._dsattr 1110 1111 # create a new object of the same type it is now and NOT only Dataset 1112 dataset = super(Dataset, self).__new__(self.__class__) 1113 1114 # now init it: to make it work all Dataset contructors have to accept 1115 # Class(data=Dict, dsattr=Dict) 1116 dataset.__init__(data=new_data, 1117 dsattr=new_dsattr, 1118 check_data=False, 1119 copy_samples=False, 1120 copy_data=False, 1121 copy_dsattr=False 1122 ) 1123 1124 # samples attributes might have changed after applying samplesmapper 1125 if samplesmapper: 1126 dataset._resetallunique(force=True) 1127 1128 return dataset

1129 1130

1131 - def selectSamples(self, ids):

1132 """Choose a subset of samples defined by samples IDs. 1133 1134 Returns a new dataset object containing the selected sample 1135 subset. 1136 1137 TODO: yoh, we might need to sort the mask if the mask is a 1138 list of ids and is not ordered. Clarify with Michael what is 1139 our intent here! 1140 """ 1141 # without having a sequence a index the masked sample array would 1142 # loose its 2d layout 1143 if not operator.isSequenceType( ids ): 1144 ids = [ids] 1145 # TODO: Reconsider crafting a slice if it can be done to don't copy 1146 # the data 1147 #try: 1148 # minmask = min(mask) 1149 # maxmask = max(mask) 1150 #except: 1151 # minmask = min(map(int,mask)) 1152 # maxmask = max(map(int,mask)) 1153 # lets see if we could get it done with cheap view/slice 1154 #(minmask, maxmask) != (0, 1) and \ 1155 #if len(mask) > 2 and \ 1156 # N.array([N.arange(minmask, maxmask+1) == N.array(mask)]).all(): 1157 # slice_ = slice(minmask, maxmask+1) 1158 # if __debug__: 1159 # debug("DS", "We can and do convert mask %s into splice %s" % 1160 # (mask, slice_)) 1161 # mask = slice_ 1162 # mask all sample attributes 1163 data = {} 1164 for k, v in self._data.iteritems(): 1165 data[k] = v[ids, ] 1166 1167 # create a new object of the same type it is now and NOT onyl Dataset 1168 dataset = super(Dataset, self).__new__(self.__class__) 1169 1170 # now init it: to make it work all Dataset contructors have to accept 1171 # Class(data=Dict, dsattr=Dict) 1172 dataset.__init__(data=data, 1173 dsattr=self._dsattr, 1174 check_data=False, 1175 copy_samples=False, 1176 copy_data=False, 1177 copy_dsattr=False) 1178 1179 dataset._resetallunique(force=True) 1180 return dataset

1181 1182 1183

1184 - def index(self, *args, **kwargs):

1185 """Universal indexer to obtain indexes of interesting samples/features. 1186 See .select() for more information 1187 1188 :Return: tuple of (samples indexes, features indexes). Each 1189 item could be also None, if no selection on samples or 1190 features was requested (to discriminate between no selected 1191 items, and no selections) 1192 """ 1193 s_indx = [] # selections for samples 1194 f_indx = [] # selections for features 1195 return_dataset = kwargs.pop('return_dataset', False) 1196 largs = len(args) 1197 1198 args = list(args) # so we could override 1199 # Figure out number of positional 1200 largs_nonstring = 0 1201 # need to go with index since we might need to override internally 1202 for i in xrange(largs): 1203 l = args[i] 1204 if isinstance(l, basestring): 1205 if l.lower() == 'all': 1206 # override with a slice 1207 args[i] = slice(None) 1208 else: 1209 break 1210 largs_nonstring += 1 1211 1212 if largs_nonstring >= 1: 1213 s_indx.append(args[0]) 1214 if __debug__ and 'CHECK_DS_SELECT' in debug.active: 1215 _validate_indexes_uniq_sorted(args[0], 'select', 'samples') 1216 if largs_nonstring == 2: 1217 f_indx.append(args[1]) 1218 if __debug__ and 'CHECK_DS_SELECT' in debug.active: 1219 _validate_indexes_uniq_sorted(args[1], 'select', 'features') 1220 elif largs_nonstring > 2: 1221 raise ValueError, "Only two positional arguments are allowed" \ 1222 ". 1st for samples, 2nd for features" 1223 1224 # process left positional arguments which must encode selections like 1225 # ('labels', [1,2,3]) 1226 1227 if (largs - largs_nonstring) % 2 != 0: 1228 raise ValueError, "Positional selections must come in pairs:" \ 1229 " e.g. ('labels', [1,2,3])" 1230 1231 for i in xrange(largs_nonstring, largs, 2): 1232 k, v = args[i:i+2] 1233 kwargs[k] = v 1234 1235 # process keyword parameters 1236 data_ = self._data 1237 for k, v in kwargs.iteritems(): 1238 if k == 'samples': 1239 s_indx.append(v) 1240 elif k == 'features': 1241 f_indx.append(v) 1242 elif data_.has_key(k): 1243 # so it is an attribute for samples 1244 # XXX may be do it not only if __debug__ 1245 if __debug__: # and 'CHECK_DS_SELECT' in debug.active: 1246 if not N.any([isinstance(v, cls) for cls in 1247 [list, tuple, slice, int]]): 1248 raise ValueError, "Trying to specify selection for %s " \ 1249 "based on unsupported '%s'" % (k, v) 1250 s_indx.append(self._getSampleIdsByAttr(v, attrib=k, sort=False)) 1251 else: 1252 raise ValueError, 'Keyword "%s" is not known, thus' \ 1253 'select() failed' % k 1254 1255 def combine_indexes(indx, nelements): 1256 """Helper function: intersect selections given in indx 1257 1258 :Parameters: 1259 indxs : list of lists or slices 1260 selections of elements 1261 nelements : int 1262 number of elements total for deriving indexes from slices 1263 """ 1264 indx_sel = None # pure list of ids for selection 1265 for s in indx: 1266 if isinstance(s, slice) or \ 1267 isinstance(s, N.ndarray) and s.dtype==bool: 1268 # XXX there might be a better way than reconstructing the full 1269 # index list. Also we are loosing ability to do simlpe slicing, 1270 # ie w.o making a copy of the selected data 1271 all_indexes = N.arange(nelements) 1272 s = all_indexes[s] 1273 elif not operator.isSequenceType(s): 1274 s = [ s ] 1275 1276 if indx_sel is None: 1277 indx_sel = Set(s) 1278 else: 1279 # To be consistent 1280 #if not isinstance(indx_sel, Set): 1281 # indx_sel = Set(indx_sel) 1282 indx_sel = indx_sel.intersection(s) 1283 1284 # if we got Set -- convert 1285 if isinstance(indx_sel, Set): 1286 indx_sel = list(indx_sel) 1287 1288 # sort for the sake of sanity 1289 indx_sel.sort() 1290 1291 return indx_sel

1292 1293 # Select samples 1294 if len(s_indx) == 1 and isinstance(s_indx[0], slice) \ 1295 and s_indx[0] == slice(None): 1296 # so no actual selection -- full slice 1297 s_indx = s_indx[0] 1298 else: 1299 # else - get indexes 1300 if len(s_indx) == 0: 1301 s_indx = None 1302 else: 1303 s_indx = combine_indexes(s_indx, self.nsamples) 1304 1305 # Select features 1306 if len(f_indx): 1307 f_indx = combine_indexes(f_indx, self.nfeatures) 1308 else: 1309 f_indx = None 1310 1311 return s_indx, f_indx 1312 1313

1314 - def select(self, *args, **kwargs):

1315 """Universal selector 1316 1317 WARNING: if you need to select duplicate samples 1318 (e.g. samples=[5,5]) or order of selected samples of features 1319 is important and has to be not ordered (e.g. samples=[3,2,1]), 1320 please use selectFeatures or selectSamples functions directly 1321 1322 Examples: 1323 Mimique plain selectSamples:: 1324 1325 dataset.select([1,2,3]) 1326 dataset[[1,2,3]] 1327 1328 Mimique plain selectFeatures:: 1329 1330 dataset.select(slice(None), [1,2,3]) 1331 dataset.select('all', [1,2,3]) 1332 dataset[:, [1,2,3]] 1333 1334 Mixed (select features and samples):: 1335 1336 dataset.select([1,2,3], [1, 2]) 1337 dataset[[1,2,3], [1, 2]] 1338 1339 Select samples matching some attributes:: 1340 1341 dataset.select(labels=[1,2], chunks=[2,4]) 1342 dataset.select('labels', [1,2], 'chunks', [2,4]) 1343 dataset['labels', [1,2], 'chunks', [2,4]] 1344 1345 Mixed -- out of first 100 samples, select only those with 1346 labels 1 or 2 and belonging to chunks 2 or 4, and select 1347 features 2 and 3:: 1348 1349 dataset.select(slice(0,100), [2,3], labels=[1,2], chunks=[2,4]) 1350 dataset[:100, [2,3], 'labels', [1,2], 'chunks', [2,4]] 1351 1352 """ 1353 s_indx, f_indx = self.index(*args, **kwargs) 1354 1355 # Select samples 1356 if s_indx == slice(None): 1357 # so no actual selection was requested among samples. 1358 # thus proceed with original dataset 1359 if __debug__: 1360 debug('DS', 'in select() not selecting samples') 1361 ds = self 1362 else: 1363 # else do selection 1364 if __debug__: 1365 debug('DS', 'in select() selecting samples given selections' 1366 + str(s_indx)) 1367 ds = self.selectSamples(s_indx) 1368 1369 # Select features 1370 if f_indx is not None: 1371 if __debug__: 1372 debug('DS', 'in select() selecting features given selections' 1373 + str(f_indx)) 1374 ds = ds.selectFeatures(f_indx) 1375 1376 return ds

1377 1378 1379

1380 - def where(self, *args, **kwargs):

1381 """Obtain indexes of interesting samples/features. See select() for more information 1382 1383 XXX somewhat obsoletes idsby... 1384 """ 1385 s_indx, f_indx = self.index(*args, **kwargs) 1386 if s_indx is not None and f_indx is not None: 1387 return s_indx, f_indx 1388 elif s_indx is not None: 1389 return s_indx 1390 else: 1391 return f_indx

1392 1393

1394 - def __getitem__(self, *args):

1395 """Convinience dataset parts selection 1396 1397 See select for more information 1398 """ 1399 # for cases like ['labels', 1] 1400 if len(args) == 1 and isinstance(args[0], tuple): 1401 args = args[0] 1402 1403 args_, args = args, () 1404 for a in args_: 1405 if isinstance(a, slice) and \ 1406 isinstance(a.start, basestring): 1407 # for the constructs like ['labels':[1,2]] 1408 if a.stop is None or a.step is not None: 1409 raise ValueError, \ 1410 "Selection must look like ['chunks':[2,3]]" 1411 args += (a.start, a.stop) 1412 else: 1413 args += (a,) 1414 return self.select(*args)

1415 1416

1417 - def permuteLabels(self, status, perchunk=True, assure_permute=False):

1418 """Permute the labels. 1419 1420 TODO: rename status into something closer in semantics. 1421 1422 :Parameters: 1423 status : bool 1424 Calling this method with set to True, the labels are 1425 permuted among all samples. If 'status' is False the 1426 original labels are restored. 1427 perchunk : bool 1428 If True permutation is limited to samples sharing the same 1429 chunk value. Therefore only the association of a certain 1430 sample with a label is permuted while keeping the absolute 1431 number of occurences of each label value within a certain 1432 chunk constant. 1433 assure_permute : bool 1434 If True, assures that labels are permutted, ie any one is 1435 different from the original one 1436 """ 1437 # local bindings 1438 _data = self._data 1439 1440 if len(self.uniquelabels)<2: 1441 raise RuntimeError, \ 1442 "Call to permuteLabels is bogus since there is insuficient" \ 1443 " number of labels: %s" % self.uniquelabels 1444 1445 if not status: 1446 # restore originals 1447 if _data.get('origlabels', None) is None: 1448 raise RuntimeError, 'Cannot restore labels. ' \ 1449 'permuteLabels() has never been ' \ 1450 'called with status == True.' 1451 self.labels = _data['origlabels'] 1452 _data.pop('origlabels') 1453 else: 1454 # store orig labels, but only if not yet done, otherwise multiple 1455 # calls with status == True will destroy the original labels 1456 if not _data.has_key('origlabels') \ 1457 or _data['origlabels'] == None: 1458 # bind old labels to origlabels 1459 _data['origlabels'] = _data['labels'] 1460 # copy labels 1461 _data['labels'] = copy.copy(_data['labels']) 1462 1463 labels = _data['labels'] 1464 # now scramble 1465 if perchunk: 1466 for o in self.uniquechunks: 1467 labels[self.chunks == o] = \ 1468 N.random.permutation(labels[self.chunks == o]) 1469 else: 1470 labels = N.random.permutation(labels) 1471 1472 self.labels = labels 1473 1474 if assure_permute: 1475 if not (_data['labels'] != _data['origlabels']).any(): 1476 if not (assure_permute is True): 1477 if assure_permute == 1: 1478 raise RuntimeError, \ 1479 "Cannot assure permutation of labels %s for " \ 1480 "some reason with chunks %s and while " \ 1481 "perchunk=%s . Should not happen" % \ 1482 (self.labels, self.chunks, perchunk) 1483 else: 1484 assure_permute = 11 # make 10 attempts 1485 if __debug__: 1486 debug("DS", "Recalling permute to assure different labels") 1487 self.permuteLabels(status, perchunk=perchunk, 1488 assure_permute=assure_permute-1)

1489 1490

1491 - def getRandomSamples(self, nperlabel):

1492 """Select a random set of samples. 1493 1494 If 'nperlabel' is an integer value, the specified number of samples is 1495 randomly choosen from the group of samples sharing a unique label 1496 value ( total number of selected samples: nperlabel x len(uniquelabels). 1497 1498 If 'nperlabel' is a list which's length has to match the number of 1499 unique label values. In this case 'nperlabel' specifies the number of 1500 samples that shall be selected from the samples with the corresponding 1501 label. 1502 1503 The method returns a Dataset object containing the selected 1504 samples. 1505 """ 1506 # if interger is given take this value for all classes 1507 if isinstance(nperlabel, int): 1508 nperlabel = [ nperlabel for i in self.uniquelabels ] 1509 1510 sample = [] 1511 # for each available class 1512 labels = self.labels 1513 for i, r in enumerate(self.uniquelabels): 1514 # get the list of pattern ids for this class 1515 sample += random.sample( (labels == r).nonzero()[0], 1516 nperlabel[i] ) 1517 1518 return self.selectSamples( sample )

1519 1520 1521 # def _setchunks(self, chunks): 1522 # """Sets chunks and recomputes uniquechunks 1523 # """ 1524 # self._data['chunks'] = N.array(chunks) 1525 # self._dsattr['uniquechunks'] = None # None!since we might not need them 1526 1527

1528 - def getNSamples( self ):

1529 """Currently available number of patterns. 1530 """ 1531 return self._data['samples'].shape[0]

1532 1533

1534 - def getNFeatures( self ):

1535 """Number of features per pattern. 1536 """ 1537 return self._data['samples'].shape[1]

1538 1539

1540 - def getLabelsMap(self):

1541 """Stored labels map (if any) 1542 """ 1543 return self._dsattr.get('labels_map', None)

1544 1545

1546 - def setLabelsMap(self, lm):

1547 """Set labels map. 1548 1549 Checks for the validity of the mapping -- values should cover 1550 all existing labels in the dataset 1551 """ 1552 values = Set(lm.values()) 1553 labels = Set(self.uniquelabels) 1554 if not values.issuperset(labels): 1555 raise ValueError, \ 1556 "Provided mapping %s has some existing labels (out of %s) " \ 1557 "missing from mapping" % (list(values), list(labels)) 1558 self._dsattr['labels_map'] = lm

1559 1560

1561 - def setSamplesDType(self, dtype):

1562 """Set the data type of the samples array. 1563 """ 1564 # local bindings 1565 _data = self._data 1566 1567 if _data['samples'].dtype != dtype: 1568 _data['samples'] = _data['samples'].astype(dtype)

1569 1570

1571 - def defineFeatureGroups(self, definition):

1572 """Assign `definition` to featuregroups 1573 1574 XXX Feature-groups was not finished to be useful 1575 """ 1576 if not len(definition) == self.nfeatures: 1577 raise ValueError, \ 1578 "Length of feature group definition %i " \ 1579 "does not match the number of features %i " \ 1580 % (len(definition), self.nfeatures) 1581 1582 self._dsattr['featuregroups'] = N.array(definition)

1583 1584

1585 - def convertFeatureIds2FeatureMask(self, ids):

1586 """Returns a boolean mask with all features in `ids` selected. 1587 1588 :Parameters: 1589 ids: list or 1d array 1590 To be selected features ids. 1591 1592 :Returns: 1593 ndarray: dtype='bool' 1594 All selected features are set to True; False otherwise. 1595 """ 1596 fmask = N.repeat(False, self.nfeatures) 1597 fmask[ids] = True 1598 1599 return fmask

1600 1601

1602 - def convertFeatureMask2FeatureIds(self, mask):

1603 """Returns feature ids corresponding to non-zero elements in the mask. 1604 1605 :Parameters: 1606 mask: 1d ndarray 1607 Feature mask. 1608 1609 :Returns: 1610 ndarray: integer 1611 Ids of non-zero (non-False) mask elements. 1612 """ 1613 return mask.nonzero()[0]

1614 1615 1616 @staticmethod

1617 - def _checkCopyConstructorArgs(**kwargs):

1618 """Common sanity check for Dataset copy constructor calls.""" 1619 # check if we have samples (somwhere) 1620 samples = None 1621 if kwargs.has_key('samples'): 1622 samples = kwargs['samples'] 1623 if samples is None and kwargs.has_key('data') \ 1624 and kwargs['data'].has_key('samples'): 1625 samples = kwargs['data']['samples'] 1626 if samples is None: 1627 raise DatasetError, \ 1628 "`samples` must be provided to copy constructor call." 1629 1630 if not len(samples.shape) == 2: 1631 raise DatasetError, \ 1632 "samples must be in 2D shape in copy constructor call."

1633 1634 1635 # read-only class properties 1636 nsamples = property( fget=getNSamples ) 1637 nfeatures = property( fget=getNFeatures ) 1638 labels_map = property( fget=getLabelsMap, fset=setLabelsMap ) 1639

1640 -def datasetmethod(func):

1641 """Decorator to easily bind functions to a Dataset class 1642 """ 1643 if __debug__: 1644 debug("DS_", "Binding function %s to Dataset class" % func.func_name) 1645 1646 # Bind the function 1647 setattr(Dataset, func.func_name, func) 1648 1649 # return the original one 1650 return func

1651 1652 1653 # Following attributes adherent to the basic dataset 1654 Dataset._registerAttribute("samples", "_data", abbr='S', hasunique=False) 1655 Dataset._registerAttribute("labels", "_data", abbr='L', hasunique=True) 1656 Dataset._registerAttribute("chunks", "_data", abbr='C', hasunique=True) 1657 # samples ids (already unique by definition) 1658 Dataset._registerAttribute("origids", "_data", abbr='I', hasunique=False) 1659

Source Code for Module mvpa.datasets.base