1
2
3
4
5
6
7
8
9 """Collection of dataset splitters.
10
11 Module Description
12 ==================
13
14 Splitters are destined to split the provided dataset varous ways to
15 simplify cross-validation analysis, implement boosting of the
16 estimates, or sample null-space via permutation testing.
17
18 Most of the splitters at the moment split 2-ways -- conventionally
19 first part is used for training, and 2nd part for testing by
20 `CrossValidatedTransferError` and `SplitClassifier`.
21
22 Brief Description of Available Splitters
23 ========================================
24
25 * `NoneSplitter` - just return full dataset as the desired part (training/testing)
26 * `OddEvenSplitter` - 2 splits: (odd samples,even samples) and (even, odd)
27 * `HalfSplitter` - 2 splits: (first half, second half) and (second, first)
28 * `NFoldSplitter` - splits for N-Fold cross validation.
29
30 Module Organization
31 ===================
32
33 .. packagetree::
34 :style: UML
35
36 """
37
38 __docformat__ = 'restructuredtext'
39
40 import operator
41
42 import numpy as N
43
44 import mvpa.misc.support as support
45 from mvpa.base.dochelpers import enhancedDocString
46 from mvpa.datasets.miscfx import coarsenChunks
47
48 if __debug__:
49 from mvpa.base import debug
50
52 """Base class of dataset splitters.
53
54 Each splitter should be initialized with all its necessary parameters. The
55 final splitting is done running the splitter object on a certain Dataset
56 via __call__(). This method has to be implemented like a generator, i.e. it
57 has to return every possible split with a yield() call.
58
59 Each split has to be returned as a sequence of Datasets. The properties
60 of the splitted dataset may vary between implementations. It is possible
61 to declare a sequence element as 'None'.
62
63 Please note, that even if there is only one Dataset returned it has to be
64 an element in a sequence and not just the Dataset object!
65 """
66
67 _STRATEGIES = ('first', 'random', 'equidistant')
68 _NPERLABEL_STR = ['equal', 'all']
69
70 - def __init__(self,
71 nperlabel='all',
72 nrunspersplit=1,
73 permute=False,
74 count=None,
75 strategy='equidistant',
76 discard_boundary=None,
77 attr='chunks'):
78 """Initialize splitter base.
79
80 :Parameters:
81 nperlabel : int or str (or list of them) or float
82 Number of dataset samples per label to be included in each
83 split. If given as a float, it must be in [0,1] range and would
84 mean the ratio of selected samples per each label.
85 Two special strings are recognized: 'all' uses all available
86 samples (default) and 'equal' uses the maximum number of samples
87 the can be provided by all of the classes. This value might be
88 provided as a sequence whos length matches the number of datasets
89 per split and indicates the configuration for the respective dataset
90 in each split.
91 nrunspersplit: int
92 Number of times samples for each split are chosen. This
93 is mostly useful if a subset of the available samples
94 is used in each split and the subset is randomly
95 selected for each run (see the `nperlabel` argument).
96 permute : bool
97 If set to `True`, the labels of each generated dataset
98 will be permuted on a per-chunk basis.
99 count : None or int
100 Desired number of splits to be output. It is limited by the
101 number of splits possible for a given splitter
102 (e.g. `OddEvenSplitter` can have only up to 2 splits). If None,
103 all splits are output (default).
104 strategy : str
105 If `count` is not None, possible strategies are possible:
106 first
107 First `count` splits are chosen
108 random
109 Random (without replacement) `count` splits are chosen
110 equidistant
111 Splits which are equidistant from each other
112 discard_boundary : None or int or sequence of int
113 If not `None`, how many samples on the boundaries between
114 parts of the split to discard in the training part.
115 If int, then discarded in all parts. If a sequence, numbers
116 to discard are given per part of the split.
117 E.g. if splitter splits only into (training, testing)
118 parts, then `discard_boundary`=(2,0) would instruct to discard
119 2 samples from training which are on the boundary with testing.
120 attr : str
121 Sample attribute used to determine splits.
122 """
123
124 self.__nperlabel = None
125 self.__runspersplit = nrunspersplit
126 self.__permute = permute
127 self.__splitattr = attr
128 self.discard_boundary = discard_boundary
129
130
131
132
133 self.count = count
134 """Number (max) of splits to output on call"""
135
136 self._setStrategy(strategy)
137
138
139 self.setNPerLabel(nperlabel)
140
141
142 __doc__ = enhancedDocString('Splitter', locals())
143
152
154 """Set the number of samples per label in the split datasets.
155
156 'equal' sets sample size to highest possible number of samples that
157 can be provided by each class. 'all' uses all available samples
158 (default).
159 """
160 if isinstance(value, basestring):
161 if not value in self._NPERLABEL_STR:
162 raise ValueError, "Unsupported value '%s' for nperlabel." \
163 " Supported ones are %s or float or int" % (value, self._NPERLABEL_STR)
164 self.__nperlabel = value
165
166
168 """Each subclass has to implement this method. It gets a sequence with
169 the unique attribte ids of a dataset and has to return a list of lists
170 containing attribute ids to split into the second dataset.
171 """
172 raise NotImplementedError
173
174
176 """Splits the dataset.
177
178 This method behaves like a generator.
179 """
180
181
182 ds_class = dataset.__class__
183 DS_permuteLabels = ds_class.permuteLabels
184 try:
185 DS_getNSamplesPerLabel = ds_class._getNSamplesPerAttr
186 except AttributeError:
187
188
189 pass
190 DS_getRandomSamples = ds_class.getRandomSamples
191
192
193 cfgs = self.splitcfg(dataset)
194
195
196 count, Ncfgs = self.count, len(cfgs)
197
198
199
200 if count is not None and count < Ncfgs:
201 if count < 1:
202
203 return
204 strategy = self.strategy
205 if strategy == 'first':
206 cfgs = cfgs[:count]
207 elif strategy in ['equidistant', 'random']:
208 if strategy == 'equidistant':
209
210
211 step = float(Ncfgs) / count
212 assert(step >= 1.0)
213 indexes = [int(round(step * i)) for i in xrange(count)]
214 elif strategy == 'random':
215 indexes = N.random.permutation(range(Ncfgs))[:count]
216
217
218 indexes.sort()
219 else:
220
221 raise RuntimeError, "Really should not happen"
222 if __debug__:
223 debug("SPL", "For %s strategy selected %s splits "
224 "from %d total" % (strategy, indexes, Ncfgs))
225 cfgs = [cfgs[i] for i in indexes]
226
227
228 for split in cfgs:
229
230
231 if not operator.isSequenceType(self.__nperlabel) \
232 or isinstance(self.__nperlabel, str):
233 nperlabelsplit = [self.__nperlabel] * len(split)
234 else:
235 nperlabelsplit = self.__nperlabel
236
237
238 split_ds = self.splitDataset(dataset, split)
239
240
241 for run in xrange(self.__runspersplit):
242
243
244 finalized_datasets = []
245
246 for ds, nperlabel in zip(split_ds, nperlabelsplit):
247
248 if self.__permute:
249 DS_permuteLabels(ds, True, perchunk=True)
250
251
252 if nperlabel == 'all' or ds is None:
253 finalized_datasets.append(ds)
254 else:
255
256
257
258
259
260 if nperlabel == 'equal':
261
262 npl = N.array(DS_getNSamplesPerLabel(
263 ds, attrib='labels').values()).min()
264 elif isinstance(nperlabel, float) or (
265 operator.isSequenceType(nperlabel) and
266 len(nperlabel) > 0 and
267 isinstance(nperlabel[0], float)):
268
269
270 counts = N.array(DS_getNSamplesPerLabel(
271 ds, attrib='labels').values())
272 npl = (counts * nperlabel).round().astype(int)
273 else:
274 npl = nperlabel
275
276
277 finalized_datasets.append(
278 DS_getRandomSamples(ds, npl))
279
280 yield finalized_datasets
281
282
284 """Split a dataset by separating the samples where the configured
285 sample attribute matches an element of `specs`.
286
287 :Parameters:
288 dataset : Dataset
289 This is this source dataset.
290 specs : sequence of sequences
291 Contains ids of a sample attribute that shall be split into the
292 another dataset.
293 :Returns: Tuple of splitted datasets.
294 """
295
296 filters = []
297 none_specs = 0
298 cum_filter = None
299
300
301 discard_boundary = self.discard_boundary
302 if isinstance(discard_boundary, int):
303 if discard_boundary != 0:
304 discard_boundary = (discard_boundary,) * len(specs)
305 else:
306 discard_boundary = None
307
308 splitattr_data = eval('dataset.' + self.__splitattr)
309 for spec in specs:
310 if spec is None:
311 filters.append(None)
312 none_specs += 1
313 else:
314 filter_ = N.array([ i in spec \
315 for i in splitattr_data])
316 filters.append(filter_)
317 if cum_filter is None:
318 cum_filter = filter_
319 else:
320 cum_filter = N.logical_and(cum_filter, filter_)
321
322
323 if none_specs > 1:
324 raise ValueError, "Splitter cannot handle more than one `None` " \
325 "split definition."
326
327 for i, filter_ in enumerate(filters):
328 if filter_ is None:
329 filters[i] = N.logical_not(cum_filter)
330
331
332
333 if discard_boundary is not None:
334 ndiscard = discard_boundary[i]
335 if ndiscard != 0:
336
337
338
339 f, lenf = filters[i], len(filters[i])
340 f_pad = N.concatenate(([True]*ndiscard, f, [True]*ndiscard))
341 for d in xrange(2*ndiscard+1):
342 f = N.logical_and(f, f_pad[d:d+lenf])
343 filters[i] = f[:]
344
345
346
347
348 split_datasets = []
349
350
351 dataset_selectSamples = dataset.selectSamples
352 for filter_ in filters:
353 if (filter_ == False).all():
354 split_datasets.append(None)
355 else:
356 split_datasets.append(dataset_selectSamples(filter_))
357
358 return split_datasets
359
360
362 """String summary over the object
363 """
364 return \
365 "SplitterConfig: nperlabel:%s runs-per-split:%d permute:%s" \
366 % (self.__nperlabel, self.__runspersplit, self.__permute)
367
368
370 """Return splitcfg for a given dataset"""
371 return self._getSplitConfig(eval('dataset.unique' + self.__splitattr))
372
373
374 strategy = property(fget=lambda self:self.__strategy,
375 fset=_setStrategy)
376
377
379 """This is a dataset splitter that does **not** split. It simply returns
380 the full dataset that it is called with.
381
382 The passed dataset is returned as the second element of the 2-tuple.
383 The first element of that tuple will always be 'None'.
384 """
385
386 _known_modes = ['first', 'second']
387
388 - def __init__(self, mode='second', **kwargs):
389 """Cheap init -- nothing special
390
391 :Parameters:
392 mode
393 Either 'first' or 'second' (default) -- which output dataset
394 would actually contain the samples
395 """
396 Splitter.__init__(self, **(kwargs))
397
398 if not mode in NoneSplitter._known_modes:
399 raise ValueError, "Unknown mode %s for NoneSplitter" % mode
400 self.__mode = mode
401
402
403 __doc__ = enhancedDocString('NoneSplitter', locals(), Splitter)
404
405
407 """Return just one full split: no first or second dataset.
408 """
409 if self.__mode == 'second':
410 return [([], None)]
411 else:
412 return [(None, [])]
413
414
416 """String summary over the object
417 """
418 return \
419 "NoneSplitter / " + Splitter.__str__(self)
420
421
422
424 """Split a dataset into odd and even values of the sample attribute.
425
426 The splitter yields to splits: first (odd, even) and second (even, odd).
427 """
428 - def __init__(self, usevalues=False, **kwargs):
429 """Cheap init.
430
431 :Parameters:
432 usevalues: Boolean
433 If True the values of the attribute used for splitting will be
434 used to determine odd and even samples. If False odd and even
435 chunks are defined by the order of attribute values, i.e. first
436 unique attribute is odd, second is even, despite the
437 corresponding values might indicate the opposite (e.g. in case
438 of [2,3].
439 """
440 Splitter.__init__(self, **(kwargs))
441
442 self.__usevalues = usevalues
443
444
445 __doc__ = enhancedDocString('OddEvenSplitter', locals(), Splitter)
446
447
449 """Huka chaka!
450 YOH: LOL XXX
451 """
452 if self.__usevalues:
453 return [(None, uniqueattrs[(uniqueattrs % 2) == True]),
454 (None, uniqueattrs[(uniqueattrs % 2) == False])]
455 else:
456 return [(None, uniqueattrs[N.arange(len(uniqueattrs)) %2 == True]),
457 (None, uniqueattrs[N.arange(len(uniqueattrs)) %2 == False])]
458
459
461 """String summary over the object
462 """
463 return \
464 "OddEvenSplitter / " + Splitter.__str__(self)
465
466
467
469 """Split a dataset into two halves of the sample attribute.
470
471 The splitter yields to splits: first (1st half, 2nd half) and second
472 (2nd half, 1st half).
473 """
478
479
480 __doc__ = enhancedDocString('HalfSplitter', locals(), Splitter)
481
482
484 """Huka chaka!
485 """
486 return [(None, uniqueattrs[:len(uniqueattrs)/2]),
487 (None, uniqueattrs[len(uniqueattrs)/2:])]
488
489
491 """String summary over the object
492 """
493 return \
494 "HalfSplitter / " + Splitter.__str__(self)
495
496
497
499 """Split a dataset into N-groups of the sample attribute.
500
501 For example, NGroupSplitter(2) is the same as the HalfSplitter and
502 yields to splits: first (1st half, 2nd half) and second (2nd half,
503 1st half).
504 """
505 - def __init__(self, ngroups=4, **kwargs):
506 """Initialize the N-group splitter.
507
508 :Parameter:
509 ngroups: Int
510 Number of groups to split the attribute into.
511 kwargs
512 Additional parameters are passed to the `Splitter` base class.
513 """
514 Splitter.__init__(self, **(kwargs))
515
516 self.__ngroups = ngroups
517
518 __doc__ = enhancedDocString('NGroupSplitter', locals(), Splitter)
519
520
522 """Huka chaka, wuka waka!
523 """
524
525
526 if len(uniqueattrs) < self.__ngroups:
527 raise ValueError, "Number of groups (%d) " % (self.__ngroups) + \
528 "must be less than " + \
529 "or equal to the number of unique attributes (%d)" % \
530 (len(uniqueattrs))
531
532
533 split_ind = coarsenChunks(uniqueattrs, nchunks=self.__ngroups)
534 split_ind = N.asarray(split_ind)
535
536
537 split_list = [(None, uniqueattrs[split_ind==i])
538 for i in range(self.__ngroups)]
539 return split_list
540
541
543 """String summary over the object
544 """
545 return \
546 "N-%d-GroupSplitter / " % self.__ngroup + Splitter.__str__(self)
547
548
549
551 """Generic N-fold data splitter.
552
553 Provide folding splitting. Given a dataset with N chunks, with
554 cvtype=1 (which is default), it would generate N splits, where
555 each chunk sequentially is taken out (with replacement) for
556 cross-validation. Example, if there is 4 chunks, splits for
557 cvtype=1 are:
558
559 [[1, 2, 3], [0]]
560 [[0, 2, 3], [1]]
561 [[0, 1, 3], [2]]
562 [[0, 1, 2], [3]]
563
564 If cvtype>1, then all possible combinations of cvtype number of
565 chunks are taken out for testing, so for cvtype=2 in previous
566 example:
567
568 [[2, 3], [0, 1]]
569 [[1, 3], [0, 2]]
570 [[1, 2], [0, 3]]
571 [[0, 3], [1, 2]]
572 [[0, 2], [1, 3]]
573 [[0, 1], [2, 3]]
574
575 """
576
577 - def __init__(self,
578 cvtype = 1,
579 **kwargs):
580 """Initialize the N-fold splitter.
581
582 :Parameter:
583 cvtype: Int
584 Type of cross-validation: N-(cvtype)
585 kwargs
586 Additional parameters are passed to the `Splitter` base class.
587 """
588 Splitter.__init__(self, **(kwargs))
589
590
591 self.__cvtype = cvtype
592
593
594 __doc__ = enhancedDocString('NFoldSplitter', locals(), Splitter)
595
596
598 """String summary over the object
599 """
600 return \
601 "N-%d-FoldSplitter / " % self.__cvtype + Splitter.__str__(self)
602
603
605 """Returns proper split configuration for N-M fold split.
606 """
607 return [(None, i) for i in \
608 support.getUniqueLengthNCombinations(uniqueattrs,
609 self.__cvtype)]
610
611
612
614 """Split a dataset using an arbitrary custom rule.
615
616 The splitter is configured by passing a custom spitting rule (`splitrule`)
617 to its constructor. Such a rule is basically a sequence of split
618 definitions. Every single element in this sequence results in excatly one
619 split generated by the Splitter. Each element is another sequence for
620 sequences of sample ids for each dataset that shall be generated in the
621 split.
622
623 Example:
624
625 * Generate two splits. In the first split the *second* dataset
626 contains all samples with sample attributes corresponding to
627 either 0, 1 or 2. The *first* dataset of the first split contains
628 all samples which are not split into the second dataset.
629
630 The second split yields three datasets. The first with all samples
631 corresponding to sample attributes 1 and 2, the second dataset
632 contains only samples with attrbiute 3 and the last dataset
633 contains the samples with attribute 5 and 6.
634
635 CustomSplitter([(None, [0, 1, 2]), ([1,2], [3], [5, 6])])
636 """
637 - def __init__(self, splitrule, **kwargs):
638 """Cheap init.
639 """
640 Splitter.__init__(self, **(kwargs))
641
642 self.__splitrule = splitrule
643
644
645 __doc__ = enhancedDocString('CustomSplitter', locals(), Splitter)
646
647
649 """Huka chaka!
650 """
651 return self.__splitrule
652
653
655 """String summary over the object
656 """
657 return "CustomSplitter / " + Splitter.__str__(self)
658