mvpa.base.externals

1 # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- 2 # vi: set ft=python sts=4 ts=4 sw=4 et: 3 ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## 4 # 5 # See COPYING file distributed along with the PyMVPA package for the 6 # copyright and license terms. 7 # 8 ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## 9 """Helper to verify presence of external libraries and modules 10 """ 11 12 __docformat__ = 'restructuredtext' 13 import os 14 15 from mvpa.base import warning 16 from mvpa import cfg 17 18 if __debug__: 19 from mvpa.base import debug 20 21 versions = {} 22 """Versions of available externals, as tuples 23 """ 24

25 -def __version_to_tuple(v):

26 """Simple helper to convert version given as a string into a tuple. 27 28 Tuple of integers constructed by splitting at '.'. 29 """ 30 if isinstance(v, basestring): 31 v = tuple([int(x) for x in v.split('.') if not x.startswith('dev')]) 32 elif isinstance(v, tuple) or isinstance(v, list): 33 # assure tuple 34 v = tuple(v) 35 else: 36 raise ValueError, "Do not know how to treat version '%s'" % str(v) 37 return v

38 39

40 -def __check_scipy():

41 """Check if scipy is present an if it is -- store its version 42 """ 43 import warnings 44 # To don't allow any crappy warning to sneak in 45 warnings.simplefilter('ignore', DeprecationWarning) 46 try: 47 import scipy as sp 48 except: 49 warnings.simplefilter('default', DeprecationWarning) 50 raise 51 warnings.simplefilter('default', DeprecationWarning) 52 # Infiltrate warnings if necessary 53 numpy_ver = versions['numpy'] 54 scipy_ver = versions['scipy'] = __version_to_tuple(sp.__version__) 55 # There is way too much deprecation warnings spit out onto the 56 # user. Lets assume that they should be fixed by scipy 0.7.0 time 57 if scipy_ver >= (0, 6, 0) and scipy_ver < (0, 7, 0) \ 58 and numpy_ver > (1, 1, 0): 59 import warnings 60 if not __debug__ or (__debug__ and not 'PY' in debug.active): 61 debug('EXT', "Setting up filters for numpy DeprecationWarnings") 62 filter_lines = [ 63 ('NumpyTest will be removed in the next release.*', 64 DeprecationWarning), 65 ('PyArray_FromDims: use PyArray_SimpleNew.', 66 DeprecationWarning), 67 ('PyArray_FromDimsAndDataAndDescr: use PyArray_NewFromDescr.', 68 DeprecationWarning), 69 # Trick re.match, since in warnings absent re.DOTALL in re.compile 70 ('[\na-z \t0-9]*The original semantics of histogram is scheduled to be.*' 71 '[\na-z \t0-9]*', Warning) ] 72 for f, w in filter_lines: 73 warnings.filterwarnings('ignore', f, w)

74 75

76 -def __check_numpy():

77 """Check if numpy is present (it must be) an if it is -- store its version 78 """ 79 import numpy as N 80 versions['numpy'] = __version_to_tuple(N.__version__)

81 82

83 -def __check_pywt(features=None):

84 """Check for available functionality within pywt 85 86 :Parameters: 87 features : list of basestring 88 List of known features to check such as 'wp reconstruct', 89 'wp reconstruct fixed' 90 """ 91 import pywt 92 import numpy as N 93 data = N.array([ 0.57316901, 0.65292526, 0.75266733, 0.67020084, 0.46505364, 94 0.76478331, 0.33034164, 0.49165547, 0.32979941, 0.09696717, 95 0.72552711, 0.4138999 , 0.54460628, 0.786351 , 0.50096306, 96 0.72436454, 0.2193098 , -0.0135051 , 0.34283984, 0.65596245, 97 0.49598417, 0.39935064, 0.26370727, 0.05572373, 0.40194438, 98 0.47004551, 0.60327258, 0.25628266, 0.32964893, 0.24009889,]) 99 mode = 'per' 100 wp = pywt.WaveletPacket(data, 'sym2', mode) 101 wp2 = pywt.WaveletPacket(data=None, wavelet='sym2', mode=mode) 102 try: 103 for node in wp.get_level(2): wp2[node.path] = node.data 104 except: 105 raise ImportError, \ 106 "Failed to reconstruct WP by specifying data in the layer" 107 108 if 'wp reconstruct fixed' in features: 109 rec = wp2.reconstruct() 110 if N.linalg.norm(rec[:len(data)] - data) > 1e-3: 111 raise ImportError, \ 112 "Failed to reconstruct WP correctly" 113 return True

114 115

116 -def __check_libsvm_verbosity_control():

117 """Check for available verbose control functionality 118 """ 119 import mvpa.clfs.libsvmc._svmc as _svmc 120 try: 121 _svmc.svm_set_verbosity(0) 122 except: 123 raise ImportError, "Provided version of libsvm has no way to control " \ 124 "its level of verbosity"

125

126 -def __check_shogun(bottom_version, custom_versions=[]):

127 """Check if version of shogun is high enough (or custom known) to 128 be enabled in the testsuite 129 130 :Parameters: 131 bottom_version : int 132 Bottom version which must be satisfied 133 custom_versions : list of int 134 Arbitrary list of versions which could got patched for 135 a specific issue 136 """ 137 import shogun.Classifier as __sc 138 ver = __sc.Version_get_version_revision() 139 if (ver in custom_versions) or (ver >= bottom_version): 140 return True 141 else: 142 raise ImportError, 'Version %s is smaller than needed %s' % \ 143 (ver, bottom_version)

144 145

146 -def __check_weave():

147 """Apparently presence of scipy is not sufficient since some 148 versions experience problems. E.g. in Sep,Oct 2008 lenny's weave 149 failed to work. May be some other converter could work (? See 150 http://lists.debian.org/debian-devel/2008/08/msg00730.html for a 151 similar report. 152 153 Following simple snippet checks compilation of the basic code using 154 weave 155 """ 156 from scipy import weave 157 from scipy.weave import converters, build_tools 158 import numpy as N 159 # to shut weave up 160 import sys 161 # we can't rely on weave at all at the restoring argv. On etch box 162 # restore_sys_argv() is apparently is insufficient 163 oargv = sys.argv[:] 164 ostdout = sys.stdout 165 if not( __debug__ and 'EXT_' in debug.active): 166 from StringIO import StringIO 167 sys.stdout = StringIO() 168 # *nix specific solution to shut weave up. 169 # Some users must complain and someone 170 # needs to fix this to become more generic. 171 cargs = [">/dev/null", "2>&1"] 172 else: 173 cargs = [] 174 fmsg = None 175 try: 176 data = N.array([1,2,3]) 177 counter = weave.inline("data[0]=fabs(-1);", ['data'], 178 type_converters=converters.blitz, 179 verbose=0, 180 extra_compile_args=cargs, 181 compiler = 'gcc') 182 except Exception, e: 183 fmsg = "Failed to build simple weave sample." \ 184 " Exception was %s" % str(e) 185 186 sys.stdout = ostdout 187 # needed to fix sweave which might "forget" to restore sysv 188 # build_tools.restore_sys_argv() 189 sys.argv = oargv 190 if fmsg is not None: 191 raise ImportError, fmsg 192 else: 193 return "Everything is cool"

194 195

196 -def __check_atlas_family(family):

197 # XXX I guess pylint will dislike it a lot 198 from mvpa.atlases.warehouse import KNOWN_ATLAS_FAMILIES 199 names, pathpattern = KNOWN_ATLAS_FAMILIES[family] 200 filename = pathpattern % {'name':names[0]} 201 if not os.path.exists(filename): 202 raise ImportError, "Cannot find file %s for atlas family %s" \ 203 % (filename, family) 204 pass

205 206

207 -def __check_stablerdist():

208 import scipy.stats 209 import numpy as N 210 ## Unfortunately 0.7.0 hasn't fixed the issue so no chance but to do 211 ## a proper numerical test here 212 try: 213 scipy.stats.rdist(1.32, 0, 1).cdf(-1.0 + N.finfo(float).eps) 214 # Actually previous test is insufficient for 0.6, so enabling 215 # elderly test on top 216 # ATM all known implementations which implement custom cdf for 217 # rdist are misbehaving, so there should be no _cdf 218 if '_cdf' in scipy.stats.distributions.rdist_gen.__dict__.keys(): 219 raise ImportError, \ 220 "scipy.stats carries misbehaving rdist distribution" 221 except ZeroDivisionError: 222 raise RuntimeError, "RDist in scipy is still unstable on the boundaries"

223 224

225 -def __check_in_ipython():

226 # figure out if ran within IPython 227 if '__IPYTHON__' in globals()['__builtins__']: 228 return 229 raise RuntimeError, "Not running in IPython session"

230 231

232 -def __check_matplotlib():

233 """Check for presence of matplotlib and set backend if requested.""" 234 import matplotlib 235 backend = cfg.get('matplotlib', 'backend') 236 if backend: 237 matplotlib.use(backend)

238

239 -def __check_pylab():

240 """Check if matplotlib is there and then pylab""" 241 exists('matplotlib', raiseException=True) 242 import pylab as P

243

244 -def __check_pylab_plottable():

245 """Simple check either we can plot anything using pylab. 246 247 Primary use in unittests 248 """ 249 try: 250 exists('pylab', raiseException=True) 251 import pylab as P 252 fig = P.figure() 253 P.plot([1,2], [1,2]) 254 P.close(fig) 255 except: 256 raise RuntimeError, "Cannot plot in pylab" 257 return True

258 259

260 -def __check_griddata():

261 """griddata might be independent module or part of mlab 262 """ 263 264 try: 265 from griddata import griddata as __ 266 return True 267 except ImportError: 268 if __debug__: 269 debug('EXT_', 'No python-griddata available') 270 271 from matplotlib.mlab import griddata as __ 272 return True

273 274 # contains list of available (optional) external classifier extensions 275 _KNOWN = {'libsvm':'import mvpa.clfs.libsvmc._svm as __; x=__.convert2SVMNode', 276 'libsvm verbosity control':'__check_libsvm_verbosity_control();', 277 'nifti':'from nifti import NiftiImage as __', 278 'nifti >= 0.20090205.1': 279 'from nifti.clib import detachDataFromImage as __', 280 'ctypes':'import ctypes as __', 281 'shogun':'import shogun as __', 282 'shogun.mpd': 'import shogun.Classifier as __; x=__.MPDSVM', 283 'shogun.lightsvm': 'import shogun.Classifier as __; x=__.SVMLight', 284 'shogun.svrlight': 'from shogun.Regression import SVRLight as __', 285 'numpy': "__check_numpy()", 286 'scipy': "__check_scipy()", 287 'good scipy.stats.rdist': "__check_stablerdist()", 288 'weave': "__check_weave()", 289 'pywt': "import pywt as __", 290 'pywt wp reconstruct': "__check_pywt(['wp reconstruct'])", 291 'pywt wp reconstruct fixed': "__check_pywt(['wp reconstruct fixed'])", 292 'rpy': "import rpy as __", 293 'lars': "import rpy; rpy.r.library('lars')", 294 'elasticnet': "import rpy; rpy.r.library('elasticnet')", 295 'glmnet': "import rpy; rpy.r.library('glmnet')", 296 'matplotlib': "__check_matplotlib()", 297 'pylab': "__check_pylab()", 298 'pylab plottable': "__check_pylab_plottable()", 299 'openopt': "import scikits.openopt as __", 300 'mdp': "import mdp as __", 301 'mdp >= 2.4': "from mdp.nodes import LLENode as __", 302 'sg_fixedcachesize': "__check_shogun(3043, [2456])", 303 # 3318 corresponds to release 0.6.4 304 'sg >= 0.6.4': "__check_shogun(3318)", 305 'hcluster': "import hcluster as __", 306 'griddata': "__check_griddata()", 307 'cPickle': "import cPickle as __", 308 'gzip': "import gzip as __", 309 'lxml': "from lxml import objectify as __", 310 'atlas_pymvpa': "__check_atlas_family('pymvpa')", 311 'atlas_fsl': "__check_atlas_family('fsl')", 312 'running ipython env': "__check_in_ipython()", 313 } 314 315

316 -def exists(dep, force=False, raiseException=False, issueWarning=None):

317 """ 318 Test whether a known dependency is installed on the system. 319 320 This method allows us to test for individual dependencies without 321 testing all known dependencies. It also ensures that we only test 322 for a dependency once. 323 324 :Parameters: 325 dep : string or list of string 326 The dependency key(s) to test. 327 force : boolean 328 Whether to force the test even if it has already been 329 performed. 330 raiseException : boolean 331 Whether to raise RuntimeError if dependency is missing. 332 issueWarning : string or None or True 333 If string, warning with given message would be thrown. 334 If True, standard message would be used for the warning 335 text. 336 """ 337 # if we are provided with a list of deps - go through all of them 338 if isinstance(dep, list) or isinstance(dep, tuple): 339 results = [ exists(dep_, force, raiseException) for dep_ in dep ] 340 return bool(reduce(lambda x,y: x and y, results, True)) 341 342 # where to look in cfg 343 cfgid = 'have ' + dep 344 345 # prevent unnecessarry testing 346 if cfg.has_option('externals', cfgid) \ 347 and not cfg.getboolean('externals', 'retest', default='no') \ 348 and not force: 349 if __debug__: 350 debug('EXT', "Skip retesting for '%s'." % dep) 351 352 # check whether an exception should be raised, even though the external 353 # was already tested previously 354 if not cfg.getboolean('externals', cfgid) \ 355 and raiseException \ 356 and cfg.getboolean('externals', 'raise exception', True): 357 raise RuntimeError, "Required external '%s' was not found" % dep 358 return cfg.getboolean('externals', cfgid) 359 360 361 # determine availability of external (non-cached) 362 363 # default to 'not found' 364 result = False 365 366 if not _KNOWN.has_key(dep): 367 raise ValueError, "%s is not a known dependency key." % (dep) 368 else: 369 # try and load the specific dependency 370 if __debug__: 371 debug('EXT', "Checking for the presence of %s" % dep) 372 373 # Exceptions which are silently caught while running tests for externals 374 _caught_exceptions = [ImportError, AttributeError, RuntimeError] 375 376 # check whether RPy is involved and catch its excpetions as well. 377 # however, try to determine whether this is really necessary, as 378 # importing RPy also involved starting a full-blown R session, which can 379 # take seconds and therefore is quite nasty... 380 if dep.count('rpy') or _KNOWN[dep].count('rpy'): 381 try: 382 from rpy import RException 383 _caught_exceptions += [RException] 384 except: 385 pass 386 387 388 estr = '' 389 try: 390 exec _KNOWN[dep] 391 result = True 392 except tuple(_caught_exceptions), e: 393 estr = ". Caught exception was: " + str(e) 394 395 if __debug__: 396 debug('EXT', "Presence of %s is%s verified%s" % 397 (dep, {True:'', False:' NOT'}[result], estr)) 398 399 if not result: 400 if raiseException \ 401 and cfg.getboolean('externals', 'raise exception', True): 402 raise RuntimeError, "Required external '%s' was not found" % dep 403 if issueWarning is not None \ 404 and cfg.getboolean('externals', 'issue warning', True): 405 if issueWarning is True: 406 warning("Required external '%s' was not found" % dep) 407 else: 408 warning(issueWarning) 409 410 411 # store result in config manager 412 if not cfg.has_section('externals'): 413 cfg.add_section('externals') 414 if result: 415 cfg.set('externals', 'have ' + dep, 'yes') 416 else: 417 cfg.set('externals', 'have ' + dep, 'no') 418 419 return result

420 421

422 -def testAllDependencies(force=False):

423 """ 424 Test for all known dependencies. 425 426 :Parameters: 427 force : boolean 428 Whether to force the test even if it has already been 429 performed. 430 431 """ 432 # loop over all known dependencies 433 for dep in _KNOWN: 434 if not exists(dep, force): 435 warning("%s is not available." % dep) 436 437 if __debug__: 438 debug('EXT', 'The following optional externals are present: %s' \ 439 % [ k[5:] for k in cfg.options('externals') 440 if k.startswith('have') \ 441 and cfg.getboolean('externals', k) == True ])

442

Source Code for Module mvpa.base.externals