Ji Zhang Ji Zhang - 26 days ago 6
Python Question

AttributeError: 'list' object has no attribute 'rfind' using petsc4py

I face an error when I using a combination of petsc4py and cython:
AttributeError: 'list' object has no attribute 'rfind'

the bellowing code is a part of .../petsc4py/demo/wrap-cython/setup.py, a example of the project, which have an error:

def configuration(parent_package='',top_path=None):
INCLUDE_DIRS = []
LIBRARY_DIRS = []
LIBRARIES = []

# PETSc
import os
PETSC_DIR = os.environ['PETSC_DIR']
PETSC_ARCH = os.environ.get('PETSC_ARCH', '')
from os.path import join, isdir
if PETSC_ARCH and isdir(join(PETSC_DIR, PETSC_ARCH)):
INCLUDE_DIRS += [join(PETSC_DIR, PETSC_ARCH, 'include'),
join(PETSC_DIR, 'include')]
LIBRARY_DIRS += [join(PETSC_DIR, PETSC_ARCH, 'lib')]
else:
if PETSC_ARCH: pass # XXX should warn ...
INCLUDE_DIRS += [join(PETSC_DIR, 'include')]
LIBRARY_DIRS += [join(PETSC_DIR, 'lib')]
LIBRARIES += [#'petscts', 'petscsnes', 'petscksp',
#'petscdm', 'petscmat', 'petscvec',
'petsc']

# PETSc for Python
import petsc4py
INCLUDE_DIRS += [petsc4py.get_include()]

# Configuration
from numpy.distutils.misc_util import Configuration
config = Configuration('', parent_package, top_path)
config.add_extension('_Bratu3D',
sources = ['Bratu3D.pyx',
'Bratu3Dimpl.c'],
depends = ['Bratu3Dimpl.h'],
include_dirs=INCLUDE_DIRS + [os.curdir],
libraries=LIBRARIES,
library_dirs=LIBRARY_DIRS,
runtime_library_dirs=LIBRARY_DIRS)
return config

if __name__ == "__main__":
from numpy.distutils.core import setup
setup(**configuration(top_path='').todict())


error messages are:

CC=/usr/local/openmpi-1.10.2/bin/mpicc F90=/usr/local/openmpi-1.10.2/bin/mpif90 LDSHARED='/usr/local/openmpi-1.10.2/bin/mpicc -fPIC -Wall -Wwrite-strings -Wno-strict-aliasing -Wno-unknown-pragmas -fvisibility=hidden -g3 -shared' \
python setup.py -q build_ext --inplace
Traceback (most recent call last):
File "setup.py", line 66, in <module>
setup(**configuration(top_path='').todict())
File "/home/zhangji/anaconda3/envs/obenv/lib/python3.5/site-packages/numpy/distutils/core.py", line 169, in setup
return old_setup(**new_attr)
File "/home/zhangji/anaconda3/envs/obenv/lib/python3.5/distutils/core.py", line 148, in setup
dist.run_commands()
File "/home/zhangji/anaconda3/envs/obenv/lib/python3.5/distutils/dist.py", line 955, in run_commands
self.run_command(cmd)
File "/home/zhangji/anaconda3/envs/obenv/lib/python3.5/distutils/dist.py", line 974, in run_command
cmd_obj.run()
File "/home/zhangji/anaconda3/envs/obenv/lib/python3.5/site-packages/numpy/distutils/command/build_ext.py", line 82, in run
self.run_command('build_src')
File "/home/zhangji/anaconda3/envs/obenv/lib/python3.5/distutils/cmd.py", line 313, in run_command
self.distribution.run_command(command)
File "/home/zhangji/anaconda3/envs/obenv/lib/python3.5/distutils/dist.py", line 974, in run_command
cmd_obj.run()
File "/home/zhangji/anaconda3/envs/obenv/lib/python3.5/site-packages/numpy/distutils/command/build_src.py", line 147, in run
self.build_sources()
File "/home/zhangji/anaconda3/envs/obenv/lib/python3.5/site-packages/numpy/distutils/command/build_src.py", line 164, in build_sources
self.build_extension_sources(ext)
File "/home/zhangji/anaconda3/envs/obenv/lib/python3.5/site-packages/numpy/distutils/command/build_src.py", line 329, in build_extension_sources
sources, py_files = self.filter_py_files(sources)
File "/home/zhangji/anaconda3/envs/obenv/lib/python3.5/site-packages/numpy/distutils/command/build_src.py", line 389, in filter_py_files
return self.filter_files(sources, ['.py'])
File "/home/zhangji/anaconda3/envs/obenv/lib/python3.5/site-packages/numpy/distutils/command/build_src.py", line 398, in filter_files
(base, ext) = os.path.splitext(source)
File "/home/zhangji/anaconda3/envs/obenv/lib/python3.5/posixpath.py", line 122, in splitext
return genericpath._splitext(p, sep, None, extsep)
File "/home/zhangji/anaconda3/envs/obenv/lib/python3.5/genericpath.py", line 118, in _splitext
sepIndex = p.rfind(sep)
AttributeError: 'list' object has no attribute 'rfind'
make: *** [Bratu3D.so] Error 1


Another example at .../petsc4py/demo/wrap-swig/setup.py have nearly the same code but without any error. The only exception is:

config.add_extension('_Bratu3D',
sources = ['Bratu3D.i',
'Bratu3D.c'],
depends = ['Bratu3D.h'],
include_dirs=INCLUDE_DIRS + [os.curdir],
libraries=LIBRARIES,
library_dirs=LIBRARY_DIRS,
runtime_library_dirs=LIBRARY_DIRS)


Thanks a lot.

Answer

I've had the same problem with the project petsc4py/demo/wrap-cython

In fact the code works well with a bit of monkey patching on this file: numpy/distutils/command/build_src.py

The method is defined on numpy 1.11.0 as:

def generate_a_pyrex_source(self, base, ext_name, source, extension):
    """Pyrex is not supported, but some projects monkeypatch this method.

    That allows compiling Cython code, see gh-6955.
    This method will remain here for compatibility reasons.
    """
    return []

You have to override this method with your own definition. Here is the updated script tested with python 3.5.1, petsc4py 2.0.0:

wrap-cython/setup.py

#!/usr/bin/env python
# $ python setup.py build_ext --inplace

from numpy.distutils.command import build_src

# a bit of monkeypatching ...
import Cython.Compiler.Main
build_src.Pyrex = Cython
build_src.have_pyrex = True


def have_pyrex():
    import sys
    try:
        import Cython.Compiler.Main
        sys.modules['Pyrex'] = Cython
        sys.modules['Pyrex.Compiler'] = Cython.Compiler
        sys.modules['Pyrex.Compiler.Main'] = Cython.Compiler.Main
        return True
    except ImportError:
        return False
build_src.have_pyrex = have_pyrex

##########################
# BEGIN additionnal code #
##########################
from numpy.distutils.misc_util import appendpath
from numpy.distutils import log
from os.path import join as pjoin, dirname
from distutils.dep_util import newer_group
from distutils.errors import DistutilsError


def generate_a_pyrex_source(self, base, ext_name, source, extension):
    ''' Monkey patch for numpy build_src.build_src method
    Uses Cython instead of Pyrex.
    Assumes Cython is present
    '''
    if self.inplace:
        target_dir = dirname(base)
    else:
        target_dir = appendpath(self.build_src, dirname(base))
    target_file = pjoin(target_dir, ext_name + '.c')
    depends = [source] + extension.depends
    if self.force or newer_group(depends, target_file, 'newer'):
        import Cython.Compiler.Main
        log.info("cythonc:> %s" % (target_file))
        self.mkpath(target_dir)
        options = Cython.Compiler.Main.CompilationOptions(
            defaults=Cython.Compiler.Main.default_options,
            include_path=extension.include_dirs,
            output_file=target_file)
        cython_result = Cython.Compiler.Main.compile(source, options=options)
        if cython_result.num_errors != 0:
            raise DistutilsError("%d errors while compiling %r with Cython" % (cython_result.num_errors, source))
    return target_file

build_src.build_src.generate_a_pyrex_source = generate_a_pyrex_source
########################
# END additionnal code #
########################


def configuration(parent_package='', top_path=None):
    INCLUDE_DIRS = []
    LIBRARY_DIRS = []
    LIBRARIES = []

    # PETSc
    import os
    PETSC_DIR = os.environ['PETSC_DIR']
    PETSC_ARCH = os.environ.get('PETSC_ARCH', '')
    from os.path import join, isdir
    if PETSC_ARCH and isdir(join(PETSC_DIR, PETSC_ARCH)):
        INCLUDE_DIRS += [join(PETSC_DIR, PETSC_ARCH, 'include'),
                         join(PETSC_DIR, 'include')]
        LIBRARY_DIRS += [join(PETSC_DIR, PETSC_ARCH, 'lib')]
    else:
        if PETSC_ARCH:
            pass  # XXX should warn ...
        INCLUDE_DIRS += [join(PETSC_DIR, 'include')]
        LIBRARY_DIRS += [join(PETSC_DIR, 'lib')]
    LIBRARIES += [  # 'petscts', 'petscsnes', 'petscksp',
        # 'petscdm', 'petscmat',  'petscvec',
        'petsc']

    # PETSc for Python
    import petsc4py
    INCLUDE_DIRS += [petsc4py.get_include()]

    # Configuration
    from numpy.distutils.misc_util import Configuration
    config = Configuration('', parent_package, top_path)
    config.add_extension('Bratu3D',
                         sources=['Bratu3D.pyx', 'Bratu3Dimpl.c'],
                         depends=['Bratu3Dimpl.h'],
                         include_dirs=INCLUDE_DIRS + [os.curdir],
                         libraries=LIBRARIES,
                         library_dirs=LIBRARY_DIRS,
                         runtime_library_dirs=LIBRARY_DIRS)
    return config

if __name__ == "__main__":
    from numpy.distutils.core import setup
    setup(**configuration(top_path='').todict())