gavwould gavwould - 5 months ago 10
Linux Question

How to know if process has truly finished with a dlclose()ed library?

I'm on Linux (Ubuntu 12.04, gcc 4.6.3), trying to bend dlopen/close to my will in order to make a plugin-based application that can reload its plugins as and when necessary (e.g. if they're recompiled).

The basic theory is simple: dlopen the plugin; use it, keeping track of all its symbols that are in use. When the time comes to reload, clean up all the symbols and dlclose the plugin.

I threw together a simple demo app, 'test.cpp':

#include <dlfcn.h>
#include <iostream>
using namespace std;
int main(int argc, char** argv)
{
if (argc > 1)
{
void* h = dlopen(argv[1], RTLD_NOW|RTLD_LOCAL);
if (!h)
{
cerr << "ERROR: " << dlerror() << endl;
return 1;
}
cin.get();
if (dlclose(h))
{
cerr << "ERROR: " << dlerror() << endl;
return 2;
}
cin.get();
}
return 0;
}


Compile with:

g++ test.cpp -o test -ldl


To make a trivial library that can be passed as a argument to the above code, use:

touch libtest.cpp && g++ -rdynamic -shared libtest.cpp -o libtest.so


Then run with:

./test ./libtest.so


Here's the problem; if, after hitting [Enter] once (i.e. after loading and supposedly unloading the library), you run 'pmap' to check which libraries are loaded in 'test', it'll tell you that libtest.so is still there! Now this is despite a valid return from dlclose() and no reasonable way that the reference count could have edged above 1 prior to that (this can be verified by attempting a second dlclose() - it will give an error return saying that it's already closed).

So either Linux never unloads a dlopen()ed library (contradicts the documentation), or 'pmap' is wrong. If it is the latter, is there a more reliable method of determining if a library is still loaded?

Answer

I don't observe the same as you with the below program:

// file soq.c
#include <dlfcn.h>
#include <iostream>
#include <cstdio>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
using namespace std;
int main(int argc, char** argv)
{
  char cmd[60];
  snprintf(cmd, sizeof(cmd), "pmap %d", getpid());
    if (argc > 1)
    {
        void* h = dlopen(argv[1], RTLD_NOW|RTLD_LOCAL);
        if (!h)
        {
            cerr << "ERROR: " << dlerror() << endl;
            return 1;
        }
    cerr << "after dlopen " << argv[1] << endl;
    system(cmd);
        cin.get();
        if (dlclose(h))
        {
            cerr << "ERROR: " << dlerror() << endl;
            return 2;
        }
        cin.get();
    cerr << "after close " << argv[1] << endl;
    system(cmd);
    }
    return 0;
}

I am getting, as expected:

% ./soq ./libempty.so
./soq ./libempty.so
after dlopen ./libempty.so
5276:   ./soq ./libempty.so
0000000000400000      8K r-x--  /home/basile/tmp/soq
0000000000601000      4K rw---  /home/basile/tmp/soq
0000000001b4d000    132K rw---    [ anon ]
00007f1dbfd01000      4K r-x--  /home/basile/tmp/libempty.so
00007f1dbfd02000   2044K -----  /home/basile/tmp/libempty.so
00007f1dbff01000      4K rw---  /home/basile/tmp/libempty.so
00007f1dbff02000   1524K r-x--  /lib/x86_64-linux-gnu/libc-2.13.so
00007f1dc007f000   2048K -----  /lib/x86_64-linux-gnu/libc-2.13.so
00007f1dc027f000     16K r----  /lib/x86_64-linux-gnu/libc-2.13.so
00007f1dc0283000      4K rw---  /lib/x86_64-linux-gnu/libc-2.13.so
00007f1dc0284000     20K rw---    [ anon ]
00007f1dc0289000     84K r-x--  /lib/x86_64-linux-gnu/libgcc_s.so.1
00007f1dc029e000   2048K -----  /lib/x86_64-linux-gnu/libgcc_s.so.1
00007f1dc049e000      4K rw---  /lib/x86_64-linux-gnu/libgcc_s.so.1
00007f1dc049f000    516K r-x--  /lib/x86_64-linux-gnu/libm-2.13.so
00007f1dc0520000   2044K -----  /lib/x86_64-linux-gnu/libm-2.13.so
00007f1dc071f000      4K r----  /lib/x86_64-linux-gnu/libm-2.13.so
00007f1dc0720000      4K rw---  /lib/x86_64-linux-gnu/libm-2.13.so
00007f1dc0721000    928K r-x--  /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.17
00007f1dc0809000   2048K -----  /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.17
00007f1dc0a09000     32K r----  /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.17
00007f1dc0a11000      8K rw---  /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.17
00007f1dc0a13000     84K rw---    [ anon ]
00007f1dc0a28000      8K r-x--  /lib/x86_64-linux-gnu/libdl-2.13.so
00007f1dc0a2a000   2048K -----  /lib/x86_64-linux-gnu/libdl-2.13.so
00007f1dc0c2a000      4K r----  /lib/x86_64-linux-gnu/libdl-2.13.so
00007f1dc0c2b000      4K rw---  /lib/x86_64-linux-gnu/libdl-2.13.so
00007f1dc0c2c000    128K r-x--  /lib/x86_64-linux-gnu/ld-2.13.so
00007f1dc0e1c000     20K rw---    [ anon ]
00007f1dc0e49000      8K rw---    [ anon ]
00007f1dc0e4b000      4K r----  /lib/x86_64-linux-gnu/ld-2.13.so
00007f1dc0e4c000      4K rw---  /lib/x86_64-linux-gnu/ld-2.13.so
00007f1dc0e4d000      4K rw---    [ anon ]
00007fff076c3000    132K rw---    [ stack ]
00007fff077b4000      4K r-x--    [ anon ]
ffffffffff600000      4K r-x--    [ anon ]
 total            15984K




after close ./libempty.so
5276:   ./soq ./libempty.so
0000000000400000      8K r-x--  /home/basile/tmp/soq
0000000000601000      4K rw---  /home/basile/tmp/soq
0000000001b4d000    132K rw---    [ anon ]
00007f1dbff02000   1524K r-x--  /lib/x86_64-linux-gnu/libc-2.13.so
00007f1dc007f000   2048K -----  /lib/x86_64-linux-gnu/libc-2.13.so
00007f1dc027f000     16K r----  /lib/x86_64-linux-gnu/libc-2.13.so
00007f1dc0283000      4K rw---  /lib/x86_64-linux-gnu/libc-2.13.so
00007f1dc0284000     20K rw---    [ anon ]
00007f1dc0289000     84K r-x--  /lib/x86_64-linux-gnu/libgcc_s.so.1
00007f1dc029e000   2048K -----  /lib/x86_64-linux-gnu/libgcc_s.so.1
00007f1dc049e000      4K rw---  /lib/x86_64-linux-gnu/libgcc_s.so.1
00007f1dc049f000    516K r-x--  /lib/x86_64-linux-gnu/libm-2.13.so
00007f1dc0520000   2044K -----  /lib/x86_64-linux-gnu/libm-2.13.so
00007f1dc071f000      4K r----  /lib/x86_64-linux-gnu/libm-2.13.so
00007f1dc0720000      4K rw---  /lib/x86_64-linux-gnu/libm-2.13.so
00007f1dc0721000    928K r-x--  /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.17
00007f1dc0809000   2048K -----  /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.17
00007f1dc0a09000     32K r----  /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.17
00007f1dc0a11000      8K rw---  /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.17
00007f1dc0a13000     84K rw---    [ anon ]
00007f1dc0a28000      8K r-x--  /lib/x86_64-linux-gnu/libdl-2.13.so
00007f1dc0a2a000   2048K -----  /lib/x86_64-linux-gnu/libdl-2.13.so
00007f1dc0c2a000      4K r----  /lib/x86_64-linux-gnu/libdl-2.13.so
00007f1dc0c2b000      4K rw---  /lib/x86_64-linux-gnu/libdl-2.13.so
00007f1dc0c2c000    128K r-x--  /lib/x86_64-linux-gnu/ld-2.13.so
00007f1dc0e1c000     20K rw---    [ anon ]
00007f1dc0e48000     12K rw---    [ anon ]
00007f1dc0e4b000      4K r----  /lib/x86_64-linux-gnu/ld-2.13.so
00007f1dc0e4c000      4K rw---  /lib/x86_64-linux-gnu/ld-2.13.so
00007f1dc0e4d000      4K rw---    [ anon ]
00007fff076c3000    132K rw---    [ stack ]
00007fff077b4000      4K r-x--    [ anon ]
ffffffffff600000      4K r-x--    [ anon ]
 total            13936K 

So you did run your pmap wrongly.

By the way, you could avoid any dlclose-ing in practice, as my manydl.c example demonstrates. In practice, not bothering dlclose-ing means only a tiny address space leak, not a big deal in practice. (You can dlopen nearly a million different shared objects without much hurt).

And to know when a shared object is unloaded, use the "destructor" functions of your dlclose-d plugin (e.g. destructor of static data in C++, or attribute((destructor)) in e.g. C code), because they are called from inside the unloading dlclose.

Comments