dep_tree2sql.sh

#!/bin/bash

############################################################################
#
# MODULE:       dep_tree2sql.sh
# AUTHOR(S):    Glynn Clements
# PURPOSE:      run "nm" on every object file, library and executable to
#               find dependencies, stored in PostgreSQL database
# COPYRIGHT:    (C) 2006 by the GRASS Development Team
#
#               This program is free software under the GNU General Public
#               License (>=v2). Read the file COPYING that comes with GRASS
#               for details.
#
#############################################################################
#
# From: Glynn Clements <glynn at gclements.plus.com>
# Date: Mon, 12 Jun 2006 13:00:25 +0100
#
# Usage: after having compiled GRASS, run "utils/dep_tree2sql.sh `pwd`" from the
# top of the GRASS source tree.
#
# Essentially, the script runs "nm" on every object file, library and
# executable it finds (and "ldd" on all of the executables), processes
# the output with egrep/sed/awk, then imports the results into a
# PostgreSQL database.
#
# "nm" lists the symbol table of an object file, library, or executable,
# indicating whether each symbol is imported into, defined in and/or
# exported from the file.
#
# Most of the database tables record which symbols are imported into or
# exported from which object files, libraries or executables. E.g. the
# "obj_imp" table lists which symbols are imported into which object
# files.
#
# You can then use simple queries such as:
#
#     grass=> SELECT object FROM obj_imp WHERE symbol = 'I_get_target' ;
#                                    object
#     --------------------------------------------------------------------
#      imagery/i.ortho.photo/photo.2image/OBJ.i686-pc-linux-gnu/target.o
#      imagery/i.ortho.photo/photo.2target/OBJ.i686-pc-linux-gnu/target.o
#      imagery/i.ortho.photo/photo.elev/OBJ.i686-pc-linux-gnu/main.o
#      imagery/i.ortho.photo/photo.rectify/OBJ.i686-pc-linux-gnu/target.o
#      imagery/i.ortho.photo/photo.target/OBJ.i686-pc-linux-gnu/main.o
#      imagery/i.rectify/OBJ.i686-pc-linux-gnu/target.o
#
# to discover which files import a given symbol, or more complex queries
# such as:
#
#     grass=> SELECT DISTINCT b.object FROM lib_exp a, obj_imp b
#     grass-> WHERE a.library = 'libgrass_form.6.1.cvs.so' AND a.symbol = b.symbol ;
#                               object
#     -----------------------------------------------------------
#      vector/v.digit/OBJ.i686-pc-linux-gnu/attr.o
#      vector/v.digit/OBJ.i686-pc-linux-gnu/line.o
#      vector/v.what/OBJ.i686-pc-linux-gnu/what.o
#      visualization/nviz/src/OBJ.i686-pc-linux-gnu/query_vect.o
#     (5 rows)
#
# to discover which files import any symbol defined in a specific
# library. And so on.
#
# For simple "which files use this function" queries, a database lookup
# is quicker and more reliable than grep-ing the source tree.
#
# Assuming that the dep_tree2sql.sh script runs successfully (some of it is
# Linux-specific, other bits are PostgreSQL-specific, but the changes
# required for a different OS or RDBMS should be quite minor), the
# easiest way to figure out what is in a given table (apart from looking
# at the name) is to just sample it, e.g.:
#
#     grass=> SELECT * FROM stlib_exp LIMIT 5 ;
#           library      |   object   |    symbol
#     -------------------+------------+---------------
#      libgrass_manage.a | add_elem.o | add_element
#      libgrass_manage.a | ask.o      | ask_in_mapset
#      libgrass_manage.a | ask.o      | ask_new
#      libgrass_manage.a | ask.o      | ask_old
#      libgrass_manage.a | copyfile.o | copyfile
#     (5 rows)
#

tmpdir=/tmp/sql-grass
dbname=grass

if [ -n "$1" ]; then
    builddir="$1"
else
    echo "Usage: del_tree2sql.sh <source directory>" >&2
    exit 1
fi

rm -rf "$tmpdir"
mkdir -m 711 "$tmpdir" || exit 1

cd "$builddir" || exit 1

(
    cd dist.* || exit 1

    #LD_LIBRARY_PATH=`pwd`/lib
    #export LD_LIBRARY_PATH

    find . -type f -perm /a+x \! -name '*.so.*' \
        | while read -r file; do ldd "$file" | sed 's!^!'"$file"'!'; done 2> /dev/null \
        | sed -e 's/^\.\///' -e 's/ (0x.*)$//' -e 's/ => \(.*\)$/	\1/' -e 's/ => .*$//' \
        | grep -F -v 'not a dynamic executable' \
        | awk -vOFS='\t' '{print $1,$2,$3 ? $3 : $2}' \
            > "$tmpdir/ldd.lst"

    find . -type f -perm /a+x \! -name '*.so' -print0 \
        | xargs -0 nm -AD 2> /dev/null \
        | grep -E ': {8}{1,2} U ' \
        | sed -e 's/:/ /g' -e 's/\.\///' \
        | awk -vOFS='\t' '{print $1,$3}' \
            > "$tmpdir/prog_imp.lst"

    find . -type f -perm /a+x \! -name '*.so' -print0 \
        | xargs -0 nm -AD 2> /dev/null \
        | grep -E ':[0-9a-f]{8}{1,2} [BCDGRSTW] ' \
        | sed -e 's/:/ /g' -e 's/\.\///' \
        | awk -vOFS='\t' '{print $1,$4}' \
            > "$tmpdir/prog_exp.lst"

)

find ./* -type f -name 'lib?*.a' -print0 \
    | xargs -0 nm -A \
    | grep -E ':[0-9a-f]{8}{1,2} [BCDGRSTW] ' \
    | sed 's/:/ /g' \
    | awk -vOFS='\t' '{print gensub("^[^ ]*/","",1,$1),$2,$5}' \
        > "$tmpdir/stlib_exp.lst"

find ./* -type f -name 'lib?*.so' -print0 \
    | xargs -0 nm -AD \
    | grep -E ':[0-9a-f]{8}{1,2} [BCDGRSTW] ' \
    | sed 's/:/ /g' \
    | awk -vOFS='\t' '{print gensub("^[^ ]*/","",1,$1),$4}' \
        > "$tmpdir/shlib_exp.lst"

find ./* -type f -name '*.o' -print0 \
    | xargs -0 nm -A \
    | grep -E ':[0-9a-f]{8}{1,2} [BCDGRSTW] ' \
    | sed 's/:/ /g' \
    | awk -vOFS='\t' '{print $1,$4}' \
        > "$tmpdir/obj_exp.lst"

find ./* -type f -name 'lib?*.a' -print0 \
    | xargs -0 nm -A \
    | grep -E ': {8}{1,2} U ' \
    | sed 's/:/ /g' \
    | awk -vOFS='\t' '{print gensub("^[^ ]*/","",1,$1),$2,$4}' \
        > "$tmpdir/stlib_imp.lst"

find ./* -type f -name 'lib?*.so' -print0 \
    | xargs -0 nm -AD \
    | grep -E ': {8}{1,2} U ' \
    | sed 's/:/ /g' \
    | awk -vOFS='\t' '{print gensub("^[^ ]*/","",1,$1),$3}' \
        > "$tmpdir/shlib_imp.lst"

find ./* -type f -name '*.o' -print0 \
    | xargs -0 nm -A \
    | grep -E ': {8}{1,2} U ' \
    | sed 's/:/ /g' \
    | awk -vOFS='\t' '{print $1,$3}' \
        > "$tmpdir/obj_imp.lst"

mapfile -t libs < <(awk '{print $3}' "$tmpdir/ldd.lst" | uniq | sort | uniq)

nm -AD "${libs[@]}" \
    | grep -E ':[0-9a-f]{8}{1,2} [TWDRC] ' \
    | sed 's/:/ /g' \
    | awk -vOFS='\t' '{print gensub("^[^ ]*/","",1,$1),$4}' \
        > "$tmpdir/libs.lst"

cat > "$tmpdir/ansi.lst" << EOF
abort
asctime
atexit
atof
atoi
atol
bsearch
calloc
clearerr
ctime
exit
fclose
feof
ferror
fflush
fgetc
fgetpos
fgets
fopen
fprintf
fputc
fputs
fread
free
freopen
fscanf
fseek
fsetpos
ftell
fwrite
getc
getchar
getenv
gets
isalnum
isalpha
isdigit
islower
isspace
ldiv
localtime
longjmp
malloc
memcmp
memcpy
memmove
memset
perror
printf
putc
putchar
puts
qsort
rand
realloc
remove
rename
rewind
scanf
setbuf
setjmp
setvbuf
signal
sprintf
srand
sscanf
stderr
stdin
stdout
strcat
strchr
strcmp
strcpy
strcspn
strerror
strftime
strlen
strncat
strncmp
strncpy
strpbrk
strrchr
strspn
strstr
strtod
strtok
strtol
time
tmpfile
tmpnam
tolower
toupper
ungetc
vfprintf
vsprintf
fgetpos64
fopen64
freopen64
fsetpos64
tmpfile64
EOF

dropdb --if-exists "$dbname"
createdb "$dbname"

psql -n -q -d "$dbname" << EOF

-- ----------------------------------------------------------------------

CREATE TABLE stlib_exp (
    library VARCHAR(80) NOT NULL,
    object VARCHAR(40) NOT NULL,
    symbol VARCHAR(256) NOT NULL
    ) ;

\copy stlib_exp FROM '$tmpdir/stlib_exp.lst'

CREATE TABLE shlib_exp (
    library VARCHAR(80) NOT NULL,
    symbol VARCHAR(256) NOT NULL
    ) ;

\copy shlib_exp FROM '$tmpdir/shlib_exp.lst'

CREATE TABLE obj_exp (
    object VARCHAR(100) NOT NULL,
    symbol VARCHAR(256) NOT NULL
    ) ;

\copy obj_exp FROM '$tmpdir/obj_exp.lst'

CREATE TABLE stlib_imp (
    library VARCHAR(80) NOT NULL,
    object VARCHAR(40) NOT NULL,
    symbol VARCHAR(256) NOT NULL
    ) ;

\copy stlib_imp FROM '$tmpdir/stlib_imp.lst'

CREATE TABLE shlib_imp (
    library VARCHAR(80) NOT NULL,
    symbol VARCHAR(256) NOT NULL
    ) ;

\copy shlib_imp FROM '$tmpdir/shlib_imp.lst'

CREATE TABLE obj_imp (
    object VARCHAR(100) NOT NULL,
    symbol VARCHAR(256) NOT NULL
    ) ;

\copy obj_imp FROM '$tmpdir/obj_imp.lst'

CREATE TABLE prog_imp (
    program VARCHAR(80) NOT NULL,
    symbol VARCHAR(256) NOT NULL
    ) ;

\copy prog_imp FROM '$tmpdir/prog_imp.lst'

CREATE TABLE prog_exp (
    program VARCHAR(80) NOT NULL,
    symbol VARCHAR(256) NOT NULL
    ) ;

\copy prog_exp FROM '$tmpdir/prog_exp.lst'

CREATE TABLE libs (
    library VARCHAR(80) NOT NULL,
    symbol VARCHAR(1000) NOT NULL
    ) ;

\copy libs FROM '$tmpdir/libs.lst'

CREATE TABLE ldd (
    program VARCHAR(80) NOT NULL,
    library VARCHAR(256) NOT NULL,
    path VARCHAR(256)
    ) ;

\copy ldd FROM '$tmpdir/ldd.lst'

CREATE TABLE ansi (
    symbol VARCHAR(256) NOT NULL
    ) ;

\copy ansi FROM '$tmpdir/ansi.lst'

-- ----------------------------------------------------------------------

SELECT DISTINCT library, symbol
    INTO TABLE lib_exp
    FROM stlib_exp
UNION
SELECT DISTINCT library, symbol
    FROM shlib_exp ;

CREATE TABLE duplicates AS
    SELECT DISTINCT symbol
    FROM lib_exp
    GROUP BY symbol
    HAVING COUNT(*) > 1 ;

CREATE TABLE duplicates2 AS
    SELECT *
    FROM lib_exp
    WHERE symbol IN (
        SELECT symbol
        FROM duplicates
    ) ;

SELECT DISTINCT library, symbol
    INTO TABLE lib_imp
    FROM stlib_imp
UNION
SELECT DISTINCT library, symbol
    FROM shlib_imp ;

CREATE TABLE imports AS
    SELECT a.library, a.symbol
    FROM lib_imp a
    WHERE NOT EXISTS (
        SELECT b.library, b.symbol
        FROM lib_exp b
        WHERE b.symbol = a.symbol
        AND b.library = a.library
    ) ;

CREATE TABLE defined AS
    SELECT DISTINCT symbol
    FROM lib_exp ;

CREATE TABLE used AS
    SELECT DISTINCT symbol
    FROM imports ;

CREATE TABLE undefined AS
    SELECT symbol
    FROM used u
    WHERE NOT EXISTS (
        SELECT *
        FROM defined d
        WHERE d.symbol = u.symbol
    ) ;

SELECT symbol
INTO TABLE undefined_1
    FROM undefined
EXCEPT
SELECT b.symbol
    FROM undefined a, libs b
    WHERE a.symbol = b.symbol ;

CREATE TABLE undefined_2 AS
    SELECT i.symbol, i.object, i.library
    FROM stlib_imp i, undefined_1 u
    WHERE i.symbol = u.symbol ;

CREATE TABLE depends AS
    SELECT  i.library AS im_lib,
        i.symbol AS symbol,
        e.library AS ex_lib
    FROM imports i, lib_exp e
    WHERE i.symbol = e.symbol ;

CREATE TABLE lib_deps AS
    SELECT DISTINCT im_lib, ex_lib
    FROM depends
    WHERE im_lib <> ex_lib ;

CREATE TABLE lib_deps_1 AS
    SELECT  a.im_lib,
        a.ex_lib AS in_lib,
        b.ex_lib
    FROM lib_deps a, lib_deps b
    WHERE a.ex_lib = b.im_lib ;

CREATE TABLE lib_deps_2 AS
    SELECT  a.im_lib,
        a.in_lib AS in1_lib,
        a.ex_lib AS in2_lib,
        b.ex_lib
    FROM lib_deps_1 a, lib_deps b
    WHERE a.ex_lib = b.im_lib
    AND a.im_lib <> a.ex_lib ;

SELECT im_lib, ex_lib
INTO TABLE lib_deps_trans
    FROM lib_deps
UNION
SELECT im_lib, ex_lib
    FROM lib_deps_1
UNION
SELECT im_lib, ex_lib
    FROM lib_deps_2 ;

CREATE TABLE prog_libs AS
SELECT DISTINCT a.program, b.library
FROM prog_imp a, lib_exp b
WHERE a.symbol = b.symbol ;

SELECT DISTINCT a.symbol
INTO TABLE libc
    FROM prog_imp a, libs b
    WHERE a.symbol = b.symbol
    AND b.library = 'libc.so.6'
UNION
    SELECT DISTINCT a.symbol
    FROM imports a, libs b
    WHERE a.symbol = b.symbol
    AND b.library = 'libc.so.6' ;

SELECT symbol
INTO nonansi
    FROM libc
    WHERE symbol !~ '_.*'
EXCEPT
SELECT symbol
    FROM ansi ;

CREATE TABLE nonansi_progs AS
    SELECT a.program, a.symbol
    FROM prog_imp a, nonansi b
    WHERE a.symbol = b.symbol ;

CREATE TABLE nonansi_libs AS
    SELECT a.library, a.symbol
    FROM imports a, nonansi b
    WHERE a.symbol = b.symbol ;

CREATE TABLE nonansi_prog_counts AS
    SELECT symbol, COUNT(*)
    FROM nonansi_progs
    GROUP BY symbol ;

CREATE TABLE nonansi_lib_counts AS
    SELECT symbol, COUNT(*)
    FROM nonansi_libs
    GROUP BY symbol ;

SELECT symbol
INTO TABLE nonansi_counts
    FROM nonansi_prog_counts
UNION
SELECT symbol
    FROM nonansi_lib_counts ;

ALTER TABLE nonansi_counts
    ADD COLUMN progs INTEGER ;

ALTER TABLE nonansi_counts
    ADD COLUMN libs INTEGER ;

UPDATE nonansi_counts
    SET progs = 0, libs = 0 ;

UPDATE nonansi_counts
    SET progs = b.count
    FROM nonansi_prog_counts b
    WHERE nonansi_counts.symbol = b.symbol ;

UPDATE nonansi_counts
    SET libs = c.count
    FROM nonansi_lib_counts c
    WHERE nonansi_counts.symbol = c.symbol;

-- SELECT a.symbol, a.program
--     FROM prog_imp a, nonansi_prog_counts b
--     WHERE a.symbol = b.symbol
--     AND a.program NOT LIKE 'bin/%'
--     ORDER BY b.count DESC, b.symbol ;

-- SELECT symbol, library
--     FROM duplicates2
--     ORDER BY symbol ;

-- SELECT a.im_lib, a.ex_lib
--     FROM lib_deps a, lib_deps b
--     WHERE   a.ex_lib = b.im_lib
--     AND     b.ex_lib = a.im_lib ;

-- SELECT * FROM lib_deps_2
--     WHERE im_lib = ex_lib ;

-- SELECT * FROM lib_deps_1
--     WHERE im_lib = ex_lib ;

-- SELECT im_lib FROM lib_deps_trans
--     WHERE im_lib = ex_lib ;

-- SELECT a.program, a.library
-- FROM ldd a
-- WHERE a.library LIKE 'libgrass_%.so'
-- AND a.library NOT IN (
-- SELECT b.library
-- FROM prog_libs b
-- WHERE b.program = a.program
-- ) ;

-- ----------------------------------------------------------------------

EOF

rm -rf "$tmpdir"