prtwash: enable the cleanup tool to recognize sources that have been renamed

2 files changed, 258 insertions, 110 deletions

diff --git a/prtwash b/prtwash
index 91bbd8f..9a69226 100755..100644
--- a/prtwash
+++ b/prtwash
@@ -4,6 +4,7 @@
 #              of the CRUX Linux distribution.
 #
 #    Copyright (c) 2003 by Simone Rota                  <sip@varlock.com>
+#    Revised 2021 by John McQuah			<jmcquah@disroot.org>
 #
 #    *************************************************************************
 #    *                                                                       *
@@ -14,17 +15,18 @@
 #    *                                                                       *
 #    *************************************************************************
 #
-#   Took some inspiration and code from Martin Opel's prtsweep v1.6 script
+#   Took some inspiration and code from Martin Opel's prtsweep v1.6 script,
+#     and comments by therealfun on FlySpray bug 1851.
 #
 #   **** USE AT YOUR OWN RISK ****
 #
 #   TODO  
-#         - a summary of files / bytes removed after the
-#           execution would be useful.
+#       - a summary of files / bytes removed after the
+#         execution would be useful.
+#	- verify signatures before sourcing each Pkgfile.
 #
 
-VERSION="0.8"
-CONFIGFILE="/etc/prt-get.conf"
+VERSION="0.9"
 
 usage() {
     echo "Usage: prtwash [-p] [-s] [-d] [-t] [-a] <path> [<path> ...]"
@@ -33,7 +35,7 @@ usage() {
 
 showversion() {
     echo "prtwash" $VERSION
-    echo "(c) 2003 by Simone Rota"
+    echo "(c) 2003 by Simone Rota, revised 2021 by John McQuah"
     echo "This program is distributed under the GNU GPL license"
 }
 
@@ -59,7 +61,7 @@ checkparams() {
             fi
             
             if [ ! -d "${prtdirs[$p]}" ]; then
-                echo "WARN: directory '${prtdirs[$p]}' not found. Skipping."
+                echo "WARN: '${prtdirs[$p]}' is not the directory for a port. Skipping."
                 unset prtdirs[$p]
                 continue
             fi
@@ -74,8 +76,9 @@ checkparams() {
 
 getoptions () {
 # Name says it all.
-    while getopts psdathv opt; do
+    while getopts bpsdathv opt; do
         case "$opt" in
+	    b) removeoldbuilds=1;;
             p) removepackage=1;;
             s) removesources=1;;
             d) removeaddonfiles=1;;
@@ -100,7 +103,7 @@ getdirs() {
 # scans /etc/prt-get.conf for port dirs to process
     current=""
     for s in `sed 's/^[ \t]*//;s/[ \t]*$//' \
-             $CONFIGFILE|grep '^prtdir.*:'|sed 's/:/ /;s/,/ /;s/prtdir//;s/#.*//'`
+             $CONF_PGET|grep '^prtdir.*:'|sed 's/:/ /;s/,/ /;s/prtdir//;s/#.*//'`
     do
         if [ "`echo $s|grep '/'`" != "" ]; then
             current=$s
@@ -112,7 +115,7 @@ getdirs() {
     done
 
     for s in `sed 's/^[ \t]*//;s/[ \t]*$//;/:/d' \
-             $CONFIGFILE|grep '^prtdir.*'|sed 's/prtdir//;s/#.*//'`
+             $CONF_PGET|grep '^prtdir.*'|sed 's/prtdir//;s/#.*//'`
     do
         basedirs=(${basedirs[*]} $s)
     done
@@ -130,25 +133,132 @@ remove() {
     fi
 }
 
+swash() {
+# There are faster and safer ways of cleaning a directory shared among all ports than 
+# to iterate through the ports collection itself.
+# Inform the admin that this script is ill-suited for their configuration.
+	echo "It looks like you have defined common directories for downloads or packages." >&2
+	echo "For such a configuration, this script is slower and less reliable" >&2
+	echo "than a custom tool would be." >&2
+	echo "If you intended to clean out your ports repositories instead," >&2
+	echo "consider the companion tool prtsweep." >&2
+	echo "Proceed anyway? (y/n)" >&2
+	read CONFIRM
+	if [ "$CONFIRM" != "y" ]; then
+		exit 0
+	fi
+}
+
+finalwarning() {
+	echo "Last chance to bail out!" >&2
+	echo "This script will remove from the common download directories more files " >&2
+	echo "than perhaps you intended." >&2
+	echo "Proceed anyway? (y/n)" >&2
+	read CONFIRM
+	if [ "$CONFIRM" != "y" ]; then
+		exit 0
+	fi
+}
+
+nwash() {
+    if [ "$auto" = 0 ]; then
+
+        for p in ${prtdirs[@]}; do
+	    echo "Washing the directory of port $p:"
+            wash $p
+        done
+
+        exit 0
+    else
+
+    if [ ! -f "$CONF_PGET" ]; then
+        echo "ERROR: cannot find configuration file '$CONF_PGET'"
+        exit -1
+    fi
+    echo "Analyzing port tree..."
+
+    getdirs
+    
+    # wash base dirs
+    for d in ${basedirs[*]}; do
+        if [ -d "$d" ]; then
+            for p in "$d"/*; do
+                if [ -d "$p" ]; then
+                    wash $p
+                fi  
+            done
+        else
+            echo "ERROR: directory '$d' not found, check your configuration file!" 
+        fi 
+    done 
+
+    # wash custom dirs (see prt-get.conf)
+    for p in ${singledirs[*]}; do
+         if [ -d "$p" ]; then
+             wash $p
+         else
+             echo "ERROR: directory '$p' not found, check your configuration file!"
+         fi      
+    done
+    exit 0
+    fi
+    exit 0
+}
+
 wash() {
 # Does the actual removal work.
-    if [ ! -f "$1"/Pkgfile ]; then
+    local tosave name version source renames pkgfiles packagename portname SAVELOC PKGLOC
+    local dir=$1
+    if [ ! -f "$dir"/Pkgfile ]; then
     # tested before, the test here is for
     # auto (-a) command
-        echo "WARN: no Pkgfile found in $1. Skipping."
+        echo "WARN: no Pkgfile found in $dir. Skipping."
     else
-        tosave=() # array of files not to be deleted
-        . "$1"/Pkgfile
-        PKGMK_COMPRESSION_MODE="gz"
-        [ -f /etc/pkgmk.conf ] && . /etc/pkgmk.conf
-        packagename="$name"#"$version"-"$release"".pkg.tar.$PKGMK_COMPRESSION_MODE"
+        declare -a tosave # array of files not to be deleted
+	pkgfiles=(`( . $dir/Pkgfile; PKGLOC=$(eval "printf '%s' $PKGGLOB"); \
+		PKGFILE=$(eval "printf '%s' $BLTGLOB"); \
+		SAVELOC=$(eval "printf '%s' $SRCGLOB"); \
+	    for (( p=0; p<${#source[@]}; p++ )); do \
+	    [ -n "${renames[$p]}" -a "${renames[$p]}" != "SKIP" ] && source[$p]="ftp://host/${renames[$p]}"; \
+	    done; \
+	printf '%s ' "${SAVELOC:-$dir}"; \
+	printf '%s ' "${PKGLOC:-$dir}/$PKGFILE"; \
+       	printf '%s ' "${source[@]}")`)
+
+	SAVELOC=${pkgfiles[0]}
+        packagename=`basename ${pkgfiles[1]}`
+	PKGLOC=`dirname ${pkgfiles[1]}`
 
-        if [ ! "$removesources" = 1 ]; then
         # keep sources
-            for src in "${source[@]}"; do 
-                tosave=( "${tosave[@]}" "`basename "$src"`" )
-            done
-        fi
+	if [ "$removesources" = 0 ]; then
+	for (( p=2; p<${#pkgfiles[@]}; p++ )) ; do 
+		tosave=( "${tosave[@]}" `get_filename "${pkgfiles[$p]}"` )
+	done
+	fi
+
+        # new in version 0.3 we (try to) always save files that are
+        # retrieved by rsync/httpup (non http(s) or ftp in source)
+        # ie patches, etc. These can be deleted with -d (extra files)
+        # option
+	if [ "$removesources" = 1 ] && [ "$removeaddonfiles" = 0 ]; then
+		for (( p=2; p<${#pkgfiles[@]}; p++ )) ; do 
+		     src="${pkgfiles[$p]}"
+		     pkgfiles[$p]=$(get_filename $src)
+		     [ "${src:0:7}" = "ftp://" \
+                     -o "${src:0:8}" = "http://" \
+                     -o "${src:0:6}" = "https://" ] ||
+		     tosave=( "${tosave[@]}" "${pkgfiles[$p]}" )
+		done
+	fi
+
+	# Now that the distinction between files obtained via rsync/httpup
+	# and files downloaded during `pkgmk -d` has been encoded in the
+	# array $tosave, clean up the filenames in the $pkgfiles array.
+	if [ "$removesources" = 0 ] || [ "$removeaddonfiles" = 1 ]; then
+		for (( p=2; p<${#pkgfiles[@]}; p++ )) ; do 
+			pkgfiles[$p]=$(get_filename ${pkgfiles[$p]})
+		done
+	fi
 
         if [ ! "$removepackage" = 1 ]; then
         # keep package
@@ -157,64 +267,85 @@ wash() {
 
         if [ ! "$removeaddonfiles" = 1 ]; then
         # keep additional and dot files
-            tosave=( "${tosave[@]}" ".signature" )
-            tosave=( "${tosave[@]}" ".32bit" )
-            tosave=( "${tosave[@]}" ".footprint" )
-            tosave=( "${tosave[@]}" ".nostrip" )
-            tosave=( "${tosave[@]}" "README" )
-            tosave=( "${tosave[@]}" "FAQ" )
-            tosave=( "${tosave[@]}" "pre-install" )
-            tosave=( "${tosave[@]}" "post-install" )
+            tosave=( "${tosave[@]}" \
+			".signature" \
+			".32bit" \
+			".footprint" \
+			".nostrip" \
+			"README" \
+			"FAQ" \
+			"pre-install" \
+			"post-install" )
         fi
 
         # keep Pkgfile. We always want to keep this nice funny file.
         tosave=( "${tosave[@]}" "Pkgfile" )
 
-          # new in version 0.3 we (try to) always save files that are
-        # retrieved by cvs (non http(s) or ftp in source)
-        # ie patches, etc. This can be deleted with -d (extra files)
-        # option
-        if [ ! "$removeaddonfiles" = 1 ]; then
-            for src in "${source[@]}"; do 
-                #echo $src
-                if [ "${src:0:7}" != "http://" \
-                     -a "${src:0:8}" != "https://" \
-                     -a "${src:0:6}" != "ftp://" ]
-                then
-                    #echo "`basename "$src"`" "will NOT be deleted"
-                    tosave=( "${tosave[@]}" "`basename "$src"`" )
-                fi
-            done
-        fi
+	unset pkgfiles[0] pkgfiles[1]
 
-        for f in "$1"/{*,.footprint,.signature,.32bit}; do
-            if ! keep $f; then
-                remove $f
-            fi  
-        done
+	# trust that our parsing of the Pkgfile has not set the wrong source directory.
+	#  But if an absolute path is specified, restrict the `ls -A` output
+	#  to avoid overzealous deletions.
+	cd $SAVELOC
+	if [ "${SAVELOC:0:1}" != "/" ]; then
+	    for f in `ls -A`; do
+		keep $f || remove $f
+            done
+	elif [ ${#pkgfiles[@]} -gt 0 ]; then
+	    for f in ${pkgfiles[@]}; do
+		for match in `ls -A ${f}* 2>/dev/null`; do
+		    keep $match || remove $match
+	        done
+            done
+	fi
+	cd - &>/dev/null
+
+	# same trust as to the location of the built packages, if different from SAVELOC.
+	# Older versions can be discarded too, if $removeoldbuilds = 1.
+	if [ "$removepackage" = 1 ]; then
+	(cd $PKGLOC; [ -f $packagename ] && keep "$packagename" || remove "$packagename")
+	fi
+
+	if [ "$removeoldbuilds" = 1 ]; then
+	portname=$(echo "$packagename" | sed 's/#.*//')
+	(cd $PKGLOC; for f in `ls ${portname}*.pkg.tar.[bgx]z* 2>/dev/null`; do keep $f || remove $f; done)
+	fi
     fi
 }
 
 keep() {
-# Chechs if filename item is in tosave[] array
-# return 1 if present, 0 if not.
+# Checks if filename item is in tosave[] array
+# return 0 if present, 1 if not.
     item=$1
+    local excode
+    excode=1 # assume not in the array until proven otherwise
+
     for i in "${tosave[@]}"; do
-        if [ "$i" = `basename "$item"` ]; then
-            return 0
+	if [ "$i" = "$item" ]; then
+	    [ "$test" = "1" ] && echo "= (t) ... keeping $item" || echo "= ... keeping $item"
+            excode=0
+	    break
         fi
     done
-      
-    return 1
+
+    return $excode
+}
+
+get_filename() {
+	if [[ $1 =~ ^(http|https|ftp|file)://.*/(.+) ]]; then
+		echo "${BASH_REMATCH[2]}"
+	else
+		echo $1
+	fi
 }
 
 printparams() {
-# used for testing.
-    echo removepackage: $removepackage
-    echo removesources: $removesources
-    echo removeaddonfiles: $removeaddonfiles
-    echo auto: $auto
-    echo test: $test
+    echo "removepackage: $removepackage"
+    echo "removeoldbuilds: $removeoldbuilds"
+    echo "removesources: $removesources"
+    echo "removeaddonfiles: $removeaddonfiles"
+    echo "auto: $auto"
+    echo "test: $test"
 }
 
 
@@ -229,58 +360,58 @@ if [ $# -lt 1 ]; then
     exit -1
 fi
 
-
 removepackage=0
 removesources=0
 removeaddonfiles=0
+removeoldbuilds=0
 test=0
 auto=0
 basedirs=()
 singledirs=()
 prtdirs=()
 
+CONF_PGET="/etc/prt-get.conf"
+CONF_PMK="/etc/pkgmk.conf"
 
 getoptions $@
 checkparams $@
 
+COMPRESSION_MODE="gz"
+if [ -f $PKGCONFIG ]; then
+	COMPRESS_CFG=$(grep "PKGMK_COMPRESSION_MODE" $CONF_PMK | \
+	sed 's/[\t w]*#.*//;s/.*=//;s/\"//g')
+	[ -n "$COMPRESS_CFG" ] && COMPRESSION_MODE=$COMPRESS_CFG
+	SRCGLOB=$(grep "PKGMK_SOURCE_DIR" $CONF_PMK | sed 's/[\t w]*#.*//; s/.*=//; s/\"//g')
+	PKGGLOB=$(grep "PKGMK_PACKAGE_DIR" $CONF_PMK | sed 's/[\t w]*#.*//; s/.*=//; s/\"//g')
+fi
 
-if [ "$auto" = 0 ]; then
-
-    for p in ${prtdirs[@]}; do
-        wash $p
-    done
+BLTGLOB='$name#$version-$release.pkg.tar.$COMPRESSION_MODE'
 
-else
+# First determine the type of fs layout the admin configured for source downloads.
+# Layouts 1 and 2: at least one directory is shared among all ports (for sources or built packages),
+#   or the admin has organized downloads in some other way outside of $PORTS_ROOT.
 
-    if [ ! -f "$CONFIGFILE" ]; then
-        echo "ERROR: cannot find configuration file '$CONFIGFILE'"
-        exit -1
-    fi
-    echo "Analyzing port tree..."
-
-    getdirs
-    
-    # wash base dirs
-    for d in ${basedirs[*]}; do
-        if [ -d "$d" ]; then
-            for p in "$d"/*; do
-                if [ -d "$p" ]; then
-                    wash $p
-                fi  
-            done
-        else
-            echo "ERROR: directory '$d' not found, check your configuration file!" 
-        fi 
-    done 
-
-    # wash custom dirs (see prt-get.conf)
-    for p in ${singledirs[*]}; do
-         if [ -d "$p" ]; then
-             wash $p
-         else
-             echo "ERROR: directory '$p' not found, check your configuration file!"
-         fi      
-    done
+if [ -n "${SRCGLOB}" ] && echo "${SRCGLOB}" | grep -q -v '\$name'; then
+     swash
+     nwash
+elif [ -n "${PKGGLOB}" ] && echo "${PKGGLOB}" | grep -q -v '\$name'; then
+     swash
+     nwash
 fi
 
-exit 0
+# Layout 3: separate directories, named according to port, are potentially cluttered with downloads.
+# Unlike prtsweep, actually try to follow these dynamically-defined paths and clean them up.
+# $PORTS_ROOT might still remain cluttered after this "wash", but the admin can just run prtsweep.
+# For such filesystem layouts, the two scripts complement each other nicely.
+if [ -z "${SRCGLOB}" -o -z "${PKGGLOB}" ]; then
+	nwash
+elif [ -n "${SRCGLOB}" ] && echo "$SRCGLOB" | grep -q '\$name'; then
+	nwash
+elif [ -n "${PKGGLOB}" ] && echo "$PKGGLOB" | grep -q '\$name'; then
+	nwash
+else
+	# Should have exited by now, but if not,
+	# offer another chance to abort.
+	finalwarning
+	nwash
+fi
diff --git a/prtwash.1 b/prtwash.1
index 6355f27..ed69623 100644
--- a/prtwash.1
+++ b/prtwash.1
@@ -1,7 +1,6 @@
-." Text automatically generated by txt2man-1.4.7
-.TH prtwash 1 "October 15, 2020" "prtwash 0.8" ""
+.TH prtwash 1 "June 15, 2021" "prtwash 0.9" ""
 .SH NAME
-\fBprtwash \fP- a script to clean the port dirs in CRUX Linux.
+\fBprtwash \fP- a script to clean the port dirs in CRUX.
 \fB
 .SH SYNOPSIS
 .nf
@@ -10,15 +9,15 @@
 .fam T
 .fi
 .SH DESCRIPTION
-\fBprtwash\fP is a bash script to clean the port tree of the CRUX
-Linux distribution. You can clean a single or multiple directories 
+\fBprtwash\fP is a bash script to clean the port tree of a CRUX
+distribution. You can clean a single or multiple directories 
 or the entire port tree. With the given options it is possible to
 choose what kind of files the program should delete.
 A test mode is provided.
 .PP
 \fBprtwash\fP was inspired by Martin Opel's prtsweep script.
 .SH OPTIONS
-By default prtsweep will NOT delete the following items
+By default prtwash will NOT delete the following items
 in a port directory:
 .IP \(bu 3
 The Pkgfile
@@ -32,7 +31,7 @@ The .signature, .footprint, .32bit and .nostrip files
 Additional files: README, FAQ, pre-install, post-install
 .PP
 All items not included in the previous list WILL be deleted
-(ie: the 'work' dir of pkgmk, if present).
+(e.g.: the 'work' dir of pkgmk, if present).
 .PP
 You can choose to delete some of the above passing the proper
 option:
@@ -68,12 +67,30 @@ Display usage information
 .B
 -v
 Display version
+.SH ENVIRONMENT
+In automatic mode, \fBprtwash\fP gets a list of repositories from
+/etc/prt-get.conf, and for each repository descends into the
+individual port directories to read the associated Pkgfiles.
+After reading the Pkgfile in light of any custom settings from
+/etc/pkgmk.conf, \fBprtwash\fP will know the source filenames,
+the package filename, and where those files are likely to be found.
+.PP
+If the directories potentially cluttered by pkgmk builds are shared
+among many ports, e.g., by setting a nonempty PKGMK_SOURCE_DIR with
+no reference to the port name, then \fBprtwash\fP will pause and
+ask the user to confirm the wash. This gives users the chance to
+consider whether a different tool like \fBprtsweep\fP is better
+suited to the task they want performed.
+.PP
+If the directories potentially cluttered by pkgmk builds are
+defined dynamically by port name (or unset), then \fBprtwash\fP
+in automatic mode will proceed without user intervention.
 .SH EXAMPLES
 .TP
 .B
 \fBprtwash\fP -a -t
 does a default wash action on the entire port tree;
-donsn't really delete the files
+doesn't really delete the files
 .TP
 .B
 \fBprtwash\fP -a -p -s
@@ -82,7 +99,7 @@ and built packages from the entire port tree
 .TP
 .B
 \fBprtwash\fP -a -p -s -d
-same as tha above, but also
+same as the above, but also
 removes .signature, .footprint, README, etc
 .TP
 .B
@@ -94,6 +111,6 @@ and built packages from the sitecopy directory
 \fBprtwash\fP /usr/ports/contrib/*
 does a default wash action on the contrib port tree
 .SH AUTHORS
-Simone Rota <sip@varlock.com>
+Simone Rota <sip at varlock dot com>, John McQuah <jmcquah at disroot dot org>
 .SH SEE ALSO
 \fBprtsweep\fP(1)