From d295b0fa38d1aff3197597334df40e407b30db62 Mon Sep 17 00:00:00 2001 From: John McQuah Date: Sat, 12 Feb 2022 14:35:12 +0100 Subject: prtwash: enable the cleanup tool to recognize sources that have been renamed --- prtwash | 331 +++++++++++++++++++++++++++++++++++++++++++------------------- prtwash.1 | 37 +++++-- 2 files changed, 258 insertions(+), 110 deletions(-) mode change 100755 => 100644 prtwash diff --git a/prtwash b/prtwash old mode 100755 new mode 100644 index 91bbd8f..9a69226 --- a/prtwash +++ b/prtwash @@ -4,6 +4,7 @@ # of the CRUX Linux distribution. # # Copyright (c) 2003 by Simone Rota +# Revised 2021 by John McQuah # # ************************************************************************* # * * @@ -14,17 +15,18 @@ # * * # ************************************************************************* # -# Took some inspiration and code from Martin Opel's prtsweep v1.6 script +# Took some inspiration and code from Martin Opel's prtsweep v1.6 script, +# and comments by therealfun on FlySpray bug 1851. # # **** USE AT YOUR OWN RISK **** # # TODO -# - a summary of files / bytes removed after the -# execution would be useful. +# - a summary of files / bytes removed after the +# execution would be useful. +# - verify signatures before sourcing each Pkgfile. # -VERSION="0.8" -CONFIGFILE="/etc/prt-get.conf" +VERSION="0.9" usage() { echo "Usage: prtwash [-p] [-s] [-d] [-t] [-a] [ ...]" @@ -33,7 +35,7 @@ usage() { showversion() { echo "prtwash" $VERSION - echo "(c) 2003 by Simone Rota" + echo "(c) 2003 by Simone Rota, revised 2021 by John McQuah" echo "This program is distributed under the GNU GPL license" } @@ -59,7 +61,7 @@ checkparams() { fi if [ ! -d "${prtdirs[$p]}" ]; then - echo "WARN: directory '${prtdirs[$p]}' not found. Skipping." + echo "WARN: '${prtdirs[$p]}' is not the directory for a port. Skipping." unset prtdirs[$p] continue fi @@ -74,8 +76,9 @@ checkparams() { getoptions () { # Name says it all. - while getopts psdathv opt; do + while getopts bpsdathv opt; do case "$opt" in + b) removeoldbuilds=1;; p) removepackage=1;; s) removesources=1;; d) removeaddonfiles=1;; @@ -100,7 +103,7 @@ getdirs() { # scans /etc/prt-get.conf for port dirs to process current="" for s in `sed 's/^[ \t]*//;s/[ \t]*$//' \ - $CONFIGFILE|grep '^prtdir.*:'|sed 's/:/ /;s/,/ /;s/prtdir//;s/#.*//'` + $CONF_PGET|grep '^prtdir.*:'|sed 's/:/ /;s/,/ /;s/prtdir//;s/#.*//'` do if [ "`echo $s|grep '/'`" != "" ]; then current=$s @@ -112,7 +115,7 @@ getdirs() { done for s in `sed 's/^[ \t]*//;s/[ \t]*$//;/:/d' \ - $CONFIGFILE|grep '^prtdir.*'|sed 's/prtdir//;s/#.*//'` + $CONF_PGET|grep '^prtdir.*'|sed 's/prtdir//;s/#.*//'` do basedirs=(${basedirs[*]} $s) done @@ -130,25 +133,132 @@ remove() { fi } +swash() { +# There are faster and safer ways of cleaning a directory shared among all ports than +# to iterate through the ports collection itself. +# Inform the admin that this script is ill-suited for their configuration. + echo "It looks like you have defined common directories for downloads or packages." >&2 + echo "For such a configuration, this script is slower and less reliable" >&2 + echo "than a custom tool would be." >&2 + echo "If you intended to clean out your ports repositories instead," >&2 + echo "consider the companion tool prtsweep." >&2 + echo "Proceed anyway? (y/n)" >&2 + read CONFIRM + if [ "$CONFIRM" != "y" ]; then + exit 0 + fi +} + +finalwarning() { + echo "Last chance to bail out!" >&2 + echo "This script will remove from the common download directories more files " >&2 + echo "than perhaps you intended." >&2 + echo "Proceed anyway? (y/n)" >&2 + read CONFIRM + if [ "$CONFIRM" != "y" ]; then + exit 0 + fi +} + +nwash() { + if [ "$auto" = 0 ]; then + + for p in ${prtdirs[@]}; do + echo "Washing the directory of port $p:" + wash $p + done + + exit 0 + else + + if [ ! -f "$CONF_PGET" ]; then + echo "ERROR: cannot find configuration file '$CONF_PGET'" + exit -1 + fi + echo "Analyzing port tree..." + + getdirs + + # wash base dirs + for d in ${basedirs[*]}; do + if [ -d "$d" ]; then + for p in "$d"/*; do + if [ -d "$p" ]; then + wash $p + fi + done + else + echo "ERROR: directory '$d' not found, check your configuration file!" + fi + done + + # wash custom dirs (see prt-get.conf) + for p in ${singledirs[*]}; do + if [ -d "$p" ]; then + wash $p + else + echo "ERROR: directory '$p' not found, check your configuration file!" + fi + done + exit 0 + fi + exit 0 +} + wash() { # Does the actual removal work. - if [ ! -f "$1"/Pkgfile ]; then + local tosave name version source renames pkgfiles packagename portname SAVELOC PKGLOC + local dir=$1 + if [ ! -f "$dir"/Pkgfile ]; then # tested before, the test here is for # auto (-a) command - echo "WARN: no Pkgfile found in $1. Skipping." + echo "WARN: no Pkgfile found in $dir. Skipping." else - tosave=() # array of files not to be deleted - . "$1"/Pkgfile - PKGMK_COMPRESSION_MODE="gz" - [ -f /etc/pkgmk.conf ] && . /etc/pkgmk.conf - packagename="$name"#"$version"-"$release"".pkg.tar.$PKGMK_COMPRESSION_MODE" + declare -a tosave # array of files not to be deleted + pkgfiles=(`( . $dir/Pkgfile; PKGLOC=$(eval "printf '%s' $PKGGLOB"); \ + PKGFILE=$(eval "printf '%s' $BLTGLOB"); \ + SAVELOC=$(eval "printf '%s' $SRCGLOB"); \ + for (( p=0; p<${#source[@]}; p++ )); do \ + [ -n "${renames[$p]}" -a "${renames[$p]}" != "SKIP" ] && source[$p]="ftp://host/${renames[$p]}"; \ + done; \ + printf '%s ' "${SAVELOC:-$dir}"; \ + printf '%s ' "${PKGLOC:-$dir}/$PKGFILE"; \ + printf '%s ' "${source[@]}")`) + + SAVELOC=${pkgfiles[0]} + packagename=`basename ${pkgfiles[1]}` + PKGLOC=`dirname ${pkgfiles[1]}` - if [ ! "$removesources" = 1 ]; then # keep sources - for src in "${source[@]}"; do - tosave=( "${tosave[@]}" "`basename "$src"`" ) - done - fi + if [ "$removesources" = 0 ]; then + for (( p=2; p<${#pkgfiles[@]}; p++ )) ; do + tosave=( "${tosave[@]}" `get_filename "${pkgfiles[$p]}"` ) + done + fi + + # new in version 0.3 we (try to) always save files that are + # retrieved by rsync/httpup (non http(s) or ftp in source) + # ie patches, etc. These can be deleted with -d (extra files) + # option + if [ "$removesources" = 1 ] && [ "$removeaddonfiles" = 0 ]; then + for (( p=2; p<${#pkgfiles[@]}; p++ )) ; do + src="${pkgfiles[$p]}" + pkgfiles[$p]=$(get_filename $src) + [ "${src:0:7}" = "ftp://" \ + -o "${src:0:8}" = "http://" \ + -o "${src:0:6}" = "https://" ] || + tosave=( "${tosave[@]}" "${pkgfiles[$p]}" ) + done + fi + + # Now that the distinction between files obtained via rsync/httpup + # and files downloaded during `pkgmk -d` has been encoded in the + # array $tosave, clean up the filenames in the $pkgfiles array. + if [ "$removesources" = 0 ] || [ "$removeaddonfiles" = 1 ]; then + for (( p=2; p<${#pkgfiles[@]}; p++ )) ; do + pkgfiles[$p]=$(get_filename ${pkgfiles[$p]}) + done + fi if [ ! "$removepackage" = 1 ]; then # keep package @@ -157,64 +267,85 @@ wash() { if [ ! "$removeaddonfiles" = 1 ]; then # keep additional and dot files - tosave=( "${tosave[@]}" ".signature" ) - tosave=( "${tosave[@]}" ".32bit" ) - tosave=( "${tosave[@]}" ".footprint" ) - tosave=( "${tosave[@]}" ".nostrip" ) - tosave=( "${tosave[@]}" "README" ) - tosave=( "${tosave[@]}" "FAQ" ) - tosave=( "${tosave[@]}" "pre-install" ) - tosave=( "${tosave[@]}" "post-install" ) + tosave=( "${tosave[@]}" \ + ".signature" \ + ".32bit" \ + ".footprint" \ + ".nostrip" \ + "README" \ + "FAQ" \ + "pre-install" \ + "post-install" ) fi # keep Pkgfile. We always want to keep this nice funny file. tosave=( "${tosave[@]}" "Pkgfile" ) - # new in version 0.3 we (try to) always save files that are - # retrieved by cvs (non http(s) or ftp in source) - # ie patches, etc. This can be deleted with -d (extra files) - # option - if [ ! "$removeaddonfiles" = 1 ]; then - for src in "${source[@]}"; do - #echo $src - if [ "${src:0:7}" != "http://" \ - -a "${src:0:8}" != "https://" \ - -a "${src:0:6}" != "ftp://" ] - then - #echo "`basename "$src"`" "will NOT be deleted" - tosave=( "${tosave[@]}" "`basename "$src"`" ) - fi - done - fi + unset pkgfiles[0] pkgfiles[1] - for f in "$1"/{*,.footprint,.signature,.32bit}; do - if ! keep $f; then - remove $f - fi - done + # trust that our parsing of the Pkgfile has not set the wrong source directory. + # But if an absolute path is specified, restrict the `ls -A` output + # to avoid overzealous deletions. + cd $SAVELOC + if [ "${SAVELOC:0:1}" != "/" ]; then + for f in `ls -A`; do + keep $f || remove $f + done + elif [ ${#pkgfiles[@]} -gt 0 ]; then + for f in ${pkgfiles[@]}; do + for match in `ls -A ${f}* 2>/dev/null`; do + keep $match || remove $match + done + done + fi + cd - &>/dev/null + + # same trust as to the location of the built packages, if different from SAVELOC. + # Older versions can be discarded too, if $removeoldbuilds = 1. + if [ "$removepackage" = 1 ]; then + (cd $PKGLOC; [ -f $packagename ] && keep "$packagename" || remove "$packagename") + fi + + if [ "$removeoldbuilds" = 1 ]; then + portname=$(echo "$packagename" | sed 's/#.*//') + (cd $PKGLOC; for f in `ls ${portname}*.pkg.tar.[bgx]z* 2>/dev/null`; do keep $f || remove $f; done) + fi fi } keep() { -# Chechs if filename item is in tosave[] array -# return 1 if present, 0 if not. +# Checks if filename item is in tosave[] array +# return 0 if present, 1 if not. item=$1 + local excode + excode=1 # assume not in the array until proven otherwise + for i in "${tosave[@]}"; do - if [ "$i" = `basename "$item"` ]; then - return 0 + if [ "$i" = "$item" ]; then + [ "$test" = "1" ] && echo "= (t) ... keeping $item" || echo "= ... keeping $item" + excode=0 + break fi done - - return 1 + + return $excode +} + +get_filename() { + if [[ $1 =~ ^(http|https|ftp|file)://.*/(.+) ]]; then + echo "${BASH_REMATCH[2]}" + else + echo $1 + fi } printparams() { -# used for testing. - echo removepackage: $removepackage - echo removesources: $removesources - echo removeaddonfiles: $removeaddonfiles - echo auto: $auto - echo test: $test + echo "removepackage: $removepackage" + echo "removeoldbuilds: $removeoldbuilds" + echo "removesources: $removesources" + echo "removeaddonfiles: $removeaddonfiles" + echo "auto: $auto" + echo "test: $test" } @@ -229,58 +360,58 @@ if [ $# -lt 1 ]; then exit -1 fi - removepackage=0 removesources=0 removeaddonfiles=0 +removeoldbuilds=0 test=0 auto=0 basedirs=() singledirs=() prtdirs=() +CONF_PGET="/etc/prt-get.conf" +CONF_PMK="/etc/pkgmk.conf" getoptions $@ checkparams $@ +COMPRESSION_MODE="gz" +if [ -f $PKGCONFIG ]; then + COMPRESS_CFG=$(grep "PKGMK_COMPRESSION_MODE" $CONF_PMK | \ + sed 's/[\t w]*#.*//;s/.*=//;s/\"//g') + [ -n "$COMPRESS_CFG" ] && COMPRESSION_MODE=$COMPRESS_CFG + SRCGLOB=$(grep "PKGMK_SOURCE_DIR" $CONF_PMK | sed 's/[\t w]*#.*//; s/.*=//; s/\"//g') + PKGGLOB=$(grep "PKGMK_PACKAGE_DIR" $CONF_PMK | sed 's/[\t w]*#.*//; s/.*=//; s/\"//g') +fi -if [ "$auto" = 0 ]; then - - for p in ${prtdirs[@]}; do - wash $p - done +BLTGLOB='$name#$version-$release.pkg.tar.$COMPRESSION_MODE' -else +# First determine the type of fs layout the admin configured for source downloads. +# Layouts 1 and 2: at least one directory is shared among all ports (for sources or built packages), +# or the admin has organized downloads in some other way outside of $PORTS_ROOT. - if [ ! -f "$CONFIGFILE" ]; then - echo "ERROR: cannot find configuration file '$CONFIGFILE'" - exit -1 - fi - echo "Analyzing port tree..." - - getdirs - - # wash base dirs - for d in ${basedirs[*]}; do - if [ -d "$d" ]; then - for p in "$d"/*; do - if [ -d "$p" ]; then - wash $p - fi - done - else - echo "ERROR: directory '$d' not found, check your configuration file!" - fi - done - - # wash custom dirs (see prt-get.conf) - for p in ${singledirs[*]}; do - if [ -d "$p" ]; then - wash $p - else - echo "ERROR: directory '$p' not found, check your configuration file!" - fi - done +if [ -n "${SRCGLOB}" ] && echo "${SRCGLOB}" | grep -q -v '\$name'; then + swash + nwash +elif [ -n "${PKGGLOB}" ] && echo "${PKGGLOB}" | grep -q -v '\$name'; then + swash + nwash fi -exit 0 +# Layout 3: separate directories, named according to port, are potentially cluttered with downloads. +# Unlike prtsweep, actually try to follow these dynamically-defined paths and clean them up. +# $PORTS_ROOT might still remain cluttered after this "wash", but the admin can just run prtsweep. +# For such filesystem layouts, the two scripts complement each other nicely. +if [ -z "${SRCGLOB}" -o -z "${PKGGLOB}" ]; then + nwash +elif [ -n "${SRCGLOB}" ] && echo "$SRCGLOB" | grep -q '\$name'; then + nwash +elif [ -n "${PKGGLOB}" ] && echo "$PKGGLOB" | grep -q '\$name'; then + nwash +else + # Should have exited by now, but if not, + # offer another chance to abort. + finalwarning + nwash +fi diff --git a/prtwash.1 b/prtwash.1 index 6355f27..ed69623 100644 --- a/prtwash.1 +++ b/prtwash.1 @@ -1,7 +1,6 @@ -." Text automatically generated by txt2man-1.4.7 -.TH prtwash 1 "October 15, 2020" "prtwash 0.8" "" +.TH prtwash 1 "June 15, 2021" "prtwash 0.9" "" .SH NAME -\fBprtwash \fP- a script to clean the port dirs in CRUX Linux. +\fBprtwash \fP- a script to clean the port dirs in CRUX. \fB .SH SYNOPSIS .nf @@ -10,15 +9,15 @@ .fam T .fi .SH DESCRIPTION -\fBprtwash\fP is a bash script to clean the port tree of the CRUX -Linux distribution. You can clean a single or multiple directories +\fBprtwash\fP is a bash script to clean the port tree of a CRUX +distribution. You can clean a single or multiple directories or the entire port tree. With the given options it is possible to choose what kind of files the program should delete. A test mode is provided. .PP \fBprtwash\fP was inspired by Martin Opel's prtsweep script. .SH OPTIONS -By default prtsweep will NOT delete the following items +By default prtwash will NOT delete the following items in a port directory: .IP \(bu 3 The Pkgfile @@ -32,7 +31,7 @@ The .signature, .footprint, .32bit and .nostrip files Additional files: README, FAQ, pre-install, post-install .PP All items not included in the previous list WILL be deleted -(ie: the 'work' dir of pkgmk, if present). +(e.g.: the 'work' dir of pkgmk, if present). .PP You can choose to delete some of the above passing the proper option: @@ -68,12 +67,30 @@ Display usage information .B -v Display version +.SH ENVIRONMENT +In automatic mode, \fBprtwash\fP gets a list of repositories from +/etc/prt-get.conf, and for each repository descends into the +individual port directories to read the associated Pkgfiles. +After reading the Pkgfile in light of any custom settings from +/etc/pkgmk.conf, \fBprtwash\fP will know the source filenames, +the package filename, and where those files are likely to be found. +.PP +If the directories potentially cluttered by pkgmk builds are shared +among many ports, e.g., by setting a nonempty PKGMK_SOURCE_DIR with +no reference to the port name, then \fBprtwash\fP will pause and +ask the user to confirm the wash. This gives users the chance to +consider whether a different tool like \fBprtsweep\fP is better +suited to the task they want performed. +.PP +If the directories potentially cluttered by pkgmk builds are +defined dynamically by port name (or unset), then \fBprtwash\fP +in automatic mode will proceed without user intervention. .SH EXAMPLES .TP .B \fBprtwash\fP -a -t does a default wash action on the entire port tree; -donsn't really delete the files +doesn't really delete the files .TP .B \fBprtwash\fP -a -p -s @@ -82,7 +99,7 @@ and built packages from the entire port tree .TP .B \fBprtwash\fP -a -p -s -d -same as tha above, but also +same as the above, but also removes .signature, .footprint, README, etc .TP .B @@ -94,6 +111,6 @@ and built packages from the sitecopy directory \fBprtwash\fP /usr/ports/contrib/* does a default wash action on the contrib port tree .SH AUTHORS -Simone Rota +Simone Rota , John McQuah .SH SEE ALSO \fBprtsweep\fP(1) -- cgit v1.2.3