diff --git a/addinfo.sh b/addinfo.sh index 9f5f656..b28c952 100644 --- a/addinfo.sh +++ b/addinfo.sh @@ -1,6 +1,6 @@ #!/bin/sh - # -# Copyright 2018, 2022 John-Mark Gurney. +# Copyright 2018, 2022-2023 John-Mark Gurney. # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -24,8 +24,6 @@ # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. # -# $Id$ -# set -e @@ -35,6 +33,9 @@ set -e # snapshot.idx, and then run "sh addinfo.sh -c YYYYMMDD" with the oldest # known snapshot. +SNAPDIR=$(dirname $0) +MAILDIR="$SNAPDIR"/mail + usage() { echo "Usage: $0 [ -m ] " echo "Usage: $0 -c " @@ -72,6 +73,10 @@ while :; do ;; esac done +if [ ! -d "$MAILDIR" ]; then + echo 'ERROR: '"$MAILDIR"' does not exist.' + exit 3 +fi if [ x"$complete" = x"1" -a x"$more" = x"1" ]; then echo '-m and -c cannot be specified at the same time.' usage 2 @@ -87,9 +92,11 @@ while ! mkdir "$0.running"; do sleep 1; done +trap 'rmdir "$0.running"' 0 + export SNAPAID_SH=source -. $(dirname $0)/snapaid.sh +. "$SNAPDIR"/snapaid.sh if [ x"$complete" = x"1" ]; then if [ ! -f snapshot.complete.idx ]; then @@ -114,7 +121,6 @@ if [ x"$complete" = x"1" ]; then mv snapshot.idx.new.xz snapshot.idx.xz mv snapshot.complete.idx.new.xz snapshot.complete.idx.xz rm snapshot.idx snapshot.complete.idx - rmdir "$0.running" exit 0 fi @@ -129,8 +135,26 @@ if ! verifygpgfile "$tmpfname"; then exit 1 fi +mid="$(awk 'tolower($1) == "message-id:" { + MID=$2 + sub(".*<", "", MID) + sub(">.*", "", MID) + + printf "%s", MID + + exit 0 +}' "$tmpfname")" + +midfile="$MAILDIR/$mid" +if [ -e "$midfile" ]; then + echo "ERROR: $midfile already exists, failed processing $tmpfname" + exit 3 +fi + # process file awk -f ./mksnapidx.awk "$tmpfname" > additional +cp "$tmpfname" "$midfile" +chmod og+rX "$midfile" rm "$tmpfname" # only check if there isn't more to come @@ -164,5 +188,3 @@ else mv snapshot.idx.new.xz snapshot.idx.xz mv snapshot.complete.idx.new.xz snapshot.complete.idx.xz fi - -rmdir "$0.running" diff --git a/snapaid.sh b/snapaid.sh index d3df4ef..091ff45 100755 --- a/snapaid.sh +++ b/snapaid.sh @@ -1,6 +1,6 @@ #!/bin/sh - # -# Copyright 2018, 2022 John-Mark Gurney. +# Copyright 2018, 2022-2023 John-Mark Gurney. # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -60,6 +60,7 @@ hostname=people.FreeBSD.org hostname=www.funkthat.com completeurl="https://${hostname}/~jmg/FreeBSD-snap/snapshot.complete.idx.xz" currenturl="https://${hostname}/~jmg/FreeBSD-snap/snapshot.idx.xz" +mailarchurl="https://www.funkthat.com/~jmg/FreeBSD-snap/mail/%s" # type release arch platform date svnrev xxx fname url mid # 1 2 3 4 5 6 7 8 9 10 @@ -121,31 +122,13 @@ get_raw() { midfile="$STOREDIR/$mid".raw if [ ! -e "$midfile" ]; then - # get the location, it's a database lookup - loc=$($WGET --max-redirect=0 --method=HEAD -S -o - -O - 'https://docs.freebsd.org/cgi/mid.cgi?'"$mid" 2>/dev/null | awk 'tolower($1) == "location:" { print $2; exit }') - - if [ x"$loc" = x"" ]; then - # Some emails are sent to both -current and -snapshot, - # such as 20160529215940.GA11785@FreeBSD.org - # try w/ some magic sed - loc=$($WGET -O - 'https://docs.freebsd.org/cgi/mid.cgi?'"$mid" 2>/dev/null | - sed -Ee '/.*(getmsg.cgi?[^"]*).*/!d;s//\/cgi\/\1/' | head -1) - fi - - # if it's host relative, add https - if [ x"$loc" != x"${loc#//}" ]; then - # add https - loc="https:$loc" - elif [ x"$loc" != x"${loc#/[^/]}" ]; then - # add https+host - loc="https://docs.freebsd.org$loc" - fi - # get the raw part tmpfile="$STOREDIR/.tmp.$$.$mid".raw # strip out everything but message id and first signed part - $WGET -O - "$loc"+raw 2>/dev/null | minimizeemail > "$tmpfile" + # minimizeemail isn't necessary anymore as the archive + # already has done it, but keep it + $WGET -O - $(printf "$mailarchurl" "$mid") 2>/dev/null | minimizeemail > "$tmpfile" if verifygpg "$tmpfile"; then mv "$tmpfile" "$STOREDIR/$mid.raw"