snapaid/addinfo.sh

#!/bin/sh -
#
# Copyright 2018, 2022-2023 John-Mark Gurney.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.
#

set -e

#
# Note: It appears that sometimes files can "disappear" from the server, so,
# uncompress snapshot.complete.idx.xz to both snapshot.complete.idx and
# snapshot.idx, and then run "sh addinfo.sh -c YYYYMMDD" with the oldest
# known snapshot.
#
# $ xzcat snapshot.idx.xz  | awk '{ print $5 }' | sort -u
# $ unxz < snapshot.complete.idx.xz > snapshot.complete.idx
# $ unxz < snapshot.complete.idx.xz > snapshot.idx


SNAPDIR=$(dirname $0)
MAILDIR="$SNAPDIR"/mail

usage() {
	echo "Usage: $0 [ -m ] <file>"
	echo "Usage: $0 -c <date>"
	echo ''
	echo 'Options:'
	echo '	-m  Do not check what files are available.  This is useful for'
	echo '	    bulk import.'
	echo '	-c  Complete a bulk import (previously using -m), and assume that'
	echo '	    any snapshots before specified data are unfetchable.'
	echo ''
	echo 'date is specified as YYYYMMDD'

	if [ x"$1" != x"" ]; then
		exit $1
	fi
}

args=`getopt cm $*`
if [ $? -ne 0 ]; then
	usage 2
fi
set -- $args
while :; do
	case "$1" in
	-c)
		complete=1
		shift
		;;
	-m)
		more=1
		shift
		;;
	--)
		shift; break
		;;
	esac
done
if [ ! -d "$MAILDIR" ]; then
	echo 'ERROR: '"$MAILDIR"' does not exist.'
	exit 3
fi
if [ x"$complete" = x"1" -a x"$more" = x"1" ]; then
	echo '-m and -c cannot be specified at the same time.'
	usage 2
elif [ x"$complete" = x"1" -a $# -ne 1 ]; then
	echo 'must only specify a date with -c'
	usage 2
elif [ x"$complete" != x"1" -a $# -ne 1 ]; then
	echo 'must specify exactly one file'
	usage 2
fi

while ! mkdir "$0.running"; do
	sleep 1;
done

trap 'rmdir "$0.running"' 0

export SNAPAID_SH=source

. "$SNAPDIR"/snapaid.sh

if [ x"$complete" = x"1" ]; then
	if [ ! -f snapshot.complete.idx ]; then
		echo 'snapshot.complete.idx does not exist, aborting...'
		rmdir "$0.running"
		exit 5
	fi

	if [ ! -f snapshot.idx ]; then
		echo 'snapshot.idx does not exist, aborting...'
		rmdir "$0.running"
		exit 5
	fi

	sort -u snapshot.complete.idx | xz > snapshot.complete.idx.new.xz
	awk '$5 == "xxx" || $5 >= "'"$1"'" {
			if (!system("wget --method=HEAD " $9))
				print
		}
		' snapshot.idx | sort -u | xz > snapshot.idx.new.xz
	chmod 644 snapshot.idx.new.xz snapshot.complete.idx.new.xz
	mv snapshot.idx.new.xz snapshot.idx.xz
	mv snapshot.complete.idx.new.xz snapshot.complete.idx.xz
	rm snapshot.idx snapshot.complete.idx
	exit 0
fi

# minimize file
tmpfname="tmp.snapinf.asc"
minimizeemail < "$1" > "$tmpfname"

if ! verifygpgfile "$tmpfname"; then
	echo 'failed verify'
	rm "$tmpfname"
	rmdir "$0.running"
	exit 1
fi

mid="$(awk 'tolower($1) == "message-id:" {
        MID=$2
        sub(".*<", "", MID)
        sub(">.*", "", MID)

	printf "%s", MID

	exit 0
}' "$tmpfname")"

midfile="$MAILDIR/$mid"
if [ -e "$midfile" ]; then
	echo "ERROR: $midfile already exists, failed processing $tmpfname"
	exit 3
fi

# process file
awk -f ./mksnapidx.awk "$tmpfname" > additional
cp "$tmpfname" "$midfile"
chmod og+rX "$midfile"
rm "$tmpfname"

# only check if there isn't more to come
if [ x"$more" = x"1" ]; then
	echo queuing $(wc -l additional) entries
	(cat snapshot.idx 2>/dev/null || xzcat snapshot.idx.xz; cat additional) > snapshot.idx.new
	(cat snapshot.complete.idx 2>/dev/null || xzcat snapshot.complete.idx.xz; cat additional) > snapshot.complete.idx.new
else
	(xzcat snapshot.idx.xz; cat additional) | sort -u | awk '
	{
		print "testing " $9 > "/dev/stderr"
		if (!system("wget --method=HEAD " $9))
			print
	}
	' > snapshot.idx.new

	xz snapshot.idx.new

	(xzcat snapshot.complete.idx.xz || :; cat additional) | sort -u > snapshot.complete.idx.new
	xz snapshot.complete.idx.new
fi

rm additional

# install new indexes
if [ x"$more" = x"1" ]; then
	mv snapshot.idx.new snapshot.idx
	mv snapshot.complete.idx.new snapshot.complete.idx
else
	chmod 644 snapshot.idx.new.xz snapshot.complete.idx.new.xz
	mv snapshot.idx.new.xz snapshot.idx.xz
	mv snapshot.complete.idx.new.xz snapshot.complete.idx.xz
fi