A utility for downloading and verifying FreeBSD releases and snapshots
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 

645 lines
15 KiB

  1. #!/bin/sh -
  2. #
  3. # Copyright 2018, 2022-2023 John-Mark Gurney.
  4. # All rights reserved.
  5. #
  6. # Redistribution and use in source and binary forms, with or without
  7. # modification, are permitted provided that the following conditions
  8. # are met:
  9. # 1. Redistributions of source code must retain the above copyright
  10. # notice, this list of conditions and the following disclaimer.
  11. # 2. Redistributions in binary form must reproduce the above copyright
  12. # notice, this list of conditions and the following disclaimer in the
  13. # documentation and/or other materials provided with the distribution.
  14. #
  15. # THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16. # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17. # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18. # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19. # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20. # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21. # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22. # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23. # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24. # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25. # SUCH DAMAGE.
  26. #
  27. # $Id$
  28. #
  29. STOREDIR="$HOME/.snapaid"
  30. KEYS="78B342BA26C7B2AC681EA7BE524F0C37A0B946A3 EAF48BBA7CC77A30FEFC0DA938CECA690C6A6A6E"
  31. KEY_URLS='https://cgit.freebsd.org/doc/plain/documentation/static/pgpkeys/gjb.key https://cgit.freebsd.org/doc/plain/documentation/static/pgpkeys/cperciva.key'
  32. setdefaults() {
  33. GPG=$(which gpg2)
  34. WGET=$(which wget)
  35. SHASUM=$(which shasum)
  36. }
  37. setdefaults
  38. if [ ! -x "$GPG" ]; then
  39. echo 'Failed to find gpg2 executable'
  40. exit 1
  41. fi
  42. if [ ! -x "$WGET" ]; then
  43. echo 'Failed to find wget executable'
  44. exit 1
  45. fi
  46. if [ ! -x "$SHASUM" ]; then
  47. echo 'Failed to find shasum executable'
  48. exit 1
  49. fi
  50. #wget:
  51. # -N for timestamps
  52. # --backups=x for backing up
  53. hostname=people.FreeBSD.org
  54. hostname=www.funkthat.com
  55. completeurl="https://${hostname}/~jmg/FreeBSD-snap/snapshot.complete.idx.xz"
  56. currenturl="https://${hostname}/~jmg/FreeBSD-snap/snapshot.idx.xz"
  57. mailarchurl="https://www.funkthat.com/~jmg/FreeBSD-snap/mail/%s"
  58. # type release arch platform date svnrev xxx fname url mid
  59. # 1 2 3 4 5 6 7 8 9 10
  60. # iso 11.1-STABLE arm-armv6 BEAGLEBONE 20180315 r330998 xxx FreeBSD-11.1-STABLE-arm-armv6-BEAGLEBONE-20180315-r330998.img.xz https://download.freebsd.org/ftp/snapshots/ISO-IMAGES/11.1/FreeBSD-11.1-STABLE-arm-armv6-BEAGLEBONE-20180315-r330998.img.xz 20180316000842.GA7399@FreeBSD.org
  61. set -e
  62. # This is used for some testing functions
  63. copy_function() {
  64. declare -F "$1" > /dev/null || return 1
  65. local func="$(declare -f "$1")"
  66. eval "${2}(${func#*\(}"
  67. }
  68. # Test function to cause a bad input
  69. cmd_failure() {
  70. exit 1
  71. }
  72. # First time fails, second time run real command
  73. gpg_first_fails() {
  74. copy_function verifygpg_orig verifygpg
  75. return 1
  76. }
  77. # When first arg is --, just touch the file to fake a bad d/l
  78. bad_file_dl() {
  79. if [ x"$1" = x"--" ]; then
  80. touch $(basename "$2")
  81. else
  82. $WGET_orig "$@"
  83. fi
  84. }
  85. # Make sure that the storage directory is present
  86. mkstore() {
  87. mkdir "$STOREDIR" 2>/dev/null || :
  88. if ! [ -x "$STOREDIR" -a -w "$STOREDIR" -a -d "$STOREDIR" ]; then
  89. echo "$STOREDIR is not a writable directory."
  90. fi
  91. }
  92. check_keys() {
  93. for i in $KEYS; do
  94. if ! $GPG --list-keys "$i" >/dev/null 2>&1; then
  95. return 1
  96. fi
  97. done
  98. return 0
  99. }
  100. # Given a message id, get the raw body and store it.
  101. get_raw() {
  102. mkstore
  103. mid="$1"
  104. midfile="$STOREDIR/$mid".raw
  105. if [ ! -e "$midfile" ]; then
  106. # get the raw part
  107. tmpfile="$STOREDIR/.tmp.$$.$mid".raw
  108. # strip out everything but message id and first signed part
  109. # minimizeemail isn't necessary anymore as the archive
  110. # already has done it, but keep it
  111. $WGET -O - $(printf "$mailarchurl" "$mid") 2>/dev/null | minimizeemail > "$tmpfile"
  112. if verifygpg "$tmpfile"; then
  113. mv "$tmpfile" "$STOREDIR/$mid.raw"
  114. else
  115. rm "$tmpfile"
  116. echo Bad signature from mail archive.
  117. return 1
  118. fi
  119. else
  120. if ! verifygpg "$midfile"; then
  121. rm "$midfile"
  122. get_raw "$mid"
  123. return $?
  124. fi
  125. fi
  126. }
  127. fetch() {
  128. mkstore
  129. if ! (cd "$STOREDIR" && $WGET -N "$1" >/dev/null 2>&1); then
  130. return 1
  131. fi
  132. }
  133. getvermid() {
  134. xzcat "$STOREDIR"/snapshot.complete.idx.xz | awk '$8 == fname {
  135. print $10
  136. }' fname="$i"
  137. }
  138. minimizeemail() {
  139. awk '
  140. tolower($1) == "message-id:" && check == 0 {
  141. print
  142. }
  143. tolower($1) == "content-type:" && tolower($2) == "multipart/signed;" && check == 0 {
  144. getboundary = 1
  145. print
  146. next
  147. }
  148. getboundary == 1 {
  149. getboundary = 0
  150. haveboundary = 1
  151. print
  152. if (substr($2, 1, 9) == "boundary=")
  153. boundary=substr($2, 11, length($2) - 11)
  154. next
  155. }
  156. haveboundary && $1 == ("--" boundary) {
  157. sigbody = 1
  158. }
  159. $0 == "-----BEGIN PGP SIGNED MESSAGE-----" {
  160. sigbody = 1
  161. }
  162. sigbody {
  163. print
  164. }
  165. $0 == "-----END PGP SIGNATURE-----" && !haveboundary {
  166. sigbody = 0
  167. }'
  168. }
  169. verifygpgfile() {
  170. local fname
  171. fname="$1"
  172. if grep "multipart/signed" "$fname" >/dev/null 2>&1; then
  173. tmpdir=$(mktemp -d -t snapaid)
  174. # Note mkfifo does not work, for some reason I got a
  175. # different order, they are small, so just write them to
  176. # disk and clean up afterward.
  177. awk -v FNAME="$tmpdir"/msg '
  178. BEGIN {
  179. if (FNAME == "") {
  180. print "FNAME not specified." > "/dev/stderr"
  181. exit 1
  182. }
  183. }
  184. END {
  185. #print "exiting" > "/dev/stderr"
  186. }
  187. {
  188. #print "raw " $0 > "/dev/stderr"
  189. }
  190. $0 ~ "protocol=\"application/pgp-signature" {
  191. if (substr($2, 1, 9) == "boundary=")
  192. boundary=substr($2, 11, length($2) - 11)
  193. #print "boundary " boundary " remaining line: " $0 > "/dev/stderr"
  194. next
  195. }
  196. boundary != "" && $1 == ("--" boundary) {
  197. if (state == 0) {
  198. output = 1
  199. outfname = FNAME
  200. printf("") > outfname
  201. state = 1
  202. } else if (state == 1) {
  203. close(outfname)
  204. outfname = FNAME ".asc"
  205. printf("") > outfname
  206. state = 2
  207. } else if (state == 2) {
  208. close(outfname)
  209. output = 0
  210. }
  211. #print "state " state ", boundary: " boundary ", output: " output > "/dev/stderr"
  212. next
  213. }
  214. # Do not print the final line ending. It belongs w/ the ending boundary,
  215. # and we do not want to prepend it first time through
  216. output {
  217. printf("%s%s", lineend, $0) >> outfname
  218. lineend = "\r\n"
  219. }' < "$fname"
  220. $GPG --verify "$tmpdir"/msg.asc "$tmpdir"/msg 2>/dev/null
  221. exitval="$?"
  222. #rm -f "$tmpdir"/msg.asc "$tmpdir"/msg
  223. #rmdir "$tmpdir"
  224. return "$exitval"
  225. else
  226. $GPG --verify "$fname" 2>/dev/null
  227. fi
  228. }
  229. # takes basename of arg, which much exist in STOREDIR, and verifies
  230. # that the signature is valid.
  231. verifygpg() {
  232. local fname
  233. fname=$(basename "$1")
  234. if ! (cd "$STOREDIR" && verifygpgfile "$fname" ); then
  235. echo 'ERROR: PGP signature verification failed!'
  236. return 1
  237. fi
  238. }
  239. # Verifies the file
  240. verifyfile() {
  241. local fname
  242. local hashinfo
  243. local algo hash
  244. fname="$STOREDIR/${1}.raw"
  245. hashinfo=$(awk -v FNAME="$2" '
  246. $0 ~ "protocol=\"application/pgp-signature" {
  247. if (substr($2, 1, 9) == "boundary=")
  248. boundary=substr($2, 11, length($2) - 11)
  249. #print "boundary " boundary " remaining line: " $0 > "/dev/stderr"
  250. next
  251. }
  252. boundary != "" && $1 == ("--" boundary) {
  253. if (check)
  254. check = 0
  255. else
  256. check = 1
  257. next
  258. }
  259. check && $2 == ("(" FNAME ")") {
  260. hashes[$1] = $4
  261. }
  262. $0 == "-----BEGIN PGP SIGNED MESSAGE-----" {
  263. check = 1
  264. }
  265. $0 == "-----BEGIN PGP SIGNATURE-----" {
  266. check = 0
  267. }
  268. END {
  269. if ("SHA512" in hashes)
  270. algo = "SHA512"
  271. else if ("SHA256" in hashes)
  272. algo = "SHA256"
  273. else {
  274. print "unkn BADHASH"
  275. exit 1
  276. }
  277. print algo " " hashes[algo]
  278. }
  279. ' "$fname")
  280. read algo hash <<-EOF
  281. ${hashinfo}
  282. EOF
  283. if [ x"$algo" == x"unkn" -o x"$algo" = x"" ]; then
  284. echo 'Unable to find hash for file.'
  285. exit 1
  286. fi
  287. echo "$hash $2" | $SHASUM -a "${algo#SHA}" -c -
  288. }
  289. dlverify() {
  290. fname="$8"
  291. dlurl="$9"
  292. vermid="${10}"
  293. # verify snap email
  294. if ! get_raw "$vermid"; then
  295. echo "Unable to fetch/verify snapshot email for: $fname"
  296. return 1
  297. fi
  298. if ! [ -f $(basename "$dlurl") ]; then
  299. # fetch link
  300. $WGET -- "$dlurl"
  301. else
  302. echo 'Image already exists, verifying...'
  303. fi
  304. if ! verifyfile "$vermid" "$fname"; then
  305. echo 'Removing bad file.'
  306. rm "$fname"
  307. return 1
  308. fi
  309. }
  310. # Special wget that doesn't fetch the large file
  311. WGET_special1() {
  312. if [ "$1" = "--" -a "$2" = "https://download.freebsd.org/ftp/snapshots/ISO-IMAGES/14.0/FreeBSD-14.0-CURRENT-amd64-20210909-58a7bf124cc-249268-bootonly.iso.xz" ]; then
  313. return 0
  314. else
  315. $WGET_orig "$@"
  316. fi
  317. }
  318. # Allow the file to be sourced so the functions for checking PGP
  319. # signatures can be used by addinfo.sh
  320. if [ x"$SNAPAID_SH" = x"source" ]; then
  321. return 0
  322. fi
  323. if ! check_keys; then
  324. echo 'Necessary keys have not been imported into key ring.'
  325. echo "Please obtain they following keyid(s):"
  326. echo $KEYS
  327. echo ""
  328. echo "The keys may be obtained from the following URLs:"
  329. for i in $KEY_URLS; do
  330. echo "$i"
  331. done
  332. echo ""
  333. echo "and imported into GPG w/ the --import option. This can be"
  334. echo "done via the command:"
  335. echo "fetch -o - $KEY_URLS | gpg --import -"
  336. echo ""
  337. echo "For extra security, additional verification should be done, such"
  338. echo "as manually verifying finger prints."
  339. exit 3
  340. fi
  341. if [ x"$1" = x"verify" ]; then
  342. shift
  343. if ! fetch "$completeurl"; then
  344. echo Failed to fetch the complete index.
  345. exit 1
  346. fi
  347. for i in "$@"; do
  348. vermid=$(getvermid "$i")
  349. if [ x"$vermid" = x"" ]; then
  350. echo "Unable to find entry for: $i"
  351. continue
  352. fi
  353. if ! get_raw "$vermid"; then
  354. echo "Unable to fetch snapshot email for: $i"
  355. continue
  356. fi
  357. verifyfile "$vermid" "$i"
  358. done
  359. elif [ x"$1" = x"find" ]; then
  360. shift
  361. fetch "$currenturl"
  362. tmpdir=$(mktemp -d -t snapaid)
  363. trap "rm -r $tmpdir" 0
  364. ( cd "$tmpdir";
  365. xzcat "$STOREDIR"/snapshot.idx.xz | sort -r -k 5 -k 2 > selection;
  366. while :; do
  367. cnt=$(wc -l < selection)
  368. if [ x"$1" = x"" ]; then
  369. # display current list
  370. awk '
  371. BEGIN {
  372. # xzcat snapshot.complete.idx.xz | ./maxcol.awk
  373. # xzcat snapshot.complete.idx.xz | awk "{ print $3}" | sort -u
  374. # note that for powerpc-* that first part is dropped
  375. fmtstr = "%2s %-3s %-15s %-14s %-18s %-8s %-11s\n"
  376. printf(fmtstr, "#", "TYP", "RELEASE", "ARCH", "PLATFORM/TYPE", "DATE", "REV")
  377. cnt = 1
  378. }
  379. {
  380. if ($3 ~ /^powerpc-/)
  381. $3 = substr($3, 9)
  382. if ($4 == "xxx")
  383. plt=$7
  384. else
  385. plt=$4
  386. printf(fmtstr, cnt, $1, $2, $3, plt, $5, $6)
  387. if (cnt >= 20)
  388. exit 0
  389. cnt += 1
  390. }
  391. ' selection
  392. fi
  393. if [ x"$1" != x"" ]; then
  394. sel="$1"
  395. shift
  396. else
  397. read -p 'Select image #, enter search term, reset, or quit: ' sel
  398. fi
  399. if [ x"$sel" = x"reset" ]; then
  400. xzcat "$STOREDIR"/snapshot.idx.xz | sort -r -k 5 > selection;
  401. continue
  402. elif [ x"$sel" = x"quit" ]; then
  403. echo "$sel" > sel
  404. break
  405. fi
  406. if [ "$cnt" -gt 20 ]; then
  407. cnt=20
  408. fi
  409. if [ "$sel" -ge 1 -a "$sel" -le "$cnt" ] 2>/dev/null; then
  410. echo $(tail -n +"$sel" selection | head -n 1) > sel
  411. break
  412. else
  413. # restrict
  414. if ! grep -- "$sel" selection > selection.new; then
  415. echo WARNING: Ignoring selection, no results.
  416. else
  417. mv selection.new selection
  418. fi
  419. fi
  420. done
  421. )
  422. sel=$(cat "$tmpdir"/sel)
  423. if [ x"$sel" = x"quit" ]; then
  424. exit 0
  425. fi
  426. set -- ${sel}
  427. echo selected image "$8"
  428. dlverify ${sel}
  429. elif [ x"$1" = x"test" ]; then
  430. # Setup test environment
  431. tmpdir=$(mktemp -d -t snapaid)
  432. trap "rm -r $tmpdir" 0
  433. cd "$tmpdir"
  434. STOREDIR="$tmpdir"/snapaid
  435. # Make sure that the check keys function works.
  436. echo 'Testing check_keys works...'
  437. # Prime the custom keyring
  438. GPG="gpg2 --no-default-keyring --keyring pubring.gpg"
  439. for i in $KEY_URLS; do
  440. $WGET -O - -- "$i" 2>/dev/null | $GPG --import 2>/dev/null
  441. done
  442. if ! check_keys; then
  443. echo failed
  444. exit 1
  445. fi
  446. KEYS_orig="$KEYS"
  447. KEYS="0x1384923867573928" # bogus key
  448. if check_keys; then
  449. echo failed
  450. exit 1
  451. fi
  452. echo passed
  453. # save WGET & SHASUM
  454. WGET_orig="$WGET"
  455. SHASUM_orig="$SHASUM"
  456. # Testing signature that is an attachment
  457. echo 'Testing email with attached signature...'
  458. # called by dlverify
  459. #get_raw "20210909215942.GH1630@FreeBSD.org"
  460. WGET=WGET_special1
  461. SHASUM="echo FreeBSD-14.0-CURRENT-amd64-20210909-58a7bf124cc-249268-bootonly.iso.xz: OK; : "
  462. if ! dlverify iso 14.0-CURRENT amd64 xxx 20210909 xxx bootonly FreeBSD-14.0-CURRENT-amd64-20210909-58a7bf124cc-249268-bootonly.iso.xz https://download.freebsd.org/ftp/snapshots/ISO-IMAGES/14.0/FreeBSD-14.0-CURRENT-amd64-20210909-58a7bf124cc-249268-bootonly.iso.xz 20210909215942.GH1630@FreeBSD.org; then
  463. echo 'failed'
  464. exit 1
  465. fi
  466. echo passed
  467. WGET="$WGET_orig"
  468. SHASUM="$SHASUM_orig"
  469. # Test a bad download fails
  470. echo 'Testing dlverify...'
  471. WGET=bad_file_dl
  472. # if dlverify is successsful, then it's a failure
  473. if dlverify iso 13.0-CURRENT sparc64 xxx 20181026 r339752 bootonly FreeBSD-13.0-CURRENT-sparc64-20181026-r339752-bootonly.iso.xz https://download.freebsd.org/ftp/snapshots/ISO-IMAGES/13.0/FreeBSD-13.0-CURRENT-sparc64-20181026-r339752-bootonly.iso.xz 20181026184443.GD75350@FreeBSD.org; then
  474. echo 'failed'
  475. exit 1
  476. fi
  477. # Make sure that a bad d/l was not left behind
  478. if [ -e FreeBSD-13.0-CURRENT-sparc64-20181026-r339752-bootonly.iso.xz ]; then
  479. echo failed
  480. exit 1
  481. fi
  482. echo passed
  483. # Test getting the raw file
  484. echo 'Testing get_raw success...'
  485. mid='20160122055622.GA87581@FreeBSD.org'
  486. get_raw "$mid"
  487. # Verify resulsts
  488. (cd "$STOREDIR" && echo '6e53df5995b6cc423c7f2d63b6df52d5d7f70e8586c25f91433fd8a1a2466e77be6a38884bde8bedd9ff6e7deb0215a66e1c2a16e4955503c20445e649a5fb47 20160122055622.GA87581@FreeBSD.org.raw' | $SHASUM -a 512 -c)
  489. echo passed
  490. # If the file already exists, but fails verification, that
  491. # it will refetch and be correct
  492. echo 'Testing get_raw with file already present that fails verification...'
  493. copy_function verifygpg verifygpg_orig
  494. copy_function gpg_first_fails verifygpg
  495. get_raw "$mid"
  496. (cd "$STOREDIR" && echo '6e53df5995b6cc423c7f2d63b6df52d5d7f70e8586c25f91433fd8a1a2466e77be6a38884bde8bedd9ff6e7deb0215a66e1c2a16e4955503c20445e649a5fb47 20160122055622.GA87581@FreeBSD.org.raw' | $SHASUM -a 512 -c)
  497. echo passed
  498. # If the file already exists, a "broken" wget won't cause
  499. # a problem
  500. echo 'Testing get_raw with file already present...'
  501. WGET=cmd_failure
  502. get_raw "$mid"
  503. echo passed
  504. # Test failure
  505. echo 'Testing get_raw fails w/ bad data...'
  506. WGET=cmd_failure
  507. rm "$STOREDIR/$mid.raw"
  508. # it should fail
  509. ! get_raw "$mid"
  510. # and the desired file should not exist
  511. if [ -e "$STOREDIR/$mid.raw" ]; then
  512. echo 'Test failed!'
  513. exit 1;
  514. fi
  515. echo passed
  516. setdefaults
  517. echo tests completed!!!
  518. else
  519. if [ $# -gt 0 ]; then
  520. echo "Unknown verb: $1"
  521. fi
  522. echo "Usage:"
  523. echo " $0 verify file ..."
  524. echo " $0 find [ termselection ... ]"
  525. echo ""
  526. echo "The verify option will attempt to verify each file specified."
  527. echo ""
  528. echo "The find option will start up an interactive session to find"
  529. echo "and select the snapshot to download and verify."
  530. fi