Re: veritysetup forward error correction failure

On 18/07/2019 2:09 pm, Milan Broz wrote:
> On 18/07/2019 13:41, Tom Eccles wrote:
>> On 17/07/2019 8:49 pm, Milan Broz wrote:
>>> If you can still reproduce it, please send version of the utility and
>>> kernel (and --debug output as suggested in another mail) and if you have some
>>> data/hash/fec images that can be used to reproduce it, let me know where I can find it.
>>
>> Attached is a bash script gen_image.sh which should reproduce the issue.
>> find_and_corrupt.c should be in the working directory when gen_image.sh is
>> run. find_and_corrupt.c corrupts the disk image by simply searching for a
>> known string and introducing an error (see the diff outputted by gen_image.sh).
>>
>> See in particular the error and dmesg sample when reading the corrupted file
>> (gen_image.sh:77) and the success when running veritysetup verify
>> (gen_image.sh:86).
>>
>> I have created an issue at https://gitlab.com/cryptsetup/cryptsetup/issues/462
> 
> Thanks.
> 
> I think I see the problem, and it is actually not FEC related.
> 
> Could you please try to manually allocate loop devices in RW mode (not read-only
> as we have during automatic loop allocation if mapping image is in file)
> over the provided images, and then use these loop device as arguments for
> the veritysetup?
> 
> Like
>  # losetup /dev/loop0 data8M.img.corrupt 
>  # losetup /dev/loop1 hash-img 
>  # losetup /dev/loop2 fec-img 
> 
> and
> 
>  # veritysetup open --fec-roots 24 --fec-device /dev/loop2 /dev/loop0 test /dev/loop1 15546e6799bb13608c77fa9cb840682a2dec3cfdb948d942ff3487f537966c01 --debug
> 
> Does it correct the data now? For me it prints
> 
>   kernel: device-mapper: verity: sha256 using implementation "sha256-generic"
>   kernel: fec_decode_bufs: 13 callbacks suppressed
>   kernel: device-mapper: verity-fec: 7:0: FEC 0: corrected 153 errors
> 
> Milan
> 

This is not working for me.

My modified version of the script is attached.

My output when running the script is

sudo dd if=/dev/zero of=data8M.img bs=8M count=1
[sudo] password for tomeccles:
1+0 records in
1+0 records out
8388608 bytes (8.4 MB, 8.0 MiB) copied, 0.0100743 s, 833 MB/s
sudo mkfs.ext4 -b 4096 -L verity-data data8M.img
mke2fs 1.44.5 (15-Dec-2018)
Discarding device blocks: done
Creating filesystem with 2048 4k blocks and 2048 inodes

Allocating group tables: done
Writing inode tables: done
Creating journal (1024 blocks): done
Writing superblocks and filesystem accounting information: done

sudo mount -t ext4 data8M.img mnt
echo "Can you spot this error?" | sudo tee "mnt/test_file.txt"
Can you spot this error?
sudo umount data8M.img
sudo veritysetup --verbose --debug --fec-roots 24 --fec-device fec-img
format "data8M.img" "hash-img" > "veritysetup_format.out"
cp data8M.img data8M.img.corrupt
./find_and_corrupt data8M.img.corrupt
Searching for "Can you spot this error?"
Found at 4497432!
Overwriting with "Cao you spot this error?"
--- /dev/fd/63	2019-07-18 14:45:50.983857200 +0100
+++ /dev/fd/62	2019-07-18 14:45:50.983857200 +0100
@@ -281086,7 +281086,7 @@
 00449fd0: 0000 0000 0000 0000 0000 0000 0000 0000  ................
 00449fe0: 0000 0000 0000 0000 0000 0000 0000 0000  ................
 00449ff0: 0000 0000 0000 0000 0000 0000 0000 0000  ................
-0044a000: 4361 6e20 796f 7520 7370 6f74 2074 6869  Can you spot thi
+0044a000: 4361 6f20 796f 7520 7370 6f74 2074 6869  Cao you spot thi
 0044a010: 7320 6572 726f 723f 0a00 0000 0000 0000  s error?........
 0044a020: 0000 0000 0000 0000 0000 0000 0000 0000  ................
 0044a030: 0000 0000 0000 0000 0000 0000 0000 0000  ................
sudo losetup /dev/loop0 data8M.img.corrupt
sudo losetup /dev/loop1 hash-img
sudo losetup /dev/loop2 fec-img
sudo veritysetup --verbose --debug --fec-roots 24 --fec-device /dev/loop2
open /dev/loop0 verity /dev/loop1
a963b029f27238283eae4b1cab58e94eebfaee29c9143e3716f992ea5c69811a
# cryptsetup 2.1.0 processing "veritysetup --verbose --debug --fec-roots 24
--fec-device /dev/loop2 open /dev/loop0 verity /dev/loop1
a963b029f27238283eae4b1cab58e94eebfaee29c9143e3716f992ea5c69811a"
# Running command open.
# Allocating context for crypt device /dev/loop1.
# Trying to open and read device /dev/loop1 with direct-io.
# Initialising device-mapper backend library.
# Setting ciphertext data device to /dev/loop0.
# Trying to open and read device /dev/loop0 with direct-io.
# Trying to load VERITY crypt type from device /dev/loop1.
# Crypto backend (OpenSSL 1.1.1c  28 May 2019) initialized in cryptsetup
library version 2.1.0.
# Detected kernel Linux 4.19.37-verity x86_64.
# Reading VERITY header of size 512 on device /dev/loop1, offset 0.
# Trying to open and read device /dev/loop2 with direct-io.
# Activating volume verity by volume key.
# dm version   [ opencount flush ]   [16384] (*1)
# dm versions   [ opencount flush ]   [16384] (*1)
# Detected dm-ioctl version 4.39.0.
# Detected dm-verity version 1.4.0.
# Device-mapper backend running with UDEV support enabled.
# dm status verity  [ opencount noflush ]   [16384] (*1)
# Trying to activate VERITY device verity using hash sha256.
# Calculated device size is 16384 sectors (RW), offset 0.
# DM-UUID is CRYPT-VERITY-ae0850832ef64e1294ed2fd0a433f5d0-verity
# Udev cookie 0xd4dbd1e (semid 0) created
# Udev cookie 0xd4dbd1e (semid 0) incremented to 1
# Udev cookie 0xd4dbd1e (semid 0) incremented to 2
# Udev cookie 0xd4dbd1e (semid 0) assigned to CREATE task(0) with flags
DISABLE_LIBRARY_FALLBACK         (0x20)
# dm create verity CRYPT-VERITY-ae0850832ef64e1294ed2fd0a433f5d0-verity [
opencount flush ]   [16384] (*1)
# dm reload verity  [ opencount flush readonly securedata ]   [16384] (*1)
# dm resume verity  [ opencount flush readonly securedata ]   [16384] (*1)
# verity: Stacking NODE_ADD (253,0) 0:6 0660 [trust_udev]
# verity: Stacking NODE_READ_AHEAD 256 (flags=1)
# Udev cookie 0xd4dbd1e (semid 0) decremented to 1
# Udev cookie 0xd4dbd1e (semid 0) waiting for zero
# Udev cookie 0xd4dbd1e (semid 0) destroyed
# verity: Skipping NODE_ADD (253,0) 0:6 0660 [trust_udev]
# verity: Processing NODE_READ_AHEAD 256 (flags=1)
# verity (253:0): read ahead is 256
# verity: retaining kernel read ahead of 256 (requested 256)
# dm status verity  [ opencount noflush ]   [16384] (*1)
# Verity volume verity status is V.
# Releasing crypt device /dev/loop1 context.
# Releasing device-mapper backend.
Command successful.
sudo mount -t ext4 -o ro /dev/mapper/verity mnt
cat: mnt/test_file.txt: Input/output error
[    2.495571] snd_hda_codec_generic hdaudioC0D0:    inputs:
[    2.495572] snd_hda_codec_generic hdaudioC0D0:      Line=0x5
[    2.510317] [drm] Initialized qxl 0.1.0 20120117 for 0000:00:02.0 on minor 0
[   13.358563] loop: module loaded
[   13.379473] EXT4-fs (loop0): mounted filesystem with ordered data mode.
Opts: (null)
[   15.773706] EXT4-fs (dm-0): mounted filesystem with ordered data mode.
Opts: (null)
[   15.799744] device-mapper: verity-fec: 7:0: FEC 0: failed to correct: -74
[   15.799772] device-mapper: verity: 7:0: data block 1098 is corrupted
[   15.847392] device-mapper: verity-fec: 7:0: FEC 0: failed to correct: -74
[   15.847418] device-mapper: verity: 7:0: data block 1098 is corrupted
sudo umount mnt
sudo veritysetup close verity
sudo losetup -d  /dev/loop1 /dev/loop2
sudo veritysetup --verbose --debug --fec-roots 24 --fec-device fec-img
verify data8M.img.corrupt hash-img
a963b029f27238283eae4b1cab58e94eebfaee29c9143e3716f992ea5c69811a
# cryptsetup 2.1.0 processing "veritysetup --verbose --debug --fec-roots 24
--fec-device fec-img verify data8M.img.corrupt hash-img
a963b029f27238283eae4b1cab58e94eebfaee29c9143e3716f992ea5c69811a"
# Running command verify.
# Allocating context for crypt device hash-img.
# Trying to open and read device hash-img with direct-io.
# Initialising device-mapper backend library.
# Setting ciphertext data device to data8M.img.corrupt.
# Trying to open and read device data8M.img.corrupt with direct-io.
# Trying to load VERITY crypt type from device hash-img.
# Crypto backend (OpenSSL 1.1.1c  28 May 2019) initialized in cryptsetup
library version 2.1.0.
# Detected kernel Linux 4.19.37-verity x86_64.
# Reading VERITY header of size 512 on device hash-img, offset 0.
# Trying to open and read device fec-img with direct-io.
# Checking volume  by volume key.
# Trying to activate VERITY device [none] using hash sha256.
# Verification of data in userspace required.
# Hash verification sha256, data device data8M.img.corrupt, data blocks
2048, hash_device hash-img, offset 1.
# Using 2 hash levels.
# Data device size required: 8388608 bytes.
# Hash device size required: 73728 bytes.
Verification failed at position 4497408.
Verification of data area failed.
# Verification failed, trying to repair with FEC device.
Found 1 repairable errors with FEC device.
# Releasing crypt device hash-img context.
# Releasing device-mapper backend.
Command successful.
Success

_______________________________________________
dm-crypt mailing list
[email protected]
https://www.saout.de/mailman/listinfo/dm-crypt

#!/bin/bash

readonly IMG_SIZE="8M"
readonly IMG_PATH="data$IMG_SIZE.img"
readonly CORRUPT_IMG_PATH="$IMG_PATH.corrupt"

readonly MOUNTPOINT="mnt"
readonly TEST_STR="Can you spot this error?"
readonly TEST_FILE="$MOUNTPOINT/test_file.txt"

readonly FEC_IMG="fec-img"
readonly HASH_IMG="hash-img"

readonly CORRUPT_LOOP="/dev/loop0"
readonly HASH_LOOP="/dev/loop1"
readonly FEC_LOOP="/dev/loop2"

readonly VERITYSETUP_ARGS="--verbose --debug --fec-roots 24"
readonly VERITYSETUP_OUT="veritysetup_format.out"
readonly DM_NAME="verity"

readonly FIND_AND_CORRUPT="find_and_corrupt"

pr_err()
{
	echo "ERROR: $*" 1>&2
	exit 1
}

assert_noerr()
{
	echo "$*"
	if ! eval "$*"; then
		pr_err "$*"
	fi
}

# set up dm-verity volume

assert_noerr sudo dd if=/dev/zero of="$IMG_PATH" bs="$IMG_SIZE" count=1

assert_noerr sudo mkfs.ext4 -b 4096 -L verity-data "$IMG_PATH"

if ! [ -d "$MOUNTPOINT" ]; then
	assert_noerr mkdir "$MOUNTPOINT"
fi

assert_noerr sudo mount -t ext4 "$IMG_PATH" "$MOUNTPOINT"

assert_noerr "echo \"$TEST_STR\" | sudo tee \"$TEST_FILE\""

assert_noerr sudo umount "$IMG_PATH"

assert_noerr sudo "veritysetup $VERITYSETUP_ARGS --fec-device $FEC_IMG format \"$IMG_PATH\" \"$HASH_IMG\" > \"$VERITYSETUP_OUT\""

# create a corrupted version of the data image

if ! [ -x "$FIND_AND_CORRUPT" ]; then
	assert_noerr cc -Wall -Wextra -Wpedantic -Werror -O2 find_and_corrupt.c -o "$FIND_AND_CORRUPT"
fi

assert_noerr cp "$IMG_PATH" "$CORRUPT_IMG_PATH"

assert_noerr "./$FIND_AND_CORRUPT" "$CORRUPT_IMG_PATH"

# "Can" -> "Cao" is a 1-bit change: 0x63 -> 0x6f
diff -u <(xxd "$IMG_PATH") <(xxd "$CORRUPT_IMG_PATH")

# set up read-*write* loop devices (Milan Broz dm-crypt mailing list)
assert_noerr sudo losetup "$CORRUPT_LOOP" "$CORRUPT_IMG_PATH"
assert_noerr sudo losetup "$HASH_LOOP" "$HASH_IMG"
assert_noerr sudo losetup "$FEC_LOOP" "$FEC_IMG"

# Access via verity open

hash=$(grep "Root hash:" "$VERITYSETUP_OUT" | cut -f 2)
if [ $? != 0 ]; then
	pr_err "Failed to get hash"
fi

assert_noerr sudo veritysetup "$VERITYSETUP_ARGS" --fec-device "$FEC_LOOP" open "$CORRUPT_LOOP" "$DM_NAME" "$HASH_LOOP" "$hash"

assert_noerr sudo mount -t ext4 -o ro "/dev/mapper/$DM_NAME" "$MOUNTPOINT"

# bug here:
cat "$TEST_FILE"
sudo dmesg | tail

assert_noerr sudo umount "$MOUNTPOINT"

assert_noerr sudo veritysetup close "$DM_NAME"

assert_noerr sudo losetup -d "$CORRUP_LOOP" "$HASH_LOOP" "$FEC_LOOP"

# demonstrate that the error is correctable by veritysetup verify

assert_noerr sudo veritysetup "$VERITYSETUP_ARGS" --fec-device "$FEC_IMG" verify "$CORRUPT_IMG_PATH" "$HASH_IMG" "$hash"

echo "Success"