Differences between version 2 and predecessor to the previous major change of CyberLeo/hashery.sh.

Other diffs: Previous Revision, Previous Author

version 2

#!/bin/sh

# Format:
# <hash> <type> <size> <filename>
#
# hash can be any supported hash, but it is up to the implementer to differentiate between them
# hash does not make much sense for certain types
# Set to all zeroes for fifos, chars, directories, blocks, sockets, whiteouts, or unknowns
# Set to the hash of the content of the symlink, with tailing newline
# (the link itself, not the file or directory to which it points)
#
# Type is one of
#  f - fifo
#  c - char
#  d - directory
#  b - block
#  - - regular
#  l - symlink
#  s - socket
#  w - whiteout
#  ? - unknown
#
# Size is expressed zero-padded in at least 8 hexadecimal digits
# more digits can be used to express filesizes larger than 4 gigabytes
#
# Name is unquoted, and extends from the character after the first space after the size to the end of the line
# Non-empty directories need not be included, since they are implicitly specified by the stuff inside them

sudo sh <<"EOF"
minsize=16777216
emptyhash="$(sha256 -qs "")"
zerohash="$(echo "${emptyhash}" | tr '[0-9a-f]' '0')"

lst="$(mktemp -t hashery)"
trap "rm -f '${lst}'" exit hup int term kill

# First, compute the number of files for reporting
listlen=$(find . | pv -cN count -l | sed -e '/^\.$/d; s/^\.\///; s/\\/\\\\/g' | sort | tee "${lst}" | wc -l)

oldifs="${IFS}"
IFS='
'
cat "${lst}" | pv -cN list -ls ${listlen} | while read file
do
  IFS="${oldifs}"
  stat -f '%p %z %Y' "./${file}" | while read mode size target
  do
    # Now we have name, target, mode, size; compute the type
    hash="${zerohash}"
    type=$(printf %07o $(( 0${mode} & 0170000 )) )
    case ${type} in
    0010000) stype="f" ;;
    0020000) stype="c" ;;
    0040000) stype="d"
      # Ignore non-empty directories
      [ -z "$(ls -1a "./${file}" | sed -e '/^\.\{1,2\}$/d')" ] || continue
      ;;
    0060000) stype="b" ;;
    0100000) stype="-"
      [ "${size}" -eq 0 ] && hash="${emptyhash}"
      [ "${size}" -le "${minsize}" ] && hash="$(sha256 -q < "${file}")"
      [ "${size}" -gt "${minsize}" ] && hash="$(pv -cN file "./${file}" | sha256 -q)"
      ;;
    0120000) stype="l"; hash="$(sha256 -qs "${target}")" ;;
    0140000) stype="s" ;;
    0160000) stype="w" ;;
    *) stype="?" ;;
    esac
    printf "%s %s %08X %s\n" "${hash}" "${stype}" "${size:-0}" "${file}"
  done
done
EOF

Here are some possibilities for getting this to work on a Linux machine:

cat ../.hash/lnk.lst | sed -e 's/^\.\///' | while read link
do
  hash="$(readlink "${link}" | sha256sum | cut -d' ' -f1)"
  type="l"
  size="$(stat -c %s "${link}")"
  name="${link}"
  printf "%s %s %08X %s\n" "${hash}" "${type}" "${size}" "${name}"
done | tee -a ../.hash/all.hash

cat ../.hash/reg.lst | sed -e 's/^\.\///' | while read file
do
  hash="$(sha256sum "${file}" | cut -d' ' -f1)"
  type="-"
  size="$(stat -c %s "${file}")"
  name="${file}"
  printf "%s %s %08X %s\n" "${hash}" "${type}" "${size}" "${name}"
done | tee -a ../.hash/all.hash

hash="$(sha256sum /dev/null | cut -d' ' -f1 | tr 'A-Za-z0-9' '0')"
type="d"
size="0"
cat ../.hash/dir.lst | sed -e '/^\.$/d; s/^\.\///' | while read dir
do
  name="${dir}"
  ls -1a "${dir}" | sed -e '/^\.\{1,2\}$/d' | grep -q '.' && continue
  printf "%s %s %08X %s\n" "${hash}" "${type}" "${size}" "${name}"
done | tee -a ../.hash/all.hash

LC_ALL=C sort -k 4,99999999 all.hash