CDNWiki - View Source: copysparse.php

**** WARNING ****
PHP has an internal 2GB file size limitation for fseek()! Use the Ruby version below for images larger than 2GB!

Originally built to copy a block device (a mac mini in firewire mass storage mode) to an image file, excluding all-null blocks (sparse file) to save space.

Also contains examples of thousands-separator number formatting (i.e. 10,000,000) and human-readable magnitude number formatting (i.e. 10.0GB) functions.

<code brush="php">
#!/usr/bin/env php
<?php

declare(ticks=1);
bcscale(3);

// Read a file and process it in chunks.
$infile = "/dev/sde";
$outfile = "/media/recover/MacMini-whole-sparse.img";
$blksz = 4096;
$blkct = 19537686;
$blkct_b = bcmul($blkct, $blksz, 0);

function ksep($number) {
	$number = (string) $number;
	$max = strlen($number);
	$recip = $max;
	for ($i = 0; $i < $max; ++$i) {
		--$recip;
		if ($i && !($i % 3))
			$out = "," . $out;
		$out = $number{$recip} . $out;
	}
	return $out;
}

function prettyGigs($bytes) {
	$powers = " kMGTPEZY";
	$power = 0;
	$mod = 1;
	while (bcdiv($bytes, $mod) > 1024) {
		$mod = bcmul($mod, 1024);
		++$power;
	}
	$num = bcdiv($bytes, $mod);
	switch (((int) $num != $num)? strpos($num, "."): 0) {
		case "0":
			$fmt = "%0.0f%s";
			break;
		case "1":
			$fmt = "%0.3f%s";
			break;
		case "2":
			$fmt = "%0.2f%s";
			break;
		case "3":
		default:
			$fmt = "%0.1f%s";
			break;
	}
	if ($power)
		$power = $powers{$power}."B";
	else	$power = "B";
	return sprintf($fmt, $num, $power);
}

function stats() {
	pcntl_alarm(1);
	$blksz = $GLOBALS['blksz'];
	$blkct = $GLOBALS['blkct'];
	$blkct_b = $GLOBALS['blkct_b'];
	$data = $GLOBALS['data'];
	$data_b = bcmul($data, $blksz, 0);
	$sparse = $GLOBALS['sparse'];
	$sparse_b = bcmul($sparse, $blksz, 0);
	$count = bcadd($GLOBALS['data'], $GLOBALS['sparse'], 0);
	$count_b = bcmul($count, $blksz, 0);
	$pct = bcmul(bcdiv($count, $GLOBALS['blkct'], 10), 100.0);
	$speed = bcmul(bcsub($count, $GLOBALS['old_count']), $blksz);

	printf("\rD: %ss (%s) S: %ss (%s) +: %ss (%s) T: %ss (%s) %%: %0.4f %s/sec    ",
		ksep($data), prettyGigs($data_b), ksep($sparse), prettyGigs($sparse_b), ksep($count), prettyGigs($count_b),
		ksep($blkct), prettyGigs($blkct_b), $pct, prettyGigs($speed));

/*	printf("\rBlk: %s Spa: %s Sum: %s Tot: %s Pct: %0.4f    ",
		thousands($GLOBALS['data']), thousands($GLOBALS['sparse']),
		thousands($count), thousands($GLOBALS['blkct']), $pct);
*/	$GLOBALS['old_count'] = $count;
}

pcntl_signal(SIGALRM, "stats", true);
pcntl_alarm(1);
echo "\n\nPrecaching...";

/* No longer required
// Precalculate crc
$bufr = str_repeat("\0", 4096);
$zero = crc32($bufr);
*/

$input = fopen($infile, "rb");
$output = fopen($outfile, "wb");

if (!is_resource($input))
	die(sprintf("%s failed!\n", $infile));
if (!is_resource($output))
	die(sprintf("%s failed!\n", $outfile));

$data = 0;
$sparse = 0;

while (!feof($input)) {
	$bufr = fread($input, $blksz);
	$chars = count_chars($bufr, 3);
	if ($chars === "\0") {
		// Nul block. Skip it.
		fseek($output, ftell($output) + $blksz);
		$sparse++;
		continue;
	}
/* Counting chars takes roughly half the time as computing the crc32.
	if (crc32($bufr) == $zero) {
		// Matching block? compare by bytes
		$chars = count_chars($bufr, 3);
		if ($chars === "\0") {
			// Null block. Skip it.
			fseek($output, ftell($output) + $blksz);
			continue;
		}
	}
*/	fwrite($output, $bufr, $blksz);
	$data++;
}

fclose($input);
fclose($output);

echo "\n\n";

?>
</code>

Ruby version:
<code brush="ruby">
#!/usr/bin/env ruby

# Fucking kludge.
require 'dl/import'
module Alarm
	extend DL::Importable
	dlload "libc.so.6"
	extern "unsigned int alarm(unsigned int)"
end

# $stdout (standard out) is buffered! $stdout.flush() if you want it to show up immediately!
# Variables:
# * global variables start with $
# * instance variables start with @
# * local variables start with _ or a lowercase
# * constants start with an uppercase.
# Chr to ord: "x"[0] <- Access by index.
# Ord to chr: 120.chr <- Access chr method
# In for, start..end is start through end, while start...end is start up to but not including end.
# $data is, apparently, a reserved word.

$blksz = 4096;
$blkct = 19537686;
$blkct_b = $blkct * $blksz;

src_file = "/dev/sde";
out_file = "/media/recover/MacMini-whole-sparse.img";

src = File.new(src_file);
out = File.new(out_file, File::CREAT|File::TRUNC|File::RDWR, 0600);

#$old_sum_b = 0;

def kSep(kSep_num)
	kSep_num = kSep_num.to_s;
	max = kSep_num.length;
	recip = max;
	out = "";
	for cur in 0...max
		recip-= 1;
		if (cur != 0 && (cur % 3) == 0)
			out = "," << out;
		end
		out = kSep_num[recip].chr << out;
	end
	return out;
end

def pMag(bytes)
	bytes = bytes.to_f;
	powers = " kMGTPEZY";
	power = 0;
	fmt = "";
	out = "";
	mod = 1;
	while ((bytes / mod) > 1024)
		mod = mod * 1024;
		power+= 1;
	end
	num = bytes / mod;
	case (num.to_s.index("."))
		when nil then fmt = "%0.0f%s";
		when 0 then fmt = "%0.0f%s";
		when 1 then fmt = "%0.3f%s";
		when 2 then fmt = "%0.2f%s";
		when 3 then fmt = "%0.1f%s";
		else fmt = "$0.0f%s";
	end
	out << sprintf(fmt, num, powers[power].chr);
	return out;
end

trap("ALRM") do
	Alarm.alarm(1);
	# Format and display stats all pretty-like
	isdata_b = $isdata * $blksz;
	issparse_b = $issparse * $blksz;
	sum = $isdata + $issparse;
	sum_b = sum * $blksz;
	pct = (sum.to_f / $blkct) * 100;
	perf = sum_b - (@old_sum_b? @old_sum_b: 0);

	$stdout.printf("\rD: %ss (%sB) S: %ss (%sB) +: %ss (%sB) T: %ss (%sB) %%: %0.3f%% %sB/sec    ",
		kSep($isdata), pMag(isdata_b), kSep($issparse), pMag(issparse_b),
		kSep(sum), pMag(sum_b), kSep($blkct), pMag($blkct_b), pct, pMag(perf));
	@old_sum_b = sum_b;
	$stdout.flush();
end

# Control vars
$isdata = 0;
$issparse = 0;
buf = "";
notnull = false;

# Start display loop
Alarm.alarm(1);

$stdout.printf("Precaching...");
$stdout.flush();

# Start processing loop
while (!src.eof?)
	buf = src.read($blksz);

# Old byte-scanning section
#	notnull = false;
#	for byte in 0...$blksz
#		if (buf[byte] != 0)
#			# Break out of the loop on non-null data
#			notnull = true;
#			break;
#		end
#	end
#	if (notnull)
#		$isdata+= 1;
#		out.write(buf);
#	else
#		$issparse+= 1;
#		out.seek($blksz, File::SEEK_CUR);
#	end

# New regex section
# !!!! Ruby's regular expressions are different! ^/$ match after/before newlines! Use \A/\Z to match start/end of string!
	if (/\A\0+\Z/ =~ buf)
		$issparse+= 1;
		out.seek($blksz, File::SEEK_CUR);
	else
		$isdata+= 1;
		out.write(buf);
	end
end

# Make everything nice and even
out.truncate($blkct * $blksz);

src.close();
out.close();

$stdout.printf("\n\n");

exit(0);
</code>