Re: Shipping the mini.iso files with the installer-images package?

[Wouter Verhelst <w@uter.be>]

So.

On Wed, Dec 13, 2023 at 10:45:36AM +0200, Wouter Verhelst wrote:
> That sounds like a much more reasonable way forward, although I'm not
> keen on extending di-netboot-assistant (it's massive already, and does
> very different things). At any rate, I'll have a look at implementing
> this. Thanks for the suggestion!

This turned out to be fairly simple in the end. First, create a apt.conf
file like so:

Acquire::IndexTargets::deb::SHA256SUMS {
	MetaKey "$(COMPONENT)/installer-$(ARCHITECTURE)/current/images/SHA256SUMS";
	ShortDescription "SHA256SUMS";
	Description "$(RELEASE)/$(COMPONENT) $(ARCHITECTURE) d-i SHA256SUMS (deb)";
};

Next, create a config file like this:

images:
  mini.iso:
    mirror_type: deb
    limit:
      Suite: stable
      Architecture: amd64
    basedir: main/installer-amd64/current/images
    relative_name: ./netboot/gtk/mini.iso
    target_filename: /var/lib/libvirt/images/bookworm-mini.iso
  netboot.iso:
    mirror_type: cd
    basedir: current/amd64/iso-cd
    filename_regex: debian-[0-9.]+-amd64-netinst.iso
    target_filename: /var/lib/libvirt/images/bookworm-netinst.iso

And then once you have that, the following perl script should do the
job.

(config file stored either as /etc/debian-isosync/config.yaml or
passed as the first argument to the script)

---
#!/usr/bin/perl -w

use v5.36;

use Crypt::Digest::SHA256 qw/sha256_file_hex/;
use YAML::XS qw/LoadFile Dump/;
use Dpkg::Control::HashCore;
use LWP::UserAgent;
use File::Temp qw/tempdir/;

my $ua = LWP::UserAgent->new(show_progress => 1);
$ua->env_proxy;

my $cfilename = shift;
if(!defined($cfilename)) {
	$cfilename = "/etc/debian-isosync/config.yaml";
}

my $cfile = LoadFile($cfilename) or die $!;

die "need list of images in config file" unless exists($cfile->{images});
die "images setting in config file is not a hash" unless ref($cfile->{images}) eq "HASH";

sub get_apt_sha256sum {
	my $rv = [];
	my $found;
	open my $indextargets, "-|", "apt-get", "indextargets";
	do {
		my $indexes = Dpkg::Control::HashCore->new;
		$found = $indexes->parse($indextargets, "index targets");
		push @$rv, $indexes if $found;
	} while $found;

	close $indextargets;

	return $rv;
}

sub ensure_file($url, $sha256_expected, $filename) {
	if(-f $filename) {
		my $sha256sum_found = sha256_file_hex($filename);
		if ($sha256sum_found eq $sha256_expected) {
			say "$filename is up-to-date, no update required";
			return;
		}
	} else {
		say "$filename not found, downloading";
	}
	$ua->get($url, ":content_file" => $filename);
}

NAME:
foreach my $name(keys %{$cfile->{images}}) {
	say "checking $name...";
	my $image = $cfile->{images}{$name};
	die "invalid entry $name in config file: missing mirror type" unless exists($image->{mirror_type});
	die "invalid entry $name in config file: missing directory with SHA256SUMS file" unless exists($image->{basedir});
	die "invalid entry $name in config file: missing target filename" unless exists($image->{target_filename});
	next if $image->{disabled};

	if($image->{mirror_type} eq "deb") {
		die "invalid entry $name in config file: missing relative filename" unless exists($image->{relative_name});
		my $indexes = get_apt_sha256sum;
		INDEX:
		foreach my $index(@$indexes) {
			next INDEX unless $index->{ShortDesc} eq "SHA256SUMS";
			if(exists($image->{limit})) {
				foreach my $limit(keys %{$image->{limit}}) {
					next INDEX unless $index->{$limit} eq $image->{limit}{$limit};
				}
			}
			open my $shafile, "<", $index->{Filename};
			my $sha256sum_expected;
			SHASUM:
			while(my $line = <$shafile>) {
				chomp $line;
				my $filename;
				($sha256sum_expected, $filename) = split /\s+/, $line;
				last SHASUM if $filename eq $image->{relative_name};
				$sha256sum_expected = undef;
			}
			next INDEX unless defined($sha256sum_expected);
			ensure_file(join('/', $index->{"Base-URI"}, $image->{basedir}, $image->{relative_name}), $sha256sum_expected, $image->{target_filename});
			last INDEX;
		}
	} elsif($image->{mirror_type} eq "cd") {
		die "invalid entry $name in config file: missing filename regex" unless exists($image->{filename_regex});
		next if $image->{disabled};

		my $dir = tempdir(CLEANUP => 1);
		$ua->get("https://cdimage.debian.org/debian-cd/"; . $image->{basedir} . "/SHA256SUMS", ":content_file" => "$dir/SHA256SUMS");
		$ua->get("https://cdimage.debian.org/debian-cd/"; . $image->{basedir} . "/SHA256SUMS.sign", ":content_file" => "$dir/SHA256SUMS.gpg");
		open my $gpgv, "-|", "gpgv", "--status-fd", "1", "--keyring", "/usr/share/keyrings/debian-role-keys.gpg", "$dir/SHA256SUMS.gpg", "$dir/SHA256SUMS";
		my $found_validsig = 0;
		while(my $line = <$gpgv>) {
			next unless $line =~ /^\[GNUPG:\] VALIDSIG/;
			$found_validsig++;
		}
		die "could not download for $name: no valid SHA256SUMS signature found" unless $found_validsig;
		close $gpgv;
		open my $shafile, "<", "$dir/SHA256SUMS";
		my $sha256sum_expected;
		my $filename;
		SHASUM:
		while(my $line = <$shafile>) {
			chomp $line;
			($sha256sum_expected, $filename) = split /\s+/, $line;
			last SHASUM if $filename =~ $image->{filename_regex};
			$sha256sum_expected = undef;
		}
		die "could not download for $name: file not found in SHA256SUMS" unless defined($sha256sum_expected);
		ensure_file("https://cdimage.debian.org/debian-cd/"; . $image->{basedir} . "/" . $filename, $sha256sum_expected, $image->{target_filename});
	} else {
		die "unknown mirror type " . $image->{mirror_type} . "\n";
	}
}
---

I'm not sure whether this is sufficiently useful to upload to the
archive... thoughts?

-- 
     w@uter.{be,co.za}
wouter@{grep.be,fosdem.org,debian.org}

I will have a Tin-Actinium-Potassium mixture, thanks.