Fix product builds with packages/files containing special characters

Author: mlschroe

PBuild/Checker.pm

@@ -519,7 +519,7 @@ sub check_product {
         next if $seen_fn{$fn};
         next if $fn =~ /^::import::(.*?)::(.*)$/ && $seen_fn{$2};
         my $b = $bininfo->{$fn};
-        push @rpms, { %{$bininfo->{$fn}}, 'package' => $apackid };
+        push @rpms, { %{$bininfo->{$fn}}, 'package' => $apackid, 'fn' => $fn };
         $seen_fn{$fn} = 1; 
         push @next_unneeded_na, $na unless $ba eq 'src' || $ba eq 'nosrc';
       }

PBuild/OBS.pm

@@ -387,7 +387,7 @@ sub fetch_gbininfo {
   my $packagebinaryversionlist = PBuild::Structured::fromxml($data, $dtd_packagebinaryversionlist, 0, 1);
   my $gbininfo = {};
   for my $binaryversionlist (@{$packagebinaryversionlist->{'binaryversionlist'} || []}) {
-    my $location = "${baseurl}build/$prp/$arch/$binaryversionlist->{'package'}/";
+    my $location = "${baseurl}build/$prp/$arch/".PBuild::Util::urlencode("$binaryversionlist->{'package'}/");
     my %bins;
     for my $binary (@{$binaryversionlist->{'binary'} || []}) {
       my $filename = $binary->{'name'};
@@ -407,7 +407,11 @@ sub fetch_gbininfo {
       $bin->{'hdrmd5'} = $binary->{'hdrmd5'} if $binary->{'hdrmd5'};
       $bin->{'leadsigmd5'} = $binary->{'leadsigmd5'} if $binary->{'leadsigmd5'};
       $bin->{'md5sum'} = $binary->{'md5sum'} if $binary->{'md5sum'};
-      $bin->{'location'} = $location . $filename;
+      if ($filename =~ /([\000-\040<>;\"#\?&\+=%[\177-\377])/s) {
+        $bin->{'location'} = $location . PBuild::Util::urlencode($filename);
+      } else {
+        $bin->{'location'} = $location . $filename;
+      }
       $bins{$filename} = $bin;
     }
     $gbininfo->{$binaryversionlist->{'package'}} = \%bins;
@@ -435,46 +439,35 @@ sub fetch_gbininfo_cookie {
 }
 
 sub fetch_productbinaries_cpioextract {
-  my ($ent, $xfile, $repodir, $packid, $files) = @_;
+  my ($ent, $xfile, $repodir, $packid, $files, $callback) = @_;
   return undef unless $ent->{'cpiotype'} == 8;
   my $name = $ent->{'name'};
-  PBuild::Verify::verify_filename("$packid-$name");
-  my $tmpname = "$repodir/.$$.$packid-$name";
+  my $binname = "$packid-$name";
+  PBuild::Verify::verify_filename($binname);
+  my $tmpname = ".$$.$binname";
   if (!defined($xfile)) {
     return undef unless $files->{$name};
-    return $tmpname;	# ok, extract this one!
+    return "$repodir/$tmpname";		# ok, extract this one!
   }
   my $bin = $files->{$name};
-  die unless $bin;
-  if ($bin->{'md5sum'}) {
-    Build::Download::checkfiledigest($tmpname, "md5:$bin->{'md5sum'}");
-  } elsif ($bin->{'hdrmd5'} || $bin->{'leadsigmd5'}) {
-    my $leadsigmd5;
-    my $hdrmd5 = Build::Rpm::queryhdrmd5($tmpname, \$leadsigmd5);
-    die("downloaded binary does not match hdrmd5\n") if $bin->{'hdrmd5'} && $bin->{'hdrmd5'} ne ($hdrmd5 || '');
-    die("downloaded binary does not match leadsigmd5\n") if $bin->{'leadsigmd5'} && $bin->{'leadsigmd5'} ne ($leadsigmd5 || '');
-  }
-  rename($tmpname, "$repodir/_gbins/$packid-$name") || die("rename $tmpname $repodir/_gbins/$packid-$name: $!\n");
-  die unless ($bin->{'package'} || '') eq $packid;
-  $bin->{'filename'} = "$packid-$name";
-  return undef;	# continue extracting
+  die unless $bin && ($bin->{'package'} || '') eq $packid;
+  $callback->($repodir, $tmpname, $binname, $bin);
+  return undef;		# continue extracting
 }
 
 sub fetch_productbinaries {
-  my ($url, $repodir, $bins) = @_;
+  my ($url, $repodir, $bins, $callback) = @_;
   # group by package
   my %packages;
   my $location;
   for my $bin (@$bins) {
     my $l = $bin->{'location'};
     die("fetch_productbinaries: missing location\n") unless $l;
     die("fetch_productbinaries: bad location $l\n")  unless $l =~ /^(.+)\/([^\/]+)\/([^\/]+)$/;
-    die("fetch_productbinaries: package mismatch$l\n")  unless $2 eq $bin->{'package'};
     $location = $1 unless defined $location;
     die("fetch_productbinaries: location conflict\n") unless $1 eq $location;
-    $packages{$2}->{$3} = $bin;
+    $packages{$bin->{'package'}}->{$bin->{'fn'}} = $bin;
   }
-  PBuild::Util::mkdir_p("$repodir/_gbins");
   my $ua = create_ua();
   for my $packid (sort keys %packages) {
     my $files = $packages{$packid};
@@ -484,9 +477,10 @@ sub fetch_productbinaries {
     $requrl .= "&binary=".PBuild::Util::urlencode($_, 1) for sort keys %$files;
     my $tmpcpio = "$repodir/.$$.binaries.cpio";
     Build::Download::download($requrl, $tmpcpio, undef, 'ua' => $ua, 'retry' => 3);
-    PBuild::Cpio::cpio_extract($tmpcpio, sub {fetch_productbinaries_cpioextract($_[0], $_[1], $repodir, $packid, $files)});
+    PBuild::Cpio::cpio_extract($tmpcpio, sub {fetch_productbinaries_cpioextract($_[0], $_[1], $repodir, $packid, $files, $callback)});
     unlink($tmpcpio);
   }
+  return $ua;
 }
 
 1;

PBuild/RemoteRepo.pm

@@ -436,6 +436,8 @@ sub is_matching_binary {
   return 0 if $b1->{'version'} ne $b2->{'version'};
   return 0 if ($b1->{'epoch'} || 0) ne ($b2->{'epoch'} || 0);
   return 0 if (defined $b1->{'release'} ? $b1->{'release'} : '__undef__') ne (defined $b2->{'release'} ? $b2->{'release'} : '__undef__');
+  return 0 if $b1->{'hdrmd5'} && $b2->{'hdrmd5'} && $b1->{'hdrmd5'} ne $b2->{'hdrmd5'};
+  return 0 if $b1->{'leadsigmd5'} && $b2->{'leadsigmd5'} && $b1->{'leadsigmd5'} ne $b2->{'leadsigmd5'};
   return 1;
 }
 
@@ -531,15 +533,59 @@ sub fetchbinaries {
   PBuild::Util::store("$repodir/._metadata.$$", "$repodir/_metadata", $meta->{'metadata'});
 }
 
+#
+# Check if the downloaded binary matches and set the local filename
+#
+sub fetchproductbinaries_replace {
+  my ($repodir, $tmpname, $binname, $bin) = @_;
+  Build::Download::checkfiledigest("$repodir/$tmpname", $bin->{'checksum'}) if $bin->{'checksum'};
+  if ($bin->{'md5sum'}) {
+    Build::Download::checkfiledigest("$repodir/$tmpname", $bin->{'checksum'}) if "md5:$bin->{'md5sum'}";
+  } elsif ($bin->{'hdrmd5'}) {
+    my $leadsigmd5;
+    my $hdrmd5 = Build::queryhdrmd5("$repodir/$tmpname", \$leadsigmd5);
+    die("downloaded product binary $binname does not match repository metadata (hdrmd5)\n") unless ($hdrmd5 || '') eq $bin->{'hdrmd5'};
+    die("downloaded product binary $binname does not match repository metadata (leadsigmd5)\n") unless !$bin->{'leadsigmd5'} || ($leadsigmd5 || '') eq $bin->{'leadsigmd5'};
+  }
+  PBuild::Verify::verify_filename($binname);
+  rename("$repodir/$tmpname", "$repodir/_gbins/$binname") || die("rename $repodir/$tmpname $repodir/_gbins/$binname: $!\n");
+  $bin->{'filename'} = $binname;
+}
+
 #
 # Download missing gbininfo product binaries from a remote repository
 #
 sub fetchproductbinaries {
   my ($meta, $bins) = @_;
   my $repodir = $meta->{'repodir'};
   print "fetching ".PBuild::Util::plural(scalar(@$bins), 'product binary')." from $meta->{'url'}\n";
-  die("unsupported repo type $meta->{'repotype'}\n") unless $meta->{'repotype'} eq 'obs';
-  PBuild::OBS::fetch_productbinaries($meta->{'url'}, $repodir, $bins);
+  my $repodir = $meta->{'repodir'};
+  PBuild::Util::mkdir_p("$repodir/_gbins");
+  my $ua;
+  $ua = PBuild::OBS::fetch_productbinaries($meta->{'url'}, $repodir, $bins, \&fetchproductbinaries_replace);
+  for my $bin (@$bins) {
+    next if $bin->{'filename'};
+    my $location = $bin->{'location'};
+    die("missing location for binary $bin->{'name'}\n") unless $location;
+    die("bad location: $location\n") unless $location =~ /^(?:https?|zypp):\/\//;
+    my $packid = $bin->{'package'};
+    my $fn = $bin->{'fn'};
+    die unless $packid && $fn;
+    my $binname = "$packid-$fn";
+    PBuild::Verify::verify_filename($binname);
+    my $tmpname = ".$$.$binname";
+    $ua ||= Build::Download::create_ua();
+    download($location, "$repodir/$tmpname", undef, undef, $ua);
+    fetchproductbinaries_replace($repodir, $tmpname, $binname, $bin);
+  }
+  # copy filename into real gbininfo as we clone the bininfo in the Checker
+  my $gbininfo = $meta->{'metadata'}->{'gbininfo'};
+  for my $bin (@{$bins}) {
+    next unless $bin->{'filename'} && $bin->{'fn'} && $bin->{'package'};
+    my $bininfo = $gbininfo->{$bin->{'package'}};
+    die unless $bininfo && $bininfo->{$bin->{'fn'}};
+    $bininfo->{$bin->{'fn'}}->{'filename'} = $bin->{'filename'};
+  }
   # updata meta data
   PBuild::Util::store("$repodir/._gbininfo.$$", "$repodir/_gbininfo", $meta->{'metadata'});
 }