::Doc handle non-ASCII docs - #545

Author: mohawk2

lib/PDL/Doc.pm

@@ -457,6 +457,7 @@ use File::Basename;
 use File::Spec::Functions qw(file_name_is_absolute abs2rel rel2abs catdir catfile);
 use Cwd (); # to help Debian packaging
 use Config;
+use Encode;
 
 our $pager = $ENV{PERLDOC_PAGER} // $ENV{PAGER} // $Config{pager};
 
@@ -781,6 +782,7 @@ sub decodedb {
   while (read $fh, my $plen, 2) {
     my ($len) = unpack "v", $plen;
     read $fh, my($txt), $len;
+    $txt = Encode::decode('UTF-8', $txt);
     my ($sym, $module, @a) = split chr(0), $txt;
     push @a, "" if @a % 2; # Add null string at end if necessary -- solves bug with missing REF section.
     $hash{$sym}{$module} = { @a, Dbfile => $filename }; # keep the origin pdldoc.db path
@@ -825,7 +827,7 @@ sub encodedb {
         #store paths to *.pm files relative to pdldoc.db
         if file_name_is_absolute($fi) && -f $fi;
       delete $val->{Dbfile}; # no need to store Dbfile
-      my $txt = join(chr(0),$name,$module,map +($_=>$val->{$_}), sort keys %$val);
+      my $txt = Encode::encode('UTF-8', join chr(0),$name,$module,map +($_=>$val->{$_}), sort keys %$val);
       print $fh pack("v",length($txt)).$txt;
     }
   }

lib/PDL/Doc/Perldl.pm

@@ -245,8 +245,8 @@ sub finddoc  {
     }
 
     # print out the matches
-
     open my $out, "| pod2text | $PDL::Doc::pager";
+    binmode $out, ':encoding(UTF-8)';
 
     if($subfield) {
       if($subfield <= @match) {

t/doc.t

@@ -218,4 +218,42 @@ PDL::Doc::getfuncdocs('funcparen', $pod_fh, $func_fh);
 $func_text =~ s#\n+\z#\n#;
 is $func_text, $m_also_text;
 
+my $canary_text = <<EOF;
+=encoding utf8
+
+=head1 NAME
+
+PDL::Trans - Trans stuff
+
+=head1 FUNCTIONS
+
+=head2 ctrsqrt
+
+=for ref
+
+Uses a recurrence of Bj\x{f6}rck and Hammarling.
+EOF
+my $canary_got = PDL::Doc::scantext(Encode::encode('UTF-8', $canary_text), 'Trans.pm');
+is_deeply $canary_got, {
+  'PDL::Trans' => {
+    'PDL::Trans' => {
+      'File' => 'Trans.pm',
+      'Ref' => 'Module: Trans stuff'
+    }
+  },
+  'ctrsqrt' => {
+    'PDL::Trans' => {
+      'File' => 'Trans.pm',
+      'Module' => 'PDL::Trans',
+      'Ref' => "Uses a recurrence of Bj\x{f6}rck and Hammarling.",
+    }
+  }
+} or diag explain $canary_got;
+open $fh, '>', \$encoded_text;
+PDL::Doc::encodedb($canary_got, $fh, 'DIRNAME');
+open $fh, '<', \$encoded_text;
+$decode_hash = PDL::Doc::decodedb($fh, 'FILENAME');
+for my $val (values %$decode_hash) { delete $_->{Dbfile} for values %$val }
+is_deeply $decode_hash, $canary_got or diag explain $decode_hash;
+
 done_testing;