Adding upstream version 0.72.upstream/0.72

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

5 files changed, 432 insertions, 0 deletions

diff --git a/testsuite/check b/testsuite/check
new file mode 100755
index 0000000..03afc3b
--- /dev/null
+++ b/testsuite/check
@@ -0,0 +1,216 @@
+#! /bin/sh
+module=man
+
+DIFF="diff -uBb"
+WDIFF="wdiff -3 -n"
+
+if [ $# = 1 ] && [ $1 = "-h" ] ; then
+    if [ "$module" = "man" ] ; then
+        echo "\
+Compare how are displayed an original man page and the po4a-normalized one.
+Usage: $0
+            Check all man pages found in /usr/share/man/man[1-8]
+       $0 <space separated list of man pages>
+            Check the specified man pages.
+       $0 -f <file>
+            Check the man pages specified in <file> (one page per line).
+Man pages can be gzipped.
+$0 erases all LISTE.* files in the current directory and creates
+the following files:
+LISTE.TOTAL List of all the checked man pages
+LISTE.OK    List of the man pages for which $DIFF does not see a difference.
+LISTE.OK2   $DIFF does not see a difference afetr converting hyphens
+            to minus signs, `` to \", and '' to \" (in both pages).
+LISTE.WOK1  $WDIFF doesn't see any difference after the same modifications
+LISTE.WOK2  This tries to detect changes in the hyphenation of words.
+LISTE.WOK3  This removes minus signs, and thus detects more changes in
+            hyphenation. It also mask font changes.
+LISTE.PBS   po4a preferred to stop processing the man page (non
+            supported,...)
+LISTE.WDIFF These are probably bugs in po4a or in the man page
+LISTE.IGN   man pages po4a refused to operate on (e.g. were generated by
+            Pod::Man)
+You may wish to use stats.sh to compare to runs of $0."
+    fi
+
+    exit 0
+fi
+
+rm -f LISTE.*
+touch LISTE.{TOTAL,OK,OK2,WDIFF,PBS,IGN,WOK1,WOK2,WOK3}
+
+if [ $# = 2 ] && [ $1 = "-f" ] ; then
+   cat $2 > LISTE.TOTAL
+elif [ $# != 0 ] ; then
+   echo $@ > LISTE.TOTAL
+else
+   if [ "$module" = "man" ] ; then
+      find /usr/share/man/man[1-9] /usr/X11R6/man/man[1-9] -type f | sort > LISTE.TOTAL
+   elif [ "$module" = "pod" ] ; then
+      locate pod|egrep '\.pod$' | sort > LISTE.TOTAL
+   fi
+fi
+
+tmp=/tmp/po4a-check-$module-$$
+mkdir -p $tmp
+if [ "$module" = "man" ] ; then
+   cmdtotxt="mantotxt"
+elif [ "$module" = "pod" ] ; then
+   cmdtotxt="podtotxt"
+fi
+
+
+mantotxt() {
+    `grog -Tutf8 $1` > $1.txt
+}
+
+podtotxt() { # $1 file to convert ; $2 name to pretend to have
+  if [ $1 != $2 ] ; then
+    mv $2 $2.old
+    mv $1 $2
+  fi
+  pod2man $2 > $2.man
+  man -Pcat -l $2.man > $2.txt
+  if [ $1 != $2 ] ; then
+    mv $2.txt $1.txt
+    mv $2 $1
+    mv $2.old $2
+  fi
+}
+
+
+for fich in `cat LISTE.TOTAL` ; do
+  if echo $fich | egrep '\.gz$' ; then
+    newfich=`basename $fich .gz`;
+    zcat $fich > $tmp/$newfich;
+  else
+    newfich=`basename $fich`
+    cat $fich > $tmp/$newfich;
+  fi
+
+  echo "####### $fich";
+  rm -f po4a-normalize.*
+  if PERLLIB=../lib ../po4a-normalize -f $module -M iso-8859-1 $tmp/$newfich \
+                    -o groff_code=translate \
+                    -o noarg=ny0,Sp,Pp,BS,BE,VE,ES,zZ,zY \
+                    -o translate_each=TA,IN,hN,AS,OP,Tp,Ip,TQ \
+                    -o no_wrap=Ds:De,Ex:Ee,CS:CE,Vb:Ve \
+                    -o untranslated=Id,VS,rm \
+                    -o translate_joined=FN,Sh,iX \
+                    -o inline=ZN,Pn \
+                    -o generated 2>&1 ; then
+      mv po4a-normalize.output $tmp/po4a-normalize.output
+      $cmdtotxt $tmp/po4a-normalize.output $tmp/$newfich;
+
+      $cmdtotxt $tmp/$newfich $tmp/$newfich;
+      echo $fich
+      echo "$DIFF $tmp/$newfich.txt $tmp/po4a-normalize.output.txt"
+      if $DIFF $tmp/$newfich.txt $tmp/po4a-normalize.output.txt ; then
+        echo ">ok"
+        echo $fich >> LISTE.OK
+      else
+        awk 'BEGIN{RS=" ";ORS=" "}
+             {gsub("\xE2\x80\x90","-"); # this is safe
+              gsub("\xE2\x88\x92","-"); # the minus sign in eqn
+              gsub("\xE2\x80\x98\xE2\x80\x98","\xE2\x80\x9C"); # this can
+              # be troublesome (e.g. two single quotes => " insteas of ""
+              gsub("\xE2\x80\x99\xE2\x80\x99","\xE2\x80\x9D");
+              print}' $tmp/$newfich.txt > $tmp/"$newfich"_ignore
+        awk 'BEGIN{RS=" ";ORS=" "}
+             {gsub("\xE2\x80\x90","-");
+              gsub("\xE2\x88\x92","-");
+              gsub("\xE2\x80\x98\xE2\x80\x98","\xE2\x80\x9C");
+              gsub("\xE2\x80\x99\xE2\x80\x99","\xE2\x80\x9D");
+              print}' $tmp/po4a-normalize.output.txt > $tmp/po4a-normalize.ignore
+        echo $fich
+        echo "$DIFF $tmp/"$newfich"_ignore $tmp/po4a-normalize.ignore"
+        if $DIFF $tmp/"$newfich"_ignore $tmp/po4a-normalize.ignore ; then
+          echo ">ok2"
+          echo $fich >> LISTE.OK2
+        else
+          echo $fich
+          echo "$WDIFF $tmp/"$newfich"_ignore $tmp/po4a-normalize.ignore"
+          if $WDIFF $tmp/"$newfich"_ignore $tmp/po4a-normalize.ignore; then
+            echo ">wok1"
+            echo $fich >> LISTE.WOK1
+          else
+            awk 'BEGIN{RS=";";ORS=";"}
+                 {gsub("-\n[[:blank:]]+",""); # remove hyphen at the end of lines
+                                    # this permits to detect some diff due
+                                    # to word rewrapping (fails if a word
+                                    # containing a hyphen was wrapped)
+                  print}' $tmp/"$newfich"_ignore > $tmp/"$newfich"_ignore2
+            awk 'BEGIN{RS=";";ORS=";"}
+                 {gsub("-\n[[:blank:]]+","");
+                  print}' $tmp/po4a-normalize.ignore > $tmp/po4a-normalize.ignore2
+            # the same, but for word wrapped on hyphen.
+            # This is useful only if there is either word wrapped on
+            # hyphen or word not wrapped on hyphen. But this may be
+            # usefull to classify the diff
+            awk 'BEGIN{RS=";";ORS=";"}
+                 {gsub("-\n[[:blank:]]+","-");
+                  print}' $tmp/"$newfich"_ignore > $tmp/"$newfich"_ignore2b
+            awk 'BEGIN{RS=";";ORS=";"}
+                 {gsub("-\n[[:blank:]]+","-");
+                  print}' $tmp/po4a-normalize.ignore > $tmp/po4a-normalize.ignore2b
+            echo $fich
+            echo "$WDIFF $tmp/"$newfich"_ignore2 $tmp/po4a-normalize.ignore2"
+            $WDIFF $tmp/"$newfich"_ignore2 $tmp/po4a-normalize.ignore2
+            ret1=$?
+            echo $fich
+            echo "$WDIFF $tmp/"$newfich"_ignore2b $tmp/po4a-normalize.ignore2b"
+            $WDIFF $tmp/"$newfich"_ignore2b $tmp/po4a-normalize.ignore2b
+            ret2=$?
+            if [ $ret1 = 0 ] || [ $ret2 = 0 ]; then
+              echo ">wok2"
+              echo $fich >> LISTE.WOK2
+            else
+              awk '{gsub(".\x08",""); # this hides font diff ?
+                    print}' $tmp/"$newfich"_ignore > $tmp/"$newfich"_ignore3_tmp
+              awk 'BEGIN{RS=";";ORS=";"}
+                   {gsub("-\n[[:blank:]]+","");
+                    gsub("-","");
+                    print}' $tmp/"$newfich"_ignore3_tmp > $tmp/"$newfich"_ignore3
+
+              awk '{gsub(".\x08",""); # this hides font diff ?
+                    print}' $tmp/po4a-normalize.ignore > $tmp/po4a-normalize.ignore3_tmp
+              awk 'BEGIN{RS=";";ORS=";"}
+                   {gsub("-\n[[:blank:]]+","");
+                    gsub("-","");
+                    print}' $tmp/po4a-normalize.ignore3_tmp > $tmp/po4a-normalize.ignore3
+
+              echo $fich
+              echo "$WDIFF $tmp/"$newfich"_ignore3 $tmp/po4a-normalize.ignore3"
+              if $WDIFF $tmp/"$newfich"_ignore3 $tmp/po4a-normalize.ignore3
+              then
+                echo ">wok3"
+                echo $fich >> LISTE.WOK3
+              else
+                echo ">wdiff"
+                echo $fich >> LISTE.WDIFF
+              fi
+            fi
+          fi
+        fi
+      fi
+  else
+    if [ $? = 254 ]; then
+      echo ">ign"
+      echo $fich >> LISTE.IGN
+    else
+      l=$(wc -l $tmp/$newfich|cut -d" " -f1)
+      egrep "^.so " $tmp/$newfich 2>&1 > /dev/null
+      if [ "$?" = "0" ] && [ "$l" = "1" ];
+      then
+        echo ">ign"
+        echo $fich >> LISTE.IGN
+      else
+        echo ">pbs"
+        echo $fich >> LISTE.PBS
+      fi
+    fi
+  fi
+  rm -f $tmp/$newfich* $tmp/po4a-normalize.*
+  echo '-------------------'
+done
+rm -r $tmp
diff --git a/testsuite/stats.man b/testsuite/stats.man
new file mode 100755
index 0000000..4305452
--- /dev/null
+++ b/testsuite/stats.man
@@ -0,0 +1,120 @@
+#! /usr/bin/perl -w
+
+# This script looks all installed man page, and report some stats about them
+#  (usefull only to guide reflexions to po-man implementation)
+
+use strict;
+
+use File::Find ();
+
+# for the convenience of &wanted calls, including -eval statements:
+#use vars qw/*name *dir *prune/;
+#*name   = *File::Find::name;
+#*dir    = *File::Find::dir;
+#*prune  = *File::Find::prune;
+
+my %page;
+$page{'pod'}{'desc'} = "generated by Pod::Man";
+$page{'help2man'}{'desc'} = "generated by help2man";
+$page{'docbook-to-man'}{'desc'} = "generated by docbook-to-man";
+$page{'docbook2man'}{'desc'} = "generated by docbook2man";
+$page{'latex2man'}{'desc'} = "generated by latex2man";
+$page{'db2man.xsl'}{'desc'} = "generated by db2man.xsl";
+$page{'mtex2man'}{'desc'} = "generated by mtex2man(1)";
+$page{'auto'}{'desc'} = "contains 'THIS FILE HAS BEEN AUTOMATICALLY GENERATED.  DO NOT EDIT.'";
+$page{'notedit'}{'desc'} = "contains 'DO NOT EDIT' or 'generated', but I dunno by who.";
+$page{'man'}{'desc'} = "seem to be a source file";
+
+my %used=();    # used macros
+my %defined=(); # self defined macros
+
+foreach my $type (keys %page) {
+    $page{$type}{'defcount'} = 0;
+}
+
+sub wanted {
+    my ($dev,$ino,$mode,$nlink,$uid,$gid);
+
+    if ((($dev,$ino,$mode,$nlink,$uid,$gid) = lstat($_)) && -f $_) {
+	my $file = $File::Find::dir."/".$_;
+	return unless $file =~ m|/usr/share/man/man|; # pass translations
+	if ($file =~ /\.gz$/) {
+	    open FILE,"zcat $file|" || die "Can't open $file\n";
+	} else {
+	    open FILE,"<$file" || die "Can't open $file\n";
+	}
+
+	my $header="";
+	while ($_=<FILE>) {
+	    last unless /^.."/;#"
+#	    chomp;
+	    $header .= "    $_";
+	}
+	$header = "\n$header" if (length $header);
+	my $define="";
+	while (<FILE>) {
+	    if (m/^\.de (.*)$/) {
+		my $macro=$1;
+		$define .= $macro.",";
+		$defined{$macro} =(defined($defined{$macro}) ? $defined{$macro}+1 : 1);
+		next;
+	    } elsif (m/^\.de/) {
+		print "DEF: $_";
+	    } elsif (m/^\.([^ ]*)/) {
+		my $macro=$1;
+		chomp($macro);
+		$macro =~ s/\s*$//g;
+		$macro =~ s/^\s*//g;
+		$macro =~ s/.".*$/|"/;
+		next unless (length($macro));
+#		unless ($define =~ m/$macro/) { # ignore self defined macro
+		    $used{$macro} = (defined($used{$macro}) ? $used{$macro}+1 : 1);
+#		}
+	    }
+	}
+	close FILE;
+	my $type="man";
+	     if ($header =~ /Pod::Man/)                 { $type="pod";
+	} elsif ($header =~ /generated by help2man/)    { $type="help2man";
+	} elsif ($header =~ /with docbook-to-man/)      { $type="docbook-to-man";
+	} elsif ($header =~ /generated by docbook2man/) { $type="docbook2man";
+	} elsif ($header =~ /created with latex2man/)   { $type="latex2man";
+	} elsif ($header =~ /Generated by db2man.xsl/)  { $type="db2man.xsl";
+	} elsif ($header =~ /generated automatically by mtex2man/)  { $type="mtex2man";
+	} elsif ($header =~ /THIS FILE HAS BEEN AUTOMATICALLY GENERATED.  DO NOT EDIT./) {
+	    $type="auto";
+	} elsif ($header =~ /DO NOT EDIT/i || $header =~ /generated/i) {
+	    $type="notedit";
+	}
+	if (length($define)) {
+	    $page{$type}{'define'}{$file} = $define;
+	    $page{$type}{'defcount'}++;
+	}
+	push @{$page{$type}{'files'}},$file;
+	push @{$page{$type}{'headers'}},"$file\n$header";
+    }
+}
+# Traverse desired filesystems
+File::Find::find({wanted => \&wanted}, '/usr/share/man');
+
+my $total=0;
+foreach my $type (sort keys %page) {
+    $page{$type}{'num'} = scalar @{$page{$type}{'files'}};
+    $total += $page{$type}{'num'};
+}
+print "Seen $total pages.\n";
+foreach my $type (sort {$page{$a}{'num'} <=> $page{$b}{'num'}} keys %page) {
+    print "  ".$page{$type}{'num'}." ".$page{$type}{'desc'}.". ".
+	$page{$type}{'defcount'}." seem to define new macro.\n";
+}
+
+print "\nHere is a list of the (regular and self defined) macros used:\n";
+foreach my $macro (sort {$used{$a} <=> $used{$b}} keys %used) {
+    print " $macro (used ".$used{$macro}." times)\n";
+}
+
+#print "\nHere is the list of pages seeming to be source page, and defining new macros:\n";
+#foreach my $page(sort { $page{'man'}{'define'}{$a} cmp $page{'man'}{'define'}{$b}
+#		      } keys %{$page{'man'}{'define'}}) {
+#    print $page{'man'}{'define'}{$page}.". in $page\n";
+#}
diff --git a/testsuite/stats.sh b/testsuite/stats.sh
new file mode 100755
index 0000000..d330106
--- /dev/null
+++ b/testsuite/stats.sh
@@ -0,0 +1,66 @@
+#!/bin/sh
+
+# see the 'check' script for the meaning of the categories.
+# You can tune the order of the categories to have all improvement in the
+# upper right or lower left corner of the table.
+types="IGN OK OK2 WOK1 WOK2 WOK3 PBS WDIFF"
+
+if [ "$#" != "2" ] ; then
+    echo "\
+Compare two executions of the 'check' script, and display the results in a
+table.
+
+Usage: $0 <directory1> <directory2>
+Were both directories contain the results (LISTE.* files) of two 'check'
+executions.
+
+The categories are configurable at the beginning of $0.
+
+$0 creates, in the stats_work directory, files named cat1_cat2, which
+indicate which man pages changed from the cat1 category to the cat2 one.
+"
+    exit 0
+fi
+
+old=$1
+new=$2
+[ -d stats_work ] || mkdir stats_work
+
+# copy the LISTE.* files in stats_work
+for i in $types
+do
+        sort $old/LISTE.$i > stats_work/old_$i
+        sort $new/LISTE.$i > stats_work/new_$i
+done
+
+# create the cat1_cat2 files which contain the common lines of two
+# categories.
+for i in $types
+do
+        for j in $types
+        do
+                cat stats_work/old_$i stats_work/new_$j | sort | uniq -d > stats_work/"$i"_"$j"
+        done
+done
+
+# display the report
+echo "dir1:$old"
+echo "dir2:$new"
+echo
+echo -n "dir1\\dir2"
+for i in $types
+do
+        printf "% 6s" $i
+done
+echo
+for i in $types
+do
+        printf "% 9s" $i
+        for j in $types
+        do
+                printf "% 6d" $(wc -l stats_work/"$i"_"$j"|cut -d" " -f1)
+        done
+        echo
+done
+echo -n "total: " $(for i in $types; do cat stats_work/old_$i; done | wc -l)
+echo    " |"      $(for i in $types; do cat stats_work/new_$i; done | wc -l)
diff --git a/testsuite/test.man b/testsuite/test.man
new file mode 100644
index 0000000..dda31a9
--- /dev/null
+++ b/testsuite/test.man
@@ -0,0 +1,23 @@
+.TH test 1 TEST test
+.SH Section1
+blibli
+.B blabla
+bloblo
+.
+.B blabla
+.
+0
+.
+1
+.fi
+.SH Section2
+Lalala
+ {
+ }
+.
+bliblu
+.fi
+.SH Section3
+.IR .i. .r. .I. .R. to\ be\ continued
+.fi
+.fi
diff --git a/testsuite/test.man.addendum b/testsuite/test.man.addendum
new file mode 100644
index 0000000..0ed5ba1
--- /dev/null
+++ b/testsuite/test.man.addendum
@@ -0,0 +1,7 @@
+PO4A-HEADER:position=Section2;mode=after;endboundary=^\.fi$
+PO4A-HEADER:position=Section2;mode=before;endboundary=^\.fi$
+PO4A-HEADER:position=Section2;mode=after;beginboundary=^\.fi$
+PO4A-HEADER:position=Section2;mode=after;beginboundary=^\.fiDFSSDGDSFGFSDGFD$
+.SH Added Section
+Some content
+.fi