Namazu-users-en(old)
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
Improvements to mhonarc.pl filter for Namazu
- From: Earl Hood <earl@xxxxxxxxxxxx>
- Date: Thu, 01 Aug 2002 00:32:00 -0500
- X-ml-name: namazu-users-en
- X-mail-count: 00322
I spent sometime today to improve the mhonarc.pl filter and to
deal with variations in message header formatting and to extract
information from the MHonArc <!--X-...--> comments. With this
filter, I now get message-id searching for my archives. A
useful feature to allow for URLs to messages that are resistant
to MHonArc message numbering.
Since MHonArc allows one to customize a message header in in radical
ways, it is possible that header fields will not be extracted (except
the <!--X-...--> will always be extracted). However, I did test the
filter against the table formatting style used by default in MHArc
and MHonArc's built-in default formatting. The key dependency for
the filter to handle the HTML message header is that field names and
field values are logically separated by a colon, ':'.
Attached is the diff of the mhonarc.pl I grabbed from CVS. I have
also attached the full modified mhonarc.pl since the changes are
significant and for those who may want to try it out directly.
All that is needed is to copy mhonarc.pl into the namazu filter
directory.
I hope this modified version can make it into the standard Namazu
release.
--ewh
Index: mhonarc.pl
===================================================================
RCS file: /storage/cvsroot/namazu/filter/mhonarc.pl,v
retrieving revision 1.24
diff -u -r1.24 mhonarc.pl
--- mhonarc.pl 2001/09/15 00:45:30 1.24
+++ mhonarc.pl 2002/08/01 05:06:19
@@ -2,7 +2,9 @@
# -*- Perl -*-
# $Id: mhonarc.pl,v 1.24 2001/09/15 00:45:30 kenji Exp $
# Copyright (C) 1997-2000 Satoru Takabayashi ,
-# 1999 NOKUBI Takatsugu All rights reserved.
+# 1999 NOKUBI Takatsugu,
+# 2002 Earl Hood
+# All rights reserved.
# This is free software with ABSOLUTELY NO WARRANTY.
#
# This program is free software; you can redistribute it and/or modify
@@ -32,8 +34,10 @@
#
# This pattern specifies MHonArc's file names.
+# NOTE: MHonArc does allow one to customize the filename for message files.
+# So we make the regex a little flexible to handle common variations.
#
-my $MHONARC_MESSAGE_FILE = 'msg\d{5}\.html(?:\.gz)?';
+my $MHONARC_MESSAGE_FILE = '\bmsg\d+\.s?html(?:\.gz)?\Z';
sub mediatype() {
return ('text/html; x-type=mhonarc');
@@ -70,12 +74,12 @@
util::vprint("Processing MHonArc file ...\n");
- unless ($cfile =~ /($MHONARC_MESSAGE_FILE)$/o)
+ if (($cfile !~ /$MHONARC_MESSAGE_FILE/o) ||
+ ($$contref !~ /\A\s*<!-- MHonArc /))
{
- return "is MHonArc's index file! skipped."; # error
+ return "is not a MHonArc message file! skipped."; # error
}
-
mhonarc_filter($contref, $weighted_str, $fields);
html::html_filter($contref, $weighted_str, $fields, $headings);
@@ -92,32 +96,93 @@
return undef;
}
-# Assume a normal message files by MHonArc v2.1.0
+# Assume a normal message files by MHonArc v2.x
sub mhonarc_filter ($$) {
- my ($contref, $weighted_str) = @_;
-
- # It's useful to handle MHonArc message files.
- $$contref =~ s/<!--X-MsgBody-End-->.*//s;
- $$contref =~ s/<!--X-TopPNI-->.*<!--X-TopPNI-End-->//s;
- $$contref =~ s/<!--X-Subject-Header-Begin-->.*<!--X-Subject-Header-End-->//s;
-
- # Separate headers and a body message.
- $$contref =~ s/<!--X-Head-Body-Sep-Begin-->/\n/;
-
- # Handle a field consists of two or more lines.
- $$contref =~ s!^(<LI>)(.*?)(</LI>$)!$1 . lftospace($2) . $3!gemsi;
+ my ($contref, $weighted_str, $fields) = @_;
+ my $mha_fields = { };
- # For plugging spaces before headers
- $$contref =~ s/^<LI>//gim;
-
- # Make header's name not to be indexed words.
- $$contref =~ s!</?EM>!!gi;
- $$contref =~ s/^\s+//;
+ my $pos = index($$contref, '<!--X-Head-End-->');
+ if ($pos > 0) {
+ load_mhonarc_fields($mha_fields, $weighted_str,
+ substr($$contref, 0, $pos));
+ }
+
+ # Strip off front-matter
+ $pos = index($$contref, '<!--X-Head-of-Message-->');
+ substr($$contref, 0, $pos + length('<!--X-Head-of-Message-->')) = "";
+
+ # Strip off end-matter
+ $pos = index($$contref, '<!--X-Body-of-Message-End-->');
+ substr($$contref, $pos) = "";
+
+ # Extract message header for separate processing, will be added back
+ my $msg_header = "";
+ $pos = index($$contref, '<!--X-Head-of-Message-End-->');
+ if ($pos >= 0) {
+ $msg_header = substr($$contref, 0, $pos);
+ substr($$contref, 0, $pos) = "";
+ }
+
+ # Strip out stuff between header and body
+ $pos = index($$contref, '<!--X-Body-of-Message-->');
+ substr($$contref, 0, $pos + length('<!--X-Body-of-Message-->')) = "";
+
+ # Reformat header to make it nice for mailnews filter
+ if ($msg_header ne "") {
+ $msg_header =~ s/\A\s+//;
+ html::remove_html_elements(\$msg_header);
+ $msg_header =~ s/^\s*([\w\-_]+:)/$1/gm;
+ $msg_header =~ s/^([\w\-_]+:)(?:[^\n\S]*\n[^\n\S]*)+/$1 /gm;
+ }
+
+ # Format MHonArc X comment extracted headers as regular headers
+ my $mha_header = "";
+ my($fld_name, $fld_value);
+ while (($fld_name, $fld_value) = each %$mha_fields) {
+ $mha_header .= join('', $fld_name, ': ', $fld_value, "\n");
+ }
+
+ # Added header back to content string.
+ $$contref = $mha_header . $msg_header . "\n" . $$contref;
+
+ # Return extract MHonArc fields
+ #$mha_fields;
+}
+
+sub load_mhonarc_fields {
+ my $fields = shift;
+ my $weighted_str = shift;
+ my $mha_head = shift;
+
+ if ($mha_head =~ /<!--X-Subject: ([^-]+) -->/) {
+ my $subject = uncommentize($1);
+ 1 while ($subject =~ s/\A\s*(re|sv|fwd|fw)[\[\]\d]*[:>-]+\s*//i);
+ $subject =~ s/\A\s*\[[^\]]+\]\s*//;
+ $fields->{'subject'} = $subject;
+ }
+ if ($mha_head =~ /<!--X-From-R13: ([^-]+) -->/) {
+ $fields->{'from'} = mrot13(uncommentize($1));
+ } elsif ($mha_head =~ /<!--X-From: ([^-]+) -->/) {
+ $fields->{'from'} = uncommentize($1);
+ }
+ if ($mha_head =~ /<!--X-Message-Id: ([^-]+) -->/) {
+ $fields->{'message-id'} = '<' . uncommentize($1). '>';
+ }
+ if ($mha_head =~ /<!--X-Date: ([^-]+) -->/) {
+ $fields->{'date'} = uncommentize($1);
+ }
+}
+
+sub uncommentize {
+ my($txt) = $_[0];
+ $txt =~ s/&#(\d+);/pack("C",$1)/ge;
+ $txt;
+}
+
+sub mrot13 {
+ my $str = shift;
+ $str =~ tr/@A-Z[a-z/N-Z[@A-Mn-za-m/;
+ $str;
}
-sub lftospace ($) {
- my ($str) = @_;
- $str =~ s/[\r\n]/ /g;
- return $str;
-}
1;
#
# -*- Perl -*-
# $Id: mhonarc.pl,v 1.24 2001/09/15 00:45:30 kenji Exp $
# Copyright (C) 1997-2000 Satoru Takabayashi ,
# 1999 NOKUBI Takatsugu,
# 2002 Earl Hood
# All rights reserved.
# This is free software with ABSOLUTELY NO WARRANTY.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either versions 2, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
# 02111-1307, USA
#
# This file must be encoded in EUC-JP encoding
#
package mhonarc;
use strict;
require 'util.pl';
require 'gfilter.pl';
require 'html.pl';
require 'mailnews.pl';
#
# This pattern specifies MHonArc's file names.
# NOTE: MHonArc does allow one to customize the filename for message files.
# So we make the regex a little flexible to handle common variations.
#
my $MHONARC_MESSAGE_FILE = '\bmsg\d+\.s?html(?:\.gz)?\Z';
sub mediatype() {
return ('text/html; x-type=mhonarc');
}
sub status() {
return 'yes';
}
sub recursive() {
return 0;
}
sub pre_codeconv() {
return 1;
}
sub post_codeconv () {
return 0;
}
sub add_magic ($) {
my ($magic) = @_;
$magic->addMagicEntry('0 string \<!--\ MHonArc text/html; x-type=mhonarc');
return;
}
sub filter ($$$$$) {
my ($orig_cfile, $contref, $weighted_str, $headings, $fields)
= @_;
my $cfile = defined $orig_cfile ? $$orig_cfile : '';
util::vprint("Processing MHonArc file ...\n");
if (($cfile !~ /$MHONARC_MESSAGE_FILE/o) ||
($$contref !~ /\A\s*<!-- MHonArc /))
{
return "is not a MHonArc message file! skipped."; # error
}
mhonarc_filter($contref, $weighted_str, $fields);
html::html_filter($contref, $weighted_str, $fields, $headings);
$$contref =~ s/^\s+//;
mailnews::uuencode_filter($contref);
mailnews::mailnews_filter($contref, $weighted_str, $fields);
mailnews::mailnews_citation_filter($contref, $weighted_str);
gfilter::line_adjust_filter($contref);
gfilter::line_adjust_filter($weighted_str);
gfilter::white_space_adjust_filter($contref);
gfilter::show_filter_debug_info($contref, $weighted_str,
$fields, $headings);
return undef;
}
# Assume a normal message files by MHonArc v2.x
sub mhonarc_filter ($$) {
my ($contref, $weighted_str, $fields) = @_;
my $mha_fields = { };
my $pos = index($$contref, '<!--X-Head-End-->');
if ($pos > 0) {
load_mhonarc_fields($mha_fields, $weighted_str,
substr($$contref, 0, $pos));
}
# Strip off front-matter
$pos = index($$contref, '<!--X-Head-of-Message-->');
substr($$contref, 0, $pos + length('<!--X-Head-of-Message-->')) = "";
# Strip off end-matter
$pos = index($$contref, '<!--X-Body-of-Message-End-->');
substr($$contref, $pos) = "";
# Extract message header for separate processing, will be added back
my $msg_header = "";
$pos = index($$contref, '<!--X-Head-of-Message-End-->');
if ($pos >= 0) {
$msg_header = substr($$contref, 0, $pos);
substr($$contref, 0, $pos) = "";
}
# Strip out stuff between header and body
$pos = index($$contref, '<!--X-Body-of-Message-->');
substr($$contref, 0, $pos + length('<!--X-Body-of-Message-->')) = "";
# Reformat header to make it nice for mailnews filter
if ($msg_header ne "") {
$msg_header =~ s/\A\s+//;
html::remove_html_elements(\$msg_header);
$msg_header =~ s/^\s*([\w\-_]+:)/$1/gm;
$msg_header =~ s/^([\w\-_]+:)(?:[^\n\S]*\n[^\n\S]*)+/$1 /gm;
}
# Format MHonArc X comment extracted headers as regular headers
my $mha_header = "";
my($fld_name, $fld_value);
while (($fld_name, $fld_value) = each %$mha_fields) {
$mha_header .= join('', $fld_name, ': ', $fld_value, "\n");
}
# Added header back to content string.
$$contref = $mha_header . $msg_header . "\n" . $$contref;
# Return extract MHonArc fields
#$mha_fields;
}
sub load_mhonarc_fields {
my $fields = shift;
my $weighted_str = shift;
my $mha_head = shift;
if ($mha_head =~ /<!--X-Subject: ([^-]+) -->/) {
my $subject = uncommentize($1);
1 while ($subject =~ s/\A\s*(re|sv|fwd|fw)[\[\]\d]*[:>-]+\s*//i);
$subject =~ s/\A\s*\[[^\]]+\]\s*//;
$fields->{'subject'} = $subject;
}
if ($mha_head =~ /<!--X-From-R13: ([^-]+) -->/) {
$fields->{'from'} = mrot13(uncommentize($1));
} elsif ($mha_head =~ /<!--X-From: ([^-]+) -->/) {
$fields->{'from'} = uncommentize($1);
}
if ($mha_head =~ /<!--X-Message-Id: ([^-]+) -->/) {
$fields->{'message-id'} = '<' . uncommentize($1). '>';
}
if ($mha_head =~ /<!--X-Date: ([^-]+) -->/) {
$fields->{'date'} = uncommentize($1);
}
}
sub uncommentize {
my($txt) = $_[0];
$txt =~ s/&#(\d+);/pack("C",$1)/ge;
$txt;
}
sub mrot13 {
my $str = shift;
$str =~ tr/@A-Z[a-z/N-Z[@A-Mn-za-m/;
$str;
}
1;