#!/usr/bin/perl
# $Easynet: group/sw/ripe/ripeget/ripeget-ng,v 1.58 2009/02/19 20:43:14 chrisy Exp $
#
# Note: This script is a little cumbersome to maintain now! At some point need
# to redo whole sections to make them easier to read and understand.
#
# Take an AS-MACRO or AS number and use a Routing Registry to find out
# the registered origin prefixes ("route" objects) that recursing into
# the object(s) shows.
#
# Understands both RIPE-181 and RPSL formats, can connect directly or
# use an external program (such as whois or ripewhois. The internal whois
# code can try persistant connections and will detect RADB's IRRd and
# use its special features.
#
# We can output to STDOUT, to a txt file, HTML or Cisco ACL. Or all of
# the above simultaneously. We can sort the output and the HTML is
# configurable.
#
# Doesn't require many CPAN libs to work. Just uses Socket and Getopt::Long.
#
# Major history highlights:
#
# Robert Kiessling <robert@de.easynet.net> 1999-08-16
#   Original version
#
# Chris Luke <chris@easynet.net> 2001-03-01
#   Major overhaul
#   Based on code used in Easynet Group Plc.
#
# Chris Luke <chris@easynet.net> 2002-12-07
#   Another pretty big shakeup
#   Added hostlist funtionality, tidied up lots of other stuff.
#
# Chris Luke <chris@easynet.net> 2009-02-16
#   Re-jig the output code and add IPv6 support
#
# Resources:
# RIPE NCC:   http://www.ripe.net/
# RIPE DB:    http://www.ripe.net/ripencc/pub-services/db/
# RIPE RPSL:  http://www.ripe.net/ripencc/pub-services/db/rpsl/index.html
# MERIT/RADB: http://www.ra.net/
# RADB IRRd:  http://www.irrd.net/
# RPSL RFC:   ftp://ftp.ripe.net/rfc/rfc2622.txt
# Easynet whois copyright:
#             http://www.noc.easynet.net/network/public/whois-copyright.html
#

use strict;
use POSIX;
use Socket;
use Getopt::Long;

# Stats. Printed with verbosity at one or more.

my $asqueries = 0;
my $asmacroqueries = 0;
my $mntnerqueries = 0;
my $prefixes = 0;
my $started = 0;
my $whois_tree = {};

# Defaults.

my $datestamp = gmtime().' UTC';

my $verbose = 0;
my $output_stdout = 2;
my $output_html = 0;
my $output_html_head =
	"<table tablespace=4>\n".
	"<tr><th colspan=2>#QUERY#</th></tr>\n".
	"<tr>\n".
	"  <th align=\"left\" width=200>Prefix</th>\n".
	"  <th align=\"left\">Originating AS</th>\n".
	"</tr>\n";
my $output_html_line =
	"<tr>\n".
	"<td>\n".
	"  <a href=\"http://www.noc.easynet.net/network/public/whois.cgi?Whois=-r%20-T%20route%20#PREFIX#\">#PREFIX#</a>\n".
	"</td>\n".
	"<td>\n".
	"  <a href=\"http://www.noc.easynet.net/network/public/whois.cgi?Whois=-r%20-T%20aut-num%20#ASN#\">#ASN#</a>\n".
	"</td>\n".
	"</tr>\n";
my $output_html_post =
	"</table>\n<hr>Generated by ripeget-ng on #DATESTAMP#";
my $output_txt = 0;
my $output_txt_preamble = 1;
my $output_txt_delimiter = 0;
my $output_cisco_filter = 0;
my $extended_cisco = 0;
my $prefixlist_cisco = 0;
my $prefixset_cisco = 0;
my $nostatement_cisco = 0;
my $cisco_filter_num = 0;
my $output_sorted = 0;
my $nooutput_sorted = 0;
my $output_sorted_asn = 1;
my $output_tree = 0;
my $output_tree_stdout = 0;
my $output_tree_txt = 0;
my $output_tree_html = 0;
my $output_tree_html_encaps = 1;
my $output_tree_html_title = "as-set tree";
my $output_tree_html_head = "";
my $output_tree_html_post = "<hr>Generated by ripeget-ng on #DATESTAMP#\n";
my $output_tree_source = 1;

# Default whois servers
# This is a hash of arrays. The index of the hash
# is sorted alphanumerically when processed.
my $whois_hostlist = {
	0 => [
		"whois.noc.easynet.net", 
	],
	1 => [
#		"whoisbackup.noc.easynet.net:43000",
		"whois.ripe.net",
	],
};

my $fallback_retry = 3;
my $whois_retry = 3;
my $whois_host = "";
my $whois_port = "";
my $whois_host2 = "";
my $whois_port2 = "";

my $whois_cmd = "/usr/local/bin/ripewhois";
my $whois_use_cmd = 0;
my $whois_direct = 1;
my $whois_persist = 1;
my $whois_irrd = 0;
my $whois_irrd_probe = 1;
my $whois_query_both = 0;
my $whois_sources = 0;
my $whois_ripe_sources = "";
my $whois_irrd_sources = "";
my $max_prefixlength = 32;
my $max_prefixlength6 = 64;
my $fetch_route4 = 1;
my $fetch_route6 = 1;
my $rewrite_ipv6 = 0;

my $opt_newripe = '';  # set to '-B ' if needed - note trailing space

# End of defaults.

my $query = 0;
my $printhelp = 0;
my $html_help = 0;
my $example_help = 0;
my $hostlist_help = 0;

# The bulk of the command line parameters.
my %cmdoptions = (
	"host=s" =>				\$whois_host,
	"port=s" =>				\$whois_port,
	"host2=s" =>				\$whois_host2,
	"port2=s" =>				\$whois_port2,
	"fallback-retry=i" =>			\$fallback_retry,
	"whois-retry=i" =>			\$whois_retry,
	"whois-sources=s" =>			\$whois_sources,
	"whois-cmd=s" =>			\$whois_cmd,
	"whois-use-cmd|use-whois-cmd!" =>	\$whois_use_cmd,
	"whois-irrd-probe!" =>			\$whois_irrd_probe,
	"whois-irrd!" =>			\$whois_irrd,
	"whois-query-both!" =>			\$whois_query_both,
	"whois-persist!" =>			\$whois_persist,
	"output-stdout!" =>			\$output_stdout, 
	"output-html-head=s" =>			\$output_html_head,
	"output-html-line=s" =>			\$output_html_line,
	"output-html-post=s" =>			\$output_html_post,
	"output-html=s" =>			\$output_html,
	"output-txt-preamble:s" =>		\$output_txt_preamble,
	"output-txt-delimiter=s" =>		\$output_txt_delimiter,
	"output-txt=s" =>			\$output_txt,
	"output-cisco-filter-num=s" =>		\$cisco_filter_num,
	"output-cisco-filter=s" =>		\$output_cisco_filter,
	"output-cisco-extended|output-cisco-filter-extended" =>	\$extended_cisco,
	"output-cisco-prefixlist!" =>		\$prefixlist_cisco,
	"output-cisco-prefixset!" =>		\$prefixset_cisco,
	"output-cisco-nocmd!" =>		\$nostatement_cisco,
	"output-tree!" =>			\$output_tree_stdout,
	"output-tree-txt=s" =>			\$output_tree_txt,
	"output-tree-html=s" =>			\$output_tree_html,
	"output-tree-html-encaps!" =>		\$output_tree_html_encaps,
	"output-tree-html-title=s" =>		\$output_tree_html_title,
	"output-tree-html-head=s" =>		\$output_tree_html_head,
	"output-tree-html-post=s" =>		\$output_tree_html_post,
	"output-tree-source!" =>		\$output_tree_source,
	"output-sorted:s" =>			\$output_sorted,
	"nooutput-sorted" =>			\$nooutput_sorted,
	"max-prefixlength=i" =>			\$max_prefixlength,
	"max-prefixlength6=i" =>		\$max_prefixlength6,
	"fetchipv4!" =>				\$fetch_route4,
	"fetchipv6!" =>				\$fetch_route6,
	"rewriteipv6!" => 			\$rewrite_ipv6,
	"query=s" =>				\$query,
	"verbose|v+" =>				\$verbose,
	"help-html!" =>				\$html_help,
	"help-examples!" =>			\$example_help,
	"help-hostlist|help-hostlists!" =>	\$hostlist_help,
	"help|h|?" =>				\$printhelp,
);

Getopt::Long::Configure("bundling", "pass_through");
GetOptions(%cmdoptions);

# Validate/process options

if($verbose) {
	# Make I/O unbuffered
	select(STDERR); $| = 1;
	select(STDOUT); $| = 1;

	print "INFO: Verbose set to $verbose\n";
}

my $selected_hostlist = 0;
my $arg = 0;
my $doexit = 0;

# We revert to primitive processing of the hostlist command line options.
while ($#ARGV >= 0) {
	$a = shift @ARGV;
	if($a =~ /^--delete-list/) {
		if($a =~ /=(.*)$/) {
			$arg = $1;
		} else {
			$arg = shift @ARGV;
		}
		if($arg eq "*") {
			$whois_hostlist = ();
		} else {
			@{$whois_hostlist->{$arg}} = ();
		}
	} elsif($a =~ /^--select-list/) {
		if($a =~ /=(.*)$/) {
			$arg = $1;
		} else {
			$arg = shift @ARGV;
		}
		$selected_hostlist = $arg;
	} elsif($a =~ /^--add-host/) {
		if($a =~ /=(.*)$/) {
			$arg = $1;
		} else {
			$arg = shift @ARGV;
		}
		push(@{$whois_hostlist->{$selected_hostlist}}, $arg);
	} elsif($a =~ /^--print-hostlist([s]?)/) {
		if($1 eq "s") {
			foreach my $idx (keys(%$whois_hostlist)) {
				print "Hostlist \"$idx\":\n";
				foreach my $h (@{$whois_hostlist->{$idx}}) {
					print"    $h\n";
				}
				print "\n";
			}
		} else {
			print "Hostlist \"$selected_hostlist\":\n";
			foreach my $h (@{$whois_hostlist->{$selected_hostlist}}) {
				print"    $h\n";
			}
			print "\n";
		}
		$doexit = 1;

	} elsif($a =~ /^-/) {
		print STDERR "ERROR: Unknown option $a specified.\n".
			"Please run with --help for valid options.\n";
		$doexit = 1;
	} else {
		if(!$query) {
			$query = $a;
		} else {
			$query .= " $a";
		}
	}
}

if($doexit) {
	exit;
}

if($hostlist_help) {
	print STDERR <<EOT;
RIPEGET and HOSTLISTS
---------------------

ripeget has a reasonably flexible mechanism for working out which whois
servers to connect to, though not terribly sophisticated.

It uses a set of hostlists. There can be as many hostlists as you care
to define. A hostlist is used until too many connection failure occur
on any one host in that list, when it will move to the next list. The
attempt limit is set with --fallback-retry=<num>. It will always try
once, regardless of what this is set to.

Hostlists do not strictly have to be numerically defined, and they
are used in alphanumeric order. Using a numeric is recommended for
clarity.

Each host within a single hostlist is queried in sequence and ripeget
merges the results from each. This is useful if, for instance, the
data you need to collect is spread across a number of whois servers.

Note that warnings will be printed if any object you specify or is
referenced is not found on any given whois server.

If you do not specify a TCP port number, then the normal whois port
number of 43 is assumed.

For example, to set a number of hosts on two lists and then print out
the resulting configuration, use this:

	ripeget --delete-list=\\* \\
		--select-list=0 \\
			--add-host=whois.noc.easynet.net \\
			--add-host=whois.cw.net \\
		--select-list=1 \\
			--add-host=whois.ripe.net \\
			--add-host=whois.ra.net \\
		--print-hostlists
		

Note that warnings will be printed if any object you specify or is
referenced is not found on any given whois server. Further, any sources
you specify with --whois-sources will be applied to all whois hosts
queried, possibly returning an error if the source is invalid. This
will be addressed in a later version.

EOT
	exit;
}

if($html_help) {
	print STDERR <<EOT;
RIPEGET and HTML
----------------

ripeget can output to html. Moreover, you can control how it outputs it,
to a degree. The pertinent options are:
  --output-html=<file>      Output HTML to <file>. Without this no HTML
                            will be generated at all

  --output-html-head=<line>
    Header printed at start of output. Available replaceable parameters are:
      #QUERY#               The query from the command line
      #DATESTAMP#           The current time, in UTC

  --output-html-line=<line>
    Output for each prefix found. Available replaceable parameters are:
      #QUERY#               The query from the command line
      #PREFIX#              The prefix this line should display
      #ASN#                 The ASN this line should display

  --output-html-post=<line>
    Printed at end of output. Available replaceable parameters are:
      #PREFIXES#            Number of prefixes found
      #DATESTAMP#           The current time, in UTC

The default values are:

  --output-html-head=
$output_html_head
  --output-html-line=
$output_html_line
  --output-html-post=
$output_html_post

EOT
	exit;
}

if($example_help) {
	print STDERR <<EOT;
RIPEGET EXAMPLES
----------------

Ripeget can be used in a number of ways. At its simplest, the following
simply queries the default whois server (\"ripeget --help\" will display
all defaults) and prints the results to STDERR:

    chrisy\@vorbis[~]> ripeget AS-EASYNET
    207.162.192.0/19     AS11341
    212.212.128.0/21     AS11341
    193.131.248.0/22     AS5607
    ...
    chrisy\@vorbis[~]>

More involved, but still simple, you can query a macro from two whois
servers and output in HTML and plain text formats (We do this from cron
for an information page):

    chrisy\@vorbis[~]> /usr/local/bin/ripeget \\
        --output-html=/data/www/network/public/networks-incl.html \\
        --output-txt=/data/www/network/public/networks-incl.txt \\
        AS-EASYNET >/dev/null 2>&1

You can add options to sort the output, such as:

    ripeget --output-sorted=prefix AS-EASYNET

Checkout --help-html to see how you can alter the format of outputted
HTML and --help has various options for controlling the plain-text output.
For example, to return a comma-seperated-file format, try something like:

    ripeget --output-txt-preamble=Prefix,ASN --output-txt-delimiter=, \\
        -output-txt=as-easynet.csv AS-EASYNET

EOT
	exit;
}

sub dodont($) { return $_[0]?"do":"don't"; }

if($printhelp || !$query) {
	my $def_persist =    dodont($whois_persist);
	my $def_irrd =       dodont($whois_irrd);
	my $def_irrd_probe = dodont($whois_irrd_probe);
	my $def_stdout =     dodont($output_stdout);
	my $def_sort =       dodont($output_sorted);
	my $def_sortby =     $output_sorted?($output_sorted_asn?", asn":", prefix"):'';
	my $def_whois_cmd =  dodont($whois_use_cmd);
	my $def_tree_source =dodont($output_tree_source);
	my $def_fetchipv4 =  dodont($fetch_route4);
	my $def_fetchipv6 =  dodont($fetch_route6);
	my $def_rewriteipv6 = dodont($rewrite_ipv6);

	print STDERR <<EOT;
usage: ripeget [options] <AS-MACRO|ASN>

  --delete-list=<num>       Empty the specified (or "*" for all) host list
  --select-list=<num>       Select a particular list for further operations
  --add-host=<host[:port]>  Add a specified host to the current list
                            (see --help-hostlist[s] for information)
  --print-hostlist[s]       Display all hosts in the the current
                            hostlist [or all hostslists] and exit.
                            Can be used after other list functions to
                            verify their operation
  --fallback-retry=<count>  Number of retries per host before moving on
                            to the next [$fallback_retry]
  --whois-cmd=<cmd>         Set command to spawn for whois queries
                            [$whois_cmd]
  --use-whois-cmd           Use the external whois command as opposed to
                            internal mechanism. This is not generally
                            recommended since it won't talk IRRd whois
                            (not tested recently) [$def_whois_cmd]

  --[no]persist             Enable [disable] connection persistance [$def_persist]
  --[no]whois-irrd          Talk RADB's IRRd protocol [$def_irrd]
  --[no]whois-irrd-probe    Probe for RADB's IRRd [$def_irrd_probe]

  --whois-sources=<srcs>    A comma-delimited list of IRR sources to query.
                            eg: --whois-sources=ripe,radb
                            [use whois server default]

  --[no]output-sorted=<as|prefix>
                            Sort output by ASN and prefix or just by prefix.
                            This applies to stdout, html file, txt file
                            and cisco-file output types [$def_sort$def_sortby]
  --max-prefixlength=<num>  Don't output IPv4 prefixes with a mask length
                            higher than <num>. This applies to stdout,
                            html file, txt file and cisco-file output
                            types [$max_prefixlength]
  --max-prefixlength6=<num> As above, but for IPv6 [$max_prefixlength6]

  --output-stdout           Output to stdout [$def_stdout]

  --output-html=<file>      Output HTML to <file>
  --output-html-head        Alter HTML output. See --help-html
  --output-html-line        Alter HTML output. See --help-html
  --output-html-post        Alter HTML output. See --help-html

  --output-txt=<file>       Output text to <file>
  --[no]output-txt-preamble[=<string>]
                            [Don't] put a 4-line header on top of txt file
                            [do] Default is a column-justified header
  --output-txt-delimiter=<string>
                            Set the delimiter between fields in txt output.
                            Default is a column-justified line

  --output-cisco-filter=<file>
                            Output Cisco ACL to <file>
  --output-cisco-filter-num=<num|name>
                            Set Cisco ACL number/name
  --output-cisco-extended   Output as a Cisco Extended IP access-list
  --output-cisco-prefixlist Output as a Cisco prefix-list
  --output-cisco-prefixset  Output as a Cisco prefix-set (IOS/XR)
  --output-cisco-nocmd      Output "no" command to delete existing ACL

  --output-tree
  --output-tree-txt=<file>
  --output-tree-html=<file> These three options print a tree (to stdout, a
                            named text file or a named HTML file) showing the
                            hierarchy of the provided ASN or AS-set. These
                            will disable the printing of a prefix list to
                            stdout by default, but any explicitly specified
                            output will happen in conjunction with these
                            options. The HTML format is just the txt output
                            wrapped in HTML and <pre> tags, for now
  --[no]output-tree-source  Print the IRR source of the object to the right
                            of it when printing the tree [$def_tree_source]

  --[no]fetchipv4           Fetch IPv4 route objects [$def_fetchipv4]
  --[no]fetchipv6           Fetch IPv6 route objects [$def_fetchipv6]
  --[no]rewriteipv6         Rewrite IPv6 addresses (compress, lowercase) [$def_rewriteipv6]

  --verbose, -v             Increase verbosity [$verbose]
  --help, -h                This message
  --help-examples           Some example usage

  <AS-MACRO|ASN>            RR objects to find prefixes of

EOT
	exit;
}

$whois_direct = !$whois_use_cmd;

# Legacy command line support for host specification
if($whois_host ne "") {
	#print STDERR "WARNING: Use of --host, --port, --host2 and --port2 are deprecated.\n";
	#print STDERR "WARNING: Consider using host lists instead.\n";

	$whois_port = "43" if(!$whois_port || $whois_port eq "");
	if($whois_query_both) {
		if($whois_host2 ne "") {
			$whois_port2 = "43" if(!$whois_port2 || $whois_port2 eq "");
		} else {
			$whois_query_both = 0;
		}
	}
	$whois_hostlist = {
		0 => [
		],
	};
	push(@{$whois_hostlist->{0}}, "$whois_host;$whois_port");
	push(@{$whois_hostlist->{0}}, "$whois_host2;$whois_port2") if($whois_query_both);
}

if($whois_sources) {
	$whois_irrd_sources = $whois_sources;
	$whois_ripe_sources = "-s $whois_sources";
}

if($nooutput_sorted) {
	$output_sorted = 0;
}
if($output_sorted) {
	if($output_sorted =~ /^as[n]?$/i) {
		$output_sorted_asn = 1;
	} elsif($output_sorted =~ /^prefix$/i) {
		$output_sorted_asn = 0;
	} else {
		print STDERR "ERROR: Sort type must be \"prefix\" or \"asn\".\n";
		exit;
	}
	$output_sorted = 1;
}

my $opt_route = '';
my $opt_irrdroute = '';
if($fetch_route4 && $fetch_route6) {
	$opt_route = '-T route,route6 ';
	$opt_irrdroute = '-i origin ';
} elsif($fetch_route4) {
	$opt_route = '-T route ';
	$opt_irrdroute = '!g';
} elsif($fetch_route6) {
	$opt_route = '-T route6 ';
	#$opt_irrdroute = '!6';
	$opt_irrdroute = '-T route6 -i origin ';
} else {
	# Bitch!
	print STDERR "ERROR: Must select one or both or --fetchipv4 and --fetchipv6!\n";
	exit 1;
}

if(($extended_cisco + $prefixlist_cisco + $prefixset_cisco) > 1) {
	print STDERR "ERROR: Specify only one of --output-cisco-extended, --output-cisco-prefixlist and --output-cisco-prefixset.\n";
	exit 1;
}

if($fetch_route4 && $fetch_route6 && $output_cisco_filter) {
	print STDERR "ERROR: Cisco output must be either IPv4 or IPv6, not both!\n";
	exit 1;
}

if($fetch_route6 && $output_cisco_filter && !($prefixlist_cisco || $prefixset_cisco)) {
	print STDERR "ERROR: Processing IPv6 routes for Cisco output requires either --output-cisco-prefixlist or --output-cisco-prefixset.\n";
	exit 1;
}

# enforce sensible prefix limits
$max_prefixlength = 0 if($max_prefixlength < 0);
$max_prefixlength = 32 if($max_prefixlength > 32);
$max_prefixlength6 = 0 if($max_prefixlength < 0);
$max_prefixlength6 = 128 if($max_prefixlength > 128);

# Go do the work.
process_asmacros();

if($verbose) {
	print STDERR "INFO: RIPE queries for AS macros: $asmacroqueries\n";
	print STDERR "INFO: RIPE reverse queries for AS origins: $asqueries\n";
	print STDERR "INFO: RIPE mbrs-by-ref maintainer queries: $mntnerqueries\n";
	print STDERR "INFO: Prefixes returned: $prefixes\n";
}

sub END {
	if($started) {
		print "INFO: Finished, cleaning up...\n" if($verbose>2);
		unlink($output_html.".tmp") if($output_html);
		unlink($output_txt.".tmp") if($output_txt);
		unlink($output_cisco_filter.".tmp") if($output_cisco_filter);
	}
}

exit;

sub expand_ip6($) {
	my $addr = shift;

	if($addr =~ /^(.*)::(.*)$/) {
		my @a = map { sprintf("%04.4x", POSIX::strtoul($_, 16)) } split(/:/, $1);
		my @b = split(/:/, $2);
		foreach (1..(8-(scalar(@a)+scalar(@b)))) {
			push(@a, '0000');
		}
		foreach (@b) {
			push(@a, sprintf("%04.4x", POSIX::strtoul($_, 16)));
		}
		$addr = join(':', @a);
	} else {
		my @a;
		foreach (split(/:/, $addr)) {
			push(@a, sprintf("%04.4x", POSIX::strtoul($_, 16)));
		}
		$addr = join(':', @a);
	}

	return $addr;
}

sub compress_ip6($) {
	my $addr = shift;
	$addr =~ s/:0000(:0000)+/:/;
	$addr .= ':' if($addr =~ /:$/);
	return $addr;
}

sub expand_addr($) {
	my $addr = shift;
	return expand_ip6($addr) if($addr =~ /:/);
	return $addr;
}

sub cmp_addr($$) {
	my $a = expand_addr(shift);
	my $b = expand_addr(shift);
	my $a6 = ($a =~ /:/);
	my $b6 = ($b =~ /:/);
	return -1 if ($a6 && !$b6);
	return 1 if ($b6 && !$a6);
	if($a6) {
		my @a = split(/:/, $a, 8);
		my @b = split(/:/, $b, 8);
		while(scalar(@a)) {
			my $c = POSIX::strtoul($a[0], 16) <=> POSIX::strtoul($b[0], 16);
			return $c if($c != 0);
			shift(@a);
			shift(@b);
		}
	} else {
		my @a = split(/\./, $a, 4);
		my @b = split(/\./, $b, 4);
		while(scalar(@a)) {
			my $c = $a[0] <=> $b[0];
			return $c if($c != 0);
			shift(@a);
			shift(@b);
		}
	}
	return 0;
}

sub listsort_asn {
	my ($amac, $aas, $aprefix) = split(/;/, $a);
	my ($bmac, $bas, $bprefix) = split(/;/, $b);

	if($aas ne $bas) {
		$aas =~ s/^AS//;
		$bas =~ s/^AS//;
		$aas <=> $bas;
	} else {
		return cmp_addr($aprefix, $bprefix);
	}		
}

sub listsort_prefix {
	my ($amac, $aas, $aprefix) = split(/;/, $a);
	my ($bmac, $bas, $bprefix) = split(/;/, $b);

	return cmp_addr($aprefix, $bprefix);
}

sub format_cisco(@) {
	my $txt = '';

	my $done_init;
	if($fetch_route6) {
		# Do IPv4 first, if any
		$done_init = 0;
		foreach (@_) {
			my ($macro, $oas, $prefix) = split(/;/, $_);
			$prefix =~ /(.*)\/(\d+)$/;
			next if($1 =~ /:/);
			next if ($2 > $max_prefixlength);

			if ($prefixset_cisco == 1 && !$done_init) {
				$txt .= "prefix-set $cisco_filter_num\n";
			} elsif ($prefixlist_cisco == 1 && !$done_init) {
				$txt .= "no ip prefix-list $cisco_filter_num\n";
			} elsif ($extended_cisco == 1 && !$done_init) {
				$txt .= "no ip access-list extended $cisco_filter_num\n";
				$txt .= "ip access-list extended $cisco_filter_num\n";
			} elsif(!$done_init) {
				$txt .= "no access-list extended $cisco_filter_num\n";
			}
			$done_init = 1;

			my($ipnet, $ipmask, $bits);
			$ipnet = $prefix;
			$ipnet =~ s/\/.*$//;
			$bits = $prefix;
			$bits =~ s/^.*\///;
			$ipmask = join(".",unpack ("CCCC", pack ("N",2**32-2**(32-$bits))));
			if($prefixset_cisco) {
				$txt .= "  $prefix,\n";
			} elsif($prefixlist_cisco) {
				$txt .= "ip prefix-list $cisco_filter_num permit $prefix\n";
			} elsif ($extended_cisco == 1) {
				$txt .= "permit ip host $ipnet host $ipmask\n";
			} else {
				$txt .= "access-list $cisco_filter_num permit ip host $ipnet host $ipmask\n";
			}
		}
		if($done_init) {
			if ($prefixset_cisco == 1) {
				chop($txt);
				chop($txt); # eat last comma
				$txt .= "end-set\n";
			}
			$txt .= "\n";
		}
	}

	if($fetch_route6) {
		# Now do IPv6, if any
		$done_init = 0;
		foreach (@_) {
			my ($macro, $oas, $prefix) = split(/;/, $_);
			$prefix =~ /(.*)\/(\d+)$/;
			my ($addr, $mlen) = ($1, $2);
			next if($addr =~ /\./);
			next if ($addr =~ /:/ && $mlen > $max_prefixlength6);

			if($rewrite_ipv6 && $addr =~ /:/) {
				$addr = compress_ip6(expand_ip6($addr));
				$prefix = $addr.'/'.$mlen;
			}

			if ($prefixset_cisco == 1 && !$done_init) {
				$txt .= "prefix-set $cisco_filter_num\n";
			} elsif ($prefixlist_cisco == 1 && !$done_init) {
				$txt .= "no ipv6 prefix-list $cisco_filter_num\n";
			} elsif ($extended_cisco == 1 && !$done_init) {
				#$txt .= "no ipv6 access-list extended $cisco_filter_num\n";
				#$txt .= "ipv6 access-list extended $cisco_filter_num\n";
			} elsif(!$done_init) {
				#$txt .= "no access-list extended $cisco_filter_num\n";
			}
			$done_init = 1;

			if($prefixset_cisco) {
				$txt .= "  $prefix,\n";
			} elsif($prefixlist_cisco) {
				$txt .= "ipv6 prefix-list $cisco_filter_num permit $prefix\n";
			}
		}
		if($done_init) {
			if ($prefixset_cisco == 1) {
				chop($txt);
				chop($txt); # eat last comma
				$txt .= "\nend-set\n";
			}
			$txt .= "\n";
		}
	}

	return $txt;
}

sub tbl($$) {
	my $r = shift;
	my $h = shift;
	my $w = 0;
	foreach ( @$r ) {
		my @c = @$_;
		$w = length($c[0]) if(length($c[0]) > $w);
	}
	$w++;
	my $fmt = "%-${w}s %s\n";
	my $hdr = ''; foreach (1..$w+$h) { $hdr .= '-'; }
	my $ret = '';
	foreach ( @$r ) {
		my @c = @$_;
		$ret .= sprintf($fmt, @c);
		if($h) {
			$ret .= $hdr."\n";
			$h = 0;
		}
	}
	return $ret;
}

sub format_txt(@) {
	my $txt = '';

	my @t = ();

	if($output_txt_preamble == 1) {
		$txt .= "$query Transited Prefixes\n\n";
		if($output_txt_delimiter) {
			$txt .= "Prefix".$output_txt_delimiter."Originating ASN";
		} else {
			push(@t, [ "Prefix", "Originating ASN" ]);
		}
	} else {
		my $out = $output_txt_preamble;
		$out =~ s/#QUERY#/$query/g;
		$out =~ s/#DATESTAMP#/$datestamp/g;
		$txt .= $out."\n";
	}

	foreach (@_) {
		my ($macro, $oas, $prefix) = split(/;/, $_);
		$prefix =~ /^(.*)\/(\d+)$/;
		my ($addr, $mlen) = ($1, $2);
		next if ($addr =~ /\./ && $mlen > $max_prefixlength);
		next if ($addr =~ /:/ && $mlen > $max_prefixlength6);

		if($rewrite_ipv6 && $addr =~ /:/) {
			$addr = compress_ip6(expand_ip6($addr));
			$prefix = $addr.'/'.$mlen;
		}
		if($output_txt_delimiter) {
			$txt .= "${prefix}${output_txt_delimiter}${oas}\n" if($output_txt);
		} else {
			push(@t, [ $prefix, $oas ]);
		}
	}
	if(!$output_txt_delimiter) {
		$txt .= tbl(\@t, 20);
	}
	return $txt;
}

sub format_html(@) {
	my $txt = '';
	foreach (@_) {
		my ($macro, $oas, $prefix) = split(/;/, $_);
		$prefix =~ /(.*)\/(\d+)$/;
		my ($addr, $mlen) = ($1, $2);
		next if ($addr =~ /\./ && $mlen > $max_prefixlength);
		next if ($addr =~ /:/ && $mlen > $max_prefixlength6);

		if($rewrite_ipv6 && $addr =~ /:/) {
			$addr = compress_ip6(expand_ip6($addr));
			$prefix = $addr.'/'.$mlen;
		}
		my $hprefix = $prefix;
		$hprefix =~ s/\//%2F/g;
		my $out = $output_html_line;
		$out =~ s/#QUERY#/$query/g;
		$out =~ s/#PREFIX#/$prefix/g;
		$out =~ s/#ASN#/$oas/g;
		$out =~ s/#DATESTAMP#/$datestamp/g;
		$txt .= $out;
	}
	return $txt;
}

# Get the work done, then worry about outputting what we found.
sub process_asmacros {
	my $as;

	$query =~ s/^\s+//;
	$query =~ s/\s+$//;
	$query =~ tr/a-z/A-Z/;

	$started++;

	$output_tree = 1 if($output_tree_stdout || $output_tree_txt || $output_tree_html);
	if($output_stdout == 2) {
		# Munge. We do STDOUT by default, but if we specify another
		# output format we don't do STDOUT unless explicitly asked for.
		# If output_stdout == 1, then it was asked for. 2 is the default.
		if($output_html || $output_txt || $output_cisco_filter ||
			$output_tree || $output_tree_txt || $output_tree_html) {
			$output_stdout = 0;
		}
	}
	if($output_stdout && $output_cisco_filter eq '-') {
		# Override - if cisco filter going to stdout, no txt out
		$output_stdout = 0;
	}

	my @lines = process_whois($query, $whois_tree);
	$prefixes = scalar(@lines); # before filtering, alas

	if($output_tree_stdout) {
		if(!open(TREETXT, ">&STDOUT")) {
			print STDERR "Can't print tree to file $output_tree_txt: $!\n";
		} else {
			print_tree_txt($whois_tree, "");
			close TREETXT;
		}
	}
	if($output_tree_txt) {
		if(!open(TREETXT, ">$output_tree_txt")) {
			print STDERR "Can't print tree to file $output_tree_txt: $!\n";
		} else {
			print_tree_txt($whois_tree, "");
			close TREETXT;
		}
	}
	if($output_tree_html) {
		if(open(TREEHTML, ">$output_tree_html")) {
			print_tree_html($whois_tree, "");
			close TREEHTML;
		} else {
			print STDERR "Can't print HTML tree to file $output_tree_html: $!\n";
		}
	}

	if($output_sorted) {
		if($output_sorted_asn) {
			@lines = sort listsort_asn @lines;
		} else {
			@lines = sort listsort_prefix @lines;
		}
	}

	if($output_cisco_filter) {
		if(${output_cisco_filter} eq '-') {
			open (OUT, ">&STDOUT");
		} else {
			open (OUT, ">${output_cisco_filter}.tmp");
		}
		print OUT format_cisco(@lines);
		close(OUT);
		if(${output_cisco_filter} ne '-') {
			rename("${output_cisco_filter}.tmp", ${output_cisco_filter});
		}
	}

	if($output_html) {
		$output_html_head =~ s/(^|[^\\])\\n/$1\n/g;
		$output_html_line =~ s/(^|[^\\])\\n/$1\n/g;
		$output_html_post =~ s/(^|[^\\])\\n/$1\n/g;
		open (OUT, ">${output_html}.tmp");
		my $out = $output_html_head;
		$out =~ s/#QUERY#/$query/g;
		$out =~ s/#DATESTAMP#/$datestamp/g;
		print OUT $out;
		print OUT format_html(@lines);
		$out = $output_html_post;
		$out =~ s/#QUERY#/$query/g;
		$out =~ s/#DATESTAMP#/$datestamp/g;
		$out =~ s/#PREFIXES#/$prefixes/g;
		print OUT $out;
		close(OUT);
		rename("${output_html}.tmp", ${output_html});
	}

	if($output_txt) {
		open (TXTFILE, ">${output_txt}.tmp");
		print OUT format_txt(@lines);
		close(OUT);
		rename("${output_txt}.tmp", ${output_txt});
	}
	if($output_stdout) {
		print format_txt(@lines);
	}
}

my $whois_connected = 0;
my $whois_connecterr = 0;
my $whois_attempts = 0;

my $whois_connected_host = "";
my $whois_connected_port = "";

sub whois_connect {
	if(!$whois_direct) {
		print STDERR "ERROR: whois_connect() called when not in direct mode!\n";
		$whois_connecterr = 1;
		return 0;
	}
	if($whois_connected) {
		if($whois_connected_host eq $whois_host && $whois_connected_port eq $whois_port) {
			return 1;
		} else {
			close(WHOIS);
			$whois_connected = 0;
			$whois_connecterr = 0;
		}
	}
	if(!$whois_port || $whois_port eq "") {
		$whois_port = "43";
	}
	my $proto = getprotobyname('tcp');
	socket(WHOIS, PF_INET, SOCK_STREAM, $proto) || die ("Can't create a socket");;
	my $sin = sockaddr_in($whois_port, inet_aton($whois_host));
	if(!connect(WHOIS, $sin)) {
		print STDERR "WARNING: Can't connect to ".$whois_host.":".$whois_port."\n" if ($verbose>1);
		$whois_connected = 0;
		$whois_connecterr = 1;
		return 0;
	}
	print STDERR "INFO: Connected to ".$whois_host.":".$whois_port."\n" if ($verbose>1);
	$whois_connected = 1;
	$whois_connecterr = 0;
	$whois_connected_host = $whois_host;
	$whois_connected_port = $whois_port;
	return 1;
}

sub whois_close {
	my $override = shift;

	if(($whois_connected && ($override || !$whois_persist)) || !$whois_direct || $whois_connecterr) {
		close(WHOIS);
		$whois_connected = 0;
	}
}

sub whois_sendline {
	my $line = shift;
	if($verbose > 2) {
		my @bits = split(/[\n]/, $line);
		foreach my $bit (@bits) {
			$bit =~ s/[\r\n]+//g;
			print STDERR "> $bit\n";
		}
	}
	my $ret = send(WHOIS, $line, 0);
	if(!defined($ret)) {
		$whois_connecterr = 1;
		return 0;
	}
	return 1;
}

sub whois_readline {
	my $line = <WHOIS>;
	if($verbose > 2 && defined($line)) {
		my @bits = split(/[\n]/, $line);
		foreach my $bit (@bits) {
			$bit =~ s/[\r\n]+//g;
			print STDERR "< $bit\n";
		}
	}
	$_ = $line;
	return $line;
}

# For sorting the hostlist, if the two elements
# are numbers, do a numeric compare, else use
# a string compare - so number of different
# lengths get compared properly.
sub hostlist_sort {
	if($a =~ /^[0-9]+$/ && $b =~ /^[0-9]+$/) {
		return $a <=> $b;
	}
	if($a =~ /^AS[0-9]+$/ && $b =~ /^AS[0-9]+$/) {
		my $aa = substr($a, 2);
		my $bb = substr($b, 2);
		return ($aa) <=> ($bb);
	}
	return $a cmp $b;
}

# Go start the work.
# We manage the hostlists - adding the results together
# for hosts within one list, and moving to the next
# list if something fails.
sub process_whois {
	my $query = shift;
	my $tree_section = shift;

	my $count = 0;
	my @idxarr = sort hostlist_sort keys(%$whois_hostlist);
	my $idx = shift @idxarr;
	my @list = @{$whois_hostlist->{$idx}};
	my @lines = ();

	# We have a number of lists.
	# Each list contains a list of whois hosts.
	# Query the whois hosts on this second list, combining results
	# from each.
	# If any one fails, retry that host. If we retry too many times,
	# Move to the next list of whois hosts, resetting everything.

	while(scalar(@list)) {
		($whois_host, $whois_port) = split(/;/, $list[0]);
		print STDERR "INFO: hostlist=$idx host=$list[0], counter=$count\n" if($verbose);
		my @ret = process_one_whois_host($tree_section);
		if(!$whois_connecterr) {
			push(@lines, @ret);
			shift @list;
			$count = 0;
		} else {
			print STDERR "WARNING: Failed on host $list[0], retrying..\n" if($verbose);
			$count++;
		}

		if($count >= $fallback_retry) {
			print STDERR "WARNING: Too many retries on host $list[0], ".
				"moving to next list.\n" if($verbose);
			$idx = shift @idxarr;
			if(defined(@{$whois_hostlist->{$idx}})) {
				@list = @{$whois_hostlist->{$idx}};
				$count = 0;
				process_reset_all();
			} else {
				@list = ();
			}
			@lines = ();
		}
	}
	return @lines;
}

# Go connect to the whois server, work out what language it speaks.
# Then go do the work.
sub process_one_whois_host {
	my $tree_section = shift;

	my @lines = ();
	process_reset();

	if($whois_direct) {
		# Check for IRRd
		# We have to do this before the persistance irobe because IRRd
		whois_close(0);
		# doesn't respond to the -k option, though it understands
		# and does the same thing with it!
		if($whois_irrd_probe || $whois_irrd) {
			my $blank = 0;
			return 0 if(!whois_connect());
			print STDERR "INFO: Probing for IRRd\n", if($verbose > 1);
			return 0 if(!whois_sendline("!!\r\n!v\r\n"));

			while(my $line = whois_readline()) {
				# Irrd will respond with something like:
				# A44
				# IRRd -- version 2.0.0 Alpha [08/09/2000] 
				# C
				# If we see the C it's valid.
				#
				# Old RIPE DB replies with: 
				# % Rights restricted by copyr...
				# <blank>
				# % No Entries
				# % ...
				# <blank>
				# <blank>
				#
				# And new ripe db says more or less the same.
				if($line =~ /^$/) {
					$blank++;
					last if($blank>=2);
				} elsif ($line =~ /^C$/) {
					# It's IRRD
					$whois_irrd = 1;
					$whois_persist = 1; # we know irrd can persist
					print STDERR "INFO: Found an IRRd whois server\n" if($verbose>1);
					# If any irrd sources are set, we send that command now
					if($whois_irrd_sources ne "") {
						return 0 if(!whois_sendline("!s$whois_irrd_sources\r\n"));
						$line = whois_readline();
						if($line =~ /^F/) {
							# Didn't like those sources!
							print STDERR "INFO: IRRD didn't like sources list $whois_irrd_sources, exiting.\n";
							return ();
						}
					}
					last;
				} elsif ($line =~ /^%/) {
					$blank = 0;
				} elsif ($line =~ /^[A#]/) {
					# IRRd banter, ignore
				} else {
					# God knows. Die.
					print STDERR "WARNING: Whois server speaks a foreign language, skipping.\n";
					return ();
				}
			}
			whois_close(1) if(!$whois_irrd);
		}

		# Open a connection to the whois server.
		# This will be our persistant connection if it
		# lasts.
		if(!$whois_irrd) {
			return 0 if(!whois_connect());
		}

		# Now ask for/verify persistance.
		if($whois_persist) {
			my $blank = 0;
			print STDERR "INFO: Probing for persistance\n", if($verbose > 1);
			if(!$whois_irrd) {
				# We search for an ob we know exists to elicit some
				# output from the DB. RIPE's version of the RPSL DB
				# doesn't send 2 blank lines after "-k" - mine does!
				whois_sendline("${opt_newripe}-k -r$whois_ripe_sources AS-EASYNET\r\n"); # persistant mode
				while(my $line = whois_readline()) {
					# Slurp initial banter. If we see a message like
					# % Server is running at low priority for -M, -m, -i and -k queries
					# fallback to non-persistant mode.
					# Otherwise we should see "blank" "%greeting" "blank"
					# Sigh. RPSL format doesn't do the above. Phooey.
					if($line =~ /^$/) {
						$blank++;
						last if($blank>=2);
					} elsif ($line =~ /^%ERROR:102: unknown source/) {
						# Doesn't like specificed sources
						print STDERR "INFO: RIPEDB didn't like sources list $whois_irrd_sources, exiting.\n";
						return ();
					} elsif ($line =~ /^% Server.*low priority.*-k.*/ || /^%.*ERROR/) {
						# poo.
						$whois_persist = 0;
						print STDERR "INFO: Falling back to non-persist mode\n"
							if($verbose>1);
						last;
					} else {
						$blank = 0;
					}
				}
			}
			whois_close(0);
			print STDERR "INFO: Connection will persist\n", if($whois_persist && $verbose>1);
		}

	}

	# This call goes and does the real work.
	$tree_section->{$query} = {} if(!defined($tree_section->{$query}));
	my @lines = process_one($query, $query, "", $tree_section->{$query});

	whois_close(1);

	return @lines;
}


# process_one
#
# Add a new as-macro or aut-num.
#
# Arguments:
# master: The as-macro which is the root of the search.
# arg:    The new as-macro or aut-num to be added to the list
#         for "master".
# Result: List of Strings "master;originating AS;network/length"
#
# $alreadydone{as-macro}{new} is 1 if the as-macro or aut-num 
# "new" was already added as part of "as-macro".
my %alreadydone;
my %asmacro_to_aslist; # Cache for previously encountered AS-Macros
my %asmacro_to_mntlist;
my %as_to_networks;
my %network_seen = ();
my %networksource = ();

# Clears the state for another run from the same hostlist
sub process_reset {
	%alreadydone = ();
	%asmacro_to_aslist = ();
	%asmacro_to_mntlist = ();
	%as_to_networks = ();
}

# Clears the state for another run from a new hostlist
sub process_reset_all {
	process_reset();
	%network_seen = ();
	$whois_tree = {};
	%networksource = ();
}

sub process_one {
	my $master = shift;
	my $arg = shift;
	my $parent = shift;
	my $tree_section = shift;

	if (!defined $alreadydone{$master}{$arg}) {
		# Ignore any aleady-added arg.
		# We delete this if we are generating a tree *after* any potential
		# recursion to avoid loops, but still produce a complete tree.
		# The non-tree functionality is to reduce the number of steps
		# required to produce a complete *prefix* list. A tree can show
		# duplicate copies of a branch, but this is lost on a prefix list.

		$alreadydone{$master}{$arg} = 1;
		print STDERR "INFO: Processing \"$arg\" on $whois_host:$whois_port...\n" if($verbose>1);

		if ($arg =~ /^AS\d+$/) {
			# $arg is an aut-num
			my @ret = map {
				$network_seen{$_} = 1;
				"$master;$arg;$_"
			} grep {
				!defined($network_seen{$_})
			} as_to_networks($arg, 0, $tree_section);
			delete $alreadydone{$master}{$arg} if ($output_tree);
			return @ret;
		}
		elsif ($arg =~ /^AS-[A-Za-z0-9\-:]+$/) {
			# $arg is an as-macro
			my @ret = grep {
				$_ ne ""
			} map {
				if(!defined($tree_section->{$_})) { $tree_section->{$_} = {}; }
				process_one($master,$_,$arg, $tree_section->{$_})
			} asmacro_to_aslist($arg, 0, $tree_section);
			delete $alreadydone{$master}{$arg} if ($output_tree);
			return @ret;
		}
		else {
			print STDERR "WARNING: Illegally formed entry \"$arg\" while ".
				"gathering networks for \"$master\", parent $parent, from ".
				"$whois_host:$whois_port\n";
	 	}
	} else {
		print STDERR "INFO: Already processed \"$arg\" on $whois_host:$whois_port...\n" if($verbose>1);
	}
}  

# asmacro_to_aslist
#
# Arguments:
# asmacro:  The name of an as-macro RIPE object.
# retry:    Number of retries. In case we cannot reach RIPE, we try it
#           thrice.
#
# Output:   List of AS-es or AS-macros contained in the "asmacro"
# 
sub asmacro_to_aslist {
	my $asmacro = shift;
	my $retry = shift;
	my $tree_section = shift;

	return () if($whois_connecterr);
	if ($retry > $whois_retry) {
		print STDERR "ERROR: Too many retries in whois query, skipping.\n";
		$whois_connecterr = 1;
		return ();
	}
	if (! defined $asmacro_to_aslist{$asmacro}) {
		$asmacroqueries++;
		my $irrsource = "";
		print STDERR "INFO:  AS macro $asmacro\n" if($verbose>1);
		my $command;
		if($whois_direct) {
			return 0 if(!whois_connect());
			if(!$whois_irrd || $output_tree) {
				$command = "${opt_newripe}-r$whois_ripe_sources $asmacro";
			} else {
				$command = "!i$asmacro,1";
			}
			return 0 if !(whois_sendline("$command\r\n"));
		} else {
			$command = "$whois_cmd -r$whois_ripe_sources -h $whois_host -p $whois_port $asmacro";
			print STDERR "> $command\n" if($verbose > 2);
			open(WHOIS, "$command |");
		}
		my $found = 0;
		my $aslist = "";
		my $doing_members = 0;
		my $mntlist = "";
		my $doing_mnt = 0;
		my $blank = 0;
		my $irrd_found = 0;
		while(whois_readline()) {
			if($irrd_found == 2) {
				# Last line from irrd
				last;
			}
			elsif ($irrd_found == 1) {
				# This line from irrd
				$irrd_found = 2;
				s/^\s+//; s/\s+$//;
				$aslist = $_;
				$irrsource = "IRR";
			}
			elsif (/^$/) {
				$blank++;
				last if($blank>=2);
			}
			elsif (/^%\s+No entries found / || /^%ERROR:101: no entries found/) {
				print STDERR "WARNING: AS macro \"$asmacro\" not found on $whois_host:$whois_port\n";
				# There will be two blank lines, which will signal the end of output
			}
			elsif (/^% Timeout... Closing connection/) {
				print STDERR "WARNING: Timeout ... retrying.\n";
				return asmacro_to_aslist($asmacro, $retry++, $tree_section);
			}
			elsif (/^%/) {
				#ignore
				$blank = 0;
			}
			elsif (/^D$/ && $whois_irrd) {
				# Nothing found.
				last;
			}
			elsif (/^A[0-9]+/ && $whois_irrd && !$output_tree) {
				# Found! Next line is answer.
				$irrd_found = 1;
			}
			elsif (/^[-a-z]+:/) {
				$doing_members = 0;
				$doing_mnt = 0;
				$blank = 0;
				if (/^as-macro:\s*$asmacro$/ || /^as-set:\s*$asmacro$/) {
					$found = 1;
				}
				if (/^as-list:\s*(.*)/) {
					$aslist .= " $1";
				}
				if (/^members:\s*([^\s][^#]*)(\s#.*)?/) {
					my $l;
					$l = $1;
					$l =~ s/\s+$//;
					$l =~ s/, / /g;
					$l =~ s/,/ /g;
					$l =~ s/,$//g;
					$aslist .= " $l";
					$doing_members = 1;
				}
				if (/^mbrs-by-ref:\s*([^\s][^#]*)(\s#.*)?/) {
					# bah :)
					my $l;
					$l =~ s/\s+$//;
					$l = $1;
					$l =~ s/, / /g;
					$l =~ s/,$//g;
					$mntlist .= " $l";
					$doing_mnt = 1;
				}
				if (/^source:\s*(\S+)/) {
					$irrsource = $1;
					last if($whois_irrd);
				}
			} elsif (/^\s+(\S.*)/) {
				$blank = 0;
				if($doing_members) {
					my $l = $1;
					$l =~ s/[\s,]*\s/ /g;
					$l =~ s/,$//g;
					$aslist .= " $l";
				}
				if($doing_mnt) {
					my $l = $1;
					$l =~ s/[\s,]*\s/ /g;
					$l =~ s/,$//g;
					$mntlist .= " $l";
				}
			} elsif (/^\+(\S.*)/) {
				$blank = 0;
				if($doing_members) {
					my $l = $1;
					$l =~ s/[\s,]*\s/ /g;
					$l =~ s/,$//g;
					$aslist .= " $l";
				}
				if($doing_mnt) {
					my $l = $1;
					$l =~ s/[\s,]*\s/ /g;
					$l =~ s/,$//g;
					$mntlist .= " $l";
				}
			}
			else {
				die "While executing $command:\nUnparsable output from whois: \"$_\"";
			}
		}
		whois_close(0);
		my @asl = grep { /^AS/ } split (/ +/, $aslist);

		my @mntlist = grep { /^AS/ } split (/ +/, $mntlist);
		foreach my $mnt (@mntlist) {
			$tree_section->{$mnt} = {};
			push(@asl, mntner_to_aslist($mnt, 0, $tree_section->{$mnt}));
		}
		$networksource{$asmacro} = $irrsource if(!defined($networksource{$asmacro}));
		@{$asmacro_to_aslist{$asmacro}} = @asl;
	} else {
		print STDERR "INFO: Using cached answer for \"$asmacro\" from prevous query.\n" if($verbose>1);
	}
	foreach my $asn (@{$asmacro_to_aslist{$asmacro}}) {
		$tree_section->{$asn} = () if(!defined($tree_section->{$asn}));
	}
	return @{$asmacro_to_aslist{$asmacro}};
}

# as_to_networks
#
# Arguments:
# as:       The name of an aut-num RIPE object.
# retry:    Number of retries. In case we cannot reach RIPE, we try it
#           thrice.
#
# Output:   List of networks (network/netlength) which this AS originates.
# 
sub as_to_networks {
	my $as = shift;
	my $retry = shift;
	my $tree_section = shift;

	return () if($whois_connecterr);
	if ($retry > $whois_retry) {
		print STDERR "ERROR: Too many retries in whois query, skipping.\n";
		$whois_connecterr = 1;
		return ();
	}
	if (! defined $as_to_networks{$as}) {
		my @networks = ();
		my $irrsource = 0;
		$asqueries++;
		print STDERR "INFO:  AS $as\n" if($verbose>1);
		my $command;
		if($whois_direct) {
			return 0 if(!whois_connect());
			if(!$whois_irrd) {
				$command = "${opt_newripe}${opt_route}-r$whois_ripe_sources -i origin $as";
			} else {
				$command = "${opt_irrdroute}$as";
			}
			return 0 if !(whois_sendline("$command\r\n"));
		} else {
			$command = "$whois_cmd -r$whois_ripe_sources -h $whois_host -p $whois_port ${opt_route}-i origin $as";
			print STDERR "> $command\n" if($verbose > 2);
			open(WHOIS, "$command |");
		}
		my $networks;
		my $blank = 0;
		my $irrd_found = 0;
		while (whois_readline()) {
			if($irrd_found == 2) {
				# Last line from irrd
				last;
			}
			elsif($irrd_found == 1) {
				# This line from irrd
				s/^\s+//; s/\s+$//;
				my @net = split(/ /, $_);
				foreach my $i (@net) {
					push(@networks, $i);
					print STDERR "INFO:    $i\n" if($verbose>1);
				}
				$irrsource = "IRR";
				$irrd_found = 2;
			}
			elsif (/^$/) {
				$blank++;
				last if ($blank >= 2);
			}
			elsif (/^%\s+No entries found / || /^%ERROR:101: no entries found/) {
				# This is followed by some explanatory output.
				# we ignore this, the blank line record delmiter is what we want
			}
			elsif (/^% Timeout... Closing connection/) {
				print STDERR "WARNING: Timeout ... retrying.\n";
				return as_to_networks($as, $retry++, $tree_section);
			}
			elsif (/^%/) {
				# ignore
				$blank = 0;
			}
			elsif (/^A[0-9]+/ && $whois_irrd) {
				# IRRD output!
				$irrd_found = 1;
			}
			elsif (/^D$/ && $whois_irrd) {
				# Duff, not found in irrd...
				last;
			}
			elsif (/^[-a-zA-Z0-9]+:/) {
				$blank = 0;
				if (/^route:\s*(\S+)/) {
					push(@networks, $1);
					print STDERR "INFO: added    $1\n" if($verbose>1);
				} elsif (/^route6:\s*(\S+)/) {
					push(@networks, $1);
					print STDERR "INFO:  added   $1\n" if($verbose>1);
				} elsif (/^source:\s*(\S+)/) {
					$irrsource = $1;
				}
			}
			elsif (/^\s+(.*)/) {
				# ignore
				$blank = 0;
			}
			else {
				die "While executing $command:\nUnparsable output from whois: \"$_\"";
			}
		}
		whois_close(0);
		@{$as_to_networks{$as}} = @networks;
		foreach my $net (@networks) {
			$networksource{$net} = $irrsource;
		}
	} else {
		print STDERR "INFO: Using cached answer for networks of \"$as\" from prevous query.\n" if($verbose>1);
	}
	foreach my $net (@{$as_to_networks{$as}}) {
		$tree_section->{$net} = $networksource{$net} if(!defined($tree_section->{$net}));
	}
	return @{$as_to_networks{$as}};
}

sub mntner_to_aslist {
	my $mntner = shift;
	my $retry = shift;
	my $tree_section = shift;

	return () if($whois_irrd);
	return () if($whois_connecterr);
	if ($retry > $whois_retry) {
		print STDERR "ERROR: Too many retries in whois query, skipping.\n";
		$whois_connecterr = 1;
		return ();
	}
	if (! defined $asmacro_to_aslist{$mntner}) {
		$mntnerqueries++;
		print STDERR "INFO:  mntner $mntner\n" if $verbose>1;
		my $command;
		if($whois_direct) {
			return 0 if(!whois_connect());
			if(!$whois_irrd) {
				$command = "${opt_newripe}-r$whois_ripe_sources -T aut-num -i mnt-by $mntner";
			}
			return 0 if !(whois_sendline("$command\r\n"));
		} else {
			$command = "$whois_cmd -r$whois_ripe_sources -h $whois_host -p $whois_port -T aut-num -i mnt-by $mntner";
			print STDERR "> $command\n" if($verbose > 2);
			open(WHOIS, "$command |");
		}
		my $found = 0;
		my $blank = 0;
		my $aslist = "";
		while (whois_readline()) {
			if (/^$/) {
				$blank++;
				last if($blank>=2);
			}
			elsif (/^%\s+No entries found /) {
				print STDERR "WARNING: Nothing by mntner \"$mntner\" found on $whois_host:$whois_port\n";
				last;
			}
			elsif (/^% Timeout... Closing connection/) {
				print STDERR "WARNING: Timeout ... retrying.\n";
				return mntner_to_aslist($mntner, $retry++, $tree_section);
			}
			elsif (/^%/) {
				#ignore
				$blank = 0;
			}
			elsif (/^[-a-z]+:/) {
				$blank = 0;
				if (/^aut-num:\s*(\S.*)/) {
					$aslist .= " $1";
				}
			} elsif (/^\s+(\S.*)/) {
				$blank = 0;
			}
			else {
				die "While executing $command:\nUnparsable output from whois: \"$_\"";
			}
		}
		whois_close(0);
		@{$asmacro_to_aslist{$mntner}} = grep { /^AS/ } split (/ +/, $aslist);
	}
	foreach my $mnt (@{$asmacro_to_aslist{$mntner}}) {
		$tree_section->{$mnt} = {} if(!defined($tree_section->{$mnt}));
	}
	return @{$asmacro_to_aslist{$mntner}};
}

sub print_tree_txt {
	my $ptree = shift;
	my $indent = shift;
	my $sep = shift;
	my $html = shift;

	my @ks = sort hostlist_sort keys %$ptree;
	my $i = 1;
	foreach my $k (@ks) {
		if($k =~ /^[0-9]/) {
			# It's a prefix - we can do a shortcut, avoid further
			# recursion and, bascially, cheat. :)
			my $exhm = "";
			my $extx = "";
			my $exsr = "";
			if($output_tree_source && defined($ptree->{$k}) && $ptree->{$k} ne "0" && $ptree->{$k} ne "") {
				if($ptree->{$k} eq "EASYNET" || $ptree->{$k} eq "RIPE" || $ptree->{$k} eq "RADB") {
					$exhm = "-s%20".$ptree->{$k}."%20";
				} else {
					# select RA and remote, to get all the RA mirrored DB's
					$exsr = "source=ra&db=remote&";
				}
				$extx = "  (".$ptree->{$k}.")";
			}

			print TREETXT "$indent".($indent eq ""?"":"  +- "). 
				($html?"<a href=\"?${exsr}Whois=-r%20${exhm}-T%20route%20$k\">$k</a>":"$k").
				"${extx}\n";
		} else {
			# Not a prefix, therefore either an as-set or an aut-num.
			my $exhm = "";
			my $extx = "";
			my $exsr = "";
			if($output_tree_source && defined($networksource{$k}) && $networksource{$k} ne "") {
				if($networksource{$k} eq "EASYNET" ||
				   $networksource{$k} eq "RIPE" ||
				   $networksource{$k} eq "RADB") {
					$exhm = "-s%20".$networksource{$k}."%20";
				} else {
					# select RA and remote, to get all the RA mirrored DB's
					$exsr = "source=ra&db=remote&";
				}
				$extx = "  (".$networksource{$k}.")";
			}

			# Set the type. If we don't recognise it, let the whois server worry about it.
			my $exty = ($k =~ /^AS-/?"-T%20as-set%20":($k =~ /^AS[0-9]+$/?"-T%20aut-num%20":""));

			print TREETXT "$indent".($indent eq ""?"":"  +- ").
				($html?"<a href=\"?${exsr}Whois=-r%20${exhm}${exty}$k\">$k</a>":"$k").
				"${extx}\n";
			print_tree_txt($ptree->{$k}, $indent.($i == scalar(@ks)?"     ":"  |  "),
				($i == scalar(@ks)), $html);
		}
		print TREETXT "$indent\n" if($i == scalar(@ks) && $indent !~ /^[\s]+$/ && !$sep);
		$i++;
	}
}

sub print_tree_html {
	my $ptree = shift;
	my $indent = shift;
	my $out;

	my $datestamp = gmtime().' UTC';

	open(TREETXT, ">&TREEHTML");
	$out = ${output_tree_html_title};
	$out =~ s/#QUERY#/$query/g;
	$out =~ s/#DATESTAMP#/$datestamp/g;
	
	print TREETXT "<html><head><title>${out}</title></head><body>\n" if($output_tree_html_encaps);

	$out = ${output_tree_html_head};
	$out =~ s/#QUERY#/$query/g;
	$out =~ s/#DATESTAMP#/$datestamp/g;

	print TREETXT "${out}\n";
	print TREETXT "<base href=\"http://www.noc.easynet.net/network/public/whois.cgi\">\n<pre>\n";
	print_tree_txt($ptree, $indent, 0, 1);
	print TREETXT "</pre>\n";

	$out = ${output_tree_html_post};
	$out =~ s/#QUERY#/$query/g;
	$out =~ s/#DATESTAMP#/$datestamp/g;

	print TREETXT "${out}\n";
	print TREETXT "</body></html>\n" if($output_tree_html_encaps);
	close TREETXT;
}

sub old_pr_tr {
	my $ptree = shift;
	foreach my $k (sort keys %$ptree) {
		print TREEHTML "<table><tr>$k<td><td></td></tr><tr><td></td><td>\n";
		old_pr_tr($ptree->{$k});
		print TREEHTML "</td></tr></table>\n";
	}
}
