# transform3.pl; Version 1.05, Apr 23, 1999 # # Perl-Script by Kajetan Hinner. This program is Mailware, every user should write to Kajetan@Hinner.com # transform.pl uses logfiles created by irc_report.pl and creates a data file which may be used # for mrtg3 for further processing. # # transform3.pl was developed using Linux 2.1.x, ircII 4.4, Perl 5.004 and KDE 4. As editor for large files I recommend nedit. # # How this script works: # # each input line is taken, processed and written to the outfile (a round robin database). # Take care to get the date right, this is only testet for Germany DST. # This is for further data handling with mrtg3. When you read the RRD, there may be a difference # between the values you stored and those you retrieve. This is because mrtg3 # calculates an average when you supply more than two values for those hours. # # # Author: Kajetan Hinner, University of Rostock, Sociology Dep., Kajetan@Hinner.com # # Notice: My aim was to write a working program which is reliable and easy to maintain. It was not intended to write # state of the art perl code, which may be hard to read for others. Some remarks are in german. Sorry for that. # # # Changes: # # 24.10.98: "U" is now also a valid input # 19.11.98: Match string for first-line-date corrected # 23. 4.99: Updated for mrtg Version 3 (alias names for Datasources, etc.) ######### User define section # rrd_step; 3600: one hour; 300: 5 minutes (300 seconds) # my $rrd_step = 3600; # hourly values my $rrd_step = 300; # RRD accepts new values every hour ############################# ######################### We start with some black magic to print on failure. # Change 1..1 below to 1..last_test_to_print . # (It may become useful if the test is moved to ./t subdirectory.) #BEGIN { $| = 1; print "1..1\n"; } #END {print "not ok 1\n" unless $loaded;} use RRDs; #$loaded = 1; #print "ok 1\n"; my $debug=2; print ("Debug Mode On ($debug)\n") if ($debug); # 1: print serious bugs (default) # 2: print something (for the interested into the internal working of the program) # 3: print everything (for debugging) use strict; use Date::Manip; # irc_report output uses german format (dd/mm/yy) &Date_Init("Language=German", "DateFormat=non-US"); my $tz=qx{date +%Z}; # Current time Zone... needed for epoch # calculation when DST is active my $offset = 0; # uncomment if there are time problems... if ($tz =~ /DST/) {$offset = -3600}; # when Daylight Saving Time, subtract one more hour # this is needed because DateCalc's %o seems not to work. print ("Current Time Zone is $tz, therefore we have an offset of $offset for epoch time calcs!\n") if ($offset); my $self = $0; $self =~ s!^.*/!!; # Name of perl-script if (@ARGV < 2 || @ARGV > 3) { die "Usage: perl $self data_in_file new|append (rrd_out_file)\n"; } # my $filevar ="/usr/home/khh/Socius_I.log"; my $filevar = $ARGV[0]; my $rrd_action=$ARGV[1]; # if "new": create new rrd; if "append": append to existing rrc my $rrd_outfile = "$filevar.rrd"; # default rrd_outfile if (scalar @ARGV == 3) {$rrd_outfile = $ARGV[2]}; # if 2nd command line argument is given, # accept it as the name of the rrd_out_file # $filevar .= ".data" unless ($filevar =~ /.*\.data$/); open (INFILE, "<".$filevar) or die ("Could not open $filevar: $!\n"); #open (OUTFILE, "> $rrd_outfile") or die "Can't open $rrd_outfile : $!"; my $counter = 0; # counts all lines which channel-user-values, for mean. my $date; my $users; # saves var my $channels; my $services; my $servers; my $inbuff; # read line by line my $zeile="# dummy"; # line just read and worked on my $incounter =0; # counts from the last line my $date_norm; #normal date, used for Date::Manip my $epsecs; # seconds since 1.1.70; for writing to rrd my $secs_diff; # difference from full hour my $ep_time; # for calculating beginning of rrd my $hu_time; # human readable time; convert to this format for output my $datec; # used for Date calculations my $rrd_error; # catch error if needed... # $incounter = @inbuff; # print ("File length: $incounter\n") if ($debug > 1); # Define the Round-Robin-Database and Round-Robin-Archives my $rrd_starttime; # first time when data is accepted for the RRD # when it starts with a comment, try second line while ($zeile =~ /^#/) {$incounter++; $zeile = }; print ("Line read: $zeile, line # $incounter\n"); # data line looks something like that # 24/10/1998 1:46 23801 6916 U 27 if (($date, $users, $channels, $services, $servers) = ($zeile =~ /^(.+:\d\d)\s+(\d+|U)\s+(\d+|U)\s+(\d+|U)\s+(\d+|U)\s*$/ )) { print ("First entry: '$date', users: $users, channels: $channels ... seems to be O.K.\n") if ($debug > 1); # $date_norm=&ParseDate($date); # change to special format $ep_time = &UnixDate($date,"%s")+$offset; # epoch seconds $hu_time=localtime($ep_time); # doing all this transformations just to get sure the # date is ok. print ("Source Date: '$date', Epoch seconds: $ep_time; newly calculated: '$hu_time'\n") if ($debug > 1); $epsecs = $ep_time%3600; # calculating difference from full hour if (!$epsecs) {$epsecs = 3600}; # if first date is already :00 sharp, one hour off $ep_time -= $epsecs; # align for full hour... $hu_time=localtime($ep_time); print ("Aligned to full hour: $hu_time\n") if ($debug > 1); $rrd_starttime = $ep_time; # calculate seconds (epoch seconds) } # if first line is not valid, exit program else {die "Not a valid input file $filevar, first line ('$zeile') contains no date\n"} if ($rrd_action eq "new") #shall we create a new rrd or just accept the old one? { print "Creating Round-Robin-Database $rrd_outfile\n"; # Round-Robin-Database define... # first defines number of users # second channels # thirs services # fourth servers # Data-Source:Data-Source-Type:Heartbeat:Min:Max, U means Unknown # Round-Robin-Archive:Consolidation_Function:steps:rows # daily: the first records every value for one week (7*24), every hour # weekly: the second records every value for five weeks (5*7*24), every hour # weekly: the third records every value for five weeks (5*7*24), every three hours # yearly: the fourth records the average for each day for one year # RRDs::create $rrd_outfile, "-b", $rrd_starttime, "-s", $rrd_step , "DS:users:GAUGE:3600:5000:U", "DS:channels:GAUGE:3600:500:U", "DS:servers:GAUGE:3600:3:U", "DS:services:GAUGE:3600:3:U", "RRA:AVERAGE:0.5:1:10080", "RRA:AVERAGE:0.5:3:30240", "RRA:AVERAGE:0.5:12:209664", "RRA:MAX:0:12:209664" ; # "RRA:AVERAGE:24:52*7*24", #"RRA:AVERAGE:3:5*7*24", #"RRA:AVERAGE:24:52*7*24" if($rrd_error = RRDs::error){ print "ERROR: $rrd_error\n" if $rrd_error ; # print error and exit die; } } # end if rrd_creation print "Transferring data from $filevar to the RRD $rrd_outfile. This will take a while\n"; while ($zeile=) { $incounter++; next if ($zeile =~ /^#/); # skip comments # input line looks like: # Datum Users Channels Services Servers print ("Working on '$zeile'") if ($debug > 2); # extracting "13/2/98 19:59 15970 4522 36 " # note: empty fields are not allowed! if (($date, $users, $channels, $services, $servers) = ($zeile =~ /^(.+:\d\d)\s+(\d+|U)\s+(\d+|U)\s+(\d+|U)\s+(\d+|U)\s*$/ )) { print ("foundat line $incounter: '$date', users: $users, channels: $channels, services: $services, servers: $servers\n") if ($debug > 2); $epsecs = &UnixDate($date,"%s")+$offset; # epoch seconds RRDs::update $rrd_outfile, "$epsecs:$users:$channels:$services:$servers"; print ("Updating $rrd_outfile, $epsecs:$users:$channels:$services:$servers\n") if ($debug > 2); if($rrd_error = RRDs::error){ print "ERROR: $rrd_error\n" if $rrd_error ; print ("Updating $rrd_outfile, -t $epsecs, DATA: $users:$channels:$services:$servers\n"); print ("Date was $date with date_norm $date_norm and epsecs $epsecs\n"); # die; } } else { print ("$counter: '$zeile' is invalid\n"); }; } close (INFILE); print "RRD written without known problems. Last updated at $date (Epoch Seconds: $epsecs), Localtime: " . localtime($epsecs) . "\n"; print "Length if infile $filevar: $incounter Lines\n";