use strict;
#use warnings;
##config
#usage: cd /var/www/html/obskml/scripts; perl genPlacemarksObsKML.pl http://nautilus.baruch.sc.edu/obskml/feeds/secoora_obskml_latest.kmz null secoora
my $temp_dir = '/tmp/ms_tmp';
my $http_return = 'http://nautilus.baruch.sc.edu/ms_tmp/gearth_';
my $additional_links = 'Related links: ObsKML Xenia http://secoora.org
';
my $graph_link = 'http://nautilus.baruch.sc.edu/xenia_sqlite/get_graph.php?sensor_id=var1&output=webpage&time_interval=-1 day&time_zone_arg=EASTERN';
##
=comment
#usage notes
Watch the server path specific literals, everything gets unzipped and worked on in $target_dir so you'll want to set that to some temporary folder area that gets periodically flushed.
The other xml files this script uses is style.xml .
style.xml provides the default observation range limits to be used and can also be ignored in favor of a user supplied style.xml via $styel_url
Also there is a literal http address returned to the calling php page which will need to be changed accordingly.
#see line time filter below to control acceptable date range used in creating output file
#my $date_yesterday = `date +%Y%m%d%H%M --date='12 hours ago'`;
=cut
use LWP::Simple;
use XML::LibXML;
my ($zip_obskml_url, $style_url, $feed_name) = @ARGV;
if ($feed_name) { $feed_name = $feed_name.'_'; }
my $date_log = `date +%Y%m%d%H%M`;
chomp($date_log);
open (LOG_FILE,">$feed_name\obskml_style.log");
open (LOG_FILE_2,">./log/$feed_name\obskml_style_$date_log.log");
open (DEBUG,">./debug_genPlacemarks.txt");
my $count = @ARGV;
if ($count < 1) {
print "usage: zip_obskml_url style_url\n";
exit;
}
#using print `date` for script time benchmarking
#print `date`;
#create temp working directory
my $random_value = int(rand(10000000));
my $target_dir = "$temp_dir/gearth_$random_value";
`mkdir $target_dir`;
#print $target_dir."\n";
##################
#read input files to temp directory
##################
#zip_obskml_url
my $zip_filepath = "$target_dir/obskml.xml.zip";
#print $zip_filepath."\n";
my $content = getstore($zip_obskml_url, $zip_filepath);
die "Couldn't get $zip_obskml_url" unless defined $content;
`cd $target_dir; unzip obskml.xml.zip`;
#style_url
if (!($style_url) || ($style_url eq 'null')) { `cp /var/www/html/obskml/scripts/style.xml $target_dir`; }
else {
my $style_filepath = "$target_dir/style.xml";
my $content = getstore($style_url, $style_filepath);
die "Couldn't get $style_url" unless defined $content;
}
##################
#global hashes
#note that the below hashing convention assumes each element contains only one type of child element except at the terminals
my %HoH = ();
my $rHoH = \%HoH;
my %HoH_stats = ();
#below hash used later for sorting by ObsTypes
my %ObsTypes = ();
my $rObsTypes = \%ObsTypes;
##################
#convert kml files to hash
##################
my $filelist = `ls $target_dir/*.kml`;
#print $filelist;
my @files = split(/\n/,$filelist);
#print @files;
#exit 0;
my $date_now = `date +%Y%m%d%H%M --date='+12 hours'`;
chomp($date_now);
#the below seems to catch 2+4(EDT) = 6 hours
my $date_yesterday = `date +%Y%m%d%H%M --date='-12 hours'`;
chomp($date_yesterday);
foreach my $file (@files) {
#print "$file\n";
#my $xp = XML::LibXML->new->parse_file("$target_dir/obskml_latest.kml");
my $xp = XML::LibXML->new->parse_file($file);
#print `date`;
foreach my $placemark ($xp->findnodes('//Placemark')) {
my $placemark_id = $placemark->getAttribute('id');
if (!($placemark_id)) { $placemark_id = 'none.none.none'; }
#print "$placemark_id\n";
my $local_platform = $placemark_id;
my ($operator,$platform,$package) = split(/\./,$placemark_id);
my $datetime = $placemark->find('TimeStamp/when');
$datetime = sprintf("%s", $datetime);
my $coordinates = $placemark->find('Point/coordinates');
my ($longitude,$latitude) = split(/,/,$coordinates);
if ($platform eq 'none') { $local_platform = "point($longitude,$latitude)"; } #lon/lat stand in for placemark id if none given
my $operator_url = $placemark->find('Metadata/obsList/operatorURL');
$operator_url = sprintf("%s", $operator_url);
my $platform_url = $placemark->find('Metadata/obsList/platformURL');
$platform_url = sprintf("%s", $platform_url);
my $platform_desc = $placemark->find('Metadata/obsList/platformDescription');
$platform_desc = sprintf("%s", $platform_desc);
$HoH{ $operator }{ $local_platform }{'metadata'}{ 'operator_url' } = $operator_url;
$HoH{ $operator }{ $local_platform }{'metadata'}{ 'platform_url' } = $platform_url;
$HoH{ $operator }{ $local_platform }{'metadata'}{ 'platform_desc' } = $platform_desc;
$HoH{ $operator }{ $local_platform }{'metadata'}{ 'longitude' } = $longitude;
$HoH{ $operator }{ $local_platform }{'metadata'}{ 'latitude' } = $latitude;
foreach my $observation ($placemark->findnodes('Metadata/obsList/obs')) {
my $obs_property = sprintf("%s",$observation->find('obsType'));
if ($obs_property eq '') { next; } #ignore obs with missing metadata
my $uom = sprintf("%s",$observation->find('uomType'));
$obs_property .= '.'.$uom;
#have to cast measurement to float using sprintf to avoid comparison confusion later
my $measurement = sprintf("%.2f",$observation->find('value'));
#using the below hash trick to get a list of unique ObsTypes
$ObsTypes{ $obs_property } = 1;
#print "$operator:$local_platform:$obs_property:$datetime:$longitude:$latitude:$measurement\n";
my $date_test = substr($datetime,0,4).substr($datetime,5,2).substr($datetime,8,2).substr($datetime,11,2).substr($datetime,14,2);
#print "date_test: $date_test \n";
#don't share erroneous 'future' observations greater than today's date or obs older than yesterday
if (($date_test >= $date_yesterday) && ($date_test <= $date_now)) {
print DEBUG "date pass: $date_yesterday $date_now $date_test \n";
#for logging purposes
#only want to increment count for just latest measurements, not earlier
#tried $HoH_stats{$operator}{$placemark_id}{$obs_property} but hash complains/fails for more than one unknown level reference ??
if (!($HoH_stats{$operator}{$placemark_id.$obs_property})) {
$HoH_stats{$operator}{$placemark_id.$obs_property} = 1;
$HoH_stats{ $operator }{ $obs_property }{ obs_count }++;
}
#if ($operator eq 'nerrs') { print "$date_test $date_now $date_yesterday\n"; }
#$HoH{ $operator }{ $local_platform }{'metadata'}{ 'operator_url' } = $operator_url;
#$HoH{ $operator }{ $local_platform }{'metadata'}{ 'platform_url' } = $platform_url;
#$HoH{ $operator }{ $local_platform }{'metadata'}{ 'platform_desc' } = $platform_desc;
#could probably optimize lon/lat to allow platform movement without articulating for each $obs_property - leaving for now to satisfy foreach loops further below
$HoH{ $operator }{ $local_platform }{'data'}{ $datetime }{ $obs_property }{ 'longitude' } = $longitude;
$HoH{ $operator }{ $local_platform }{'data'}{ $datetime }{ $obs_property }{ 'latitude' } = $latitude;
$HoH{ $operator }{ $local_platform }{'metadata'}{ $obs_property }{ 'data_url' } = $observation->find('dataURL');
$HoH{ $operator }{ $local_platform }{'metadata'}{ $obs_property }{ 'sensor_id' } = $observation->find('sensorID');
#$HoH{ $operator }{ $local_platform }{'data'}{ $datetime }{ $obs_property }{ 'datetime' } = $datetime;
$HoH{ $operator }{ $local_platform }{'data'}{ $datetime }{ $obs_property }{ 'measurement' } = $measurement;
$HoH{ $operator }{ $local_platform }{'data'}{ $datetime }{ $obs_property }{ 'elev' } = $observation->find('elev');
}
else { print DEBUG "date fail: $date_yesterday $date_now $date_test \n"; }
} #foreach obs
#only want to increment count for just latest measurements, not earlier
if ($HoH_stats{$operator}{$placemark_id} != 1) {
$HoH_stats{$operator}{$placemark_id} = 1;
$HoH_stats{ $operator }{ platform_count }++;
}
} #foreach Placemark
} #foreach file
#################
#kml content
##################
my $kml_content = <<"END_OF_FILE";
Ocean/Coastal Observing Platform Data
0
ObsKML Here are links to the original source ObsKML data and the styling script used to generate this KML file. Please email jeremy.cothran\@gmail.com regarding questions or comments on this kml product or sharing/registering your observation data using these tools.]]>
END_OF_FILE
#generating the kml content below if not directly output to $kml_content is sent to 'buffer' variables so that the buffer can be used
# or cleared depending on the available data
##################
#list by operator
##################
#print "by operator: ";
#print `date`;
$kml_content .= "List by operator0";
foreach my $operator ( sort keys %{$rHoH} ) {
#print "operator:$operator\n";
print LOG_FILE "operator $operator $HoH_stats{$operator}{platform_count}\n";
print LOG_FILE_2 "operator $operator $HoH_stats{$operator}{platform_count}\n";
foreach my $obs_type ( sort keys %{$rObsTypes} ) {
my $obs_count = $HoH_stats{$operator}{$obs_type}{obs_count};
if ($obs_count > 0) {
print LOG_FILE "obs_type $obs_type $HoH_stats{$operator}{$obs_type}{obs_count}\n";
print LOG_FILE_2 "obs_type $obs_type $HoH_stats{$operator}{$obs_type}{obs_count}\n";
}
}
print LOG_FILE "\n\n";
print LOG_FILE_2 "\n\n";
$kml_content .= "$operator0";
foreach my $local_platform ( sort keys %{$rHoH->{$operator}} ) {
#print "localPlatformName:$local_platform\n";
$kml_content .= "$local_platform0";
my $platform_not_active = 1;
my $desc_header = '';
my $desc_table = '';
my $description = '';
my $operator_url = '';
my $platform_url = '';
my $platform_desc = '';
my $longitude = '';
my $latitude = '';
$platform_desc = $HoH{$operator}{$local_platform}{'metadata'}{'platform_desc'};
if ($platform_desc) { $desc_header .= 'Description: '.$platform_desc.'
'; }
$operator_url = $HoH{$operator}{$local_platform}{'metadata'}{'operator_url'};
if ($operator_url) { $desc_header .= 'OperatorURL
'; }
$platform_url = $HoH{$operator}{$local_platform}{'metadata'}{'platform_url'};
if ($platform_url) { $desc_header .= 'PlatformURL
'; }
$desc_header .= "$additional_links
";
foreach my $datetime ( sort keys %{$rHoH->{$operator}{$local_platform}{'data'}} ) {
#print "datetime:$datetime\n";
$platform_not_active = 0;
my $measurement = '';
my $data_url = '';
my $sensor_id = '';
my $elev = '';
my $datetime_label = '';
foreach my $obs_property ( sort keys %{$rHoH->{$operator}{$local_platform}{'data'}{ $datetime }} ) {
#print "obs_property:$obs_property\n";
#only need to do this initially once per platform
if (!($datetime_label)) {
$datetime_label = $datetime;
$datetime_label =~ s/T/ /g;
#$datetime_label = substr($datetime_label,0,16).' GMT';
#print "$datetime_label\n";
$data_url = $HoH{$operator}{$local_platform}{'metadata'}{$obs_property}{'data_url'};
$sensor_id = $HoH{$operator}{$local_platform}{'metadata'}{$obs_property}{'sensor_id'};
if (($data_url) || ($sensor_id)) {
$desc_header .= "Click on the latest observation reading to view graphs of previous observations.
";
}
$desc_table .= '';
}
$longitude = $HoH{$operator}{$local_platform}{'data'}{ $datetime }{$obs_property}{'longitude'};
$latitude = $HoH{$operator}{$local_platform}{'data'}{ $datetime }{$obs_property}{'latitude'};
$measurement = $HoH{$operator}{$local_platform}{'data'}{ $datetime }{$obs_property}{'measurement'};
#print "$measurement\n";
$elev = $HoH{$operator}{$local_platform}{'data'}{ $datetime }{$obs_property}{'elev'};
$data_url = $HoH{$operator}{$local_platform}{'metadata'}{$obs_property}{'data_url'};
$sensor_id = $HoH{$operator}{$local_platform}{'metadata'}{$obs_property}{'sensor_id'};
#$sensor_id link overwrites $data_url if present, could convert next block to function since repeated
if ($sensor_id) {
$data_url = $graph_link;
$data_url =~ s/var1/$sensor_id/g;
}
$desc_table .= unit_convert_table($obs_property,$measurement,$data_url,$elev);
}
#only closing table if exists
#note that could sort table entries by elevation at this point if wanted
if ($desc_table ne '') { $desc_table .= '
'; }
$description = '".$desc_header.$desc_table.']]>';
$kml_content .= <<"END_OF_FILE";
$datetime
$local_platform
1
$description
$longitude,$latitude,0
END_OF_FILE
} #foreach $datetime
if ($platform_not_active == 1) {
$longitude = $HoH{$operator}{$local_platform}{'metadata'}{'longitude'};
$latitude = $HoH{$operator}{$local_platform}{'metadata'}{'latitude'};
$description = '".$desc_header.']]>';
$kml_content .= <<"END_OF_FILE";
#PlatformNotActive
$local_platform
1
$description
$longitude,$latitude,0
END_OF_FILE
}
$kml_content .= "";
} #foreach $local_platform
$kml_content .= "";
} #foreach $operator
$kml_content .= "";
##################
#list by observation
##################
#print "by observation: ";
#print `date`;
#the below is a cut and paste for the most part of the 'list be operator' except running through the ObsTypes list and comparing to determine #whether kml is output or not and the style application of colored ranges on the icons
#BETTER: the below code could be improved where the 'metadata' type elements do not need re-referencing. The placemarks could also style like 'SensorNotActive' to give a more sensor inventory where sensor not currently active/measuring (null measurements)
my $xp_style = XML::LibXML->new->parse_file("$target_dir/style.xml");
$kml_content .= "List by observation0";
$kml_content .= "";
$kml_content .= "none0";
my $kml_temp_content;
foreach my $obs_type ( sort keys %{$rObsTypes} ) {
#print "obs_type:$obs_type\n";
$kml_content .= "$obs_type0";
#must convert below to int, otherwise errors out on pass to subroutines and improper type handline
my $range_high = int($xp_style->findvalue('//style[@id="'.$obs_type.'"]/range_high'));
#print "range_high:$range_high\n";
my $range_low = int($xp_style->findvalue('//style[@id="'.$obs_type.'"]/range_low'));
#print "range_low:$range_low\n";
my $yellow_range = 'yellow = below '.$range_low;
my $orange_range = 'orange = above '.$range_high;
my $color_span = ($range_high - $range_low)/3;
my $blue_range = 'blue = '.$range_low.' to '.int($range_low+$color_span);
my $green_range = 'green = '.int($range_low+$color_span).' to '.int($range_low+$color_span*2);
my $red_range = 'red = '.int($range_low+$color_span*2).' to '.$range_high;
$kml_content .= "The following color ranges apply to this observation
violet range not defined
$yellow_range
$blue_range
$green_range
$red_range
$orange_range";
$kml_content .= "Toggle all on/off0";
foreach my $operator ( sort keys %{$rHoH} ) {
#print "operator:$operator\n";
#my $kml_temp_content = '';
$kml_content .= "$operator0";
foreach my $local_platform ( sort keys %{$rHoH->{$operator}} ) {
#print "localPlatformName:$local_platform\n";
#$kml_content .= "$local_platform0";
foreach my $datetime ( sort keys %{$rHoH->{$operator}{$local_platform}{'data'}} ) {
$kml_temp_content = '';
my $desc_header = '';
my $desc_table = '';
my $desc_footer = '';
my $operator_url = '';
my $platform_url = '';
my $platform_desc = '';
my $longitude = '';
my $latitude = '';
my $measurement = '';
my $datetime_label = '';
my $data_url = '';
my $sensor_id = '';
my $elev = '';
my $color = '';
my $obs_property = $obs_type;
#print "obs_property:$obs_property:$operator:$local_platform\n";
#only need to do this initially once per platform
if (!($datetime_label)) {
$datetime_label = $datetime;
$datetime_label =~ s/T/ /g;
#$datetime_label = substr($datetime_label,0,16).' GMT';
#print "$datetime_label\n";
$desc_header .= '";
$platform_desc = $HoH{$operator}{$local_platform}{'metadata'}{'platform_desc'};
if ($platform_desc) { $desc_header .= 'Description: '.$platform_desc.'
'; }
$operator_url = $HoH{$operator}{$local_platform}{'metadata'}{'operator_url'};
if ($operator_url) { $desc_header .= 'OperatorURL
'; }
$platform_url = $HoH{$operator}{$local_platform}{'metadata'}{'platform_url'};
if ($platform_url) { $desc_header .= 'PlatformURL
'; }
$desc_header .= "$additional_links
";
$data_url = $HoH{$operator}{$local_platform}{'metadata'}{$obs_property}{'data_url'};
$sensor_id = $HoH{$operator}{$local_platform}{'metadata'}{$obs_property}{'sensor_id'};
if (($data_url) || ($sensor_id)) {
$desc_header .= "Click on the latest observation reading to view graphs of previous observations.
";
}
$desc_table .= '';
}
$longitude = $HoH{$operator}{$local_platform}{'data'}{ $datetime }{$obs_property}{'longitude'};
$latitude = $HoH{$operator}{$local_platform}{'data'}{ $datetime }{$obs_property}{'latitude'};
$measurement = $HoH{$operator}{$local_platform}{'data'}{ $datetime }{$obs_property}{'measurement'};
#print "$measurement\n";
$color = conv_measurement_to_color($range_high,$range_low,$measurement);
#print "color:$color\n";
$elev = $HoH{$operator}{$local_platform}{'data'}{ $datetime }{$obs_property}{'elev'};
$data_url = $HoH{$operator}{$local_platform}{'metadata'}{$obs_property}{'data_url'};
$sensor_id = $HoH{$operator}{$local_platform}{'metadata'}{$obs_property}{'sensor_id'};
#$sensor_id link overwrites $data_url if present, could convert next block to function since repeated
if ($sensor_id) {
$data_url = $graph_link;
$data_url =~ s/var1/$sensor_id/g;
}
$desc_table .= unit_convert_table($obs_property,$measurement,$data_url,$elev);
#only closing table if exists
#note that could sort table entries by elevation at this point if wanted
if ($desc_table ne '') { $desc_table .= '
'; }
$desc_footer .= ']]>';
my $description = $desc_header.$desc_table.$desc_footer;
if ($latitude) { #testing for content before output kml - using latitude, but could be any of above content vars
#print "debug\n";
$kml_temp_content .= <<"END_OF_FILE";
$datetime
$local_platform
0
$description
$longitude,$latitude,0
END_OF_FILE
}
if ($kml_temp_content) {
#print "debug\n";
$kml_content .= "$local_platform0";
$kml_content .= $kml_temp_content;
$kml_content .= "";
}
} #foreach $datetime
} #foreach $local_platform
$kml_content .= "";
} #foreach $operator
$kml_content .= "";
$kml_content .= "";
} #foreach $obs_type
$kml_content .= "";
##################
#kml close and return file handle
##################
$kml_content .= <<"END_OF_FILE";
END_OF_FILE
open (FILE_KML,">$target_dir/latest_placemarks.kml");
print FILE_KML $kml_content;
close (FILE_KML);
`cd $target_dir; zip latest_placemarks.kmz latest_placemarks.kml`;
my $kml_url = $http_return.$random_value.'/latest_placemarks.kmz';
print $kml_url;
`rm -f $target_dir/*.kml ; rm -f $target_dir/*.xml`;
#print `date`;
close (LOG_FILE);
close (LOG_FILE_2);
close (DEBUG);
exit 0;
##subroutines################################
sub unit_convert_table
{
#this sub takes $parameter_id, $parameter_value and uses an xml lookup to convert them to another unit of measure returned as an html table row
my ($parameter_id, $parameter_value, $graph, $elev) = @_;
#print "$parameter_id $parameter_value\n";
$elev = sprintf("%.2f", $elev);
#print ":$elev:\n";
if ($elev ne '0.00' && $elev ne '-99999.00') { $parameter_id .= " at $elev m"; }
my $string;
if ($graph) {
$string = "
| $parameter_id | $parameter_value |
";
}
else {
$string = "| $parameter_id | $parameter_value |
";
}
#print "$string\n";
return $string;
}
sub conv_measurement_to_color
{
#note this function biased to use integer ranges and numbers (so avoid narrow ranges or recode to work with better)
my ($range_high, $range_low, $measurement) = @_;
#$measurement = int($measurement);
#if ($measurement == 1037) { print ":$measurement:$range_low:$range_high:"; }
if ($range_high == $range_low ) { return 'FF00FF'; } #violet
if ($measurement <= $range_low) { return '00FFFF'; } #yellow
if ($measurement >= $range_high) { return '0099FF'; } #orange
#NOTE: google kml color scheme is bgr(blue green red) instead of rgb(red green blue) preceeded by alpha/transparency - I've hardcoded the
#transparency as not transparent (ff) in the element and the below are the bgr hex that are substituted from the color scale.
#The color scale is low to high: light blue->dark blue, light green->dark green, light red->dark red
#anything which falls outside the range low is yellow and outside the range high is orange
#anything where the range or measurement is not defined is violet
#found the following website helpful also: http://www.yvg.com/twrs/RGBConverter.html
my $color_scale_count = 21;
my @color_scale = qw(FFBEBE FF9E9E FF7E7E FF5F5F FF3F3F FF1F1F FF0000 BEFFBE 9EFF9E 7EFF7E 5FFF5F 3FFF3F 1FFF1F 00FF00 BEBEFF 9E9EFF 7E7EFF 5F5FFF 3F3FFF 1F1FFF 0000FF);
my $increment = ($range_high - $range_low) / $color_scale_count;
my $choice = int(($measurement - $range_low)/$increment);
#print "$measurement:$increment:$choice\n";
return @color_scale[$choice];
}