use strict; #use warnings; ##config #usage: cd /var/www/html/obskml/scripts; perl genPlacemarksObsKML.pl http://nautilus.baruch.sc.edu/obskml/feeds/secoora_obskml_latest.kmz null secoora my $temp_dir = '/tmp/ms_tmp'; my $http_return = 'http://nautilus.baruch.sc.edu/ms_tmp/gearth_'; my $additional_links = 'Related links: ObsKML Xenia http://secoora.org
'; my $graph_link = 'http://nautilus.baruch.sc.edu/xenia_sqlite/get_graph.php?sensor_id=var1&output=webpage&time_interval=-1 day&time_zone_arg=EASTERN'; ## =comment #usage notes Watch the server path specific literals, everything gets unzipped and worked on in $target_dir so you'll want to set that to some temporary folder area that gets periodically flushed. The other xml files this script uses is style.xml . style.xml provides the default observation range limits to be used and can also be ignored in favor of a user supplied style.xml via $styel_url Also there is a literal http address returned to the calling php page which will need to be changed accordingly. #see line time filter below to control acceptable date range used in creating output file #my $date_yesterday = `date +%Y%m%d%H%M --date='12 hours ago'`; =cut use LWP::Simple; use XML::LibXML; my ($zip_obskml_url, $style_url, $feed_name) = @ARGV; if ($feed_name) { $feed_name = $feed_name.'_'; } my $date_log = `date +%Y%m%d%H%M`; chomp($date_log); open (LOG_FILE,">$feed_name\obskml_style.log"); open (LOG_FILE_2,">./log/$feed_name\obskml_style_$date_log.log"); open (DEBUG,">./debug_genPlacemarks.txt"); my $count = @ARGV; if ($count < 1) { print "usage: zip_obskml_url style_url\n"; exit; } #using print `date` for script time benchmarking #print `date`; #create temp working directory my $random_value = int(rand(10000000)); my $target_dir = "$temp_dir/gearth_$random_value"; `mkdir $target_dir`; #print $target_dir."\n"; ################## #read input files to temp directory ################## #zip_obskml_url my $zip_filepath = "$target_dir/obskml.xml.zip"; #print $zip_filepath."\n"; my $content = getstore($zip_obskml_url, $zip_filepath); die "Couldn't get $zip_obskml_url" unless defined $content; `cd $target_dir; unzip obskml.xml.zip`; #style_url if (!($style_url) || ($style_url eq 'null')) { `cp /var/www/html/obskml/scripts/style.xml $target_dir`; } else { my $style_filepath = "$target_dir/style.xml"; my $content = getstore($style_url, $style_filepath); die "Couldn't get $style_url" unless defined $content; } ################## #global hashes #note that the below hashing convention assumes each element contains only one type of child element except at the terminals my %HoH = (); my $rHoH = \%HoH; my %HoH_stats = (); #below hash used later for sorting by ObsTypes my %ObsTypes = (); my $rObsTypes = \%ObsTypes; ################## #convert kml files to hash ################## my $filelist = `ls $target_dir/*.kml`; #print $filelist; my @files = split(/\n/,$filelist); #print @files; #exit 0; my $date_now = `date +%Y%m%d%H%M --date='+12 hours'`; chomp($date_now); #the below seems to catch 2+4(EDT) = 6 hours my $date_yesterday = `date +%Y%m%d%H%M --date='-12 hours'`; chomp($date_yesterday); foreach my $file (@files) { #print "$file\n"; #my $xp = XML::LibXML->new->parse_file("$target_dir/obskml_latest.kml"); my $xp = XML::LibXML->new->parse_file($file); #print `date`; foreach my $placemark ($xp->findnodes('//Placemark')) { my $placemark_id = $placemark->getAttribute('id'); if (!($placemark_id)) { $placemark_id = 'none.none.none'; } #print "$placemark_id\n"; my $local_platform = $placemark_id; my ($operator,$platform,$package) = split(/\./,$placemark_id); my $datetime = $placemark->find('TimeStamp/when'); $datetime = sprintf("%s", $datetime); my $coordinates = $placemark->find('Point/coordinates'); my ($longitude,$latitude) = split(/,/,$coordinates); if ($platform eq 'none') { $local_platform = "point($longitude,$latitude)"; } #lon/lat stand in for placemark id if none given my $operator_url = $placemark->find('Metadata/obsList/operatorURL'); $operator_url = sprintf("%s", $operator_url); my $platform_url = $placemark->find('Metadata/obsList/platformURL'); $platform_url = sprintf("%s", $platform_url); my $platform_desc = $placemark->find('Metadata/obsList/platformDescription'); $platform_desc = sprintf("%s", $platform_desc); $HoH{ $operator }{ $local_platform }{'metadata'}{ 'operator_url' } = $operator_url; $HoH{ $operator }{ $local_platform }{'metadata'}{ 'platform_url' } = $platform_url; $HoH{ $operator }{ $local_platform }{'metadata'}{ 'platform_desc' } = $platform_desc; $HoH{ $operator }{ $local_platform }{'metadata'}{ 'longitude' } = $longitude; $HoH{ $operator }{ $local_platform }{'metadata'}{ 'latitude' } = $latitude; foreach my $observation ($placemark->findnodes('Metadata/obsList/obs')) { my $obs_property = sprintf("%s",$observation->find('obsType')); if ($obs_property eq '') { next; } #ignore obs with missing metadata my $uom = sprintf("%s",$observation->find('uomType')); $obs_property .= '.'.$uom; #have to cast measurement to float using sprintf to avoid comparison confusion later my $measurement = sprintf("%.2f",$observation->find('value')); #using the below hash trick to get a list of unique ObsTypes $ObsTypes{ $obs_property } = 1; #print "$operator:$local_platform:$obs_property:$datetime:$longitude:$latitude:$measurement\n"; my $date_test = substr($datetime,0,4).substr($datetime,5,2).substr($datetime,8,2).substr($datetime,11,2).substr($datetime,14,2); #print "date_test: $date_test \n"; #don't share erroneous 'future' observations greater than today's date or obs older than yesterday if (($date_test >= $date_yesterday) && ($date_test <= $date_now)) { print DEBUG "date pass: $date_yesterday $date_now $date_test \n"; #for logging purposes #only want to increment count for just latest measurements, not earlier #tried $HoH_stats{$operator}{$placemark_id}{$obs_property} but hash complains/fails for more than one unknown level reference ?? if (!($HoH_stats{$operator}{$placemark_id.$obs_property})) { $HoH_stats{$operator}{$placemark_id.$obs_property} = 1; $HoH_stats{ $operator }{ $obs_property }{ obs_count }++; } #if ($operator eq 'nerrs') { print "$date_test $date_now $date_yesterday\n"; } #$HoH{ $operator }{ $local_platform }{'metadata'}{ 'operator_url' } = $operator_url; #$HoH{ $operator }{ $local_platform }{'metadata'}{ 'platform_url' } = $platform_url; #$HoH{ $operator }{ $local_platform }{'metadata'}{ 'platform_desc' } = $platform_desc; #could probably optimize lon/lat to allow platform movement without articulating for each $obs_property - leaving for now to satisfy foreach loops further below $HoH{ $operator }{ $local_platform }{'data'}{ $datetime }{ $obs_property }{ 'longitude' } = $longitude; $HoH{ $operator }{ $local_platform }{'data'}{ $datetime }{ $obs_property }{ 'latitude' } = $latitude; $HoH{ $operator }{ $local_platform }{'metadata'}{ $obs_property }{ 'data_url' } = $observation->find('dataURL'); $HoH{ $operator }{ $local_platform }{'metadata'}{ $obs_property }{ 'sensor_id' } = $observation->find('sensorID'); #$HoH{ $operator }{ $local_platform }{'data'}{ $datetime }{ $obs_property }{ 'datetime' } = $datetime; $HoH{ $operator }{ $local_platform }{'data'}{ $datetime }{ $obs_property }{ 'measurement' } = $measurement; $HoH{ $operator }{ $local_platform }{'data'}{ $datetime }{ $obs_property }{ 'elev' } = $observation->find('elev'); } else { print DEBUG "date fail: $date_yesterday $date_now $date_test \n"; } } #foreach obs #only want to increment count for just latest measurements, not earlier if ($HoH_stats{$operator}{$placemark_id} != 1) { $HoH_stats{$operator}{$placemark_id} = 1; $HoH_stats{ $operator }{ platform_count }++; } } #foreach Placemark } #foreach file ################# #kml content ################## my $kml_content = <<"END_OF_FILE"; Ocean/Coastal Observing Platform Data 0 ObsKML Here are links to the original source ObsKML data and the styling script used to generate this KML file. Please email jeremy.cothran\@gmail.com regarding questions or comments on this kml product or sharing/registering your observation data using these tools.]]> END_OF_FILE #generating the kml content below if not directly output to $kml_content is sent to 'buffer' variables so that the buffer can be used # or cleared depending on the available data ################## #list by operator ################## #print "by operator: "; #print `date`; $kml_content .= "List by operator0"; foreach my $operator ( sort keys %{$rHoH} ) { #print "operator:$operator\n"; print LOG_FILE "operator $operator $HoH_stats{$operator}{platform_count}\n"; print LOG_FILE_2 "operator $operator $HoH_stats{$operator}{platform_count}\n"; foreach my $obs_type ( sort keys %{$rObsTypes} ) { my $obs_count = $HoH_stats{$operator}{$obs_type}{obs_count}; if ($obs_count > 0) { print LOG_FILE "obs_type $obs_type $HoH_stats{$operator}{$obs_type}{obs_count}\n"; print LOG_FILE_2 "obs_type $obs_type $HoH_stats{$operator}{$obs_type}{obs_count}\n"; } } print LOG_FILE "\n\n"; print LOG_FILE_2 "\n\n"; $kml_content .= "$operator0"; foreach my $local_platform ( sort keys %{$rHoH->{$operator}} ) { #print "localPlatformName:$local_platform\n"; $kml_content .= "$local_platform0"; my $platform_not_active = 1; my $desc_header = ''; my $desc_table = ''; my $description = ''; my $operator_url = ''; my $platform_url = ''; my $platform_desc = ''; my $longitude = ''; my $latitude = ''; $platform_desc = $HoH{$operator}{$local_platform}{'metadata'}{'platform_desc'}; if ($platform_desc) { $desc_header .= 'Description: '.$platform_desc.'
'; } $operator_url = $HoH{$operator}{$local_platform}{'metadata'}{'operator_url'}; if ($operator_url) { $desc_header .= 'OperatorURL
'; } $platform_url = $HoH{$operator}{$local_platform}{'metadata'}{'platform_url'}; if ($platform_url) { $desc_header .= 'PlatformURL
'; } $desc_header .= "$additional_links
"; foreach my $datetime ( sort keys %{$rHoH->{$operator}{$local_platform}{'data'}} ) { #print "datetime:$datetime\n"; $platform_not_active = 0; my $measurement = ''; my $data_url = ''; my $sensor_id = ''; my $elev = ''; my $datetime_label = ''; foreach my $obs_property ( sort keys %{$rHoH->{$operator}{$local_platform}{'data'}{ $datetime }} ) { #print "obs_property:$obs_property\n"; #only need to do this initially once per platform if (!($datetime_label)) { $datetime_label = $datetime; $datetime_label =~ s/T/ /g; #$datetime_label = substr($datetime_label,0,16).' GMT'; #print "$datetime_label\n"; $data_url = $HoH{$operator}{$local_platform}{'metadata'}{$obs_property}{'data_url'}; $sensor_id = $HoH{$operator}{$local_platform}{'metadata'}{$obs_property}{'sensor_id'}; if (($data_url) || ($sensor_id)) { $desc_header .= "Click on the latest observation reading to view graphs of previous observations.

"; } $desc_table .= ''; } $longitude = $HoH{$operator}{$local_platform}{'data'}{ $datetime }{$obs_property}{'longitude'}; $latitude = $HoH{$operator}{$local_platform}{'data'}{ $datetime }{$obs_property}{'latitude'}; $measurement = $HoH{$operator}{$local_platform}{'data'}{ $datetime }{$obs_property}{'measurement'}; #print "$measurement\n"; $elev = $HoH{$operator}{$local_platform}{'data'}{ $datetime }{$obs_property}{'elev'}; $data_url = $HoH{$operator}{$local_platform}{'metadata'}{$obs_property}{'data_url'}; $sensor_id = $HoH{$operator}{$local_platform}{'metadata'}{$obs_property}{'sensor_id'}; #$sensor_id link overwrites $data_url if present, could convert next block to function since repeated if ($sensor_id) { $data_url = $graph_link; $data_url =~ s/var1/$sensor_id/g; } $desc_table .= unit_convert_table($obs_property,$measurement,$data_url,$elev); } #only closing table if exists #note that could sort table entries by elevation at this point if wanted if ($desc_table ne '') { $desc_table .= '
'; } $description = '".$desc_header.$desc_table.']]>'; $kml_content .= <<"END_OF_FILE"; $datetime $local_platform 1 $description $longitude,$latitude,0 END_OF_FILE } #foreach $datetime if ($platform_not_active == 1) { $longitude = $HoH{$operator}{$local_platform}{'metadata'}{'longitude'}; $latitude = $HoH{$operator}{$local_platform}{'metadata'}{'latitude'}; $description = '".$desc_header.']]>'; $kml_content .= <<"END_OF_FILE"; #PlatformNotActive $local_platform 1 $description $longitude,$latitude,0 END_OF_FILE } $kml_content .= "
"; } #foreach $local_platform $kml_content .= "
"; } #foreach $operator $kml_content .= "
"; ################## #list by observation ################## #print "by observation: "; #print `date`; #the below is a cut and paste for the most part of the 'list be operator' except running through the ObsTypes list and comparing to determine #whether kml is output or not and the style application of colored ranges on the icons #BETTER: the below code could be improved where the 'metadata' type elements do not need re-referencing. The placemarks could also style like 'SensorNotActive' to give a more sensor inventory where sensor not currently active/measuring (null measurements) my $xp_style = XML::LibXML->new->parse_file("$target_dir/style.xml"); $kml_content .= "List by observation0"; $kml_content .= ""; $kml_content .= "none0"; my $kml_temp_content; foreach my $obs_type ( sort keys %{$rObsTypes} ) { #print "obs_type:$obs_type\n"; $kml_content .= "$obs_type0"; #must convert below to int, otherwise errors out on pass to subroutines and improper type handline my $range_high = int($xp_style->findvalue('//style[@id="'.$obs_type.'"]/range_high')); #print "range_high:$range_high\n"; my $range_low = int($xp_style->findvalue('//style[@id="'.$obs_type.'"]/range_low')); #print "range_low:$range_low\n"; my $yellow_range = 'yellow = below '.$range_low; my $orange_range = 'orange = above '.$range_high; my $color_span = ($range_high - $range_low)/3; my $blue_range = 'blue = '.$range_low.' to '.int($range_low+$color_span); my $green_range = 'green = '.int($range_low+$color_span).' to '.int($range_low+$color_span*2); my $red_range = 'red = '.int($range_low+$color_span*2).' to '.$range_high; $kml_content .= "The following color ranges apply to this observation
violet range not defined
$yellow_range
$blue_range
$green_range
$red_range
$orange_range
"; $kml_content .= "Toggle all on/off0"; foreach my $operator ( sort keys %{$rHoH} ) { #print "operator:$operator\n"; #my $kml_temp_content = ''; $kml_content .= "$operator0"; foreach my $local_platform ( sort keys %{$rHoH->{$operator}} ) { #print "localPlatformName:$local_platform\n"; #$kml_content .= "$local_platform0"; foreach my $datetime ( sort keys %{$rHoH->{$operator}{$local_platform}{'data'}} ) { $kml_temp_content = ''; my $desc_header = ''; my $desc_table = ''; my $desc_footer = ''; my $operator_url = ''; my $platform_url = ''; my $platform_desc = ''; my $longitude = ''; my $latitude = ''; my $measurement = ''; my $datetime_label = ''; my $data_url = ''; my $sensor_id = ''; my $elev = ''; my $color = ''; my $obs_property = $obs_type; #print "obs_property:$obs_property:$operator:$local_platform\n"; #only need to do this initially once per platform if (!($datetime_label)) { $datetime_label = $datetime; $datetime_label =~ s/T/ /g; #$datetime_label = substr($datetime_label,0,16).' GMT'; #print "$datetime_label\n"; $desc_header .= '"; $platform_desc = $HoH{$operator}{$local_platform}{'metadata'}{'platform_desc'}; if ($platform_desc) { $desc_header .= 'Description: '.$platform_desc.'
'; } $operator_url = $HoH{$operator}{$local_platform}{'metadata'}{'operator_url'}; if ($operator_url) { $desc_header .= 'OperatorURL
'; } $platform_url = $HoH{$operator}{$local_platform}{'metadata'}{'platform_url'}; if ($platform_url) { $desc_header .= 'PlatformURL
'; } $desc_header .= "$additional_links
"; $data_url = $HoH{$operator}{$local_platform}{'metadata'}{$obs_property}{'data_url'}; $sensor_id = $HoH{$operator}{$local_platform}{'metadata'}{$obs_property}{'sensor_id'}; if (($data_url) || ($sensor_id)) { $desc_header .= "Click on the latest observation reading to view graphs of previous observations.

"; } $desc_table .= ''; } $longitude = $HoH{$operator}{$local_platform}{'data'}{ $datetime }{$obs_property}{'longitude'}; $latitude = $HoH{$operator}{$local_platform}{'data'}{ $datetime }{$obs_property}{'latitude'}; $measurement = $HoH{$operator}{$local_platform}{'data'}{ $datetime }{$obs_property}{'measurement'}; #print "$measurement\n"; $color = conv_measurement_to_color($range_high,$range_low,$measurement); #print "color:$color\n"; $elev = $HoH{$operator}{$local_platform}{'data'}{ $datetime }{$obs_property}{'elev'}; $data_url = $HoH{$operator}{$local_platform}{'metadata'}{$obs_property}{'data_url'}; $sensor_id = $HoH{$operator}{$local_platform}{'metadata'}{$obs_property}{'sensor_id'}; #$sensor_id link overwrites $data_url if present, could convert next block to function since repeated if ($sensor_id) { $data_url = $graph_link; $data_url =~ s/var1/$sensor_id/g; } $desc_table .= unit_convert_table($obs_property,$measurement,$data_url,$elev); #only closing table if exists #note that could sort table entries by elevation at this point if wanted if ($desc_table ne '') { $desc_table .= '
'; } $desc_footer .= ']]>'; my $description = $desc_header.$desc_table.$desc_footer; if ($latitude) { #testing for content before output kml - using latitude, but could be any of above content vars #print "debug\n"; $kml_temp_content .= <<"END_OF_FILE"; $datetime $local_platform 0 $description $longitude,$latitude,0 END_OF_FILE } if ($kml_temp_content) { #print "debug\n"; $kml_content .= "$local_platform0"; $kml_content .= $kml_temp_content; $kml_content .= ""; } } #foreach $datetime } #foreach $local_platform $kml_content .= "
"; } #foreach $operator $kml_content .= "
"; $kml_content .= "
"; } #foreach $obs_type $kml_content .= "
"; ################## #kml close and return file handle ################## $kml_content .= <<"END_OF_FILE";
END_OF_FILE open (FILE_KML,">$target_dir/latest_placemarks.kml"); print FILE_KML $kml_content; close (FILE_KML); `cd $target_dir; zip latest_placemarks.kmz latest_placemarks.kml`; my $kml_url = $http_return.$random_value.'/latest_placemarks.kmz'; print $kml_url; `rm -f $target_dir/*.kml ; rm -f $target_dir/*.xml`; #print `date`; close (LOG_FILE); close (LOG_FILE_2); close (DEBUG); exit 0; ##subroutines################################ sub unit_convert_table { #this sub takes $parameter_id, $parameter_value and uses an xml lookup to convert them to another unit of measure returned as an html table row my ($parameter_id, $parameter_value, $graph, $elev) = @_; #print "$parameter_id $parameter_value\n"; $elev = sprintf("%.2f", $elev); #print ":$elev:\n"; if ($elev ne '0.00' && $elev ne '-99999.00') { $parameter_id .= " at $elev m"; } my $string; if ($graph) { $string = "$parameter_id$parameter_value"; } else { $string = "$parameter_id$parameter_value"; } #print "$string\n"; return $string; } sub conv_measurement_to_color { #note this function biased to use integer ranges and numbers (so avoid narrow ranges or recode to work with better) my ($range_high, $range_low, $measurement) = @_; #$measurement = int($measurement); #if ($measurement == 1037) { print ":$measurement:$range_low:$range_high:"; } if ($range_high == $range_low ) { return 'FF00FF'; } #violet if ($measurement <= $range_low) { return '00FFFF'; } #yellow if ($measurement >= $range_high) { return '0099FF'; } #orange #NOTE: google kml color scheme is bgr(blue green red) instead of rgb(red green blue) preceeded by alpha/transparency - I've hardcoded the #transparency as not transparent (ff) in the element and the below are the bgr hex that are substituted from the color scale. #The color scale is low to high: light blue->dark blue, light green->dark green, light red->dark red #anything which falls outside the range low is yellow and outside the range high is orange #anything where the range or measurement is not defined is violet #found the following website helpful also: http://www.yvg.com/twrs/RGBConverter.html my $color_scale_count = 21; my @color_scale = qw(FFBEBE FF9E9E FF7E7E FF5F5F FF3F3F FF1F1F FF0000 BEFFBE 9EFF9E 7EFF7E 5FFF5F 3FFF3F 1FFF1F 00FF00 BEBEFF 9E9EFF 7E7EFF 5F5FFF 3F3FFF 1F1FFF 0000FF); my $increment = ($range_high - $range_low) / $color_scale_count; my $choice = int(($measurement - $range_low)/$increment); #print "$measurement:$increment:$choice\n"; return @color_scale[$choice]; }