#!/usr/bin/perl 

use LWP::Simple;

#add data sources to below listing with corresponding xml source

$data_sources{'http://www.dnal.gatech.edu/wavebuoy/status'} = 
	{ xml_source => 'http://trident.baruch.sc.edu/storm_surge_data/seacoos_xml/seacoos_GTSAV1_buoy_status_hdr.xml' 
	, filter_type_1 => 'time_filter_1' 
	, filter_args_1 => 'breakoutYYYYMMDDHHMM'
	, common => 'status_2005.*.csv'
	};
$data_sources{'http://www.dnal.gatech.edu/wavebuoy/wave'} = 
	{ xml_source => 'http://trident.baruch.sc.edu/storm_surge_data/seacoos_xml/seacoos_GTSAV1_buoy_wave_hdr.xml' 
	, filter_type_1 => 'time_filter_1' 
	, filter_args_1 => 'breakoutYYYYMMDDHHMM'
	, common => 'wave_2005.*.csv'
	};
$data_sources{'http://trident.baruch.sc.edu/storm_surge_data/latest/test'} = 
	{ xml_source => 'http://trident.baruch.sc.edu/storm_surge_data/latest/test/carocoops_CAP2_buoy_hdr.xml' 
	, filter_type_1 => 'time_filter_1' 
	, filter_args_1 => 'breakoutNormal'
	, common => '.*.dat'
	};

foreach $k (keys (%data_sources))
{
  $this_dir_url = $k;

  print `date`;
  print "$this_dir_url\n";

  $xml_source = $data_sources{$k}{xml_source};
  #print "$xml_source\n";
  $filter_type_1 = $data_sources{$k}{filter_type_1};
  #print "$filter_type_1\n";
  $filter_args_1 = $data_sources{$k}{filter_args_1};
  #print "$filter_args_1\n";
  $this_common = $data_sources{$k}{common};

  do('get_latest_listing.pl');

  foreach (@latest) {
    $latest_file = $_;

    $file_longname = $this_dir_url.'/'.$latest_file;
    #print "  $file_longname\n";
    $file_longname_filter = $file_longname;
    $file_longname_filter =~ s/http:\/\///g;
    $file_longname_filter =~ s/\//SL/g;

    if (open(LAST_FETCH,"./fetch_logs/$file_longname_filter")) {
        #print "  not processed\n";
    }
    else {
        #print "  processed\n";
        `/usr/bin/perl mk_netcdf_latest.pl $xml_source $file_longname $file_longname_filter $filter_type_1 $filter_args_1`;
    	$cmd = "touch ./fetch_logs/$file_longname_filter";
    	`$cmd`;
    }
  }
}