/** transform 56-character raw weather data to csv file
fields are split as follows:
0-7   year,month,day,hr
8     sky brightness indicator
9-13  latitude x 100
14-18 longitude x 100
19-23 station id
24    land indicator (all the data processed was from land observations so this field was omitted from the cube)
25-26 present weather
27    total cloud cover
28-29 lower cloud amount
30-31 lower cloud base height
32-33 low cloud type
34-35 middle cloud type
36-37 high cloud type
38-40 middle cloud amount x 100
41-43 high cloud amount x 100
44    middle cloud amount (non overlapped)
45    high cloud amount (non overlapped)
46-47 change code
48-51 solar altitude (deg x10)
52-55 relative lunar illuminance
updated Dec 8 2009 to extract weather 12d cubes using dims:
{0,1,2,3,4,5,9,10,11,16,17,18}
**/
import java.io.*;

public class WeatherETL {

    public static void main(String[] args) {
        BufferedReader raw;
        PrintWriter pw;
        int countBadData = 0;
        String output = "";
        int k = 0;
        try {
            pw = new PrintWriter("data/weather/SEP83L.csv");
            raw = new BufferedReader(new InputStreamReader(new FileInputStream("data/weather/SEP83L.DAT")));

            String line = raw.readLine();

            while (line!=null) {
                if (line.length() != 56) {
                    countBadData++;
                    line = raw.readLine();//get the next cell
                    break;
                }


                for ( k = 0; k < 8; ++k)
                    output = output+ line.charAt(k);
                output = output +","+ line.charAt(8) +",";
                for (k =9; k < 14; ++k)
                    output = output+line.charAt(k);
                output = output +",";
                for (k =14; k < 19; ++k)
                    output = output+line.charAt(k);
                output = output +",";
                for (k = 19; k < 24; ++k)
                    output = output+line.charAt(k);
                output = output +",";
                output = output+line.charAt(25);
                output = output+line.charAt(26);
                output = output +",";
                output = output+line.charAt(32);
                output = output+line.charAt(33);
                output = output +",";
                output = output+line.charAt(34);
                output = output+line.charAt(35);
                output = output +",";
                output = output+line.charAt(36);
                output = output+line.charAt(37);
                output = output +",";
                output = output+line.charAt(46);
                output = output+line.charAt(47);
                output = output +",";
                for (k = 48; k < 52; ++k)
                    output = output+line.charAt(k);
                output = output +",";
                for (k = 52; k <56; ++k)
                    output = output+line.charAt(k);

                pw.println(output);
                output = "";//reset the output data
                line = raw.readLine();
            }
            pw.close();
            raw.close();
            System.out.printf("There were %d bad lines of data\n\n",countBadData);
        } catch (Exception e) {
            System.out.println("lazy exception caught");
            System.out.println(e.getMessage());
        }
    }
}