Add generation_data_source column.

Add a new column to the output database - `generation_data_source`,
which describes the source of data for the `generation_gwh_YYYY`
columns. This field is required for any plant that has a reported
annual generation value. In the full data processing workflow, if a
plant has generation data but this data is not tied to a source, none of
the reported generation will be included in the final database.

For this reason, this update removes 51 instances of annual generation
which are no longer in the database. It is hoped that these will return
sometime in the future once the data sources are identified.

This database update does not change the total plant count or total
capacity of the Global Power Plant Database.

Database version 1.2.3

build_databases/build_database_AUS.py

@@ -21,6 +21,7 @@
 SAVE_CODE = u"AUS"
 SOURCE_NAME = u"Australian Renewable Energy Mapping Infrastructure"
 SOURCE_URL = u"http://services.ga.gov.au/site_3/rest/services/Electricity_Infrastructure/MapServer"
+GENERATION_SOURCE = u"Australia Clean Energy Regulator"
 
 NGER_URL_1617 = u"http://www.cleanenergyregulator.gov.au/DocumentAssets/Documents/Greenhouse%20and%20energy%20information%20for%20designated%20generation%20facilities%202016-17.csv"
 NGER_FILENAME_1617 = pw.make_file_path(fileType="raw", subFolder=SAVE_CODE, filename="NGER_2016-2017.csv")
@@ -167,7 +168,7 @@
                 else:
                     gwh += gen_gwh
             # TODO: give proper time bounds
-            generation.append(pw.PlantGenerationObject.create(gwh, yr))
+            generation.append(pw.PlantGenerationObject.create(gwh, yr, source=GENERATION_SOURCE))
 
 
         # assign ID number

build_databases/build_database_IND.py

@@ -65,14 +65,14 @@
 # set up fuel type thesaurus
 fuel_thesaurus = pw.make_fuel_thesaurus()
 
-def get_CEA_generation(row, col, year, source_url):
+def get_CEA_generation(row, col, year, source_name):
     """Extract a generation data point from CEA data."""
     try:
         if row[col] == u'-':
             generation = pw.PlantGenerationObject()
         else:
             gen_gwh = float(row[col])
-            generation = pw.PlantGenerationObject.create(gen_gwh, year, source=source_url)
+            generation = pw.PlantGenerationObject.create(gen_gwh, year, source=source_name)
     except:
         generation = pw.PlantGenerationObject()
     return generation
@@ -192,11 +192,11 @@ def get_CEA_generation(row, col, year, source_url):
 
     # try to load generation data
     # TODO: organize this into fiscal year (april through march)
-    generation_13 = get_CEA_generation(rv, gen_13_14_col, 2013, SOURCE_URL)
-    generation_14 = get_CEA_generation(rv, gen_14_15_col, 2014, SOURCE_URL)
-    generation_15 = get_CEA_generation(rv, gen_15_16_col, 2015, SOURCE_URL)
-    generation_16 = get_CEA_generation(rv, gen_16_17_col, 2016, SOURCE_URL)
-    generation_17 = get_CEA_generation(rv, gen_17_18_col, 2017, SOURCE_URL)
+    generation_13 = get_CEA_generation(rv, gen_13_14_col, 2013, SOURCE_NAME)
+    generation_14 = get_CEA_generation(rv, gen_14_15_col, 2014, SOURCE_NAME)
+    generation_15 = get_CEA_generation(rv, gen_15_16_col, 2015, SOURCE_NAME)
+    generation_16 = get_CEA_generation(rv, gen_16_17_col, 2016, SOURCE_NAME)
+    generation_17 = get_CEA_generation(rv, gen_17_18_col, 2017, SOURCE_NAME)
     generation = [generation_13, generation_14, generation_15, generation_16, generation_17]
 
     try:

build_databases/build_database_USA.py

@@ -181,7 +181,7 @@
 	idnr = pw.make_id(SAVE_CODE, int(rv[COLS_923_2_2017['idnr']]))
 	if idnr in plants_dictionary:
 		if not plants_dictionary[idnr].generation[-1]:
-			generation = pw.PlantGenerationObject.create(0.0, 2017, source=SOURCE_URL)
+			generation = pw.PlantGenerationObject.create(0.0, 2017, source=SOURCE_NAME)
 			plants_dictionary[idnr].generation[-1] = generation
 		plants_dictionary[idnr].generation[-1].gwh += float(rv[COLS_923_2_2017['generation']]) * GENERATION_CONVERSION_TO_GWH
 	else:
@@ -194,7 +194,7 @@
 	idnr = pw.make_id(SAVE_CODE, int(rv[COLS_923_2_2016['idnr']]))
 	if idnr in plants_dictionary:
 		if not pw.annual_generation(plants_dictionary[idnr].generation, 2016):
-			generation = pw.PlantGenerationObject.create(0.0, 2016, source=SOURCE_URL)
+			generation = pw.PlantGenerationObject.create(0.0, 2016, source=SOURCE_NAME)
 			plants_dictionary[idnr].generation.append(generation)
 		plants_dictionary[idnr].generation[-1].gwh += float(rv[COLS_923_2_2016['generation']]) * GENERATION_CONVERSION_TO_GWH
 	else:
@@ -207,7 +207,7 @@
 	idnr = pw.make_id(SAVE_CODE, int(rv[COLS_923_2_2015['idnr']]))
 	if idnr in plants_dictionary:
 		if not pw.annual_generation(plants_dictionary[idnr].generation, 2015):
-			generation = pw.PlantGenerationObject.create(0.0, 2015, source=SOURCE_URL)
+			generation = pw.PlantGenerationObject.create(0.0, 2015, source=SOURCE_NAME)
 			plants_dictionary[idnr].generation.append(generation)
 		plants_dictionary[idnr].generation[-1].gwh += float(rv[COLS_923_2_2015['generation']]) * GENERATION_CONVERSION_TO_GWH
 	else:
@@ -220,7 +220,7 @@
 	idnr = pw.make_id(SAVE_CODE, int(rv[COLS_923_2_2014['idnr']]))
 	if idnr in plants_dictionary:
 		if not pw.annual_generation(plants_dictionary[idnr].generation, 2014):
-			generation = pw.PlantGenerationObject.create(0.0, 2014, source=SOURCE_URL)
+			generation = pw.PlantGenerationObject.create(0.0, 2014, source=SOURCE_NAME)
 			plants_dictionary[idnr].generation.append(generation)
 		plants_dictionary[idnr].generation[-1].gwh += float(rv[COLS_923_2_2014['generation']]) * GENERATION_CONVERSION_TO_GWH
 	else:
@@ -233,7 +233,7 @@
 	idnr = pw.make_id(SAVE_CODE, int(rv[COLS_923_2_2013['idnr']]))
 	if idnr in plants_dictionary:
 		if not pw.annual_generation(plants_dictionary[idnr].generation, 2013):
-			generation = pw.PlantGenerationObject.create(0.0, 2013, source=SOURCE_URL)
+			generation = pw.PlantGenerationObject.create(0.0, 2013, source=SOURCE_NAME)
 			plants_dictionary[idnr].generation.append(generation)
 		plants_dictionary[idnr].generation[-1].gwh += float(rv[COLS_923_2_2013['generation']]) * GENERATION_CONVERSION_TO_GWH
 	else:

output_database/DATABASE_VERSION

@@ -1,2 +1,2 @@
-1.2.2
+1.2.3