Merge branch 'datacommonsorg:master' into pep_annual_population

scripts/us_cdc/500_places/manifest.json

@@ -2,7 +2,7 @@
     "import_specifications": [
         {
             "import_name": "CDC500",
-            "curator_emails": [],
+            "curator_emails": ["[email protected]"],
             "provenance_url": "https://www.cdc.gov/places/index.html",
             "provenance_description": "Variables related to health from the CDC",
             "scripts": [

scripts/us_cdc/cdc500_state/manifest.json

@@ -2,7 +2,7 @@
     "import_specifications": [
         {
             "import_name": "CDC500_States",
-            "curator_emails": [],
+            "curator_emails": ["[email protected]"],
             "provenance_url": "https://www.cdc.gov/places/index.html",
             "provenance_description": "Variables related to health from the CDC",
             "scripts": [

scripts/us_hud/income/hud.tmcf

@@ -0,0 +1,143 @@
+Node: E:hud->E0
+typeOf: dcs:StatVarObservation
+variableMeasured: dcs:Median_Income_Household_With1Person_80Percentile
+observationDate: C:hud->year
+observationAbout: C:hud->fips
+observationPeriod: "P1Y"
+value: C:hud->l80_1
+unit: dcs:USDollar
+
+Node: E:hud->E1
+typeOf: dcs:StatVarObservation
+variableMeasured: dcs:Median_Income_Household_With2Person_80Percentile
+observationDate: C:hud->year
+observationAbout: C:hud->fips
+observationPeriod: "P1Y"
+value: C:hud->l80_2
+unit: dcs:USDollar
+
+Node: E:hud->E2
+typeOf: dcs:StatVarObservation
+variableMeasured: dcs:Median_Income_Household_With3Person_80Percentile
+observationDate: C:hud->year
+observationAbout: C:hud->fips
+observationPeriod: "P1Y"
+value: C:hud->l80_3
+unit: dcs:USDollar
+
+Node: E:hud->E3
+typeOf: dcs:StatVarObservation
+variableMeasured: dcs:Median_Income_Household_With4Person_80Percentile
+observationDate: C:hud->year
+observationAbout: C:hud->fips
+observationPeriod: "P1Y"
+value: C:hud->l80_4
+unit: dcs:USDollar
+
+Node: E:hud->E4
+typeOf: dcs:StatVarObservation
+variableMeasured: dcs:Median_Income_Household_With5Person_80Percentile
+observationDate: C:hud->year
+observationAbout: C:hud->fips
+observationPeriod: "P1Y"
+value: C:hud->l80_5
+unit: dcs:USDollar
+
+Node: E:hud->E5
+typeOf: dcs:StatVarObservation
+variableMeasured: dcs:Median_Income_Household_With6Person_80Percentile
+observationDate: C:hud->year
+observationAbout: C:hud->fips
+observationPeriod: "P1Y"
+value: C:hud->l80_6
+unit: dcs:USDollar
+
+Node: E:hud->E6
+typeOf: dcs:StatVarObservation
+variableMeasured: dcs:Median_Income_Household_With7Person_80Percentile
+observationDate: C:hud->year
+observationAbout: C:hud->fips
+observationPeriod: "P1Y"
+value: C:hud->l80_7
+unit: dcs:USDollar
+
+Node: E:hud->E7
+typeOf: dcs:StatVarObservation
+variableMeasured: dcs:Median_Income_Household_With8Person_80Percentile
+observationDate: C:hud->year
+observationAbout: C:hud->fips
+observationPeriod: "P1Y"
+value: C:hud->l80_8
+unit: dcs:USDollar
+
+Node: E:hud->E8
+typeOf: dcs:StatVarObservation
+variableMeasured: dcs:Median_Income_Household_With1Person_150Percentile
+observationDate: C:hud->year
+observationAbout: C:hud->fips
+observationPeriod: "P1Y"
+value: C:hud->l150_1
+unit: dcs:USDollar
+
+Node: E:hud->E9
+typeOf: dcs:StatVarObservation
+variableMeasured: dcs:Median_Income_Household_With2Person_150Percentile
+observationDate: C:hud->year
+observationAbout: C:hud->fips
+observationPeriod: "P1Y"
+value: C:hud->l150_2
+unit: dcs:USDollar
+
+Node: E:hud->E10
+typeOf: dcs:StatVarObservation
+variableMeasured: dcs:Median_Income_Household_With3Person_150Percentile
+observationDate: C:hud->year
+observationAbout: C:hud->fips
+observationPeriod: "P1Y"
+value: C:hud->l150_3
+unit: dcs:USDollar
+
+Node: E:hud->E11
+typeOf: dcs:StatVarObservation
+variableMeasured: dcs:Median_Income_Household_With4Person_150Percentile
+observationDate: C:hud->year
+observationAbout: C:hud->fips
+observationPeriod: "P1Y"
+value: C:hud->l150_4
+unit: dcs:USDollar
+
+Node: E:hud->E12
+typeOf: dcs:StatVarObservation
+variableMeasured: dcs:Median_Income_Household_With5Person_150Percentile
+observationDate: C:hud->year
+observationAbout: C:hud->fips
+observationPeriod: "P1Y"
+value: C:hud->l150_5
+unit: dcs:USDollar
+
+Node: E:hud->E13
+typeOf: dcs:StatVarObservation
+variableMeasured: dcs:Median_Income_Household_With6Person_150Percentile
+observationDate: C:hud->year
+observationAbout: C:hud->fips
+observationPeriod: "P1Y"
+value: C:hud->l150_6
+unit: dcs:USDollar
+
+Node: E:hud->E14
+typeOf: dcs:StatVarObservation
+variableMeasured: dcs:Median_Income_Household_With7Person_150Percentile
+observationDate: C:hud->year
+observationAbout: C:hud->fips
+observationPeriod: "P1Y"
+value: C:hud->l150_7
+unit: dcs:USDollar
+
+Node: E:hud->E15
+typeOf: dcs:StatVarObservation
+variableMeasured: dcs:Median_Income_Household_With8Person_150Percentile
+observationDate: C:hud->year
+observationAbout: C:hud->fips
+observationPeriod: "P1Y"
+value: C:hud->l150_8
+unit: dcs:USDollar

scripts/us_hud/income/manifest.json

@@ -9,7 +9,7 @@
       "import_inputs": [
         {
           "template_mcf": "hud.tmcf",
-          "cleaned_csv": "csv/output_2006.csv"
+          "cleaned_csv": "output/output_all_years.csv"
         }
       ],
       "cron_schedule": "15 23 * * 7"

scripts/us_hud/income/process.py

@@ -32,7 +32,9 @@
 from retry import retry
 
 FLAGS = flags.FLAGS
-flags.DEFINE_string('income_output_dir', 'csv', 'Path to write cleaned CSVs.')
+flags.DEFINE_string('income_output_dir', 'output',
+                    'Path to write cleaned CSVs.')
+flags.DEFINE_string('input_files', 'input', 'Path to download input CSVs.')
 flags.DEFINE_string('mode', '',
                     'Mode: "download", "process", or both (default: both).')
 
@@ -133,7 +135,7 @@ def process_all():
         matches = {'dcs:' + row['fips']: 'dcs:' + row['city'] for row in reader}
 
     today = datetime.date.today()
-    input_folder = 'input'
+    input_folder = FLAGS.input_files
     output_data = []
 
     if FLAGS.mode == "" or FLAGS.mode == "download":