Merge branch 'master' of https://github.com/sunlightlabs/openstates

openstates/mn/bills.py

@@ -59,7 +59,6 @@ class MNBillScraper(BillScraper, LXMLMixin):
         ("Received from", "bill:introduced"),
     )
 
-
     def scrape(self, chamber, session):
         """
         Scrape all bills for a given chamber and a given session.
@@ -92,7 +91,6 @@ def scrape(self, chamber, session):
         for b in bills:
             self.get_bill_info(chamber, session, b['bill_url'], b['version_url'])
 
-
     def get_full_bill_list(self, chamber, session):
         """
         Uses the legislator search to get a full list of bills.  Search page
@@ -116,7 +114,6 @@ def get_full_bill_list(self, chamber, session):
                 # bill: Range start-end (e.g. 1-10)
                 url = BILL_SEARCH_URL % (search_chamber, search_session, start,
                     start + stride, bill_type)
-
                 # Parse HTML
                 html = self.get(url).text
                 doc = lxml.html.fromstring(html)
@@ -147,7 +144,6 @@ def get_full_bill_list(self, chamber, session):
 
         return bills
 
-
     def get_bill_info(self, chamber, session, bill_detail_url, version_list_url):
         """
         Extracts all the requested info for a given bill.
@@ -160,6 +156,12 @@ def get_bill_info(self, chamber, session, bill_detail_url, version_list_url):
         # Get html and parse
         doc = self.lxmlize(bill_detail_url)
 
+        # Check if bill hasn't been transmitted to the other chamber yet
+        transmit_check = self.get_node(doc, '//h1[text()[contains(.,"Bills")]]/following-sibling::ul/li/text()')
+        if transmit_check is not None and 'has not been transmitted' in transmit_check.strip():
+            self.logger.debug('Bill has not been transmitted to other chamber ... skipping {0}'.format(bill_detail_url))
+            return
+
         # Get the basic parts of the bill
         bill_id = self.get_node(doc, '//h1/text()')
         self.logger.debug(bill_id)
@@ -208,7 +210,6 @@ def get_bill_info(self, chamber, session, bill_detail_url, version_list_url):
 
         self.save_bill(bill)
 
-
     def get_bill_topics(self, chamber, session):
         """
         Uses the leg search to map topics to bills.
@@ -241,7 +242,6 @@ def get_bill_topics(self, chamber, session):
                 bill = self.make_bill_id(bill)
                 self._subject_mapping[bill].append(subject)
 
-
     def extract_actions(self, bill, doc, current_chamber):
         """
         Extract the actions taken on a bill.
@@ -343,7 +343,6 @@ def extract_actions(self, bill, doc, current_chamber):
 
         return bill
 
-
     def extract_sponsors(self, bill, doc, chamber):
         """
         Extracts sponsors from bill page.
@@ -362,7 +361,6 @@ def extract_sponsors(self, bill, doc, chamber):
 
         return bill
 
-
     def extract_versions(self, bill, doc, chamber, version_list_url):
       """
       Versions of a bill are on a separate page, linked to from the column
@@ -384,7 +382,6 @@ def extract_versions(self, bill, doc, chamber, version_list_url):
 
       return bill
 
-
     # def extract_vote_from_action(self, bill, action, chamber, action_row):
     #     """
     #     Gets vote data.  For the Senate, we can only get yes and no
@@ -437,7 +434,6 @@ def make_bill_id(self, bill):
 
         return re.sub(r'(\w+?)0*(\d+)', r'\1 \2', bill)
 
-
     def chamber_from_bill(self, bill):
         """
         Given a bill id, determine chamber.
@@ -447,28 +443,24 @@ def chamber_from_bill(self, bill):
 
         return 'lower' if bill.lower().startswith('hf') else 'upper'
 
-
     def other_chamber(self, chamber):
         """
         Given a chamber, get the other.
         """
         return 'lower' if chamber == 'upper' else 'upper'
 
-
     def search_chamber(self, chamber):
         """
         Given chamber, like lower, make into MN site friendly search chamber.
         """
         return { 'lower':'House', 'upper':'Senate' }[chamber]
 
-
     def search_session(self, session):
         """
         Given session ID, make into MN site friendly search.
         """
         return self.metadata['session_details'][session]['site_id']
 
-
     def is_testing(self):
         """
         Determine if this is test mode.

openstates/mn/legislators.py

@@ -149,11 +149,12 @@ def _scrape_upper_chamber(self, term):
             row['Address2'] = next((a for a in address2_fields if a is not
                 None), False)
 
-            if (a in row['Address2'] for a in ['95 University Avenue W',
-                '100 Rev. Dr. Martin Luther King']):
+            if (a in row['Address2'] for a in ['95 University Avenue W', '100 Rev. Dr. Martin Luther King']):
+                address = '{Address}\n{Address2}\n{City}, {State} {Zipcode}'.format(**row)
+                if 'Rm. Number' in row:
+                    address = '{0} {1}'.format(row['Rm. Number'], address)
                 leg.add_office('capitol', 'Capitol Office',
-                    address='{Room} {Address}\n{Address2}\n{City}, {State} '\
-                        '{Zipcode}'.format(Room=row['Rm. Number'], **row),
+                    address=address,
                     email=email, phone=leg.get('office_phone'))
             elif row['Address2']:
                 leg.add_office('district', 'District Office',

openstates/nj/events.py

@@ -39,7 +39,8 @@ def initialize_committees(self, year_abr):
             'JEJ': 'Joint Committee on Economic Justice and Equal Employment Opportunity',
             'LSC': 'Legislative Services Commission',
             'THIE': 'Senate Task Force on Health Insurance Exchange Implementation Committee',
-            'CASC': 'College Affordability Study Commission'
+            'CASC': 'College Affordability Study Commission',
+            'CIR': 'State Commission of Investigation'
         }
         self._committees = overlay