Skip to content

Commit

Permalink
GA: Implemented proper bill type detection.
Browse files Browse the repository at this point in the history
  • Loading branch information
Andy Lo committed Jul 25, 2016
1 parent 3614b9a commit f911c67
Showing 1 changed file with 123 additions and 114 deletions.
237 changes: 123 additions & 114 deletions openstates/ga/bills.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,17 +23,17 @@


member_cache = {}
SOURCE_URL = "http://www.legis.ga.gov/Legislation/en-US/display/{session}/{bid}"
SOURCE_URL = 'http://www.legis.ga.gov/Legislation/en-US/display/{session}/{bid}'


class GABillScraper(BillScraper):
jurisdiction = 'ga'
lservice = get_client("Legislation").service
vservice = get_client("Votes").service
mservice = get_client("Members").service
lsource = get_url("Legislation")
msource = get_url("Members")
vsource = get_url("Votes")
lservice = get_client('Legislation').service
vservice = get_client('Votes').service
mservice = get_client('Members').service
lsource = get_url('Legislation')
msource = get_url('Members')
vsource = get_url('Votes')

def get_member(self, member_id):
if member_id in member_cache:
Expand All @@ -44,33 +44,110 @@ def get_member(self, member_id):
return mem

def scrape(self, session, chambers):
bill_type_map = {
'B': 'bill',
'R': 'resolution',
'JR': 'joint resolution',
'CR': 'concurrent resolution',
}

chamber_map = {
'H': 'lower',
'S': 'upper',
'J': 'joint',
'E': 'other', # Effective date
}

action_code_map = {
'HI': ['other'],
'SI': ['other'],
'HH': ['other'],
'SH': ['other'],
'HPF': ['bill:introduced'],
'HDSAS': ['other'],
'SPF': ['bill:introduced'],
'HSR': ['bill:reading:2'],
'SSR': ['bill:reading:2'],
'HFR': ['bill:reading:1'],
'SFR': ['bill:reading:1'],
'HRECM': ['bill:withdrawn', 'committee:referred'],
'SRECM': ['bill:withdrawn', 'committee:referred'],
'SW&C': ['bill:withdrawn', 'committee:referred'],
'HW&C': ['bill:withdrawn', 'committee:referred'],
'HRA': ['bill:passed'],
'SRA': ['bill:passed'],
'HPA': ['bill:passed'],
'HRECO': ['other'],
'SPA': ['bill:passed'],
'HTABL': ['other'], # 'House Tabled' - what is this?
'SDHAS': ['other'],
'HCFR': ['committee:passed:favorable'],
'SCFR': ['committee:passed:favorable'],
'HRAR': ['committee:referred'],
'SRAR': ['committee:referred'],
'STR': ['bill:reading:3'],
'SAHAS': ['other'],
'SE': ['bill:passed'],
'SR': ['committee:referred'],
'HTRL': ['bill:reading:3', 'bill:failed'],
'HTR': ['bill:reading:3'],
'S3RLT': ['bill:reading:3', 'bill:failed'],
'HASAS': ['other'],
'S3RPP': ['other'],
'STAB': ['other'],
'SRECO': ['other'],
'SAPPT': ['other'],
'HCA': ['other'],
'HNOM': ['other'],
'HTT': ['other'],
'STT': ['other'],
'SRECP': ['other'],
'SCRA': ['other'],
'SNOM': ['other'],
'S2R': ['bill:reading:2'],
'H2R': ['bill:reading:2'],
'SENG': ['bill:passed'],
'HENG': ['bill:passed'],
'HPOST': ['other'],
'HCAP': ['other'],
'SDSG': ['governor:signed'],
'SSG': ['governor:received'],
'Signed Gov': ['governor:signed'],
'HDSG': ['governor:signed'],
'HSG': ['governor:received'],
'EFF': ['other'],
'HRP': ['other'],
'STH': ['other'],
'HTS': ['other'],
}

sid = self.metadata['session_details'][session]['_guid']
legislation = backoff(
self.lservice.GetLegislationForSession,
sid
)['LegislationIndex']

for leg in legislation:
lid = leg['Id']
instrument = backoff(self.lservice.GetLegislationDetail, lid)
history = [x for x in instrument['StatusHistory'][0]]

actions = reversed([{
"code": x['Code'],
"action": x['Description'],
"_guid": x['Id'],
"date": x['Date']
'code': x['Code'],
'action': x['Description'],
'_guid': x['Id'],
'date': x['Date']
} for x in history])

guid = instrument['Id']

bill_type = instrument['DocumentType']
chamber = {
"H": "lower",
"S": "upper",
"J": "joint"
}[bill_type[0]] # XXX: This is a bit of a hack.
# A little bit hacky.
bill_prefix = instrument['DocumentType']
bill_chamber = chamber_map[bill_prefix[0]]
bill_type = bill_type_map[bill_prefix[1:]]

bill_id = "%s %s" % (
bill_type,
bill_id = '%s %s' % (
bill_prefix,
instrument['Number'],
)
if instrument['Suffix']:
Expand All @@ -82,35 +159,29 @@ def scrape(self, session, chambers):
if title is None:
continue

bill = Bill(
session,
chamber,
bill_id,
title,
description=description,
_guid=guid
)
bill = Bill(session, bill_chamber, bill_id, title, type=bill_type,
description=description, _guid=guid)

if instrument['Votes']:
for vote_ in instrument['Votes']:
_, vote_ = vote_
vote_ = backoff(self.vservice.GetVote, vote_[0]['VoteId'])

vote = Vote(
{"House": "lower", "Senate": "upper"}[vote_['Branch']],
{'House': 'lower', 'Senate': 'upper'}[vote_['Branch']],
vote_['Date'],
vote_['Caption'] or "Vote on Bill",
vote_['Caption'] or 'Vote on Bill',
(vote_['Yeas'] > vote_['Nays']),
vote_['Yeas'],
vote_['Nays'],
(vote_['Excused'] + vote_['NotVoting']),
session=session,
bill_id=bill_id,
bill_chamber=chamber)
bill_chamber=bill_chamber)

vote.add_source(self.vsource)

methods = {"Yea": vote.yes, "Nay": vote.no,}
methods = {'Yea': vote.yes, 'Nay': vote.no,}

for vdetail in vote_['Votes'][0]:
whom = vdetail['Member']
Expand All @@ -123,100 +194,38 @@ def scrape(self, session, chambers):

bill.add_vote(vote)


types = {
"HI": ["other"],
"SI": ["other"],
"HH": ["other"],
"SH": ["other"],
"HPF": ["bill:introduced"],
"HDSAS": ["other"],
"SPF": ["bill:introduced"],
"HSR": ["bill:reading:2"],
"SSR": ["bill:reading:2"],
"HFR": ["bill:reading:1"],
"SFR": ["bill:reading:1"],
"HRECM": ["bill:withdrawn", "committee:referred"],
"SRECM": ["bill:withdrawn", "committee:referred"],
"SW&C": ["bill:withdrawn", "committee:referred"],
"HW&C": ["bill:withdrawn", "committee:referred"],
"HRA": ["bill:passed"],
"SRA": ["bill:passed"],
"HPA": ["bill:passed"],
"HRECO": ["other"],
"SPA": ["bill:passed"],
"HTABL": ["other"], # "House Tabled" - what is this?
"SDHAS": ["other"],
"HCFR": ["committee:passed:favorable"],
"SCFR": ["committee:passed:favorable"],
"HRAR": ["committee:referred"],
"SRAR": ["committee:referred"],
"STR": ["bill:reading:3"],
"SAHAS": ["other"],
"SE": ["bill:passed"],
"SR": ["committee:referred"],
"HTRL": ["bill:reading:3", "bill:failed"],
"HTR": ["bill:reading:3"],
"S3RLT": ["bill:reading:3", "bill:failed"],
"HASAS": ["other"],
"S3RPP": ["other"],
"STAB": ["other"],
"SRECO": ["other"],
"SAPPT": ["other"],
"HCA": ["other"],
"HNOM": ["other"],
"HTT": ["other"],
"STT": ["other"],
"SRECP": ["other"],
"SCRA": ["other"],
"SNOM": ["other"],
"S2R": ["bill:reading:2"],
"H2R": ["bill:reading:2"],
"SENG": ["bill:passed"],
"HENG": ["bill:passed"],
"HPOST": ["other"],
"HCAP": ["other"],
"SDSG": ["governor:signed"],
"SSG": ["governor:received"],
"Signed Gov": ["governor:signed"],
"HDSG": ["governor:signed"],
"HSG": ["governor:received"],
"EFF": ["other"],
"HRP": ["other"],
"STH": ["other"],
"HTS": ["other"],
}

ccommittees = defaultdict(list)
committees = instrument['Committees']
if committees:
for committee in committees[0]:
ccommittees[{
"House": "lower",
"Senate": "upper",
'House': 'lower',
'Senate': 'upper',
}[committee['Type']]].append(committee['Name'])

for action in actions:
chamber = {
"H": "lower",
"S": "upper",
"E": "other", # Effective Date
}[action['code'][0]]
action_chamber = chamber_map[action['code'][0]]

try:
_types = types[action['code']]
action_types = action_code_map[action['code']]
except KeyError:
self.debug(action)
_types = ["other"]
error_msg = ('Code {code} for action {action} not '
'recognized.'.format(
code=action['code'],
action=action['action']))

self.logger.warning(error_msg)

action_types = ['other']

committees = []
if any(('committee' in x for x in _types)):
committees = [str(x) for x in ccommittees.get(chamber, [])]
if any(('committee' in x for x in action_types)):
committees = [str(x) for x in ccommittees.get(
action_chamber, [])]

bill.add_action(chamber, action['action'], action['date'], _types,
committees=committees,
_code=action['code'],
_code_id=action['_guid'])
bill.add_action(action_chamber, action['action'],
action['date'], action_types, committees=committees,
_code=action['code'], _code_id=action['_guid'])

sponsors = []
if instrument['Authors']:
Expand All @@ -229,7 +238,7 @@ def scrape(self, session, chambers):
]

for typ, sponsor in sponsors:
name = "{First} {Last}".format(**dict(sponsor['Name']))
name = '{First} {Last}'.format(**dict(sponsor['Name']))
bill.add_sponsor(
'primary' if 'Author' in typ else 'seconday',
name
Expand Down Expand Up @@ -261,7 +270,7 @@ def scrape(self, session, chambers):
bill.add_source(self.msource)
bill.add_source(self.lsource)
bill.add_source(SOURCE_URL.format(**{
"session": session,
"bid": guid,
'session': session,
'bid': guid,
}))
self.save_bill(bill)

0 comments on commit f911c67

Please sign in to comment.