From 119d06b9eb00a905bfb3496756261d4943cf273d Mon Sep 17 00:00:00 2001 From: Tim Showers Date: Sat, 1 Oct 2016 12:03:03 -0400 Subject: [PATCH 1/6] Get NM 2016 ss scraping again --- openstates/nm/__init__.py | 10 +++++++++- openstates/nm/bills.py | 6 ++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/openstates/nm/__init__.py b/openstates/nm/__init__.py index 3e88b64ba1..454617fd65 100644 --- a/openstates/nm/__init__.py +++ b/openstates/nm/__init__.py @@ -39,7 +39,7 @@ 'name': '2015-2016', 'start_year': 2015, 'end_year': 2016, - 'sessions': ['2015', '2015S', '2016'], + 'sessions': ['2015', '2015S', '2016', '2016S'], } ], 'session_details': { @@ -86,6 +86,14 @@ 'slug': '16%20Regular', '_scraped_name': '2016 Regular', }, + '2016S': { + 'type': 'special', + 'start_date': datetime.date(2016, 9, 30), + 'end_date': datetime.date(2016, 10, 1), + 'display_name': '2016 2nd Special Session', + 'slug': '16%20Special', + '_scraped_name': '2016 2nd Special', + }, }, 'feature_flags': ['subjects', 'influenceexplorer'], '_ignored_scraped_sessions': [ diff --git a/openstates/nm/bills.py b/openstates/nm/bills.py index 6f2c3aac38..38ef284c70 100644 --- a/openstates/nm/bills.py +++ b/openstates/nm/bills.py @@ -97,8 +97,10 @@ class NMBillScraper(BillScraper): categorizer = Categorizer() def _init_mdb(self, session): + print session[2:] ftp_base = 'ftp://www.nmlegis.gov/other/' - fname = 'LegInfo{}'.format(session[2:]) + #session slug here? + fname = 'LegInfo{}'.format('16 Special') fname_re = '(\d{{2}}-\d{{2}}-\d{{2}} \d{{2}}:\d{{2}}(?:A|P)M) .* '\ '({fname}.*zip)'.format(fname=fname) @@ -114,7 +116,7 @@ def _init_mdb(self, session): remote_file = ftp_base + matches[-1][1] # all of the data is in this Access DB, download & retrieve it - mdbfile = '%s.mdb' % fname + mdbfile = '%s.accdb' % fname # if a new mdbfile or it has changed if getattr(self, 'mdbfile', None) != mdbfile: From 0d82f122f8f6b44874549175a72bf06de72f0ec4 Mon Sep 17 00:00:00 2001 From: Tim Showers Date: Tue, 18 Oct 2016 10:17:57 -0400 Subject: [PATCH 2/6] Ignore 1800s PA sessions --- openstates/pa/__init__.py | 90 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/openstates/pa/__init__.py b/openstates/pa/__init__.py index ee6c8ef29d..b7b4c4bc9a 100644 --- a/openstates/pa/__init__.py +++ b/openstates/pa/__init__.py @@ -74,6 +74,96 @@ }, }, '_ignored_scraped_sessions': [ + '1965-1966 Special Session #1', + '1965-1966 Special Session #2', + '1965-1966 Special Session #3', + '1963-1964 Regular Session', + '1963-1964 Special Session #1', + '1963-1964 Special Session #2', + '1961-1962 Regular Session', + '1961-1962 Special Session #1', + '1959-1960 Regular Session', + '1957-1958 Regular Session', + '1955-1956 Regular Session', + '1953-1954 Regular Session', + '1951-1952 Regular Session', + '1949-1950 Regular Session', + '1947-1948 Regular Session', + '1945-1946 Regular Session', + '1943-1944 Regular Session', + '1941-1942 Regular Session', + '1939-1940 Regular Session', + '1937-1938 Regular Session', + '1935-1936 Regular Session', + '1933-1934 Regular Session', + '1931-1932 Regular Session', + '1929-1930 Regular Session', + '1927-1928 Regular Session', + '1925-1926 Regular Session', + '1923-1924 Regular Session', + '1921-1922 Regular Session', + '1919-1920 Regular Session', + '1917-1918 Regular Session', + '1915-1916 Regular Session', + '1913-1914 Regular Session', + '1911-1912 Regular Session', + '1909-1910 Regular Session', + '1907-1908 Regular Session', + '1905-1906 Regular Session', + '1903-1904 Regular Session', + '1901-1902 Regular Session', + '1899-1900 Regular Session', + '1897-1898 Regular Session', + '1895-1896 Regular Session', + '1893-1894 Regular Session', + '1891-1892 Regular Session', + '1889-1890 Regular Session', + '1887-1888 Regular Session', + '1885-1886 Regular Session', + '1883-1884 Regular Session', + '1881-1882 Regular Session', + '1879-1880 Regular Session', + '1877-1878 Regular Session', + '1875-1876 Regular Session', + '1873-1874 Regular Session', + '1871-1872 Regular Session', + '1869-1870 Regular Session', + '1867-1868 Regular Session', + '1865-1866 Regular Session', + '1863-1864 Regular Session', + '1861-1862 Regular Session', + '1859-1860 Regular Session', + '1857-1858 Regular Session', + '1855-1856 Regular Session', + '1853-1854 Regular Session', + '1851-1852 Regular Session', + '1849-1850 Regular Session', + '1847-1848 Regular Session', + '1845-1846 Regular Session', + '1843-1844 Regular Session', + '1841-1842 Regular Session', + '1839-1840 Regular Session', + '1837-1838 Regular Session', + '1835-1836 Regular Session', + '1833-1834 Regular Session', + '1831-1832 Regular Session', + '1829-1830 Regular Session', + '1827-1828 Regular Session', + '1825-1826 Regular Session', + '1823-1824 Regular Session', + '1821-1822 Regular Session', + '1819-1820 Regular Session', + '1817-1818 Regular Session', + '1815-1816 Regular Session', + '1813-1814 Regular Session', + '1811-1812 Regular Session', + '1809-1810 Regular Session', + '1807-1808 Regular Session', + '1805-1806 Regular Session', + '1803-1804 Regular Session', + '1801-1802 Regular Session', + '1965-1966 Regular Session', + '1967-1968 Regular Session', '1969-1970 Regular Session', '1971-1972 Regular Session', '1971-1972 Special Session #1', From e8c3dd745a80903b938182650f36968c9fabacd5 Mon Sep 17 00:00:00 2001 From: Tim Showers Date: Tue, 18 Oct 2016 10:18:38 -0400 Subject: [PATCH 3/6] ignore new irrelevant CT ftp folder --- openstates/ct/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openstates/ct/__init__.py b/openstates/ct/__init__.py index a73173b06c..dd2392b9c4 100644 --- a/openstates/ct/__init__.py +++ b/openstates/ct/__init__.py @@ -83,7 +83,7 @@ def session_list(): text = scrapelib.Scraper().get('ftp://ftp.cga.ct.gov').text sessions = [line.split()[-1] for line in text.splitlines()] - for not_session_name in ('incoming', 'pub', 'CGAAudio', 'rba', 'NCSL',"apaac"): + for not_session_name in ('incoming', 'pub', 'CGAAudio', 'rba', 'NCSL',"apaac", 'FOI_1'): sessions.remove(not_session_name) return sessions From d68da693a4609c08bc94ef121b616d377a1cc4f2 Mon Sep 17 00:00:00 2001 From: Tim Showers Date: Tue, 18 Oct 2016 10:20:10 -0400 Subject: [PATCH 4/6] Allow for DC bills without sponsor info --- openstates/dc/bills.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/openstates/dc/bills.py b/openstates/dc/bills.py index b7f2d54cb7..ece13978e5 100644 --- a/openstates/dc/bills.py +++ b/openstates/dc/bills.py @@ -50,6 +50,7 @@ def scrape(self, session, chambers): while len(data) > 0: for bill in data: + bill_versions = [] #sometimes they're in there more than once, so we'll keep track bill_id = bill["Title"] @@ -76,12 +77,18 @@ def scrape(self, session, chambers): bill = Bill(session,"upper", bill_id, title, type=bill_type) #sponsors and cosponsors - introducers = legislation_info["Introducer"] + try: + #sometimes there are sponsors, sometimes not. + introducers = legislation_info["Introducer"] + except KeyError: + introducers = [] + try: #sometimes there are cosponsors, sometimes not. cosponsors = legislation_info["CoSponsor"] except KeyError: cosponsors = [] + for i in introducers: sponsor_name = i["Name"] #they messed up Phil Mendelson's name From d30c3571c80becb0c8f12cd6aee9d70af965705f Mon Sep 17 00:00:00 2001 From: Tim Showers Date: Tue, 18 Oct 2016 10:20:48 -0400 Subject: [PATCH 5/6] Added veto and override actions to MO --- openstates/mo/bills.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/openstates/mo/bills.py b/openstates/mo/bills.py index f523770308..3c42f4ae09 100644 --- a/openstates/mo/bills.py +++ b/openstates/mo/bills.py @@ -54,6 +54,8 @@ def _get_action(self, actor, action): 'Third Read and Passed' : 'bill:passed', 'Signed by Governor' : 'governor:signed', 'Approved by Governor' : 'governor:signed', + 'Vetoed by Governor' : 'governor:vetoed', + 'Legislature voted to override Governor\'s veto': 'bill:veto_override:passed', } found_action = 'other' for flag in flags: From e9e200a9b84f29f3a38f3f19e1ea09052b023823 Mon Sep 17 00:00:00 2001 From: Tim Showers Date: Tue, 18 Oct 2016 10:21:35 -0400 Subject: [PATCH 6/6] Added NV 2016 special and 2017 sessions --- openstates/nv/__init__.py | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/openstates/nv/__init__.py b/openstates/nv/__init__.py index c5c096c796..c03944bf39 100644 --- a/openstates/nv/__init__.py +++ b/openstates/nv/__init__.py @@ -50,8 +50,14 @@ 'name': '2015-2016', 'start_year': 2015, 'end_year': 2016, - 'sessions': ['78', '2015Special29'], + 'sessions': ['78', '2015Special29','2016Special30'], }, + { + 'name': '2017-2018', + 'start_year': 2017, + 'end_year': 2018, + 'sessions': ['79'], + }, ], 'session_details': { '2010Special26': { @@ -92,7 +98,7 @@ '_committee_session': '28th2014Special', 'slug': '28th2014Special', }, - '78': { + '78': { 'type': 'primary', 'start_date': datetime.date(2015, 2, 15), 'end_date': datetime.date(2015, 6, 1), @@ -109,6 +115,23 @@ '_committee_session': '29th2015Special', 'slug': '29th2015Special', }, + '2016Special30': { + 'type': 'special', + 'start_date': datetime.date(2016, 10, 10), + 'end_date': datetime.date(2016, 10, 14), + 'display_name': '30th Special Session (2016)', + '_scraped_name': u'30th (2016) Special Session\xa0\xa0(click to close)', + '_committee_session': '30th2016Special', + 'slug': '30th2016Special', + }, + '79':{ + 'type': 'primary', + 'start_date': datetime.date(2017, 2, 15), + 'end_date': datetime.date(2017, 6, 1), + 'display_name': '2016 Regular Session', + '_scraped_name': u'79th (2017) Session\xa0\xa0(click to close)', + 'slug': '78th2015', + }, }, 'feature_flags': ['subjects', 'capitol_maps', 'influenceexplorer'], 'capitol_maps': [