Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Sainte anne de bellevue scraper #447

Closed
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 41 additions & 16 deletions ca_qc_sainte_anne_de_bellevue/people.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
import re

from utils import CanadianPerson as Person
from utils import CanadianScraper

Expand All @@ -8,24 +6,51 @@

class SainteAnneDeBellevuePersonScraper(CanadianScraper):
def scrape(self):
def decode_email(e):
jpmckinney marked this conversation as resolved.
Show resolved Hide resolved
de = ""
k = int(e[:2], 16)

for i in range(2, len(e) - 1, 2):
de += chr(int(e[i : i + 2], 16) ^ k)

return de

page = self.lxmlize(COUNCIL_PAGE)

councillors = page.xpath('//div[@class="block text"]')
councillors = page.xpath('//div[@class="col-md-12"]')[0]
assert len(councillors), "No councillors found"
for councillor in councillors:
name = councillor.xpath('.//div[@class="content-writable"]//strong/text()')[0]
district = councillor.xpath(".//h2/text()")[0]

if "Maire" in district:
district = "Sainte-Anne-de-Bellevue"
role = "Maire"
else:
district = "District {}".format(re.search(r"\d+", district)[0])
role = "Conseiller"
roles_and_districts = councillors.xpath(".//h2/text()")
roles = []
jpmckinney marked this conversation as resolved.
Show resolved Hide resolved
districts = []
names = []
emails = []

p = Person(primary_org="legislature", name=name, district=district, role=role)
p.add_source(COUNCIL_PAGE)
# Fill in roles and districts via h2 tags
for role in roles_and_districts:
role_and_district = role.split()

roles.append(role_and_district[0])

p.image = councillor.xpath(".//@src")[0]
p.add_contact("email", self.get_email(councillor))
if len(role_and_district) == 1:
districts.append("Sainte-Anne-de-Bellevue")
else:
districts.append("District " + role_and_district[2])

# Fill in contact info via p tags.
contact_info = councillors.xpath('.//p[a[contains(@href, "@")]]')
for contact in contact_info:
contact = contact.text_content().split()
name = " ".join(contact[:2])
names.append(name)

email = contact[3]
email = email.replace("Président", "")
emails.append(email)

assert len(roles) == len(districts) == len(names) == len(emails), "Lists are not of equal length"
for i in range(len(roles)):
p = Person(primary_org="legislature", name=names[i], district=districts[i], role=roles[i])
p.add_source(COUNCIL_PAGE)
p.add_contact("email", emails[i])
yield p