From 7a3192ba0e61386ad8be06ae7f3060be7f6219ae Mon Sep 17 00:00:00 2001 From: bdillo Date: Tue, 13 May 2025 19:43:40 -0700 Subject: [PATCH] update meetup scripts with minor fixes, allow location overriding, support hybrid events going in two sections, remove stale meetup groups --- .../country_code_to_continent.py | 1 + tools/events/meetup-automation/event.py | 33 ++- .../generate_events_meetup.py | 26 +- tools/events/meetup-automation/main.py | 14 +- tools/events/meetup-automation/utils.py | 25 +- tools/events/rust-meetups.json | 241 +++++++++--------- 6 files changed, 191 insertions(+), 149 deletions(-) diff --git a/tools/events/meetup-automation/country_code_to_continent.py b/tools/events/meetup-automation/country_code_to_continent.py index 1c2f482e2..2908a930f 100644 --- a/tools/events/meetup-automation/country_code_to_continent.py +++ b/tools/events/meetup-automation/country_code_to_continent.py @@ -230,6 +230,7 @@ 'TZ': 'Africa', 'UA': 'Europe', 'UG': 'Africa', + 'UK': 'Europe', # not an actual country code, but using it here 'US': 'North America', 'UY': 'South America', 'UZ': 'Asia', diff --git a/tools/events/meetup-automation/event.py b/tools/events/meetup-automation/event.py index 14b08b8cc..a090fcced 100644 --- a/tools/events/meetup-automation/event.py +++ b/tools/events/meetup-automation/event.py @@ -2,6 +2,9 @@ import string from dataclasses import dataclass from datetime import datetime +from typing import Optional + +from utils import LocationOverride logger = logging.getLogger(__name__) @@ -37,6 +40,8 @@ def __post_init__(self): # looks like in GB meetup considers part of the post code as the "state", which is not a common way to write # locations in GB (or that's my understanding at least) self.state = None + # and "UK" is more accurate it seems, so replace "GB" + self.country = "UK" def fields_present(self) -> int: @@ -76,6 +81,7 @@ class Event: virtual: bool organizer_name: str organizer_url: str + hybrid: bool def __post_init__(self): """ Normalize the event data here """ @@ -91,11 +97,17 @@ def to_dict(self) -> dict: "url": self.url, "virtual": self.virtual, "organizer_name": self.organizer_name, - "organizer_url": self.organizer_url + "organizer_url": self.organizer_url, + "hybrid": self.hybrid } def to_markdown_string(self) -> str: - location = f"Virtual ({self.location.to_str()})" if self.virtual else self.location.to_str() + if self.hybrid: + location = f"Hybrid ({self.location.to_str()})" + elif self.virtual: + location = f"Virtual ({self.location.to_str()})" + else: + location = self.location.to_str() return f'* {self.date.date()} | {location} | [{self.organizer_name}]({self.organizer_url})\n * [**{self.name}**]({self.url})' @@ -112,8 +124,6 @@ class RawGqlEvent: event_url_str: str venue_type: None | str event_location: Location - lat: float - long: float def __init__(self, **kwargs) -> None: logger.debug(f"Constructing RawGqlEvent from: {kwargs}") @@ -130,14 +140,18 @@ def __init__(self, **kwargs) -> None: venue = node["venue"] self.venue_type = venue["venueType"] - # TODO: do we need these lat longs? - self.lat = venue["lat"] - self.long = venue["lng"] self.event_location = Location(venue["city"], venue["state"], venue["country"]) - def to_event(self, group_url: str) -> Event: + def to_event(self, group_url: str, location_override: Optional[LocationOverride]) -> Event: + is_hybrid = False is_virtual = self.venue_type == "online" + if location_override: + if location_override == LocationOverride.VIRTUAL: + is_virtual = True + elif location_override == LocationOverride.HYBRID: + is_hybrid = True + # this is a bit weird because we want a naive datetime object that just contains the year/month/day because we get # timestamps with tz info like "2025-01-16T19:00+01:00", just strip the time and tz info before parsing date = datetime.strptime(self.date_time_str.split('T')[0], '%Y-%m-%d') @@ -155,6 +169,7 @@ def to_event(self, group_url: str) -> Event: url=self.event_url_str, virtual=is_virtual, organizer_name=self.group_name, - organizer_url=group_url + organizer_url=group_url, + hybrid=is_hybrid ) diff --git a/tools/events/meetup-automation/generate_events_meetup.py b/tools/events/meetup-automation/generate_events_meetup.py index c6b1f565b..d5d92b6f3 100644 --- a/tools/events/meetup-automation/generate_events_meetup.py +++ b/tools/events/meetup-automation/generate_events_meetup.py @@ -4,8 +4,8 @@ from jwt_auth import generate_signed_jwt from event import Event, RawGqlEvent -from utils import MeetupGroupUrl -from typing import List +from utils import LocationOverride, MeetupGroupUrl +from typing import List, Optional logger = logging.getLogger(__name__) @@ -96,15 +96,27 @@ def _build_event_listing_gql_query(self, group_url_name: str) -> dict: } } - def _parse_event_listing_gql_response(self, response: dict) -> List[RawGqlEvent]: + def _parse_event_listing_gql_response(self, response: dict, location_override: Optional[LocationOverride]) -> List[RawGqlEvent]: edges = response["groupByUrlname"]["upcomingEvents"]["edges"] events = [] # TODO: maybe move this validation somewhere else? for edge_kwargs in edges: if not edge_kwargs["node"]["venue"]: - logger.error(f"Event response missing venue: {edge_kwargs}") - continue + # for events where there's no venue in the response and we have a virtual override, inherit the location from the group here + # a bit gross but oh well + if location_override == LocationOverride.VIRTUAL: + logger.info(f"Overriding location for event: {edge_kwargs}") + venue = {} + venue["city"] = edge_kwargs["node"]["group"]["city"] + venue["state"] = edge_kwargs["node"]["group"]["state"] + venue["country"] = edge_kwargs["node"]["group"]["country"] + venue["venueType"] = "online" + + edge_kwargs["node"]["venue"] = venue + else: + logger.error(f"Event response missing venue: {edge_kwargs}") + continue events.append(RawGqlEvent(**edge_kwargs)) @@ -206,7 +218,7 @@ def get_raw_events_for_group(self, group: MeetupGroupUrl) -> List[RawGqlEvent]: "Content-Type": "application/json", } - logger.info(f"Fetching events for group {group}") + logger.debug(f"Fetching events for group {group}") query = self._build_event_listing_gql_query(group.url_name) response = requests.post(url=self.GQL_ENDPOINT, headers=headers, json=query) data = response.json()["data"] @@ -216,5 +228,5 @@ def get_raw_events_for_group(self, group: MeetupGroupUrl) -> List[RawGqlEvent]: logger.error(f"Group {group} not valid, skipping") return [] - return self._parse_event_listing_gql_response(data) + return self._parse_event_listing_gql_response(data, group.location_override) diff --git a/tools/events/meetup-automation/main.py b/tools/events/meetup-automation/main.py index 7d06414ef..4527e2071 100644 --- a/tools/events/meetup-automation/main.py +++ b/tools/events/meetup-automation/main.py @@ -35,7 +35,7 @@ def main(): for group_url in group_urls: group_raw_events = meetup_client.get_raw_events_for_group(group_url) - events += [raw_event.to_event(group_url.url) for raw_event in group_raw_events] + events += [raw_event.to_event(group_url.url, group_url.location_override) for raw_event in group_raw_events] # Remove events outside of date range. events = date_window_filter(events, args.weeks) @@ -74,7 +74,7 @@ def output_to_screen(event_list): if len(value) == 0: continue else: - print(f'### {key}:\n') + print(f'### {key}') # Output event details for event in value: @@ -104,9 +104,13 @@ def group_virtual_continent(event_list): separated_event_list = {} for event in event_list: - # Separates Events by Virtual or by Continent - key = "Virtual" if event.virtual else country_code_to_continent(event.location.country) - separated_event_list.setdefault(key, []).append(event) + if event.hybrid: + keys = "Virtual", country_code_to_continent(event.location.country) + for key in keys: + separated_event_list.setdefault(key, []).append(event) + else: + key = "Virtual" if event.virtual else country_code_to_continent(event.location.country) + separated_event_list.setdefault(key, []).append(event) return separated_event_list diff --git a/tools/events/meetup-automation/utils.py b/tools/events/meetup-automation/utils.py index df53ae60e..de8eb90b6 100644 --- a/tools/events/meetup-automation/utils.py +++ b/tools/events/meetup-automation/utils.py @@ -1,16 +1,23 @@ from dataclasses import dataclass -from typing import List +from enum import Enum +from typing import List, Optional from urllib.parse import urlparse import json +class LocationOverride(str, Enum): + HYBRID = "hybrid" + VIRTUAL = "virtual" + + @dataclass class MeetupGroupUrl: MEETUP_HOSTNAME = "www.meetup.com" url: str url_name: str + location_override: Optional[LocationOverride] - def __init__(self, url_str: str) -> None: + def __init__(self, url_str: str, location_override: Optional[str]) -> None: parsed = urlparse(url_str) if parsed.hostname != self.MEETUP_HOSTNAME: @@ -23,11 +30,21 @@ def __init__(self, url_str: str) -> None: self.url = url_str self.url_name = path_split[1] - + + if location_override: + self.location_override = LocationOverride(location_override) + else: + self.location_override = None + def read_meetup_group_urls(meetups_json: str) -> List[MeetupGroupUrl]: with open(meetups_json, "r") as f: group_urls = json.loads(f.read()) + parsed_groups = [] + + for url, metadata in group_urls.items(): + location_override = metadata.get("location_override") + parsed = MeetupGroupUrl(url, location_override) + parsed_groups.append(parsed) - parsed_groups = [MeetupGroupUrl(url) for url in group_urls] return parsed_groups diff --git a/tools/events/rust-meetups.json b/tools/events/rust-meetups.json index a751f2a3f..f7b8997c6 100644 --- a/tools/events/rust-meetups.json +++ b/tools/events/rust-meetups.json @@ -1,124 +1,117 @@ -[ - "https://www.meetup.com/bcnrust/events/", - "https://www.meetup.com/belgium-rust-user-group/events/", - "https://www.meetup.com/belgrade-rust-meetup-group/events/", - "https://www.meetup.com/bevy-game-development/events/", - "https://www.meetup.com/bostonrust/events/", - "https://www.meetup.com/boulder-rust-meetup/events/", - "https://www.meetup.com/bratislava-rust-meetup-group/events/", - "https://www.meetup.com/buffalo-rust-meetup/events/", - "https://www.meetup.com/cambridge-rust-meetup/events/", - "https://www.meetup.com/charlottesville-rust-meetup/events/", - "https://www.meetup.com/chicago-rust-meetup/events/", - "https://www.meetup.com/christchurch-rustlang-meetup-group/events/", - "https://www.meetup.com/columbus-rs/events/", - "https://www.meetup.com/copenhagen-rust-community/events/", - "https://www.meetup.com/dallasrust/events/", - "https://www.meetup.com/desert-rustaceans/events/", - "https://www.meetup.com/detroitrust/events/", - "https://www.meetup.com/dutch-rust-meetup/events/", - "https://www.meetup.com/finland-rust-meetup/events/", - "https://www.meetup.com/helsinki-rust-meetup-group/events/", - "https://www.meetup.com/indyrs/events/", - "https://www.meetup.com/johannesburg-rust-meetup/events/", - "https://www.meetup.com/join-srug/events/", - "https://www.meetup.com/london-rust-project-group/events/", - "https://www.meetup.com/mad-rs/events/", - "https://www.meetup.com/madrust/events/", - "https://www.meetup.com/meetup-group-zgphbyet/events/", - "https://www.meetup.com/minneapolis-rust-meetup/events/", - "https://www.meetup.com/montpellier-rust-meetup/events/", - "https://www.meetup.com/music-city-rust-developers/events/", - "https://www.meetup.com/mv-rust-meetup/events/", - "https://www.meetup.com/oc-rust/events/", - "https://www.meetup.com/oxford-rust-meetup-group/events/", - "https://www.meetup.com/paessler-rust-camp-2024/events/", - "https://www.meetup.com/pdxrust/events/", - "https://www.meetup.com/perth-rust-meetup-group/events/", - "https://www.meetup.com/reading-rust-workshop/events/", - "https://www.meetup.com/rust-aarhus/events/", - "https://www.meetup.com/rust-aarhus-organizers/events/", - "https://www.meetup.com/rust-akl/events/", - "https://www.meetup.com/rust-amsterdam/events/", - "https://www.meetup.com/rust-amsterdam-group/events/", - "https://www.meetup.com/rust-and-c-plus-plus-in-cardiff/events/", - "https://www.meetup.com/rust-argentina/events/", - "https://www.meetup.com/rust-atl/events/", - "https://www.meetup.com/rust-atx/events/", - "https://www.meetup.com/rust-basel/events/", - "https://www.meetup.com/rust-bay-area/events/", - "https://www.meetup.com/rust-berlin/events/", - "https://www.meetup.com/rust-bern/events/", - "https://www.meetup.com/rust-boulder-denver/events/", - "https://www.meetup.com/rust-breakfast-learn/events/", - "https://www.meetup.com/rust-brisbane/events/", - "https://www.meetup.com/rust-canberra/events/", - "https://www.meetup.com/rust-chinese-group/events/", - "https://www.meetup.com/rust-cologne-bonn/events/", - "https://www.meetup.com/rust-czech-republic/events/", - "https://www.meetup.com/rust-dublin/events/", - "https://www.meetup.com/rust-edi/events/", - "https://www.meetup.com/rust-frankfurt/events/", - "https://www.meetup.com/rust-gdansk/events/", - "https://www.meetup.com/rust-getting-started/events/", - "https://www.meetup.com/rust-hack-learn-karlsruhe/events/", - "https://www.meetup.com/rust-hyderabad/events/", - "https://www.meetup.com/rust-kw/events/", - "https://www.meetup.com/rust-language-milano/events/", - "https://www.meetup.com/rust-linz/events/", - "https://www.meetup.com/rust-london-user-group/events/", - "https://www.meetup.com/rust-los-angeles/events/", - "https://www.meetup.com/rust-lyon/events/", - "https://www.meetup.com/rust-manchester/events/", - "https://www.meetup.com/rust-medellin/events/", - "https://www.meetup.com/rust-meetup-augsburg/events/", - "https://www.meetup.com/rust-meetup-hamburg/events/", - "https://www.meetup.com/rust-melbourne/events/", - "https://www.meetup.com/rust-modern-systems-programming-in-leipzig/events/", - "https://www.meetup.com/rust-montreal/events/", - "https://www.meetup.com/rust-moravia/events/", - "https://www.meetup.com/rust-munich/events/", - "https://www.meetup.com/rust-mx/events/", - "https://www.meetup.com/rust-nederland/events/", - "https://www.meetup.com/rust-noris/events/", - "https://www.meetup.com/rust-nyc/events/", - "https://www.meetup.com/rust-oslo/events/", - "https://www.meetup.com/rust-paris/events/", - "https://www.meetup.com/rust-prague/events/", - "https://www.meetup.com/rust-pune/events/", - "https://www.meetup.com/rust-rhein-main/events/", - "https://www.meetup.com/rust-roma/events/", - "https://www.meetup.com/rust-saar/events/", - "https://www.meetup.com/rust-sao-paulo-meetup/events/", - "https://www.meetup.com/rust-seoul-meetup/events/", - "https://www.meetup.com/rust-singapore/events/", - "https://www.meetup.com/rust-study-group/events/", - "https://www.meetup.com/rust-sydney/events/", - "https://www.meetup.com/rust-tlv/events/", - "https://www.meetup.com/rust-toronto/events/", - "https://www.meetup.com/rust-trondheim/events/", - "https://www.meetup.com/rust-uruguay/events/", - "https://www.meetup.com/rust-vienna/events/", - "https://www.meetup.com/rust-warsaw/events/", - "https://www.meetup.com/rust-wellington/events/", - "https://www.meetup.com/rust-wroclaw/events/", - "https://www.meetup.com/rust-zurich/events/", - "https://www.meetup.com/rustcologne/events/", - "https://www.meetup.com/rustdc/events/", - "https://www.meetup.com/rustdelhi/events/", - "https://www.meetup.com/rustgbg/events/", - "https://www.meetup.com/rustox/events/", - "https://www.meetup.com/ruststhlm/events/", - "https://www.meetup.com/san-diego-rust/events/", - "https://www.meetup.com/san-francisco-rust-study-group/events/", - "https://www.meetup.com/spokane-rust/events/", - "https://www.meetup.com/stl-rust/events/", - "https://www.meetup.com/stockholm-rust/events/", - "https://www.meetup.com/tokyo-rust-meetup/events/", - "https://www.meetup.com/triangle-rust/events/", - "https://www.meetup.com/utah-rust/events/", - "https://www.meetup.com/vancouver-rust/events/", - "https://www.meetup.com/wellington-rust-meetup/events/", - "https://www.meetup.com/women-in-rust/events/", - "https://www.meetup.com/zagreb-rust-meetup/events/" -] +{ + "https://www.meetup.com/bcnrust": {}, + "https://www.meetup.com/belgium-rust-user-group": {}, + "https://www.meetup.com/belgrade-rust-meetup-group": {}, + "https://www.meetup.com/bevy-game-development": {}, + "https://www.meetup.com/bostonrust": {}, + "https://www.meetup.com/boulder-rust-meetup": {}, + "https://www.meetup.com/bratislava-rust-meetup-group": {}, + "https://www.meetup.com/buffalo-rust-meetup": {}, + "https://www.meetup.com/cambridge-rust-meetup": {}, + "https://www.meetup.com/charlottesville-rust-meetup": {}, + "https://www.meetup.com/chicago-rust-meetup": {}, + "https://www.meetup.com/christchurch-rustlang-meetup-group": {}, + "https://www.meetup.com/columbus-rs": {}, + "https://www.meetup.com/copenhagen-rust-community": {}, + "https://www.meetup.com/dallasrust": {}, + "https://www.meetup.com/desert-rustaceans": {}, + "https://www.meetup.com/detroitrust": {}, + "https://www.meetup.com/dutch-rust-meetup": {}, + "https://www.meetup.com/indyrs": {}, + "https://www.meetup.com/johannesburg-rust-meetup": {}, + "https://www.meetup.com/join-srug": {"location_override": "hybrid"}, + "https://www.meetup.com/london-rust-project-group": {}, + "https://www.meetup.com/madrust": {}, + "https://www.meetup.com/meetup-group-zgphbyet": {}, + "https://www.meetup.com/minneapolis-rust-meetup": {}, + "https://www.meetup.com/montpellier-rust-meetup": {}, + "https://www.meetup.com/music-city-rust-developers": {}, + "https://www.meetup.com/mv-rust-meetup": {}, + "https://www.meetup.com/oc-rust": {}, + "https://www.meetup.com/oxford-rust-meetup-group": {}, + "https://www.meetup.com/paessler-rust-camp-2024": {}, + "https://www.meetup.com/pdxrust": {}, + "https://www.meetup.com/perth-rust-meetup-group": {}, + "https://www.meetup.com/reading-rust-workshop": {}, + "https://www.meetup.com/rust-aarhus": {}, + "https://www.meetup.com/rust-aarhus-organizers": {}, + "https://www.meetup.com/rust-amsterdam": {}, + "https://www.meetup.com/rust-amsterdam-group": {}, + "https://www.meetup.com/rust-and-c-plus-plus-in-cardiff": {}, + "https://www.meetup.com/rust-argentina": {}, + "https://www.meetup.com/rust-atl": {}, + "https://www.meetup.com/rust-atx": {}, + "https://www.meetup.com/rust-basel": {}, + "https://www.meetup.com/rust-bay-area": {}, + "https://www.meetup.com/rust-berlin": {}, + "https://www.meetup.com/rust-bern": {}, + "https://www.meetup.com/rust-boulder-denver": {}, + "https://www.meetup.com/rust-breakfast-learn": {}, + "https://www.meetup.com/rust-brisbane": {}, + "https://www.meetup.com/rust-canberra": {}, + "https://www.meetup.com/rust-cologne-bonn": {}, + "https://www.meetup.com/rust-czech-republic": {}, + "https://www.meetup.com/rust-dublin": {}, + "https://www.meetup.com/rust-edi": {}, + "https://www.meetup.com/rust-frankfurt": {}, + "https://www.meetup.com/rust-gdansk": {}, + "https://www.meetup.com/rust-getting-started": {}, + "https://www.meetup.com/rust-hack-learn-karlsruhe": {}, + "https://www.meetup.com/rust-hyderabad": {}, + "https://www.meetup.com/rust-kw": {}, + "https://www.meetup.com/rust-language-milano": {}, + "https://www.meetup.com/rust-linz": {}, + "https://www.meetup.com/rust-london-user-group": {}, + "https://www.meetup.com/rust-los-angeles": {}, + "https://www.meetup.com/rust-lyon": {}, + "https://www.meetup.com/rust-manchester": {}, + "https://www.meetup.com/rust-medellin": {}, + "https://www.meetup.com/rust-meetup-hamburg": {}, + "https://www.meetup.com/rust-melbourne": {}, + "https://www.meetup.com/rust-modern-systems-programming-in-leipzig": {}, + "https://www.meetup.com/rust-montreal": {}, + "https://www.meetup.com/rust-moravia": {}, + "https://www.meetup.com/rust-munich": {}, + "https://www.meetup.com/rust-mx": {}, + "https://www.meetup.com/rust-nederland": {}, + "https://www.meetup.com/rust-noris": {"location_override": "virtual"}, + "https://www.meetup.com/rust-nyc": {}, + "https://www.meetup.com/rust-oslo": {}, + "https://www.meetup.com/rust-paris": {}, + "https://www.meetup.com/rust-prague": {}, + "https://www.meetup.com/rust-pune": {}, + "https://www.meetup.com/rust-rhein-main": {}, + "https://www.meetup.com/rust-roma": {}, + "https://www.meetup.com/rust-sao-paulo-meetup": {}, + "https://www.meetup.com/rust-seoul-meetup": {}, + "https://www.meetup.com/rust-singapore": {}, + "https://www.meetup.com/rust-study-group": {}, + "https://www.meetup.com/rust-sydney": {}, + "https://www.meetup.com/rust-tlv": {}, + "https://www.meetup.com/rust-toronto": {}, + "https://www.meetup.com/rust-trondheim": {}, + "https://www.meetup.com/rust-uruguay": {}, + "https://www.meetup.com/rust-vienna": {}, + "https://www.meetup.com/rust-warsaw": {}, + "https://www.meetup.com/rust-wellington": {}, + "https://www.meetup.com/rust-wroclaw": {}, + "https://www.meetup.com/rust-zurich": {}, + "https://www.meetup.com/rustcologne": {}, + "https://www.meetup.com/rustdc": {}, + "https://www.meetup.com/rustdelhi": {}, + "https://www.meetup.com/rustgbg": {}, + "https://www.meetup.com/rustox": {}, + "https://www.meetup.com/ruststhlm": {}, + "https://www.meetup.com/san-diego-rust": {}, + "https://www.meetup.com/san-francisco-rust-study-group": {}, + "https://www.meetup.com/spokane-rust": {}, + "https://www.meetup.com/stl-rust": {}, + "https://www.meetup.com/stockholm-rust": {}, + "https://www.meetup.com/tokyo-rust-meetup": {}, + "https://www.meetup.com/triangle-rust": {}, + "https://www.meetup.com/utah-rust": {}, + "https://www.meetup.com/vancouver-rust": {"location_override": "hybrid"}, + "https://www.meetup.com/wellington-rust-meetup": {}, + "https://www.meetup.com/women-in-rust": {}, + "https://www.meetup.com/zagreb-rust-meetup": {} +}