#!/usr/bin/env python # coding: utf-8 import os.path from googleapiclient.discovery import build from google_auth_oauthlib.flow import InstalledAppFlow from google.auth.transport.requests import Request from google.oauth2.credentials import Credentials import zipfile from dateutil.parser import isoparse from datetime import datetime from typing import List, Optional, Tuple from io import BytesIO import re import json import string import requests from dataclasses import dataclass from pytz import timezone import logging DP_SETTINGS = {"RETURN_AS_TIMEZONE_AWARE": True, "TIMEZONE": "Europe/Amsterdam", "PREFER_DATES_FROM": "future", } CONFIG = json.load(open('config.json', 'r')) SCOPES = ['https://www.googleapis.com/auth/drive', 'https://www.googleapis.com/auth/documents'] TRELLO_BOARD_ID = "604b3d9c6745546d9e8be6f1" TRELLO_LIST2ID = {'todo': '604b3da85aa63a1340fc8d9d', 'promemorie': '604b3da6fbaeef4d725f1e02', 'inprogress': '604b3daae0b0202099873f3d', 'done': '604b3dac6709b0875760d298', 'cancelled': '604b3db0d55d2e854bbece55'} TRELLO_ID2LIST = {v:k for k, v in TRELLO_LIST2ID.items()} TRELLO_ID2USER = {'6207aa8ca50ed014ea86a841': 'Frederick', '604b41197bcdec22c5caeb91': 'Maxim', '5eea09dbfac2c6072b623132': 'Robin', '510a1d94181035dc6a0019cd': 'Sam', '608077885d1ae137115c9cf4': 'Anniek', '5de634800015827d346548df': 'Mund'} TRELLO_USER2ID = {v:k for k, v in TRELLO_ID2USER.items()} EVERYONE = "Sam Robin Anniek Mund Frederick".split() CF_TASK_ID = "605cce8b29fb595d45fe065c" TASK_REGEX = re.compile(r"\s*(?:ACTIE)(?P [^\n:]+)?:\s*(?P.*?)\s*$") PROMEMORIE_REGEX = re.compile(r"\s*(?:PRO ?MEMORIE)(?P [^\n:]+)?:\s*(?P.*?)\s*$") TASKS_REGEX = re.compile(r"\s*(?:ACTIES)(?P [^\n:]*)?:\s*(?P.*?)\s*$") DONE_REGEX = re.compile(r"\s*(?:DONE):? ([^\n\r]*)") CANCEL_REGEX = re.compile(r"\s*(?:CANCEL):? ([^\n\r]*)") CURTASK_REGEX = re.compile(r"^\s*(?P[-~*+^])\s*\[(?P[A-Z]{4})\]\s*(?P[^\n:]+):\s*(?P.*?)\s*$") LOCAL = timezone('Europe/Amsterdam') logger = logging.getLogger("notulenscraper") # we want to get some kind of semi-random-looking item from a set of items # how do we do that? # idea from group theory: take a group and use a generator to generate items # group: Z*p with p = 456959 (largest prime under 26**4). # generator: 19 # task id for integer x: g^(x + g) mod p as a four-character uppercase A-Z string # why add g in the exponent? because the first exponents are going to be small # (less than our modulus) and therefore not very random-looking. # this algorithm was brought to you by doing a little too much number theory based crypto def int_to_task_id(n): p, g = 456959, 19 res = pow(g, n + g, p) task_id = "" for _ in range(4): task_id += string.ascii_uppercase[res % 26] res //= 26 return task_id[::-1] def card_to_task_id(card): """ Extract the task ID from a Trello API card structure """ fields = card['customFieldItems'] assert len(fields) > 0 assert fields[0]['idCustomField'] == CF_TASK_ID # field id of field 'Task ID' return fields[0]['value']['text'] def get_trello(get_card_cache=True): with open('trello_credentials.json') as f: data = json.load(f) trello = Trello(data['key'], data['token'], get_card_cache) return trello def get_service(): creds = None if os.path.exists('token.json'): creds = Credentials.from_authorized_user_file('token.json', SCOPES) if not creds or not creds.valid: if creds and creds.expired and creds.refresh_token: creds.refresh(Request()) else: flow = InstalledAppFlow.from_client_secrets_file('credentials.json', SCOPES) creds = flow.run_local_server(port=0) with open('token.json', 'w') as token: token.write(creds.to_json()) service = build('drive', 'v3', credentials=creds) return service def get_docs_service(): creds = None if os.path.exists('token.json'): creds = Credentials.from_authorized_user_file('token.json', SCOPES) if not creds or not creds.valid: if creds and creds.expired and creds.refresh_token: creds.refresh(Request()) else: flow = InstalledAppFlow.from_client_secrets_file('credentials.json', SCOPES) creds = flow.run_local_server(port=0) with open('token.json', 'w') as token: token.write(creds.to_json()) service = build('docs', 'v1', credentials=creds) return service class TaskIdSource: def __init__(self): self._trello = get_trello(False) def __enter__(self): desc = self._trello.get(f'/boards/{TRELLO_BOARD_ID}')['desc'] self._id = int(desc.split('next_id=')[1]) def __next__(self): id = self._id self._id += 1 return int_to_task_id(id) def __exit__(self, *args): new_desc = f"# do not edit!\nnext_id={self._id}" self._trello.put(f'/boards/{TRELLO_BOARD_ID}', desc=new_desc) @dataclass class Card: name: str list: str assignees: List[str] description: str = None due_date: Optional[datetime] = None task_id: Optional[str] = None trello_id: Optional[str] = None last_modified: Optional[datetime] = None @staticmethod def deserialize(data): return Card(name=data['name'], list=TRELLO_ID2LIST[data['idList']], assignees=Card._ids_to_assignees(data['idMembers']), description=data['desc'], due_date=isoparse(data['due']).astimezone(LOCAL) if data['due'] else None, task_id=card_to_task_id(data), trello_id=data['id'], last_modified=isoparse(data['dateLastActivity']).astimezone(LOCAL)) @staticmethod def _ids_to_assignees(ids): return [*map(TRELLO_ID2USER.get, ids)] def _assignees_to_ids(self): return [*map(TRELLO_USER2ID.get, self.assignees)] def serialize(self): if not self.task_id: with task_id_source: self.task_id = next(task_id_source) return {'name': self.name, 'idList': TRELLO_LIST2ID[self.list], 'idMembers': self._assignees_to_ids(), 'due': self.due_date.isoformat() if self.due_date else None, 'desc': self.description, 'taskId': self.task_id, 'id': self.trello_id} def _format_assignees(self): num_assignees = len(self.assignees) if num_assignees == 0: return "" if num_assignees == len(TRELLO_USER2ID): return "Iedereen: " return f"{', '.join(self.assignees)}: " def __str__(self): output = "" if self.task_id: output += f"[{self.task_id}] " output += self._format_assignees() + self.name if self.due_date: output += f" (op: {self.due_date.strftime('%Y-%m-%d')})" return output class Trello: def __init__(self, key, token, get_card_cache=True): self._key = key self._token = token self._base = "https://api.trello.com/1" self._card_cache = {} if get_card_cache: self.invalidate_card_cache() def invalidate_card_cache(self): """ Update the list of cards this instance holds. """ self._card_cache.clear() cards = self.get(f"/boards/{TRELLO_BOARD_ID}/cards", customFieldItems="true") for card in cards: task_id = card_to_task_id(card) if task_id in self._card_cache: logger.warning(f"Duplicate task with ID {task_id}") self._card_cache[task_id] = card def get(self, endpoint, **kwargs): kwargs.update({'key': self._key, 'token': self._token}) return requests.get(self._base + endpoint, params=kwargs, data=None).json() def post(self, endpoint, **kwargs): kwargs.update({'key': self._key, 'token': self._token}) return requests.post(self._base + endpoint, params=kwargs, data=None).json() def put(self, endpoint, data=None, **kwargs): kwargs.update({'key': self._key, 'token': self._token}) return requests.put(self._base + endpoint, params=kwargs, json=data).json() def delete(self, endpoint, **kwargs): kwargs.update({'key': self._key, 'token': self._token}) return requests.delete(self._base + endpoint, params=kwargs, data=None).json() def get_card(self, task_id) -> Optional[Card]: card_data = self._card_cache.get(task_id) return Card.deserialize(card_data) if card_data else None def get_cards(self, fresh=False) -> List[Card]: if fresh: self.invalidate_card_cache() return [Card.deserialize(data) for data in self._card_cache.values()] def insert_card(self, card: Card): data = card.serialize() if data['id']: # this is not a new card self.put(f"/cards/{data['id']}", data=None, **data) else: res = self.post(f"/cards", **data) self.put(f"/cards/{res['id']}/customfield/{CF_TASK_ID}/item", data={'value': {'text': card.task_id}}) def get_minutes(service): """ Get a list of all minutes ordered by date last modified. """ res = service.files().list(fields="nextPageToken, files(id, name, modifiedTime)", pageSize=100, supportsAllDrives=True, q="name contains 'Notulen'", orderBy='modifiedTime desc', corpora="drive", driveId='0AP7Ynv3qJ_CwUk9PVA', includeItemsFromAllDrives=True).execute() file_list = res.get('files', []) if not file_list: raise ValueError("Unable to get file list?") file_list.sort(key=lambda f: isoparse(f['modifiedTime']), reverse=True) return [f for f in file_list if f['name'].endswith('Notulen')] def get_file_text(service, file_id): res = service.files().export(fileId=file_id, mimeType='text/plain').execute() return res.decode('utf-8')[1:] class Transaction: """ A set of modifications to make to the Trello state. """ def __init__(self): self.edits = [] self.moves = [] self.creates = [] def edit_task(self, line, task_id, **kwargs): """ Add an edit, modifying the Card identified by task_id by setting fields to values as specified by kwargs. """ self.edits.append((line, task_id, kwargs)) def move_to_list(self, line, task_id, list): self.moves.append((line, task_id, list)) def create_task(self, line, **kwargs): self.creates.append((line, Card(**kwargs))) def validate(self, trello): """ Validate this transaction w.r.t. the given Trello instance. """ errors = [] trello.invalidate_card_cache() for (line, task_id, field_changes) in self.edits: if task_id not in trello._card_cache: errors += [(line, f"Unknown card {task_id}")] if any(field not in Card.__dataclass_fields__.keys() for field in field_changes.keys()): errors += [(line, "Updating unknown field!")] for (line, task_id, list) in self.moves: if task_id not in trello._card_cache: errors += [(line, f"Unknown card {task_id}")] return errors def changes(self, trello): """ Returns the set of changes as (type, card_to_insert) pairs. """ if self.validate(trello): raise ValueError("There are validation errors, you can't perform this transaction!") for (_, task_id, field_changes) in self.edits: card = trello.get_card(task_id) for k, v in field_changes.items(): setattr(card, k, v) yield ('edit', card) for (_, task_id, new_list) in self.moves: card = trello.get_card(task_id) card.list = new_list yield ('move', card) for (_, card) in self.creates: yield ('create', card) def split_out_date(string) -> Tuple[str, Optional[List[datetime]]]: dates = re.findall(r"((?:deadline:? )?\b((?:\d{4}-)?\d{2}-\d{2})\b.?)\.?", string, flags=re.IGNORECASE) if not dates: return string, None now = datetime.now() dtimes = [] for _, date in dates: if date.count('-') == 1: month, day = map(int, date.split('-')) year = now.year if (month < now.month and day < now.day): year += 1 else: year, month, day = map(int, date.split('-')) dtime = datetime(year=year, month=month, day=day, hour=23, minute=59) dtimes.append(LOCAL.localize(dtime)) for source_date, _ in dates: string = string.replace(source_date, '') string = string.replace(' ', ' ') string = string.strip() return string, dtimes[0] def parse_minutes(content) -> Transaction: """ Parse the specified minutes for tasks and return a transaction. """ transaction = Transaction() desugar_users = lambda string: EVERYONE if string.strip().lower() == 'iedereen' else [s.strip().title() for s in string.split(',')] for i, line in enumerate(content.splitlines()): if 'veranderingen actiepunten' in line.lower(): # stop here break for flag, task_id, names, task in CURTASK_REGEX.findall(line): task, due_date = split_out_date(task) if flag == '-': transaction.move_to_list(line, task_id, 'cancelled') elif flag == '~': transaction.move_to_list(line, task_id, 'done') elif flag == '*': transaction.edit_task(line, task_id, name=task, assignees=desugar_users(names), due_date=due_date) elif flag == '+': transaction.move_to_list(line, task_id, 'todo') elif flag == '^': transaction.move_to_list(line, task_id, 'promemorie') # Pro memorie for names, task in PROMEMORIE_REGEX.findall(line): task, due_date = split_out_date(task) transaction.create_task(line, name=task, assignees=desugar_users(names), due_date=due_date, list='promemorie') # Single task for multiple users. for names, task in TASK_REGEX.findall(line): task, due_date = split_out_date(task) transaction.create_task(line, name=task, assignees=desugar_users(names), due_date=due_date, list='todo') # Single task for individual users. for names, task in TASKS_REGEX.findall(line): users = desugar_users(names) task, due_date = split_out_date(task) for user in users: transaction.create_task(line, name=task, assignees=[user], due_date=due_date, list='todo') # Mark a comma separated list as done. for task_id_list in DONE_REGEX.findall(line): for task_id in task_id_list.strip().split(","): task_id = task_id.strip() transaction.move_to_list(line, task_id, 'done') # Mark a comma separated list as removed. for task_id_list in CANCEL_REGEX.findall(line): for task_id in task_id_list.strip().split(","): task_id = task_id.strip() transaction.move_to_list(line, task_id, 'cancelled') return transaction task_id_source = TaskIdSource()