From 7a31c3380ef6aefdba895df678879251df483846 Mon Sep 17 00:00:00 2001 From: Justin Warren Date: Sat, 14 Jan 2023 10:44:08 +1100 Subject: [PATCH] Added support for allowlists. Updated docstring for merge_blocklists() --- src/fediblockhole/__init__.py | 39 +++++++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/src/fediblockhole/__init__.py b/src/fediblockhole/__init__.py index 6cdf143..fdb3a5e 100755 --- a/src/fediblockhole/__init__.py +++ b/src/fediblockhole/__init__.py @@ -35,6 +35,9 @@ API_CALL_DELAY = 5 * 60 / 300 # 300 calls per 5 minutes # We always import the domain and the severity IMPORT_FIELDS = ['domain', 'severity'] +# Allowlists always import these fields +ALLOWLIST_IMPORT_FIELDS = ['domain', 'severity', 'public_comment', 'private_comment', 'reject_media', 'reject_reports', 'obfuscate'] + # We always export the domain and the severity EXPORT_FIELDS = ['domain', 'severity'] @@ -69,6 +72,22 @@ def sync_blocklists(conf: dict): # Merge blocklists into an update dict merged = merge_blocklists(blocklists, conf.mergeplan) + + # Apply allows specified on the commandline + for domain in conf.allow_domains: + log.info(f"Allowing domain '{domain}' specified on commandline.") + merged[domain] = DomainBlock(domain, 'noop') + + # Apply allows from URLs lists + if conf.allowlist_url_sources: + log.info("Adding allows from URL lists...") + allowlists = fetch_from_urls({}, conf.allowlist_url_sources, ALLOWLIST_IMPORT_FIELDS) + for key, alist in allowlists.items(): + log.debug(f"Processing allows from '{key}'...") + for allowed in alist: + merged[allowed.domain] = allowed + log.debug(f"Allowed domain '{allowed.domain}' from allowlist: {allowed}") + if conf.blocklist_savefile: log.info(f"Saving merged blocklist to {conf.blocklist_savefile}") save_blocklist_to_file(merged.values(), conf.blocklist_savefile, export_fields) @@ -140,9 +159,12 @@ def fetch_from_instances(blocklists: dict, sources: dict, def merge_blocklists(blocklists: dict, mergeplan: str='max') -> dict: """Merge fetched remote blocklists into a bulk update + @param blocklists: A dict of lists of DomainBlocks, keyed by source. + Each value is a list of DomainBlocks @param mergeplan: An optional method of merging overlapping block definitions 'max' (the default) uses the highest severity block found 'min' uses the lowest severity block found + @param returns: A dict of DomainBlocks keyed by domain """ merged = {} @@ -433,7 +455,7 @@ def update_known_block(token: str, host: str, block: DomainBlock): response = requests.put(url, headers=requests_headers(token), - data=blockdata, + json=blockdata._asdict(), timeout=REQUEST_TIMEOUT ) if response.status_code != 200: @@ -442,14 +464,14 @@ def update_known_block(token: str, host: str, block: DomainBlock): def add_block(token: str, host: str, blockdata: DomainBlock): """Block a domain on Mastodon host """ - log.debug(f"Blocking domain {blockdata.domain} at {host}...") + log.debug(f"Adding block entry for {blockdata.domain} at {host}...") api_path = "/api/v1/admin/domain_blocks" url = f"https://{host}{api_path}" response = requests.post(url, headers=requests_headers(token), - data=blockdata._asdict(), + json=blockdata._asdict(), timeout=REQUEST_TIMEOUT ) if response.status_code == 422: @@ -515,6 +537,8 @@ def push_blocklist(token: str, host: str, blocklist: list[dict], log.info(f"Pushing new block definition: {newblock}") blockdata = oldblock.copy() blockdata.update(newblock) + log.debug(f"Block as dict: {blockdata._asdict()}") + if not dryrun: update_known_block(token, host, blockdata) # add a pause here so we don't melt the instance @@ -530,6 +554,7 @@ def push_blocklist(token: str, host: str, blocklist: list[dict], # This is a new block for the target instance, so we # need to add a block rather than update an existing one log.info(f"Adding new block: {newblock}...") + log.debug(f"Block as dict: {newblock._asdict()}") # Make sure the new block doesn't clobber a domain with followers newblock.severity = check_followed_severity(host, token, newblock.domain, newblock.severity, max_followed_severity) @@ -615,9 +640,10 @@ def augment_args(args): if not args.import_fields: args.import_fields = conf.get('import_fields', []) - args.blocklist_url_sources = conf.get('blocklist_url_sources') - args.blocklist_instance_sources = conf.get('blocklist_instance_sources') - args.blocklist_instance_destinations = conf.get('blocklist_instance_destinations') + args.blocklist_url_sources = conf.get('blocklist_url_sources', None) + args.blocklist_instance_sources = conf.get('blocklist_instance_sources', None) + args.allowlist_url_sources = conf.get('allowlist_url_sources', None) + args.blocklist_instance_destinations = conf.get('blocklist_instance_destinations', None) return args @@ -637,6 +663,7 @@ def main(): ap.add_argument('-I', '--import-field', dest='import_fields', action='append', help="Extra blocklist fields to import.") ap.add_argument('-E', '--export-field', dest='export_fields', action='append', help="Extra blocklist fields to export.") + ap.add_argument('-A', '--allow', dest="allow_domains", action='append', default=[], help="Override any blocks to allow this domain.") ap.add_argument('--no-fetch-url', dest='no_fetch_url', action='store_true', help="Don't fetch from URLs, even if configured.") ap.add_argument('--no-fetch-instance', dest='no_fetch_instance', action='store_true', help="Don't fetch from instances, even if configured.")