Added support for allowlists.

Updated docstring for merge_blocklists()

src/fediblockhole/__init__.py

@@ -35,6 +35,9 @@ API_CALL_DELAY = 5 * 60 / 300 # 300 calls per 5 minutes
 # We always import the domain and the severity
 IMPORT_FIELDS = ['domain', 'severity']
 
+# Allowlists always import these fields
+ALLOWLIST_IMPORT_FIELDS = ['domain', 'severity', 'public_comment', 'private_comment', 'reject_media', 'reject_reports', 'obfuscate']
+
 # We always export the domain and the severity
 EXPORT_FIELDS = ['domain', 'severity']
 
@@ -69,6 +72,22 @@ def sync_blocklists(conf: dict):
 
     # Merge blocklists into an update dict
     merged = merge_blocklists(blocklists, conf.mergeplan)
+
+    # Apply allows specified on the commandline
+    for domain in conf.allow_domains:
+        log.info(f"Allowing domain '{domain}' specified on commandline.")
+        merged[domain] = DomainBlock(domain, 'noop')
+
+    # Apply allows from URLs lists
+    if conf.allowlist_url_sources:
+        log.info("Adding allows from URL lists...")
+        allowlists = fetch_from_urls({}, conf.allowlist_url_sources, ALLOWLIST_IMPORT_FIELDS)
+        for key, alist in allowlists.items():
+            log.debug(f"Processing allows from '{key}'...")
+            for allowed in alist:
+                merged[allowed.domain] = allowed
+                log.debug(f"Allowed domain '{allowed.domain}' from allowlist: {allowed}")
+
     if conf.blocklist_savefile:
         log.info(f"Saving merged blocklist to {conf.blocklist_savefile}")
         save_blocklist_to_file(merged.values(), conf.blocklist_savefile, export_fields)
@@ -140,9 +159,12 @@ def fetch_from_instances(blocklists: dict, sources: dict,
 
 def merge_blocklists(blocklists: dict, mergeplan: str='max') -> dict:
     """Merge fetched remote blocklists into a bulk update
+    @param blocklists: A dict of lists of DomainBlocks, keyed by source.
+        Each value is a list of DomainBlocks
     @param mergeplan: An optional method of merging overlapping block definitions
         'max' (the default) uses the highest severity block found
         'min' uses the lowest severity block found
+    @param returns: A dict of DomainBlocks keyed by domain
     """
     merged = {}
 
@@ -433,7 +455,7 @@ def update_known_block(token: str, host: str, block: DomainBlock):
 
     response = requests.put(url,
         headers=requests_headers(token),
-        data=blockdata,
+        json=blockdata._asdict(),
         timeout=REQUEST_TIMEOUT
     )
     if response.status_code != 200:
@@ -442,14 +464,14 @@ def update_known_block(token: str, host: str, block: DomainBlock):
 def add_block(token: str, host: str, blockdata: DomainBlock):
     """Block a domain on Mastodon host
     """
-    log.debug(f"Blocking domain {blockdata.domain} at {host}...")
+    log.debug(f"Adding block entry for {blockdata.domain} at {host}...")
     api_path = "/api/v1/admin/domain_blocks"
 
     url = f"https://{host}{api_path}"
 
     response = requests.post(url,
         headers=requests_headers(token),
-        data=blockdata._asdict(),
+        json=blockdata._asdict(),
         timeout=REQUEST_TIMEOUT
     )
     if response.status_code == 422:
@@ -515,6 +537,8 @@ def push_blocklist(token: str, host: str, blocklist: list[dict],
                 log.info(f"Pushing new block definition: {newblock}")
                 blockdata = oldblock.copy()
                 blockdata.update(newblock)
+                log.debug(f"Block as dict: {blockdata._asdict()}")
+
                 if not dryrun:
                     update_known_block(token, host, blockdata)
                     # add a pause here so we don't melt the instance
@@ -530,6 +554,7 @@ def push_blocklist(token: str, host: str, blocklist: list[dict],
             # This is a new block for the target instance, so we
             # need to add a block rather than update an existing one
             log.info(f"Adding new block: {newblock}...")
+            log.debug(f"Block as dict: {newblock._asdict()}")
 
             # Make sure the new block doesn't clobber a domain with followers
             newblock.severity = check_followed_severity(host, token, newblock.domain, newblock.severity, max_followed_severity)
@@ -615,9 +640,10 @@ def augment_args(args):
     if not args.import_fields:
         args.import_fields = conf.get('import_fields', [])
 
-    args.blocklist_url_sources = conf.get('blocklist_url_sources')
-    args.blocklist_instance_sources = conf.get('blocklist_instance_sources')
-    args.blocklist_instance_destinations = conf.get('blocklist_instance_destinations')
+    args.blocklist_url_sources = conf.get('blocklist_url_sources', None)
+    args.blocklist_instance_sources = conf.get('blocklist_instance_sources', None)
+    args.allowlist_url_sources = conf.get('allowlist_url_sources', None)
+    args.blocklist_instance_destinations = conf.get('blocklist_instance_destinations', None)
 
     return args
 
@@ -637,6 +663,7 @@ def main():
 
     ap.add_argument('-I', '--import-field', dest='import_fields', action='append', help="Extra blocklist fields to import.")
     ap.add_argument('-E', '--export-field', dest='export_fields', action='append', help="Extra blocklist fields to export.")
+    ap.add_argument('-A', '--allow', dest="allow_domains", action='append', default=[], help="Override any blocks to allow this domain.")
 
     ap.add_argument('--no-fetch-url', dest='no_fetch_url', action='store_true', help="Don't fetch from URLs, even if configured.")
     ap.add_argument('--no-fetch-instance', dest='no_fetch_instance', action='store_true', help="Don't fetch from instances, even if configured.")