README.md

@@ -386,14 +386,6 @@ Skip the fetching of blocklists from any URLs that are configured.
 
 Skip the fetching of blocklists from any remote instances that are configured.
 
-### override_private_comment
-
-Defaults to None.
-
-Stamp all *new* blocks pushed to a remote server with this comment or code. 
-Helps to identify blocks you've created on a server via Fediblockhole versus ones that
-already existed.
-
 ### mergeplan
 
 If two (or more) blocklists define blocks for the same domain, but they're

etc/sample.fediblockhole.conf.toml

@@ -77,9 +77,8 @@ blocklist_instance_destinations = [
 # merge_threshold_type = 'count'
 # merge_threshold = 0
 
-## set an override private comment to be added when pushing a NEW block to an instance
+## set an override private comment
-# this does not require importing private comments
+# override_private_comment = 'Updated by Fediblockhole'
-# override_private_comment = 'Added by Fediblock Sync'
 
 ## Set which fields we import
 ## 'domain' and 'severity' are always imported, these are additional

src/fediblockhole/__init__.py

@@ -5,7 +5,6 @@ import argparse
 import toml
 import csv
 import requests
-import re
 import json
 import time
 import os.path
@@ -375,141 +374,7 @@ def requests_headers(token: str=None):
 
     return headers
 
-def get_nodeinfo(server, token: str=None, host_meta_fallback = False):
-    headers = requests_headers(token)
-    url = f'https://{server}/.well-known/nodeinfo'
-    try:
-        resp = requests.get(url, headers=headers, timeout=REQUEST_TIMEOUT)
-    except Exception as ex:
-        log.error(f"Error getting host node info for {server}. Exception: {ex}")
-        return None
-
-    # if well-known nodeinfo isn't found, try to check host-meta for a webfinger URL
-    # needed on servers where the display domain is different than the web domain
-    if resp.status_code != 200 and not host_meta_fallback:
-        # not found, try to check host-meta as a fallback
-        log.warning(f'nodeinfo for {server} not found, checking host-meta')
-        new_server = get_server_from_host_meta(server)
-        if new_server is not None:
-            if new_server == server:
-                log.debug(f'host-meta for {server} did not get a new server.')
-                return None
-            else:
-                return get_nodeinfo(new_server, True)
-        else:
-            return None
-
-    if resp.status_code == 200:
-        nodeLoc = None
-        try:
-            nodeInfo = resp.json()
-            for link in nodeInfo['links']:
-                if link['rel'] in [
-                    'http://nodeinfo.diaspora.software/ns/schema/2.0',
-                    'http://nodeinfo.diaspora.software/ns/schema/2.1',
-                ]:
-                    nodeLoc = link['href']
-                    break
-        except Exception as ex:
-            log.error(f'error getting server {server} info from well-known node info. Exception: {ex}')
-            return None
-    else:
-        log.error(f'Error getting well-known host node info for {server}. Status Code: {resp.status_code}')
-        return None
-
-    if nodeLoc is None:
-        log.warning(f'could not find link to node info in well-known nodeinfo of {server}')
-        return None
-
-    # regrab server from nodeLoc, again in the case of different display and web domains
-    match = re.match(
-        r"https://(?P<server>[^/]+)/", nodeLoc
-    )
-    if match is None:
-        log.error(f"Error getting web server name from {server}.")
-        return None
-
-    server = match.group('server')
-
-    try:
-        resp = requests.get(nodeLoc, headers=headers, timeout=REQUEST_TIMEOUT)
-    except Exception as ex:
-        log.error(f"Error getting host node info for {server}. Exception: {ex}")
-        return None
-
-    if resp.status_code == 200:
-        try:
-            nodeInfo = resp.json()
-            if 'activitypub' not in nodeInfo['protocols']:
-                log.warning(f'server {server} does not support activitypub, skipping')
-                return None
-            return {
-                'webserver': server,
-                'software': nodeInfo['software']['name'],
-                'version': nodeInfo['software']['version'],
-                'rawnodeinfo': nodeInfo,
-            }
-        except Exception as ex:
-            log.error(f'error getting server {server} info from nodeinfo. Exception: {ex}')
-            return None
-    else:
-        log.error(f'Error getting host node info for {server}. Status Code: {resp.status_code}')
-        return None
-
-def set_server_apis(server):
-    # support for new server software should be added here
-    software_apis = {
-        'mastodonApiSupport': ['mastodon', 'pleroma', 'akkoma', 'pixelfed', 'hometown', 'iceshrimp'],
-        'misskeyApiSupport': ['misskey', 'calckey', 'firefish', 'foundkey', 'sharkey'],
-        'lemmyApiSupport': ['lemmy']
-    }
-
-    for api, softwareList in software_apis.items():
-        server[api] = server['software'] in softwareList
-
-    # search `features` list in metadata if available
-    if 'metadata' in server['rawnodeinfo'] and 'features' in server['rawnodeinfo']['metadata'] and type(server['rawnodeinfo']['metadata']['features']) is list:
-        features = server['rawnodeinfo']['metadata']['features']
-        if 'mastodon_api' in features:
-            server['mastodonApiSupport'] = True
-
-def fetch_server_info(server, token: str=None):
-    nodeinfo = get_nodeinfo(server, token=token)
-    if nodeinfo is None:
-        return None
-    else:
-        set_server_apis(nodeinfo)
-        return nodeinfo
-
-def fetch_api_type(host: str, token: str=None) -> str:
-    server_info = fetch_server_info(host, token)
-    # Many Misskey supports also support Mastodon API to varying
-    # extents, so we should check Misskey API support before checking
-    # Mastodon API support.
-    if server_info['misskeyApiSupport']:
-        return "misskey"
-    elif server_info['mastodonApiSupport']:
-        return "mastodon"
-    else:
-        sys.exit(f"Unknown API type for {host}")
-
-
 def fetch_instance_blocklist(host: str, token: str=None, admin: bool=False,
-    import_fields: list=['domain', 'severity'],
-    scheme: str='https') -> list[DomainBlock]:
-    """
-    Fetch existing block from server (Misskey or Mastodon)
-    """
-    api_type = fetch_api_type(host)
-
-    if api_type == 'mastodon':
-        return fetch_instance_blocklist_mastodon(host, token, admin, import_fields, scheme)
-    elif api_type == 'misskey':
-        return fetch_instance_blocklist_misskey(host, token, admin, import_fields, scheme)
-    else:
-        sys.exit(f"Unknown API type for server {host}")
-
-def fetch_instance_blocklist_mastodon(host: str, token: str=None, admin: bool=False,
     import_fields: list=['domain', 'severity'],
     scheme: str='https') -> list[DomainBlock]:
     """Fetch existing block list from server
@@ -520,7 +385,7 @@ def fetch_instance_blocklist_mastodon(host: str, token: str=None, admin: bool=Fa
     @param import_fields: A list of fields to import from the remote instance.
     @returns: A list of the domain blocks from the instance.
     """
-    log.info(f"Fetching instance blocklist (Mastodon) from {host} ...")
+    log.info(f"Fetching instance blocklist from {host} ...")
 
     if admin:
         api_path = "/api/v1/admin/domain_blocks"
@@ -567,54 +432,6 @@ def fetch_instance_blocklist_mastodon(host: str, token: str=None, admin: bool=Fa
 
     return blocklist
 
-def fetch_instance_blocklist_misskey(host: str, token: str=None, admin: bool=False,
-    import_fields: list=['domain', 'severity'],
-    scheme: str='https') -> list[DomainBlock]:
-    """Fetch existing block list from server
-
-    @param host: The remote host to connect to.
-    @param token: The (required) OAuth Bearer token to authenticate with.
-    @param admin: Boolean flag to use the admin API if True.
-    @param import_fields: A list of fields to import from the remote instance.
-    @returns: A list of the domain blocks from the instance.
-    """
-    log.info(f"Fetching instance blocklist (Misskey) from {host} ...")
-
-    if admin:
-        api_path = "/api/admin/meta"
-        parse_format = 'json'
-    else:
-        sys.exit("Must be an admin to use the meta API")
-
-    headers = requests_headers(token)
-
-    url = f"{scheme}://{host}{api_path}"
-
-    blockdata = []
-
-    response = requests.post(url, headers=headers, timeout=REQUEST_TIMEOUT, json={})
-    if response.status_code != 200:
-        log.error(f"{response.status_code} Cannot fetch remote blocklist: {response.content}")
-        raise ValueError("Unable to fetch domain block list: %s", response)
-
-    # Convert the Misskey block (which are just domains) into
-    # something similar to Mastodon response that Fediblockhole
-    # can understand.
-    def map_block(domain, severity):
-        return { "domain": domain, "digest": "", "severity": severity, "comment": "" }
-
-    meta = json.loads(response.content.decode('utf-8'))
-
-    blocked_hosts = [ map_block(domain, "suspend") for domain in meta['blockedHosts'] ]
-    silenced_hosts = [ map_block(domain, "silence") for domain in meta['silencedHosts'] ]
-
-    blockdata.extend(blocked_hosts)
-    blockdata.extend(silenced_hosts)
-
-    blocklist = parse_blocklist(blockdata, url, parse_format, import_fields)
-
-    return blocklist
-
 def delete_block(token: str, host: str, id: int, scheme: str='https'):
     """Remove a domain block"""
     log.debug(f"Removing domain block {id} at {host}...")
@@ -762,10 +579,11 @@ def push_blocklist(token: str, host: str, blocklist: list[DomainBlock],
     serverblocks = fetch_instance_blocklist(host, token, True, import_fields, scheme)
 
     # # Convert serverblocks to a dictionary keyed by domain name
-    for newblock in blocklist.values():
+    # knownblocks = {row.domain: row for row in serverblocks}
-        log.debug(f"Processing block: {newblock}")
 
-        # Already existing block
+    for newblock in blocklist.values():
+
+        log.debug(f"Processing block: {newblock}")
         if newblock.domain in serverblocks:
             log.debug(f"Block already exists for {newblock.domain}, checking for differences...")
 
@@ -794,12 +612,19 @@ def push_blocklist(token: str, host: str, blocklist: list[DomainBlock],
                 blockdata = oldblock.copy()
                 blockdata.update(newblock)
                 log.debug(f"Block as dict: {blockdata._asdict()}")
+
+                if not dryrun:
+                    update_known_block(token, host, blockdata, scheme)
+                    # add a pause here so we don't melt the instance
+                    time.sleep(API_CALL_DELAY)
+                else:
+                    log.info("Dry run selected. Not applying changes.")
+
             else:
                 log.debug("No differences detected. Not updating.")
                 pass
 
         else:
-            # New block
             # stamp this record with a private comment, since we're the ones adding it
             if override_private_comment:
                 newblock.private_comment = override_private_comment
@@ -811,52 +636,15 @@ def push_blocklist(token: str, host: str, blocklist: list[DomainBlock],
 
             # Make sure the new block doesn't clobber a domain with followers
             newblock.severity = check_followed_severity(host, token, newblock.domain, newblock.severity, max_followed_severity, scheme)
+            if not dryrun:
+                add_block(token, host, newblock, scheme)
+                # add a pause here so we don't melt the instance
+                time.sleep(API_CALL_DELAY)
+            else:
+                log.info("Dry run selected. Not adding block.")
 
-    # Append existing blocks to the new blocks (needed because Misskey
+    for block in serverblocks:
-    # is all at once update operation)
+        log.debug(f"Checking block: {block}")
-    for existing_block in serverblocks.values():
-        if existing_block.domain not in blocklist:
-            blocklist.blocks[existing_block.domain] = existing_block
-
-    apply_blocks_misskey(blocklist, host, scheme, token)
-
-def apply_blocks_misskey(blocklist: Blocklist, host:str, scheme: str="https", token: str=None):
-    # instead of being cool and trying to update everything, split
-    # into lists of suspends and silences, then call update meta with
-    # these new lists.
-    suspended_hosts = []
-    silenced_hosts = []
-
-    suspend_level = BlockSeverity("suspend")
-    silence_level = BlockSeverity("silence")
-
-    for block in blocklist.values():
-        if block.severity == suspend_level:
-            suspended_hosts.append(block.domain)
-        elif block.severity == silence_level:
-            silenced_hosts.append(block.domain)
-
-    # Make one big call to update-meta with new blocks and suspends
-    api_path = "/api/admin/update-meta"
-    parse_format = 'json'
-
-    headers = requests_headers(token)
-
-    url = f"{scheme}://{host}{api_path}"
-
-    update_data = {
-        "blockedHosts": suspended_hosts,
-        "silencedHosts": silenced_hosts
-    }
-
-    response = requests.post(url, headers=headers, timeout=REQUEST_TIMEOUT, json=update_data)
-    if response.status_code != 200 and response.status_code != 204:
-        log.error(f"{response.status_code} Unable to apply blocklist: {response.content}")
-        raise ValueError("Unable to apply block list: %s", response)
-
-    log.info(f"Updated meta with {len(suspended_hosts)} blocks and {len(silenced_hosts)} silences")
-    return True
-
 
 def load_config(configfile: str):
     """Augment commandline arguments with config file parameters
@@ -1010,7 +798,7 @@ def setup_argparse():
     ap.add_argument('-b', '--block-audit-file', dest="blocklist_auditfile", help="Save blocklist auditfile to this location.")
     ap.add_argument('--merge-threshold', type=int, help="Merge threshold value")
     ap.add_argument('--merge-threshold-type', choices=['count', 'pct'], help="Type of merge threshold to use.")
-    ap.add_argument('--override-private-comment', dest='override_private_comment', help="Override private_comment with this string for new blocks when pushing blocklists.")
+    ap.add_argument('--override-private-comment', dest='override_private_comment', help="Enforces a private comment for all blocks.")
 
     ap.add_argument('-I', '--import-field', dest='import_fields', action='append', help="Extra blocklist fields to import.")
     ap.add_argument('-E', '--export-field', dest='export_fields', action='append', help="Extra blocklist fields to export.")