From 9ca042ab74a3802b5c35684896770dce38cf21cf Mon Sep 17 00:00:00 2001 From: Shawn Grigson Date: Sun, 10 Sep 2023 17:21:46 -0500 Subject: [PATCH 01/15] Override the private comment --- etc/sample.fediblockhole.conf.toml | 3 +++ src/fediblockhole/__init__.py | 15 ++++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/etc/sample.fediblockhole.conf.toml b/etc/sample.fediblockhole.conf.toml index 44cd6ff..a7a7b96 100644 --- a/etc/sample.fediblockhole.conf.toml +++ b/etc/sample.fediblockhole.conf.toml @@ -77,6 +77,9 @@ blocklist_instance_destinations = [ # merge_threshold_type = 'count' # merge_threshold = 0 +## set an override private comment +# override_private_comment = 'Updated by Fediblockhole' + ## Set which fields we import ## 'domain' and 'severity' are always imported, these are additional ## diff --git a/src/fediblockhole/__init__.py b/src/fediblockhole/__init__.py index aec0a08..7949cea 100755 --- a/src/fediblockhole/__init__.py +++ b/src/fediblockhole/__init__.py @@ -90,7 +90,7 @@ def sync_blocklists(conf: argparse.Namespace): token = dest['token'] scheme = dest.get('scheme', 'https') max_followed_severity = BlockSeverity(dest.get('max_followed_severity', 'silence')) - push_blocklist(token, target, merged, conf.dryrun, import_fields, max_followed_severity, scheme) + push_blocklist(token, target, merged, conf.dryrun, import_fields, max_followed_severity, scheme, conf.override_private_comment) def apply_allowlists(merged: Blocklist, conf: argparse.Namespace, allowlists: dict): """Apply allowlists @@ -560,6 +560,7 @@ def push_blocklist(token: str, host: str, blocklist: list[DomainBlock], import_fields: list=['domain', 'severity'], max_followed_severity:BlockSeverity=BlockSeverity('silence'), scheme: str='https', + override_private_comment: str=None ): """Push a blocklist to a remote instance. @@ -575,6 +576,10 @@ def push_blocklist(token: str, host: str, blocklist: list[DomainBlock], # Force use of the admin API, and add 'id' to the list of fields if 'id' not in import_fields: import_fields.append('id') + + # if we're overriding the private comment, we need to include it in the import + if override_private_comment: + import_fields.append('private_comment') serverblocks = fetch_instance_blocklist(host, token, True, import_fields, scheme) # # Convert serverblocks to a dictionary keyed by domain name @@ -582,6 +587,10 @@ def push_blocklist(token: str, host: str, blocklist: list[DomainBlock], for newblock in blocklist.values(): + # stamp this record with a private comment + if override_private_comment: + newblock.private_comment = override_private_comment + log.debug(f"Processing block: {newblock}") if newblock.domain in serverblocks: log.debug(f"Block already exists for {newblock.domain}, checking for differences...") @@ -741,6 +750,9 @@ def augment_args(args, tomldata: str=None): if not args.save_intermediate: args.save_intermediate = conf.get('save_intermediate', False) + + if not args.override_private_comment: + args.override_private_comment = conf.get('override_private_comment', None) if not args.savedir: args.savedir = conf.get('savedir', '/tmp') @@ -787,6 +799,7 @@ def setup_argparse(): ap.add_argument('-b', '--block-audit-file', dest="blocklist_auditfile", help="Save blocklist auditfile to this location.") ap.add_argument('--merge-threshold', type=int, help="Merge threshold value") ap.add_argument('--merge-threshold-type', choices=['count', 'pct'], help="Type of merge threshold to use.") + ap.add_argument('--override-private-comment', dest='override_private_comment', help="Enforces a private comment for all blocks.") ap.add_argument('-I', '--import-field', dest='import_fields', action='append', help="Extra blocklist fields to import.") ap.add_argument('-E', '--export-field', dest='export_fields', action='append', help="Extra blocklist fields to export.") From 9f0c36e7518a153c07880bb0f69a8d0488dc4d86 Mon Sep 17 00:00:00 2001 From: Shawn Grigson Date: Sun, 10 Sep 2023 17:27:01 -0500 Subject: [PATCH 02/15] fix, only add private comment for new blocks --- src/fediblockhole/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/fediblockhole/__init__.py b/src/fediblockhole/__init__.py index 7949cea..bfac6fd 100755 --- a/src/fediblockhole/__init__.py +++ b/src/fediblockhole/__init__.py @@ -587,10 +587,6 @@ def push_blocklist(token: str, host: str, blocklist: list[DomainBlock], for newblock in blocklist.values(): - # stamp this record with a private comment - if override_private_comment: - newblock.private_comment = override_private_comment - log.debug(f"Processing block: {newblock}") if newblock.domain in serverblocks: log.debug(f"Block already exists for {newblock.domain}, checking for differences...") @@ -638,6 +634,10 @@ def push_blocklist(token: str, host: str, blocklist: list[DomainBlock], log.info(f"Adding new block: {newblock}...") log.debug(f"Block as dict: {newblock._asdict()}") + # stamp this record with a private comment, since we're the ones adding it + if override_private_comment: + newblock.private_comment = override_private_comment + # Make sure the new block doesn't clobber a domain with followers newblock.severity = check_followed_severity(host, token, newblock.domain, newblock.severity, max_followed_severity, scheme) if not dryrun: From b297e390f7f2ed02bf997b1846549f7032da8f5a Mon Sep 17 00:00:00 2001 From: Shawn Grigson Date: Sun, 10 Sep 2023 17:29:57 -0500 Subject: [PATCH 03/15] remove import fields update --- src/fediblockhole/__init__.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/fediblockhole/__init__.py b/src/fediblockhole/__init__.py index bfac6fd..ce805cf 100755 --- a/src/fediblockhole/__init__.py +++ b/src/fediblockhole/__init__.py @@ -576,10 +576,6 @@ def push_blocklist(token: str, host: str, blocklist: list[DomainBlock], # Force use of the admin API, and add 'id' to the list of fields if 'id' not in import_fields: import_fields.append('id') - - # if we're overriding the private comment, we need to include it in the import - if override_private_comment: - import_fields.append('private_comment') serverblocks = fetch_instance_blocklist(host, token, True, import_fields, scheme) # # Convert serverblocks to a dictionary keyed by domain name From b833ad6d418a890bff5a1dc1c2ad29f2d4e17d4e Mon Sep 17 00:00:00 2001 From: Shawn Grigson Date: Sun, 10 Sep 2023 18:00:43 -0500 Subject: [PATCH 04/15] try with import field --- src/fediblockhole/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/fediblockhole/__init__.py b/src/fediblockhole/__init__.py index ce805cf..8f696b9 100755 --- a/src/fediblockhole/__init__.py +++ b/src/fediblockhole/__init__.py @@ -576,6 +576,8 @@ def push_blocklist(token: str, host: str, blocklist: list[DomainBlock], # Force use of the admin API, and add 'id' to the list of fields if 'id' not in import_fields: import_fields.append('id') + if override_private_comment: + import_fields.append('private_comment') serverblocks = fetch_instance_blocklist(host, token, True, import_fields, scheme) # # Convert serverblocks to a dictionary keyed by domain name From a4be5ff67510f823aefe5973409cec99e651e04e Mon Sep 17 00:00:00 2001 From: Shawn Grigson Date: Sun, 10 Sep 2023 18:03:06 -0500 Subject: [PATCH 05/15] Revert "try with import field" This reverts commit b833ad6d418a890bff5a1dc1c2ad29f2d4e17d4e. --- src/fediblockhole/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/fediblockhole/__init__.py b/src/fediblockhole/__init__.py index 8f696b9..ce805cf 100755 --- a/src/fediblockhole/__init__.py +++ b/src/fediblockhole/__init__.py @@ -576,8 +576,6 @@ def push_blocklist(token: str, host: str, blocklist: list[DomainBlock], # Force use of the admin API, and add 'id' to the list of fields if 'id' not in import_fields: import_fields.append('id') - if override_private_comment: - import_fields.append('private_comment') serverblocks = fetch_instance_blocklist(host, token, True, import_fields, scheme) # # Convert serverblocks to a dictionary keyed by domain name From 2a96ea724520712144ead86d3d866ebf487877fb Mon Sep 17 00:00:00 2001 From: Shawn Grigson Date: Sun, 10 Sep 2023 18:06:51 -0500 Subject: [PATCH 06/15] add private comment to proper place for debugging --- src/fediblockhole/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/fediblockhole/__init__.py b/src/fediblockhole/__init__.py index ce805cf..778c587 100755 --- a/src/fediblockhole/__init__.py +++ b/src/fediblockhole/__init__.py @@ -625,15 +625,15 @@ def push_blocklist(token: str, host: str, blocklist: list[DomainBlock], pass else: + # stamp this record with a private comment, since we're the ones adding it + if override_private_comment: + newblock.private_comment = override_private_comment + # This is a new block for the target instance, so we # need to add a block rather than update an existing one log.info(f"Adding new block: {newblock}...") log.debug(f"Block as dict: {newblock._asdict()}") - # stamp this record with a private comment, since we're the ones adding it - if override_private_comment: - newblock.private_comment = override_private_comment - # Make sure the new block doesn't clobber a domain with followers newblock.severity = check_followed_severity(host, token, newblock.domain, newblock.severity, max_followed_severity, scheme) if not dryrun: From 705965a17cf2eb83de45528932ab45a5adf76976 Mon Sep 17 00:00:00 2001 From: Shawn Grigson Date: Sun, 10 Sep 2023 19:20:17 -0500 Subject: [PATCH 07/15] audit update --- src/fediblockhole/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/fediblockhole/__init__.py b/src/fediblockhole/__init__.py index 778c587..148c9b4 100755 --- a/src/fediblockhole/__init__.py +++ b/src/fediblockhole/__init__.py @@ -628,7 +628,7 @@ def push_blocklist(token: str, host: str, blocklist: list[DomainBlock], # stamp this record with a private comment, since we're the ones adding it if override_private_comment: newblock.private_comment = override_private_comment - + # This is a new block for the target instance, so we # need to add a block rather than update an existing one log.info(f"Adding new block: {newblock}...") @@ -643,6 +643,9 @@ def push_blocklist(token: str, host: str, blocklist: list[DomainBlock], else: log.info("Dry run selected. Not adding block.") + for block in serverblocks: + log.debug(f"Checking block: {block}") + def load_config(configfile: str): """Augment commandline arguments with config file parameters From c372b210d85618a5374bebded8d4243b6b6310b5 Mon Sep 17 00:00:00 2001 From: Shawn Grigson Date: Sun, 10 Sep 2023 19:34:50 -0500 Subject: [PATCH 08/15] tweak --- src/fediblockhole/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/fediblockhole/__init__.py b/src/fediblockhole/__init__.py index 148c9b4..b6d43a4 100755 --- a/src/fediblockhole/__init__.py +++ b/src/fediblockhole/__init__.py @@ -643,6 +643,7 @@ def push_blocklist(token: str, host: str, blocklist: list[DomainBlock], else: log.info("Dry run selected. Not adding block.") + log.debug(f"Checking server blocks: {serverblocks}") for block in serverblocks: log.debug(f"Checking block: {block}") From bda8bf5ebb5a44cf9610c9a354c44107061137f8 Mon Sep 17 00:00:00 2001 From: Shawn Grigson Date: Sun, 10 Sep 2023 19:38:09 -0500 Subject: [PATCH 09/15] meh --- src/fediblockhole/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/fediblockhole/__init__.py b/src/fediblockhole/__init__.py index b6d43a4..97e6f51 100755 --- a/src/fediblockhole/__init__.py +++ b/src/fediblockhole/__init__.py @@ -581,6 +581,10 @@ def push_blocklist(token: str, host: str, blocklist: list[DomainBlock], # # Convert serverblocks to a dictionary keyed by domain name # knownblocks = {row.domain: row for row in serverblocks} + log.debug(f"Checking server blocks: {serverblocks}") + for block in serverblocks: + log.debug(f"Checking block: {block}") + for newblock in blocklist.values(): log.debug(f"Processing block: {newblock}") @@ -643,10 +647,6 @@ def push_blocklist(token: str, host: str, blocklist: list[DomainBlock], else: log.info("Dry run selected. Not adding block.") - log.debug(f"Checking server blocks: {serverblocks}") - for block in serverblocks: - log.debug(f"Checking block: {block}") - def load_config(configfile: str): """Augment commandline arguments with config file parameters From d1fe11abf51df251d073ede72ceff4d143ccb986 Mon Sep 17 00:00:00 2001 From: Shawn Grigson Date: Tue, 12 Sep 2023 02:34:56 -0500 Subject: [PATCH 10/15] readme updates also 401 status_code becomes skippable --- README.md | 8 ++++++++ etc/sample.fediblockhole.conf.toml | 5 +++-- src/fediblockhole/__init__.py | 3 +++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index bdd295c..6d02789 100644 --- a/README.md +++ b/README.md @@ -386,6 +386,14 @@ Skip the fetching of blocklists from any URLs that are configured. Skip the fetching of blocklists from any remote instances that are configured. +### override_private_comment + +Defaults to None. + +Stamp all *new* blocks pushed to a remote server with this comment or code. +Helps to identify blocks you've created on a server via this process versus ones that +already existed. + ### mergeplan If two (or more) blocklists define blocks for the same domain, but they're diff --git a/etc/sample.fediblockhole.conf.toml b/etc/sample.fediblockhole.conf.toml index a7a7b96..3844512 100644 --- a/etc/sample.fediblockhole.conf.toml +++ b/etc/sample.fediblockhole.conf.toml @@ -77,8 +77,9 @@ blocklist_instance_destinations = [ # merge_threshold_type = 'count' # merge_threshold = 0 -## set an override private comment -# override_private_comment = 'Updated by Fediblockhole' +## set an override private comment to be added when pushing a NEW block to an instance +# this does not require importing private comments +# override_private_comment = 'Added by Fediblock Sync' ## Set which fields we import ## 'domain' and 'severity' are always imported, these are additional diff --git a/src/fediblockhole/__init__.py b/src/fediblockhole/__init__.py index 97e6f51..995fd7e 100755 --- a/src/fediblockhole/__init__.py +++ b/src/fediblockhole/__init__.py @@ -402,6 +402,9 @@ def fetch_instance_blocklist(host: str, token: str=None, admin: bool=False, link = True while link: response = requests.get(url, headers=headers, timeout=REQUEST_TIMEOUT) + if response.status_code == 401: + log.error(f"Cannot fetch remote blocklist. Access token has been revoked for {host}, skipping....") + break if response.status_code != 200: log.error(f"Cannot fetch remote blocklist: {response.content}") raise ValueError("Unable to fetch domain block list: %s", response) From a1f81d197faf3e1356f9ee56ff47ac0dd6f7faa9 Mon Sep 17 00:00:00 2001 From: Shawn Grigson Date: Tue, 12 Sep 2023 02:39:16 -0500 Subject: [PATCH 11/15] Fix 401 for add block, too --- src/fediblockhole/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/fediblockhole/__init__.py b/src/fediblockhole/__init__.py index 995fd7e..c9a7531 100755 --- a/src/fediblockhole/__init__.py +++ b/src/fediblockhole/__init__.py @@ -554,6 +554,9 @@ def add_block(token: str, host: str, blockdata: DomainBlock, scheme: str='https' err = json.loads(response.content) log.warning(err['error']) + elif response.status_code == 401: + log.warning(f"Access token has been revoked for {host}, no blocks being updated.") + elif response.status_code != 200: raise ValueError(f"Something went wrong: {response.status_code}: {response.content}") From 4d360e6b53311f8030abbb68beddbbe82923a4b2 Mon Sep 17 00:00:00 2001 From: Shawn Grigson Date: Tue, 12 Sep 2023 02:43:02 -0500 Subject: [PATCH 12/15] block more errors for 401 --- src/fediblockhole/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/fediblockhole/__init__.py b/src/fediblockhole/__init__.py index c9a7531..d78e7c2 100755 --- a/src/fediblockhole/__init__.py +++ b/src/fediblockhole/__init__.py @@ -485,7 +485,8 @@ def fetch_instance_follows(token: str, host: str, domain: str, scheme: str='http if response.status_code == 403: log.error(f"Cannot fetch follow information for {domain} from {host}: {response.content}") - raise ValueError(f"Something went wrong: {response.status_code}: {response.content}") + if response.status_code != 401: + raise ValueError(f"Something went wrong: {response.status_code}: {response.content}") # Get the total returned follows = int(response.json()[0]['total']) From 3592c97627851c8f4155658cebdb6025fb80d078 Mon Sep 17 00:00:00 2001 From: Shawn Grigson Date: Tue, 12 Sep 2023 08:13:17 -0500 Subject: [PATCH 13/15] revert --- src/fediblockhole/__init__.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/fediblockhole/__init__.py b/src/fediblockhole/__init__.py index d78e7c2..97e6f51 100755 --- a/src/fediblockhole/__init__.py +++ b/src/fediblockhole/__init__.py @@ -402,9 +402,6 @@ def fetch_instance_blocklist(host: str, token: str=None, admin: bool=False, link = True while link: response = requests.get(url, headers=headers, timeout=REQUEST_TIMEOUT) - if response.status_code == 401: - log.error(f"Cannot fetch remote blocklist. Access token has been revoked for {host}, skipping....") - break if response.status_code != 200: log.error(f"Cannot fetch remote blocklist: {response.content}") raise ValueError("Unable to fetch domain block list: %s", response) @@ -485,8 +482,7 @@ def fetch_instance_follows(token: str, host: str, domain: str, scheme: str='http if response.status_code == 403: log.error(f"Cannot fetch follow information for {domain} from {host}: {response.content}") - if response.status_code != 401: - raise ValueError(f"Something went wrong: {response.status_code}: {response.content}") + raise ValueError(f"Something went wrong: {response.status_code}: {response.content}") # Get the total returned follows = int(response.json()[0]['total']) @@ -555,9 +551,6 @@ def add_block(token: str, host: str, blockdata: DomainBlock, scheme: str='https' err = json.loads(response.content) log.warning(err['error']) - elif response.status_code == 401: - log.warning(f"Access token has been revoked for {host}, no blocks being updated.") - elif response.status_code != 200: raise ValueError(f"Something went wrong: {response.status_code}: {response.content}") From 0a63a47ff1d74308059aca30da1327d2fbd3306e Mon Sep 17 00:00:00 2001 From: Shawn Grigson Date: Tue, 12 Sep 2023 08:14:53 -0500 Subject: [PATCH 14/15] more revert --- src/fediblockhole/__init__.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/fediblockhole/__init__.py b/src/fediblockhole/__init__.py index 97e6f51..5bd6cf4 100755 --- a/src/fediblockhole/__init__.py +++ b/src/fediblockhole/__init__.py @@ -581,10 +581,6 @@ def push_blocklist(token: str, host: str, blocklist: list[DomainBlock], # # Convert serverblocks to a dictionary keyed by domain name # knownblocks = {row.domain: row for row in serverblocks} - log.debug(f"Checking server blocks: {serverblocks}") - for block in serverblocks: - log.debug(f"Checking block: {block}") - for newblock in blocklist.values(): log.debug(f"Processing block: {newblock}") From 58dbed0fa88482cd9001d9fcbaf354d1923bf294 Mon Sep 17 00:00:00 2001 From: Shawn Grigson Date: Mon, 2 Oct 2023 13:48:13 -0500 Subject: [PATCH 15/15] some updates to wording/help for override_private_comment --- README.md | 2 +- src/fediblockhole/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6d02789..1fdec29 100644 --- a/README.md +++ b/README.md @@ -391,7 +391,7 @@ Skip the fetching of blocklists from any remote instances that are configured. Defaults to None. Stamp all *new* blocks pushed to a remote server with this comment or code. -Helps to identify blocks you've created on a server via this process versus ones that +Helps to identify blocks you've created on a server via Fediblockhole versus ones that already existed. ### mergeplan diff --git a/src/fediblockhole/__init__.py b/src/fediblockhole/__init__.py index 5bd6cf4..c97816f 100755 --- a/src/fediblockhole/__init__.py +++ b/src/fediblockhole/__init__.py @@ -795,7 +795,7 @@ def setup_argparse(): ap.add_argument('-b', '--block-audit-file', dest="blocklist_auditfile", help="Save blocklist auditfile to this location.") ap.add_argument('--merge-threshold', type=int, help="Merge threshold value") ap.add_argument('--merge-threshold-type', choices=['count', 'pct'], help="Type of merge threshold to use.") - ap.add_argument('--override-private-comment', dest='override_private_comment', help="Enforces a private comment for all blocks.") + ap.add_argument('--override-private-comment', dest='override_private_comment', help="Override private_comment with this string for new blocks when pushing blocklists.") ap.add_argument('-I', '--import-field', dest='import_fields', action='append', help="Extra blocklist fields to import.") ap.add_argument('-E', '--export-field', dest='export_fields', action='append', help="Extra blocklist fields to export.")