Repository: nanos/FediFetcher

Issue:
FediFetcher fails on checking followings with 401
Hi, I'm having this issue where FediFetcher fails fetching posts of profiles I've followed.
I am running a gotosocial instance and the API token has read permissions. Fetching posts of my timeline works well.

```
2024-05-29 22:16:14.702945 CEST: Getting posts from last 80 followings
2024-05-29 22:16:14.984905 CEST: Job failed after 0:15:59.932115.
Traceback (most recent call last):
  File "/usr/bin/fedi-fetcher", line 1417, in <module>
    followings = get_new_followings(arguments.server, user_id, arguments.max_followings, all_known_users)
                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/bin/fedi-fetcher", line 274, in get_new_followings
    following = get_paginated_mastodon(f"https://{server}/api/v1/accounts/{user_id}/following", max)
                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/bin/fedi-fetcher", line 919, in get_paginated_mastodon
    raise Exception(
Exception: Error getting URL https://sir-photch.xyz/api/v1/accounts/REDACTED/following. Status code: 401. Ensure your access token is correct
```

Does my token need any more permissions, like `write` or `admin`? (As defined in: https://docs.gotosocial.org/en/latest/api/authentication/#getting-your-token)

Discussion:
@nanos: Is your home instance running gotosocial?@shtrophic: Yes@nanos: In that case I'm afraid I personally can only offer very limited assistance, as I have never used GTS. Whilst FediFetcher can fetch posts from various fediverse platforms, it officially only supports running against Mastodon. 

I believe that GTS has support for the Mastodon API, and thus you should be able to run FediFetcher against GTS, but I haven't tested this. And when fetching posts from GTS I have in the past noticed that GTS' implementation of the Mastodon API differs in some subtle and unexpected ways from Mastodon's implementation itself.

What I can tell you is that with mastodon a `401` response code (as seen here) means that the token you provided is not valid (e.g it's been revoked since, or you made an error when copy and pasting). Mastodon returns a `403` response instead, when the token is valid but needs more permissions. Whether GTS follows the same conventions is something I simply don't know.

You can find which permissions are required for each configuration option at the bottom of the readme file here https://github.com/nanos/FediFetcher/tree/main?tab=readme-ov-file#required-access-token-scopes, but again I don't know whether GTS requires additional scopes.

I'm happy to leave this issue open, in case anyone else can help you. If you find out the cause, please feed back here, so that we can close the issue, and potentially help others in the future.
@nanos: Closing this as stale now. Feel free to reopen if needed. 

Solution patch:
diff --git a/find_posts.py b/find_posts.py
index ff725682..aaf214cc 100644
--- a/find_posts.py
+++ b/find_posts.py
@@ -301,9 +301,11 @@ def filter_known_users(users, known_users):
         users
     ))
 
-def get_new_followers(server, user_id, max, known_followers):
+def get_new_followers(server, user_id, access_token, max, known_followers):
     """Get any new followings for the specified user, up to the max number provided"""
-    followers = get_paginated_mastodon(f"https://{server}/api/v1/accounts/{user_id}/followers", max)
+    followers = get_paginated_mastodon(f"https://{server}/api/v1/accounts/{user_id}/followers", max, {
+        "Authorization": f"Bearer {access_token}",
+    })
 
     # Remove any we already know about
     new_followers = filter_known_users(followers, known_followers)
@@ -312,9 +314,11 @@ def get_new_followers(server, user_id, max, known_followers):
 
     return new_followers
 
-def get_new_followings(server, user_id, max, known_followings):
+def get_new_followings(server, user_id, access_token, max, known_followings):
     """Get any new followings for the specified user, up to the max number provided"""
-    following = get_paginated_mastodon(f"https://{server}/api/v1/accounts/{user_id}/following", max)
+    following = get_paginated_mastodon(f"https://{server}/api/v1/accounts/{user_id}/following", max, {
+        "Authorization": f"Bearer {access_token}",
+    })
 
     # Remove any we already know about
     new_followings = filter_known_users(following, known_followings)
@@ -1698,13 +1702,13 @@ def fetch_timeline_context(timeline_posts, token, parsed_urls, seen_hosts, seen_
             if arguments.max_followings > 0:
                 logger.info(f"Getting posts from last {arguments.max_followings} followings")
                 user_id = get_user_id(arguments.server, arguments.user, token)
-                followings = get_new_followings(arguments.server, user_id, arguments.max_followings, all_known_users)
+                followings = get_new_followings(arguments.server, user_id, token, arguments.max_followings, all_known_users)
                 add_user_posts(arguments.server, token, followings, known_followings, all_known_users, seen_urls, seen_hosts)
 
             if arguments.max_followers > 0:
                 logger.info(f"Getting posts from last {arguments.max_followers} followers")
                 user_id = get_user_id(arguments.server, arguments.user, token)
-                followers = get_new_followers(arguments.server, user_id, arguments.max_followers, all_known_users)
+                followers = get_new_followers(arguments.server, user_id, token, arguments.max_followers, all_known_users)
                 add_user_posts(arguments.server, token, followers, recently_checked_users, all_known_users, seen_urls, seen_hosts)
 
             if arguments.max_follow_requests > 0:


Test patch:
diff --git a/tests/test_find_posts.py b/tests/test_find_posts.py
index 6e964ae5..6e633d53 100644
--- a/tests/test_find_posts.py
+++ b/tests/test_find_posts.py
@@ -448,14 +448,17 @@ def test_get_new_followers(
 
     server = "server"
     user_id = 1
+    access_token = "access_token"
     max = 50
     known_followers = ["follower1"]
 
     expected_result = ["follower2", "follower3"]
-    result = find_posts.get_new_followers(server, user_id, max, known_followers)
+    result = find_posts.get_new_followers(server, user_id, access_token, max, known_followers)
 
     mock_get_paginated_mastodon.assert_called_once_with(
-        f"https://{server}/api/v1/accounts/{user_id}/followers", max
+        f"https://{server}/api/v1/accounts/{user_id}/followers", max, {
+            "Authorization": f"Bearer {access_token}",
+        },
     )
     mock_filter_known_users.assert_called_once_with(
         ["follower1", "follower2", "follower3"], known_followers
@@ -473,9 +476,11 @@ def test_get_new_followings(
 ):
     mock_get_paginated_mastodon.return_value = ["user1", "user2", "user3"]
     mock_filter_known_users.return_value = ["user1", "user2"]
-    result = get_new_followings("server", "100", 5, "known_users")
+    result = get_new_followings("server", "100", "access_token", 5, "known_users")
     mock_get_paginated_mastodon.assert_called_with(
-        "https://server/api/v1/accounts/100/following", 5
+        "https://server/api/v1/accounts/100/following", 5, {
+            "Authorization": "Bearer access_token",
+        }
     )
     mock_filter_known_users.assert_called_with(
         ["user1", "user2", "user3"], "known_users"