Compare commits

...

18 commits

Author SHA1 Message Date
Jamie Bliss ed820ba0c7
Merge e46992448d into 7c34ac78ed 2024-02-09 16:03:04 -05:00
Andrew Godwin 7c34ac78ed Write a release checklist and do a couple things on it 2024-02-06 14:49:35 -07:00
Jamie Bliss e46992448d
Handle socket errors, sign Digest header
It turns out, signing the header about the content is pretty important.
2024-01-14 19:06:13 +00:00
Jamie Bliss 3c6820cfe3
Add header allowlist 2024-01-14 18:46:02 +00:00
Jamie Bliss 1dec02f89c
Fix follow_redirects handling 2024-01-12 00:19:49 -05:00
Jamie Bliss cd160050ac
Update files, emoji, fan_out 2024-01-12 00:05:30 -05:00
Jamie Bliss d8acdf4005
Add Client extensions to get, request, post2 2024-01-12 00:02:44 -05:00
Jamie Bliss bbc5cd989f
Use httpx_mock instead of httpbin 2024-01-11 23:20:15 -05:00
Jamie Bliss e4b2ec5d0d
Test request-based signing 2024-01-11 23:07:37 -05:00
Jamie Bliss 28bf2540fc
Add test for blocking localhost 2024-01-11 21:49:14 -05:00
Jamie Bliss a13d023750
Fix calling the wrapped transport 2024-01-10 23:28:44 -05:00
Jamie Bliss 17f109176e
Can't unify Header and dict
And that makes me sad
2024-01-11 04:07:09 +00:00
Jamie Bliss 0f033832d6
Merge branch 'main' into httpy 2024-01-10 23:02:45 -05:00
Jamie Bliss 5b1edda5a0
Correctly reference BaseClient 2024-01-11 04:01:54 +00:00
Jamie Bliss 2404a2a3e8
Add some basic tests. 2024-01-11 03:57:37 +00:00
Jamie Bliss c2ecb53bb3
Actually implement range blocking. 2024-01-11 03:45:04 +00:00
Jamie Bliss 6ebc23e24b
Actually implement signing in the httpy client. 2023-12-12 20:20:55 +00:00
Jamie Bliss d4058faf55
Start framing out httpy 2023-12-12 19:11:38 +00:00
11 changed files with 618 additions and 87 deletions

View file

@ -2,7 +2,6 @@ import mimetypes
from functools import partial
from typing import ClassVar
import httpx
import urlman
from cachetools import TTLCache, cached
from django.conf import settings
@ -12,6 +11,7 @@ from django.db import models
from django.utils.safestring import mark_safe
from PIL import Image
from core import httpy
from core.files import get_remote_file
from core.html import FediverseHtmlParser
from core.ld import format_ld_date
@ -45,7 +45,7 @@ class EmojiStates(StateGraph):
timeout=settings.SETUP.REMOTE_TIMEOUT,
max_size=settings.SETUP.EMOJI_MAX_IMAGE_FILESIZE_KB * 1024,
)
except httpx.RequestError:
except httpy.RequestError:
return
if file:

View file

@ -1,7 +1,7 @@
import httpx
from django.db import models
from activities.models.timeline_event import TimelineEvent
from core import httpy
from core.ld import canonicalise
from stator.models import State, StateField, StateGraph, StatorModel
from users.models import Block, FollowStates
@ -75,33 +75,33 @@ class FanOutStates(StateGraph):
case (FanOut.Types.post, False):
post = instance.subject_post
# Sign it and send it
try:
post.author.signed_request(
method="post",
uri=(
instance.identity.shared_inbox_uri
or instance.identity.inbox_uri
),
body=canonicalise(post.to_create_ap()),
)
except httpx.RequestError:
return
with httpy.Client(actor=post.author) as client:
try:
client.post2(
(
instance.identity.shared_inbox_uri
or instance.identity.inbox_uri
),
activity=canonicalise(post.to_create_ap()),
)
except httpy.RequestError:
return
# Handle sending remote posts update
case (FanOut.Types.post_edited, False):
post = instance.subject_post
# Sign it and send it
try:
post.author.signed_request(
method="post",
uri=(
instance.identity.shared_inbox_uri
or instance.identity.inbox_uri
),
body=canonicalise(post.to_update_ap()),
)
except httpx.RequestError:
return
with httpy.Client(actor=post.author) as client:
try:
client.post2(
(
instance.identity.shared_inbox_uri
or instance.identity.inbox_uri
),
activity=canonicalise(post.to_update_ap()),
)
except httpy.RequestError:
return
# Handle deleting local posts
case (FanOut.Types.post_deleted, True):
@ -117,17 +117,17 @@ class FanOutStates(StateGraph):
case (FanOut.Types.post_deleted, False):
post = instance.subject_post
# Send it to the remote inbox
try:
post.author.signed_request(
method="post",
uri=(
instance.identity.shared_inbox_uri
or instance.identity.inbox_uri
),
body=canonicalise(post.to_delete_ap()),
)
except httpx.RequestError:
return
with httpy.Client(actor=post.author) as client:
try:
client.post2(
(
instance.identity.shared_inbox_uri
or instance.identity.inbox_uri
),
activity=canonicalise(post.to_delete_ap()),
)
except httpy.RequestError:
return
# Handle local boosts/likes
case (FanOut.Types.interaction, True):
@ -164,23 +164,24 @@ class FanOutStates(StateGraph):
case (FanOut.Types.interaction, False):
interaction = instance.subject_post_interaction
# Send it to the remote inbox
try:
with httpy.Client(actor=interaction.identity) as client:
if interaction.type == interaction.Types.vote:
body = interaction.to_create_ap()
elif interaction.type == interaction.Types.pin:
body = interaction.to_add_ap()
else:
body = interaction.to_ap()
interaction.identity.signed_request(
method="post",
uri=(
instance.identity.shared_inbox_uri
or instance.identity.inbox_uri
),
body=canonicalise(body),
)
except httpx.RequestError:
return
try:
client.post2(
(
instance.identity.shared_inbox_uri
or instance.identity.inbox_uri
),
activity=canonicalise(body),
)
except httpy.RequestError:
return
# Handle undoing local boosts/likes
case (FanOut.Types.undo_interaction, True): # noqa:F841
@ -196,51 +197,55 @@ class FanOutStates(StateGraph):
case (FanOut.Types.undo_interaction, False): # noqa:F841
interaction = instance.subject_post_interaction
# Send an undo to the remote inbox
try:
if interaction.type == interaction.Types.pin:
body = interaction.to_remove_ap()
else:
body = interaction.to_undo_ap()
interaction.identity.signed_request(
method="post",
uri=(
instance.identity.shared_inbox_uri
or instance.identity.inbox_uri
),
body=canonicalise(body),
)
except httpx.RequestError:
return
with httpy.Client(actor=interaction.identity) as client:
try:
if interaction.type == interaction.Types.pin:
body = interaction.to_remove_ap()
else:
body = interaction.to_undo_ap()
client.post2(
(
instance.identity.shared_inbox_uri
or instance.identity.inbox_uri
),
activity=canonicalise(body),
)
except httpy.RequestError:
return
# Handle sending identity edited to remote
case (FanOut.Types.identity_edited, False):
identity = instance.subject_identity
try:
identity.signed_request(
method="post",
uri=(
instance.identity.shared_inbox_uri
or instance.identity.inbox_uri
),
body=canonicalise(instance.subject_identity.to_update_ap()),
)
except httpx.RequestError:
return
with httpy.Client(actor=identity) as client:
try:
client.post2(
(
instance.identity.shared_inbox_uri
or instance.identity.inbox_uri
),
activity=canonicalise(
instance.subject_identity.to_update_ap()
),
)
except httpy.RequestError:
return
# Handle sending identity deleted to remote
case (FanOut.Types.identity_deleted, False):
identity = instance.subject_identity
try:
identity.signed_request(
method="post",
uri=(
instance.identity.shared_inbox_uri
or instance.identity.inbox_uri
),
body=canonicalise(instance.subject_identity.to_delete_ap()),
)
except httpx.RequestError:
return
with httpy.Client(actor=identity) as client:
try:
client.post2(
(
instance.identity.shared_inbox_uri
or instance.identity.inbox_uri
),
activity=canonicalise(
instance.subject_identity.to_delete_ap()
),
)
except httpy.RequestError:
return
# Handle sending identity moved to remote
case (FanOut.Types.identity_moved, False):

View file

@ -1,12 +1,13 @@
import io
import blurhash
import httpx
from django.conf import settings
from django.core.files import File
from django.core.files.base import ContentFile
from PIL import Image, ImageOps
from . import httpy
class ImageFile(File):
image: Image
@ -70,7 +71,7 @@ def get_remote_file(
"User-Agent": settings.TAKAHE_USER_AGENT,
}
with httpx.Client(headers=headers) as client:
with httpy.Client(headers=headers) as client:
with client.stream(
"GET", url, timeout=timeout, follow_redirects=True
) as stream:

298
core/httpy.py Normal file
View file

@ -0,0 +1,298 @@
"""
Wrapper around HTTPX that provides some fedi-specific features.
The API is identical to httpx, but some features has been added:
* Fedi-compatible HTTP signatures
* Blocked IP ranges
(Because Y is next after X).
"""
import asyncio
import ipaddress
import logging
import socket
import typing
from ssl import SSLCertVerificationError, SSLError
from types import EllipsisType
import httpx
from django.conf import settings
from httpx import RequestError
from httpx._types import TimeoutTypes, URLTypes
from idna.core import InvalidCodepoint
from .signatures import HttpSignature
__all__ = (
"SigningActor",
"Client",
"AsyncClient",
"RequestError",
)
logger = logging.getLogger(__name__)
class SigningActor(typing.Protocol):
"""
An AP Actor with keys, that can sign requests.
Both :class:`users.models.identity.Identity`, and
:class:`users.models.system_actor.SystemActor` implement this protocol.
"""
#: The private key used for signing, in PEM format
private_key: str
# This is pretty much part of the interface, but we don't need it when
# making requests.
# public_key: str
#: The URL we should use to advertise this key
public_key_id: str
class SignedAuth(httpx.Auth):
"""
Handles signing the request.
"""
# Doing it this way so we get automatic sync/async handling
requires_request_body = True
def __init__(self, actor: SigningActor):
self.actor = actor
def auth_flow(self, request: httpx.Request):
HttpSignature.sign_request(
request, self.actor.private_key, self.actor.public_key_id
)
yield request
class BlockedIPError(Exception):
"""
Attempted to make a request that might have hit a blocked IP range.
"""
class IpFilterWrapperTransport(httpx.BaseTransport, httpx.AsyncBaseTransport):
def __init__(
self,
blocked_ranges: list[ipaddress.IPv4Network | ipaddress.IPv6Network | str],
wrappee: httpx.BaseTransport,
):
self.blocked_ranges = blocked_ranges
self.wrappee = wrappee
def __enter__(self):
self.wrappee.__enter__()
return self
def __exit__(self, *exc):
self.wrappee.__exit__(*exc)
def close(self):
self.wrappee.close()
async def __aenter__(self):
await self.wrappee.__aenter__()
return self
async def __aexit__(self, *exc):
await self.wrappee.__aexit__(self, *exc)
async def aclose(self):
await self.wrappee.close()
def _request_to_addrinfo(self, request) -> tuple:
return (
request.url.raw_host.decode("ascii"),
request.url.port or request.url.scheme,
)
def _check_addrinfo(self, req: httpx.Request, ai: typing.Sequence[tuple]):
"""
Compare an IP to the blocked ranges
"""
addr: ipaddress._BaseAddress
for info in ai:
match info:
case (socket.AF_INET, _, _, _, (addr, _)):
addr = ipaddress.IPv4Address(addr)
case (socket.AF_INET6, _, _, _, (addr, _, _, _)):
addr = ipaddress.IPv6Address(addr) # TODO: Do we need the flowinfo?
case _:
continue
for net in self.blocked_ranges:
if addr in net:
raise BlockedIPError(
"Attempted to make a connection to {addr} as {request.url.host} (blocked by {net})"
)
# It would have been nicer to do this at a lower level, so we know what
# IPs we're _actually_ connecting to, but:
# * That's really deep in httpcore and ughhhhhh
# * httpcore just passes the string hostname to the socket API anyway,
# and nobody wants to reimplement happy eyeballs, address fallback, etc
# * If any public name resolves to one of these ranges anyway, it's either
# misconfigured or malicious
def handle_request(self, request: httpx.Request) -> httpx.Response:
try:
self._check_addrinfo(
request, socket.getaddrinfo(*self._request_to_addrinfo(request))
)
except socket.gaierror:
# Some kind of look up error. Gonna assume safe and let farther
# down the stack handle it.
pass
return self.wrappee.handle_request(request)
async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
try:
self._check_addrinfo(
request,
await asyncio.get_running_loop().getaddrinfo(
*self._request_to_addrinfo(request)
),
)
except socket.gaierror:
# Some kind of look up error. Gonna assume safe and let farther
# down the stack handle it.
pass
return await self.wrappee.handle_await_request(request)
def _wrap_transport(
blocked_ranges: list[ipaddress.IPv4Network | ipaddress.IPv6Network | str]
| None
| EllipsisType,
transport,
):
"""
Gets an (Async)Transport that blocks the given IP ranges
"""
if blocked_ranges is ...:
blocked_ranges = settings.HTTP_BLOCKED_RANGES
if not blocked_ranges:
return transport
blocked_ranges = [
ipaddress.ip_network(net) if isinstance(net, str) else net
for net in typing.cast(typing.Iterable, blocked_ranges)
]
return IpFilterWrapperTransport(blocked_ranges, transport)
class BaseClient(httpx._client.BaseClient):
def __init__(
self,
*,
actor: SigningActor | None = None,
blocked_ranges: list[ipaddress.IPv4Network | ipaddress.IPv6Network | str]
| None
| EllipsisType = ...,
timeout: TimeoutTypes = settings.SETUP.REMOTE_TIMEOUT,
**opts,
):
"""
Params:
actor: Actor to sign requests as, or None to not sign requests.
blocked_ranges: IP address to refuse to connect to. Either a list of
Networks, None to disable the feature, or Ellipsis to
pull the Django setting.
"""
if actor:
opts["auth"] = SignedAuth(actor)
self._blocked_ranges = blocked_ranges
super().__init__(timeout=timeout, **opts)
def _init_transport(self, *p, **kw):
transport = super()._init_transport(*p, **kw)
return _wrap_transport(self._blocked_ranges, transport)
def build_request(self, *pargs, **kwargs):
request = super().build_request(*pargs, **kwargs)
# GET requests get implicit accept headers added
if request.method == "GET" and "Accept" not in request.headers:
request.headers["Accept"] = "application/ld+json"
# TODO: Move this to __init__
request.headers["User-Agent"] = settings.TAKAHE_USER_AGENT
return request
# BaseClient before (Async)Client because __init__
class Client(BaseClient, httpx.Client):
def request(self, method: str, url: URLTypes, **params) -> httpx.Response:
"""
Wraps some errors up nicer
"""
if method.lower == "get":
if params["follow_redirects"] is httpx._client.USE_CLIENT_DEFAULT:
params["follow_redirects"] = True
try:
response = super().request(method, url, **params)
except SSLError as invalid_cert:
# Not our problem if the other end doesn't have proper SSL
logger.info("Invalid cert on %s %s", url, invalid_cert)
raise SSLCertVerificationError(invalid_cert) from invalid_cert
except InvalidCodepoint as ex:
# Convert to a more generic error we handle
raise httpx.HTTPError(f"InvalidCodepoint: {str(ex)}") from None
else:
return response
# Deliberately not doing the above to stream() because those use cases don't
# want that handling
def get(
self, url: URLTypes, *, accept: str | None = "application/ld+json", **params
):
"""
Args:
accept: Accept header, set to None to get the open option
"""
if accept:
params.setdefault("headers", {})["Accept"] = accept
return super().get(url, **params)
def post2(self, url: URLTypes, *, activity=None, **params):
"""
Like .post() but:
* Adds activity which is like json but for activities
* Handles response errors a bit
"""
if activity is not None:
params["json"] = activity
params.setdefault("headers", {}).setdefault(
"Content-Type", "application/activity+json"
)
response = self.post(url, **params)
if (
response.status_code >= 400
and response.status_code < 500
and response.status_code != 404
):
raise ValueError(
f"POST error to {url}: {response.status_code} {response.content!r}"
)
return response
class AsyncClient(BaseClient, httpx.AsyncClient):
# FIXME: Add the fancy methods the sync version has.
# (I'm being lazy because I don't think anyone's making async requests)
pass

View file

@ -69,6 +69,16 @@ class HttpSignature:
Allows for calculation and verification of HTTP signatures
"""
#: Headers we should consider when producing signatures
HEADERS_FOR_SIGNING = {
"date",
"host",
"(request-target)",
"content-type",
"content-length",
"digest",
}
@classmethod
def calculate_digest(cls, data, algorithm="sha-256") -> str:
"""
@ -185,6 +195,64 @@ class HttpSignature:
public_key,
)
@classmethod
def sign_request(
cls,
request: httpx.Request,
private_key: str,
key_id: str,
):
"""
Adds a signature to a Request.
"""
if not request.url.scheme:
raise ValueError("URI does not contain a scheme")
# Create the core header field set
date_string = http_date()
request.headers[
"(request-target)"
] = f"{request.method.lower()} {request.url.path}"
request.headers["Host"] = request.url.host
request.headers["Date"] = date_string
# If we have a body, add a digest and content type
body_bytes = request.content
if body_bytes:
request.headers["Digest"] = cls.calculate_digest(body_bytes)
# Sign the headers
signing_headers = [
key
for key in request.headers.keys()
if key.lower() in cls.HEADERS_FOR_SIGNING
]
signed_string = "\n".join(
f"{name.lower()}: {value}"
for name, value in request.headers.items()
if name in signing_headers
)
private_key_instance: rsa.RSAPrivateKey = cast(
rsa.RSAPrivateKey,
serialization.load_pem_private_key(
private_key.encode("ascii"),
password=None,
),
)
signature = private_key_instance.sign(
signed_string.encode("utf8"),
padding.PKCS1v15(),
hashes.SHA256(),
)
request.headers["Signature"] = cls.compile_signature(
{
"keyid": key_id,
"headers": list(signing_headers),
"signature": signature,
"algorithm": "rsa-sha256",
}
)
del request.headers["(request-target)"]
@classmethod
def signed_request(
cls,

View file

@ -172,3 +172,37 @@ We use `HTMX <https://htmx.org/>`_ for dynamically loading content, and
`Hyperscript <https://hyperscript.org/>`_ for most interactions rather than raw
JavaScript. If you can accomplish what you need with these tools, please use them
rather than adding JS.
Cutting a release
-----------------
In order to make a release of Takahē, follow these steps:
* Create or update the release document (in ``/docs/releases``) for the
release; major versions get their own document, minor releases get a
subheading in the document for their major release.
* Go through the git commit history since the last release in order to write
a reasonable summary of features.
* Be sure to include the little paragraphs at the end about contributing and
the docker tag, and an Upgrade Notes section that at minimum mentions
migrations and if they're normal or weird (even if there aren't any, it's
nice to call that out).
* If it's a new doc, make sure you include it in ``docs/releases/index.rst``!
* Update the version number in ``/takahe/__init__.py``
* Update the version number in ``README.md``
* Make a commit containing these changes called ``Releasing 1.23.45``.
* Tag that commit with a tag in the format ``1.23.45``.
* Wait for the GitHub Actions to run and publish the docker images (around 20
minutes as the ARM build is a bit slow)
* Post on the official account announcing the relase and linking to the
now-published release notes.

View file

@ -35,3 +35,20 @@ In additions, there's many bugfixes and minor changes, including:
* Perform some basic domain validity
* Correctly reject more operations when the identity is deleted
* Post edit fanouts for likers/boosters
If you'd like to help with code, design, or other areas, see
:doc:`/contributing` to see how to get in touch.
You can download images from `Docker Hub <https://hub.docker.com/r/jointakahe/takahe>`_,
or use the image name ``jointakahe/takahe:0.11``.
Upgrade Notes
-------------
Migrations
~~~~~~~~~~
There are new database migrations; they are backwards-compatible and should
not present any major database load.

View file

@ -1 +1 @@
__version__ = "0.10.1"
__version__ = "0.11.0"

View file

@ -1,3 +1,4 @@
import ipaddress
import os
import secrets
import sys
@ -476,6 +477,22 @@ TAKAHE_USER_AGENT = (
f"(Takahe/{__version__}; +https://{SETUP.MAIN_DOMAIN}/)"
)
HTTP_BLOCKED_RANGES = list(
map(
ipaddress.ip_network,
[
# All of these are RFC reserved ranges
# Pulled from Wikipedia
"0.0.0.0/8", # Current network
"10.0.0.0/8", # Private, local network
"100.64.0.0/10", # Private, CGNAT
"127.0.0.0/8", # Localhost
"169.254.0.0/16", # Link-local address, zeroconf
"172.16.0.0/12", # Private, local network
],
)
)
if SETUP.LOCAL_SETTINGS:
# Let any errors bubble up
from .local_settings import * # noqa

48
tests/core/test_httpy.py Normal file
View file

@ -0,0 +1,48 @@
import dataclasses
import pytest
from pytest_httpx import HTTPXMock
from core.httpy import BlockedIPError, Client # TODO: Test async client
@dataclasses.dataclass
class MockActor:
private_key: str
public_key: str
public_key_id: str
@pytest.fixture
def signing_actor(keypair):
return MockActor(
private_key=keypair["private_key"],
public_key=keypair["public_key_id"],
public_key_id="https://example.com/test-actor",
)
def test_basics(httpx_mock: HTTPXMock):
httpx_mock.add_response()
with Client() as client:
resp = client.get("https://httpbin.org/status/200")
assert resp.status_code == 200
def test_signature_exists(httpx_mock: HTTPXMock, signing_actor):
httpx_mock.add_response()
with Client(actor=signing_actor) as client:
resp = client.get("https://httpbin.org/headers")
resp.raise_for_status()
request = httpx_mock.get_request()
assert request is not None
assert "Signature" in request.headers
def test_ip_block():
# httpx_mock actually really hates not being called, so don't use it.
with pytest.raises(BlockedIPError), Client() as client:
client.get("http://localhost/")

View file

@ -1,3 +1,4 @@
import httpx
import pytest
from django.test.client import RequestFactory
from pytest_httpx import HTTPXMock
@ -99,6 +100,48 @@ def test_sign_http(httpx_mock: HTTPXMock, keypair):
HttpSignature.verify_request(fake_request, keypair["public_key"])
def test_sign_request(keypair):
"""
Tests signing HTTP requests by round-tripping them through our verifier
"""
# Create document
document = {
"id": "https://example.com/test-create",
"type": "Create",
"actor": "https://example.com/test-actor",
"object": {
"id": "https://example.com/test-object",
"type": "Note",
},
}
request = httpx.Request(
"POST",
"https://example.com/test-actor",
json=document,
)
# Send the signed request to the mock library
HttpSignature.sign_request(
request=request,
private_key=keypair["private_key"],
key_id=keypair["public_key_id"],
)
# Retrieve it and construct a fake request object
fake_request = RequestFactory().post(
path="/test-actor",
data=request.content,
**{
(
"content_type"
if name.lower() == "content-type"
else f"HTTP_{name.upper()}"
): value
for name, value in request.headers.items()
},
)
# Verify that
HttpSignature.verify_request(fake_request, keypair["public_key"])
def test_verify_http(keypair):
"""
Tests verifying HTTP requests against a known good example