mirror of
https://git.pleroma.social/pleroma/pleroma.git
synced 2024-12-23 00:26:30 +00:00
Merge branch 'issue/1936' into 'develop'
[#1936] Ability to search for banned MediaProxy URLs  See merge request pleroma/pleroma!2873
This commit is contained in:
commit
34b099fffa
6 changed files with 121 additions and 79 deletions
|
@ -1266,11 +1266,14 @@ Loads json generated from `config/descriptions.exs`.
|
||||||
- Params:
|
- Params:
|
||||||
- *optional* `page`: **integer** page number
|
- *optional* `page`: **integer** page number
|
||||||
- *optional* `page_size`: **integer** number of log entries per page (default is `50`)
|
- *optional* `page_size`: **integer** number of log entries per page (default is `50`)
|
||||||
|
- *optional* `query`: **string** search term
|
||||||
|
|
||||||
- Response:
|
- Response:
|
||||||
|
|
||||||
``` json
|
``` json
|
||||||
{
|
{
|
||||||
|
"page_size": integer,
|
||||||
|
"count": integer,
|
||||||
"urls": [
|
"urls": [
|
||||||
"http://example.com/media/a688346.jpg",
|
"http://example.com/media/a688346.jpg",
|
||||||
"http://example.com/media/fb1f4d.jpg"
|
"http://example.com/media/fb1f4d.jpg"
|
||||||
|
@ -1290,12 +1293,7 @@ Loads json generated from `config/descriptions.exs`.
|
||||||
- Response:
|
- Response:
|
||||||
|
|
||||||
``` json
|
``` json
|
||||||
{
|
{ }
|
||||||
"urls": [
|
|
||||||
"http://example.com/media/a688346.jpg",
|
|
||||||
"http://example.com/media/fb1f4d.jpg"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
|
|
||||||
```
|
```
|
||||||
|
|
||||||
|
@ -1311,11 +1309,6 @@ Loads json generated from `config/descriptions.exs`.
|
||||||
- Response:
|
- Response:
|
||||||
|
|
||||||
``` json
|
``` json
|
||||||
{
|
{ }
|
||||||
"urls": [
|
|
||||||
"http://example.com/media/a688346.jpg",
|
|
||||||
"http://example.com/media/fb1f4d.jpg"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
|
|
||||||
```
|
```
|
||||||
|
|
|
@ -26,29 +26,40 @@ defmodule Pleroma.Web.AdminAPI.MediaProxyCacheController do
|
||||||
defdelegate open_api_operation(action), to: Spec.MediaProxyCacheOperation
|
defdelegate open_api_operation(action), to: Spec.MediaProxyCacheOperation
|
||||||
|
|
||||||
def index(%{assigns: %{user: _}} = conn, params) do
|
def index(%{assigns: %{user: _}} = conn, params) do
|
||||||
cursor =
|
entries = fetch_entries(params)
|
||||||
:banned_urls_cache
|
urls = paginate_entries(entries, params.page, params.page_size)
|
||||||
|> :ets.table([{:traverse, {:select, Cachex.Query.create(true, :key)}}])
|
|
||||||
|> :qlc.cursor()
|
|
||||||
|
|
||||||
urls =
|
render(conn, "index.json",
|
||||||
case params.page do
|
urls: urls,
|
||||||
1 ->
|
page_size: params.page_size,
|
||||||
:qlc.next_answers(cursor, params.page_size)
|
count: length(entries)
|
||||||
|
)
|
||||||
_ ->
|
|
||||||
:qlc.next_answers(cursor, (params.page - 1) * params.page_size)
|
|
||||||
:qlc.next_answers(cursor, params.page_size)
|
|
||||||
end
|
end
|
||||||
|
|
||||||
:qlc.delete_cursor(cursor)
|
defp fetch_entries(params) do
|
||||||
|
MediaProxy.cache_table()
|
||||||
|
|> Cachex.stream!(Cachex.Query.create(true, :key))
|
||||||
|
|> filter_entries(params[:query])
|
||||||
|
end
|
||||||
|
|
||||||
render(conn, "index.json", urls: urls)
|
defp filter_entries(stream, query) when is_binary(query) do
|
||||||
|
regex = ~r/#{query}/i
|
||||||
|
|
||||||
|
stream
|
||||||
|
|> Enum.filter(fn url -> String.match?(url, regex) end)
|
||||||
|
|> Enum.to_list()
|
||||||
|
end
|
||||||
|
|
||||||
|
defp filter_entries(stream, _), do: Enum.to_list(stream)
|
||||||
|
|
||||||
|
defp paginate_entries(entries, page, page_size) do
|
||||||
|
offset = page_size * (page - 1)
|
||||||
|
Enum.slice(entries, offset, page_size)
|
||||||
end
|
end
|
||||||
|
|
||||||
def delete(%{assigns: %{user: _}, body_params: %{urls: urls}} = conn, _) do
|
def delete(%{assigns: %{user: _}, body_params: %{urls: urls}} = conn, _) do
|
||||||
MediaProxy.remove_from_banned_urls(urls)
|
MediaProxy.remove_from_banned_urls(urls)
|
||||||
render(conn, "index.json", urls: urls)
|
json(conn, %{})
|
||||||
end
|
end
|
||||||
|
|
||||||
def purge(%{assigns: %{user: _}, body_params: %{urls: urls, ban: ban}} = conn, _) do
|
def purge(%{assigns: %{user: _}, body_params: %{urls: urls, ban: ban}} = conn, _) do
|
||||||
|
@ -58,6 +69,6 @@ defmodule Pleroma.Web.AdminAPI.MediaProxyCacheController do
|
||||||
MediaProxy.put_in_banned_urls(urls)
|
MediaProxy.put_in_banned_urls(urls)
|
||||||
end
|
end
|
||||||
|
|
||||||
render(conn, "index.json", urls: urls)
|
json(conn, %{})
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
|
@ -5,7 +5,11 @@
|
||||||
defmodule Pleroma.Web.AdminAPI.MediaProxyCacheView do
|
defmodule Pleroma.Web.AdminAPI.MediaProxyCacheView do
|
||||||
use Pleroma.Web, :view
|
use Pleroma.Web, :view
|
||||||
|
|
||||||
def render("index.json", %{urls: urls}) do
|
def render("index.json", %{urls: urls, page_size: page_size, count: count}) do
|
||||||
%{urls: urls}
|
%{
|
||||||
|
urls: urls,
|
||||||
|
count: count,
|
||||||
|
page_size: page_size
|
||||||
|
}
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
|
@ -21,6 +21,12 @@ defmodule Pleroma.Web.ApiSpec.Admin.MediaProxyCacheOperation do
|
||||||
operationId: "AdminAPI.MediaProxyCacheController.index",
|
operationId: "AdminAPI.MediaProxyCacheController.index",
|
||||||
security: [%{"oAuth" => ["read:media_proxy_caches"]}],
|
security: [%{"oAuth" => ["read:media_proxy_caches"]}],
|
||||||
parameters: [
|
parameters: [
|
||||||
|
Operation.parameter(
|
||||||
|
:query,
|
||||||
|
:query,
|
||||||
|
%Schema{type: :string, default: nil},
|
||||||
|
"Page"
|
||||||
|
),
|
||||||
Operation.parameter(
|
Operation.parameter(
|
||||||
:page,
|
:page,
|
||||||
:query,
|
:query,
|
||||||
|
@ -36,7 +42,26 @@ defmodule Pleroma.Web.ApiSpec.Admin.MediaProxyCacheOperation do
|
||||||
| admin_api_params()
|
| admin_api_params()
|
||||||
],
|
],
|
||||||
responses: %{
|
responses: %{
|
||||||
200 => success_response()
|
200 =>
|
||||||
|
Operation.response(
|
||||||
|
"Array of banned MediaProxy URLs in Cachex",
|
||||||
|
"application/json",
|
||||||
|
%Schema{
|
||||||
|
type: :object,
|
||||||
|
properties: %{
|
||||||
|
count: %Schema{type: :integer},
|
||||||
|
page_size: %Schema{type: :integer},
|
||||||
|
urls: %Schema{
|
||||||
|
type: :array,
|
||||||
|
items: %Schema{
|
||||||
|
type: :string,
|
||||||
|
format: :uri,
|
||||||
|
description: "MediaProxy URLs"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
end
|
end
|
||||||
|
@ -61,7 +86,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.MediaProxyCacheOperation do
|
||||||
required: true
|
required: true
|
||||||
),
|
),
|
||||||
responses: %{
|
responses: %{
|
||||||
200 => success_response(),
|
200 => empty_object_response(),
|
||||||
400 => Operation.response("Error", "application/json", ApiError)
|
400 => Operation.response("Error", "application/json", ApiError)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -88,25 +113,9 @@ defmodule Pleroma.Web.ApiSpec.Admin.MediaProxyCacheOperation do
|
||||||
required: true
|
required: true
|
||||||
),
|
),
|
||||||
responses: %{
|
responses: %{
|
||||||
200 => success_response(),
|
200 => empty_object_response(),
|
||||||
400 => Operation.response("Error", "application/json", ApiError)
|
400 => Operation.response("Error", "application/json", ApiError)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
end
|
end
|
||||||
|
|
||||||
defp success_response do
|
|
||||||
Operation.response("Array of banned MediaProxy URLs in Cachex", "application/json", %Schema{
|
|
||||||
type: :object,
|
|
||||||
properties: %{
|
|
||||||
urls: %Schema{
|
|
||||||
type: :array,
|
|
||||||
items: %Schema{
|
|
||||||
type: :string,
|
|
||||||
format: :uri,
|
|
||||||
description: "MediaProxy URLs"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
end
|
|
||||||
end
|
end
|
||||||
|
|
|
@ -9,28 +9,31 @@ defmodule Pleroma.Web.MediaProxy do
|
||||||
alias Pleroma.Web.MediaProxy.Invalidation
|
alias Pleroma.Web.MediaProxy.Invalidation
|
||||||
|
|
||||||
@base64_opts [padding: false]
|
@base64_opts [padding: false]
|
||||||
|
@cache_table :banned_urls_cache
|
||||||
|
|
||||||
|
def cache_table, do: @cache_table
|
||||||
|
|
||||||
@spec in_banned_urls(String.t()) :: boolean()
|
@spec in_banned_urls(String.t()) :: boolean()
|
||||||
def in_banned_urls(url), do: elem(Cachex.exists?(:banned_urls_cache, url(url)), 1)
|
def in_banned_urls(url), do: elem(Cachex.exists?(@cache_table, url(url)), 1)
|
||||||
|
|
||||||
def remove_from_banned_urls(urls) when is_list(urls) do
|
def remove_from_banned_urls(urls) when is_list(urls) do
|
||||||
Cachex.execute!(:banned_urls_cache, fn cache ->
|
Cachex.execute!(@cache_table, fn cache ->
|
||||||
Enum.each(Invalidation.prepare_urls(urls), &Cachex.del(cache, &1))
|
Enum.each(Invalidation.prepare_urls(urls), &Cachex.del(cache, &1))
|
||||||
end)
|
end)
|
||||||
end
|
end
|
||||||
|
|
||||||
def remove_from_banned_urls(url) when is_binary(url) do
|
def remove_from_banned_urls(url) when is_binary(url) do
|
||||||
Cachex.del(:banned_urls_cache, url(url))
|
Cachex.del(@cache_table, url(url))
|
||||||
end
|
end
|
||||||
|
|
||||||
def put_in_banned_urls(urls) when is_list(urls) do
|
def put_in_banned_urls(urls) when is_list(urls) do
|
||||||
Cachex.execute!(:banned_urls_cache, fn cache ->
|
Cachex.execute!(@cache_table, fn cache ->
|
||||||
Enum.each(Invalidation.prepare_urls(urls), &Cachex.put(cache, &1, true))
|
Enum.each(Invalidation.prepare_urls(urls), &Cachex.put(cache, &1, true))
|
||||||
end)
|
end)
|
||||||
end
|
end
|
||||||
|
|
||||||
def put_in_banned_urls(url) when is_binary(url) do
|
def put_in_banned_urls(url) when is_binary(url) do
|
||||||
Cachex.put(:banned_urls_cache, url(url), true)
|
Cachex.put(@cache_table, url(url), true)
|
||||||
end
|
end
|
||||||
|
|
||||||
def url(url) when is_nil(url) or url == "", do: nil
|
def url(url) when is_nil(url) or url == "", do: nil
|
||||||
|
|
|
@ -48,6 +48,9 @@ defmodule Pleroma.Web.AdminAPI.MediaProxyCacheControllerTest do
|
||||||
|> get("/api/pleroma/admin/media_proxy_caches?page_size=2")
|
|> get("/api/pleroma/admin/media_proxy_caches?page_size=2")
|
||||||
|> json_response_and_validate_schema(200)
|
|> json_response_and_validate_schema(200)
|
||||||
|
|
||||||
|
assert response["page_size"] == 2
|
||||||
|
assert response["count"] == 5
|
||||||
|
|
||||||
assert response["urls"] == [
|
assert response["urls"] == [
|
||||||
"http://localhost:4001/media/fb1f4d.jpg",
|
"http://localhost:4001/media/fb1f4d.jpg",
|
||||||
"http://localhost:4001/media/a688346.jpg"
|
"http://localhost:4001/media/a688346.jpg"
|
||||||
|
@ -63,6 +66,9 @@ defmodule Pleroma.Web.AdminAPI.MediaProxyCacheControllerTest do
|
||||||
"http://localhost:4001/media/tb13f47.jpg"
|
"http://localhost:4001/media/tb13f47.jpg"
|
||||||
]
|
]
|
||||||
|
|
||||||
|
assert response["page_size"] == 2
|
||||||
|
assert response["count"] == 5
|
||||||
|
|
||||||
response =
|
response =
|
||||||
conn
|
conn
|
||||||
|> get("/api/pleroma/admin/media_proxy_caches?page_size=2&page=3")
|
|> get("/api/pleroma/admin/media_proxy_caches?page_size=2&page=3")
|
||||||
|
@ -70,6 +76,30 @@ defmodule Pleroma.Web.AdminAPI.MediaProxyCacheControllerTest do
|
||||||
|
|
||||||
assert response["urls"] == ["http://localhost:4001/media/wb1f46.jpg"]
|
assert response["urls"] == ["http://localhost:4001/media/wb1f46.jpg"]
|
||||||
end
|
end
|
||||||
|
|
||||||
|
test "search banned MediaProxy URLs", %{conn: conn} do
|
||||||
|
MediaProxy.put_in_banned_urls([
|
||||||
|
"http://localhost:4001/media/a688346.jpg",
|
||||||
|
"http://localhost:4001/media/ff44b1f4d.jpg"
|
||||||
|
])
|
||||||
|
|
||||||
|
MediaProxy.put_in_banned_urls("http://localhost:4001/media/gb1f44.jpg")
|
||||||
|
MediaProxy.put_in_banned_urls("http://localhost:4001/media/tb13f47.jpg")
|
||||||
|
MediaProxy.put_in_banned_urls("http://localhost:4001/media/wb1f46.jpg")
|
||||||
|
|
||||||
|
response =
|
||||||
|
conn
|
||||||
|
|> get("/api/pleroma/admin/media_proxy_caches?page_size=2&query=F44")
|
||||||
|
|> json_response_and_validate_schema(200)
|
||||||
|
|
||||||
|
assert response["urls"] == [
|
||||||
|
"http://localhost:4001/media/gb1f44.jpg",
|
||||||
|
"http://localhost:4001/media/ff44b1f4d.jpg"
|
||||||
|
]
|
||||||
|
|
||||||
|
assert response["page_size"] == 2
|
||||||
|
assert response["count"] == 2
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
describe "POST /api/pleroma/admin/media_proxy_caches/delete" do
|
describe "POST /api/pleroma/admin/media_proxy_caches/delete" do
|
||||||
|
@ -79,7 +109,6 @@ defmodule Pleroma.Web.AdminAPI.MediaProxyCacheControllerTest do
|
||||||
"http://localhost:4001/media/fb1f4d.jpg"
|
"http://localhost:4001/media/fb1f4d.jpg"
|
||||||
])
|
])
|
||||||
|
|
||||||
response =
|
|
||||||
conn
|
conn
|
||||||
|> put_req_header("content-type", "application/json")
|
|> put_req_header("content-type", "application/json")
|
||||||
|> post("/api/pleroma/admin/media_proxy_caches/delete", %{
|
|> post("/api/pleroma/admin/media_proxy_caches/delete", %{
|
||||||
|
@ -87,7 +116,6 @@ defmodule Pleroma.Web.AdminAPI.MediaProxyCacheControllerTest do
|
||||||
})
|
})
|
||||||
|> json_response_and_validate_schema(200)
|
|> json_response_and_validate_schema(200)
|
||||||
|
|
||||||
assert response["urls"] == ["http://localhost:4001/media/a688346.jpg"]
|
|
||||||
refute MediaProxy.in_banned_urls("http://localhost:4001/media/a688346.jpg")
|
refute MediaProxy.in_banned_urls("http://localhost:4001/media/a688346.jpg")
|
||||||
assert MediaProxy.in_banned_urls("http://localhost:4001/media/fb1f4d.jpg")
|
assert MediaProxy.in_banned_urls("http://localhost:4001/media/fb1f4d.jpg")
|
||||||
end
|
end
|
||||||
|
@ -106,14 +134,11 @@ defmodule Pleroma.Web.AdminAPI.MediaProxyCacheControllerTest do
|
||||||
purge: fn _, _ -> {"ok", 0} end
|
purge: fn _, _ -> {"ok", 0} end
|
||||||
]}
|
]}
|
||||||
] do
|
] do
|
||||||
response =
|
|
||||||
conn
|
conn
|
||||||
|> put_req_header("content-type", "application/json")
|
|> put_req_header("content-type", "application/json")
|
||||||
|> post("/api/pleroma/admin/media_proxy_caches/purge", %{urls: urls, ban: false})
|
|> post("/api/pleroma/admin/media_proxy_caches/purge", %{urls: urls, ban: false})
|
||||||
|> json_response_and_validate_schema(200)
|
|> json_response_and_validate_schema(200)
|
||||||
|
|
||||||
assert response["urls"] == urls
|
|
||||||
|
|
||||||
refute MediaProxy.in_banned_urls("http://example.com/media/a688346.jpg")
|
refute MediaProxy.in_banned_urls("http://example.com/media/a688346.jpg")
|
||||||
refute MediaProxy.in_banned_urls("http://example.com/media/fb1f4d.jpg")
|
refute MediaProxy.in_banned_urls("http://example.com/media/fb1f4d.jpg")
|
||||||
end
|
end
|
||||||
|
@ -126,17 +151,14 @@ defmodule Pleroma.Web.AdminAPI.MediaProxyCacheControllerTest do
|
||||||
]
|
]
|
||||||
|
|
||||||
with_mocks [{MediaProxy.Invalidation.Script, [], [purge: fn _, _ -> {"ok", 0} end]}] do
|
with_mocks [{MediaProxy.Invalidation.Script, [], [purge: fn _, _ -> {"ok", 0} end]}] do
|
||||||
response =
|
|
||||||
conn
|
conn
|
||||||
|> put_req_header("content-type", "application/json")
|
|> put_req_header("content-type", "application/json")
|
||||||
|> post("/api/pleroma/admin/media_proxy_caches/purge", %{
|
|> post(
|
||||||
urls: urls,
|
"/api/pleroma/admin/media_proxy_caches/purge",
|
||||||
ban: true
|
%{urls: urls, ban: true}
|
||||||
})
|
)
|
||||||
|> json_response_and_validate_schema(200)
|
|> json_response_and_validate_schema(200)
|
||||||
|
|
||||||
assert response["urls"] == urls
|
|
||||||
|
|
||||||
assert MediaProxy.in_banned_urls("http://example.com/media/a688346.jpg")
|
assert MediaProxy.in_banned_urls("http://example.com/media/a688346.jpg")
|
||||||
assert MediaProxy.in_banned_urls("http://example.com/media/fb1f4d.jpg")
|
assert MediaProxy.in_banned_urls("http://example.com/media/fb1f4d.jpg")
|
||||||
end
|
end
|
||||||
|
|
Loading…
Reference in a new issue