From 1ac27aeb8270158403a644d5b1c685218bbbff6e Mon Sep 17 00:00:00 2001
From: Piero Toffanin <pt@masseranolabs.com>
Date: Sun, 16 May 2021 11:50:22 -0400
Subject: [PATCH 1/2] Flood protection

---
 app/app.py       | 37 ++++++++++++++++++++++++++++++++++---
 app/flood.py     | 36 ++++++++++++++++++++++++++++++++++++
 app/main.py      |  2 ++
 requirements.txt |  1 +
 4 files changed, 73 insertions(+), 3 deletions(-)
 create mode 100644 app/flood.py

diff --git a/app/app.py b/app/app.py
index d0723e1..e0af8de 100644
--- a/app/app.py
+++ b/app/app.py
@@ -5,8 +5,7 @@ from flask_swagger_ui import get_swaggerui_blueprint
 from pkg_resources import resource_filename
 from .api_keys import Database
 from app.language import detect_languages, transliterate
-
-api_keys_db = None
+from app import flood
 
 def get_json_dict(request):
     d = request.get_json()
@@ -88,6 +87,9 @@ def create_app(args):
     else:
       from .no_limiter import Limiter
       limiter = Limiter()
+    
+    if args.req_flood_threshold > 0:
+        flood.setup(args.req_flood_threshold)
 
     @app.errorhandler(400)
     def invalid_api(e):
@@ -99,8 +101,14 @@ def create_app(args):
 
     @app.errorhandler(429)
     def slow_down_error(e):
+        flood.report(get_remote_address())
         return jsonify({"error": "Slowdown: " + str(e.description)}), 429
 
+    @app.errorhandler(403)
+    def denied(e):
+        return jsonify({"error": str(e.description)}), 403
+
+
     @app.route("/")
     @limiter.exempt
     def index():
@@ -236,7 +244,18 @@ def create_app(args):
                 error:
                   type: string
                   description: Reason for slow down
+          403:
+            description: Banned
+            schema:
+              id: error-response
+              type: object
+              properties:
+                error:
+                  type: string
+                  description: Error message
         """
+        if flood.is_banned(get_remote_address()):
+            abort(403, description="Too many request limits violations")
 
         if request.is_json:
             json = get_json_dict(request)
@@ -350,7 +369,7 @@ def create_app(args):
               properties:
                 error:
                   type: string
-                  description: Error message
+                  description: Error message          
           500:
             description: Detection error
             schema:
@@ -369,7 +388,19 @@ def create_app(args):
                 error:
                   type: string
                   description: Reason for slow down
+          403:
+            description: Banned
+            schema:
+              id: error-response
+              type: object
+              properties:
+                error:
+                  type: string
+                  description: Error message
         """
+        if flood.is_banned(get_remote_address()):
+            abort(403, description="Too many request limits violations")
+
         if request.is_json:
             json = get_json_dict(request)
             q = json.get('q')
diff --git a/app/flood.py b/app/flood.py
new file mode 100644
index 0000000..377d85e
--- /dev/null
+++ b/app/flood.py
@@ -0,0 +1,36 @@
+import time
+import atexit
+
+from apscheduler.schedulers.background import BackgroundScheduler
+
+banned = {}
+active = False
+threshold = -1
+
+def clear_banned():
+    global banned
+    print(banned)
+    banned = {}
+
+def setup(violations_threshold = 100):
+    global active
+    global threshold
+
+    active = True
+    threshold = violations_threshold
+
+    scheduler = BackgroundScheduler()
+    scheduler.add_job(func=clear_banned, trigger="interval", weeks=4)
+    scheduler.start()
+
+    # Shut down the scheduler when exiting the app
+    atexit.register(lambda: scheduler.shutdown())
+
+
+def report(request_ip):
+    banned[request_ip] = banned.get(request_ip, 0)
+    banned[request_ip] += 1
+
+def is_banned(request_ip):
+    # More than X offences?
+    return active and banned.get(request_ip, 0) >= threshold
\ No newline at end of file
diff --git a/app/main.py b/app/main.py
index 55a7d62..ce1d5f9 100644
--- a/app/main.py
+++ b/app/main.py
@@ -14,6 +14,8 @@ def main():
                         help='Set the default maximum number of requests per minute per client (%(default)s)')
     parser.add_argument('--daily-req-limit', default=-1, type=int, metavar="<number>",
                         help='Set the default maximum number of requests per day per client, in addition to req-limit. (%(default)s)')
+    parser.add_argument('--req-flood-threshold', default=-1, type=int, metavar="<number>",
+                        help='Set the maximum number of request limit offences per 4 weeks that a client can exceed before being banned. (%(default)s)')
     parser.add_argument('--batch-limit', default=-1, type=int, metavar="<number of texts>",
                         help='Set maximum number of texts to translate in a batch request (%(default)s)')
     parser.add_argument('--ga-id', type=str, default=None, metavar="<GA ID>",
diff --git a/requirements.txt b/requirements.txt
index e13d5d9..f44a9c6 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -10,3 +10,4 @@ pycld2==0.41
 morfessor==2.0.6
 polyglot==16.7.4
 appdirs==1.4.4
+APScheduler==3.7.0

From 5c76febaa458df58522f893e7ef245481882e666 Mon Sep 17 00:00:00 2001
From: Piero Toffanin <pt@masseranolabs.com>
Date: Sun, 16 May 2021 11:52:39 -0400
Subject: [PATCH 2/2] Fix memory leak

---
 app/flood.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/app/flood.py b/app/flood.py
index 377d85e..31ee416 100644
--- a/app/flood.py
+++ b/app/flood.py
@@ -28,8 +28,9 @@ def setup(violations_threshold = 100):
 
 
 def report(request_ip):
-    banned[request_ip] = banned.get(request_ip, 0)
-    banned[request_ip] += 1
+    if active:
+        banned[request_ip] = banned.get(request_ip, 0)
+        banned[request_ip] += 1
 
 def is_banned(request_ip):
     # More than X offences?