{{? !master.no_address}}
|
{{=addressString(server)}}
| {{?}}
{{? !master.no_clients}}
-
- {{=constantWidth(server.clients + '/' + server.clients_max, 3.4)}}
+ |
+ {{=constantWidth(server.clients_list.length + '/' + server.clients_max, 3.4)}}
{{? !master.no_avgtop}} {{=constantWidth(Math.floor(server.pop_v) + '/' + server.clients_top, 3.4)}}{{?}}
{{=hoverList("Clients", server.clients_list)}}
| {{?}}
diff --git a/static/style.css b/server_list/static/style.css
similarity index 100%
rename from static/style.css
rename to server_list/static/style.css
diff --git a/server_list/tasks.py b/server_list/tasks.py
new file mode 100644
index 0000000..afd52ed
--- /dev/null
+++ b/server_list/tasks.py
@@ -0,0 +1,100 @@
+import asyncio
+import json
+import os
+from datetime import datetime
+
+from .app import app, celery, db
+from .models import Server, Stats
+from .ping import ping_servers_async, ping_server_addresses
+from .util import get_geo_continent, server_ranking
+
+
+@celery.task
+def update_server(obj):
+ geo_continent = get_geo_continent(obj["addr_info"][-1][4][0])
+ if geo_continent is not None:
+ obj["geo_continent"] = geo_continent
+
+ # Ensure that a Minetest server is actually reachable on all addresses
+ pings = ping_server_addresses(obj["address"], obj["port"])
+
+ if pings is None:
+ return
+
+ # Use average ping
+ obj["ping"] = sum(pings) / len(pings)
+
+ Server.create_or_update(obj)
+
+ db.session.commit()
+
+
+def update_ping():
+ servers = Server.query.filter_by(online=True).all()
+
+ addresses = [(s.address, s.port) for s in servers]
+ pings = []
+
+ async def do_ping():
+ pings.extend(await ping_servers_async(addresses))
+ asyncio.run(do_ping())
+
+ for i, server in enumerate(servers):
+ if pings[i] is None:
+ server.set_offline()
+ else:
+ server.ping = pings[i]
+
+
+def update_list_json():
+ online_servers = Server.query.filter_by(online=True).all()
+ online_servers.sort(key=server_ranking, reverse=True)
+ server_list = [s.as_json() for s in online_servers]
+
+ num_clients = 0
+ for server in server_list:
+ num_clients += len(server["clients_list"])
+
+ stats = Stats.get()
+ stats.max_servers = max(len(server_list), stats.max_servers)
+ stats.max_clients = max(num_clients, stats.max_clients)
+
+ list_path = os.path.join(app.static_folder, "list.json")
+ # Write to temporary file, then do an atomic replace so that clients don't
+ # see a truncated file if they load the list just as it's being updated.
+ with open(list_path + "~", "w") as fd:
+ debug = app.config["DEBUG"]
+ json.dump({
+ "total": {"servers": len(server_list), "clients": num_clients},
+ "total_max": {"servers": stats.max_servers, "clients": stats.max_clients},
+ "list": server_list,
+ },
+ fd,
+ indent="\t" if debug else None,
+ separators=(',', ': ') if debug else (',', ':')
+ )
+ os.replace(list_path + "~", list_path)
+
+
+@celery.task
+def maintenance():
+ cutoff = datetime.utcnow() - app.config["PURGE_TIME"]
+ Server.query.filter(
+ Server.online == True,
+ Server.last_update < cutoff
+ ).update({
+ "online": False,
+ "total_uptime": Server.total_uptime + (Server.last_update - Server.start_time).total_seconds(),
+ "down_time": datetime.utcnow(),
+ })
+
+ update_ping()
+
+ update_list_json()
+
+ db.session.commit()
+
+
+@celery.on_after_configure.connect
+def setup_periodic_tasks(sender, **kwargs):
+ sender.add_periodic_task(60, maintenance.s(), name='Server list maintenance')
diff --git a/server_list/templates/address_verification_failed.txt b/server_list/templates/address_verification_failed.txt
new file mode 100644
index 0000000..e1e1a04
--- /dev/null
+++ b/server_list/templates/address_verification_failed.txt
@@ -0,0 +1,10 @@
+Server address does not include a DNS record for the IP that the announcement was sent from.
+Announce IP: {{ announce_ip }}
+Address records: {{ valid_addresses | join ", " }}
+
+Help: This is usually because your server is only listening on IPv4 but your announcement is being sent over IPv6.
+There are two ways to fix this:
+ 1. (preferred) Set bind_address = :: in your server config to listen on IPv6 and add your IPv6 address to DNS as an AAAA record.
+ This allows both IPv4 and IPv6 because the linux kernel will forward IPv4 requests to the IPv6 socket by default using an IPv4-mapped IPv6 address (look up IPV6_V6ONLY for more information).
+ On other operating systems this option may not work and you'll have to use the second option.
+ 2. Set bind_address = 0.0.0.0 in your server config to force IPv4 only, the announce will then be sent from the IPv4 address.
diff --git a/server_list/util.py b/server_list/util.py
new file mode 100755
index 0000000..67aa08c
--- /dev/null
+++ b/server_list/util.py
@@ -0,0 +1,199 @@
+import re
+import socket
+import time
+
+from datetime import datetime, timedelta
+
+from .app import app
+
+try:
+ import maxminddb
+ MAXMIND_DB = app.config.get("MAXMIND_DB", None)
+ if MAXMIND_DB is not None:
+ geoip_reader = maxminddb.open_database(MAXMIND_DB, maxminddb.MODE_AUTO)
+ else:
+ app.logger.warning(
+ "For working GeoIP download the database from "
+ "https://db-ip.com/db/download/ip-to-country-lite and point "
+ "the MAXMIND_DB setting to the .mmdb file."
+ )
+ geoip_reader = None
+except ImportError:
+ app.logger.warning("maxminddb not available, GeoIP will not work.")
+
+
+UUID_RE = re.compile('^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$')
+
+
+def check_ban(announce_ip, address, port):
+ if "%s/%d" % (announce_ip, port) in app.config["BANNED_SERVERS"]:
+ return True
+
+ if address != announce_ip:
+ # Normalize address for ban checks
+ address = address.lower().rstrip(".")
+ if f"{address}/{port}" in app.config["BANNED_SERVERS"] or \
+ address in app.config["BANNED_SERVERS"]:
+ return True
+
+ return False
+
+
+def get_addr_info(address, port):
+ try:
+ return socket.getaddrinfo(
+ address,
+ port,
+ type=socket.SOCK_DGRAM,
+ proto=socket.SOL_UDP)
+ except socket.gaierror:
+ app.logger.warning("Unable to get address info for [%s]:%d.",
+ address, port)
+ return None
+
+
+def verify_announce(addr_info, address, announce_ip):
+ if address == announce_ip:
+ return True
+
+ addresses = set(data[4][0] for data in addr_info)
+ if not announce_ip in addresses:
+ app.logger.warning(
+ "Server address %r does not resolve to announce IP %r (address valid for %r).",
+ address, announce_ip, addresses)
+ return False
+
+ return True
+
+
+def get_geo_continent(ip):
+ if ip.startswith("::ffff:"):
+ ip = ip[7:]
+
+ if reader is None:
+ return
+
+ try:
+ geo = geoip_reader.get(ip)
+ except ValueError:
+ return
+
+ if geo and "continent" in geo:
+ return geo["continent"]["code"]
+ else:
+ app.logger.warning("Unable to get GeoIP Continent data for %s.", ip)
+ return None
+
+
+# fieldName: (Required, Type, SubType)
+fields = {
+ "action": (True, "str"),
+
+ "world_uuid": (False, "str"),
+
+ "address": (False, "str"),
+ "port": (False, "int"),
+
+ "clients_max": (True, "int"),
+ "uptime": (True, "int"),
+ "game_time": (True, "int"),
+ "lag": (False, "float"),
+
+ "clients_list": (True, "list", "str"),
+ "mods": (False, "list", "str"),
+
+ "version": (True, "str"),
+ "proto_min": (True, "int"),
+ "proto_max": (True, "int"),
+
+ "gameid": (True, "str"),
+ "mapgen": (False, "str"),
+ "url": (False, "str"),
+ "privs": (False, "str"),
+ "name": (True, "str"),
+ "description": (True, "str"),
+
+ # Flags
+ "creative": (False, "bool"),
+ "dedicated": (False, "bool"),
+ "damage": (False, "bool"),
+ "pvp": (False, "bool"),
+ "password": (False, "bool"),
+ "rollback": (False, "bool"),
+ "can_see_far_names": (False, "bool"),
+}
+
+def check_request_json(obj):
+ """Checks the types and values of fields in the request.
+
+ Returns error string or None.
+ """
+ for name, data in fields.items():
+ # Delete optional string fields sent as empty strings
+ if not data[0] and data[1] == "str" and obj.get(name) == "":
+ del obj[name]
+
+ if not name in obj:
+ if data[0]:
+ return f"Required field '{name}' is missing."
+ continue
+
+ type_str = type(obj[name]).__name__
+ if type_str != data[1]:
+ return f"Field '{name}'' has incorrect type (expected {data[1]} found {type_str})."
+
+
+ if len(data) >= 3:
+ for item in obj[name]:
+ subtype_str = type(item).__name__
+ if subtype_str != data[2]:
+ return f"Entry in field '{name}' has incorrect type (expected {data[2]} found {subtype_str})."
+
+ if "url" in obj:
+ url = obj["url"]
+ if not any(url.startswith(p) for p in ["http://", "https://", "//"]):
+ return "Field 'url' does not match expected format."
+
+ if "world_uuid" in obj and not UUID_RE.match(obj["world_uuid"]):
+ return "Field 'world_uuid' does not match expected format."
+
+ return None
+
+
+def server_ranking(server):
+ now = datetime.utcnow()
+ points = 0
+
+ clients = server.clients.split('\n')
+
+ # 1 per client, capped to CLIENT_LIMIT or clients_max * 0.9
+ cap = min(server.clients_max * 0.9, app.config["CLIENT_LIMIT"])
+ points += min(len(clients), cap)
+
+ # 1/2 per week of age, limited to 8
+ points += min(8, (now - server.first_seen) / timedelta(weeks=2))
+
+ # 1/2 per average client, limited to 4
+ points += min(4, server.popularity / 2)
+
+ # -8 per second of ping over 0.3s
+ if server.ping > 0.3:
+ points -= (server.ping - 0.3) * 8
+
+ # Up to -4 for less than an hour of uptime (penalty linearly decreasing)
+ ONE_HOUR = timedelta(hours=1)
+ uptime = now - server.start_time
+ if uptime < ONE_HOUR:
+ # Only apply penalty if the server was down for more than an hour
+ down_too_long = True
+ if server.down_time is not None:
+ down_too_long = (server.start_time - server.down_time) > ONE_HOUR
+
+ if down_too_long:
+ points -= ((ONE_HOUR - uptime) / ONE_HOUR) * 4
+
+ # Reduction to 40% for servers that support both legacy (v4) and v5 clients
+ if server.proto_min <= 32 and server.proto_max > 36:
+ points *= 0.4
+
+ return points
diff --git a/server_list/views.py b/server_list/views.py
new file mode 100755
index 0000000..2d38d85
--- /dev/null
+++ b/server_list/views.py
@@ -0,0 +1,107 @@
+import json
+
+from flask import render_template, request, send_from_directory, make_response
+
+from .app import app, db
+from .models import Server
+from .tasks import update_server
+from .util import check_ban, check_request_json, get_addr_info, get_geo_continent, verify_announce
+
+
+@app.route("/")
+def index():
+ return app.send_static_file("index.html")
+
+
+@app.route("/list")
+def server_list():
+ # We have to make sure that the list isn't cached,
+ # since the list isn't really static.
+ return send_from_directory(app.static_folder, "list.json", max_age=0)
+
+
+@app.route("/geoip")
+def geoip():
+ continent = get_geo_continent(request.remote_addr)
+
+ resp = make_response({
+ "continent": continent, # null on error
+ })
+ resp.cache_control.max_age = 7 * 86400
+ resp.cache_control.private = True
+
+ return resp
+
+
+@app.route("/announce", methods=["GET", "POST"])
+def announce():
+ announce_ip = request.remote_addr
+ if announce_ip.startswith("::ffff:"):
+ announce_ip = announce_ip[7:]
+
+ if announce_ip in app.config["BANNED_IPS"]:
+ return "Banned.", 403
+
+ data = request.values["json"]
+
+ if len(data) > 8192:
+ return "JSON data is too big.", 413
+
+ try:
+ obj = json.loads(data)
+ except json.JSONDecodeError as e:
+ return "Failed to decode JSON: " + e.msg, 400
+
+ if not isinstance(obj, dict):
+ return "JSON data is not an object.", 400
+
+ action = obj.get("action")
+ if action not in ("start", "update", "delete"):
+ return "Action field is invalid or missing.", 400
+
+ obj["ip"] = announce_ip
+ if not obj.get("address"):
+ obj["address"] = announce_ip
+ obj.setdefault("port", 30000)
+
+ if check_ban(announce_ip, obj["address"], obj["port"]):
+ return "Banned", 403
+
+ server = Server.find_from_json(obj)
+
+ if action == "delete":
+ if not server:
+ return "Server not found."
+ server.set_offline()
+ db.session.commit()
+ return "Removed from server list."
+
+ # Delete message does not require most fields
+ error_str = check_request_json(obj)
+ if error_str is not None:
+ return "Invalid JSON data: " + error_str, 400
+
+ if action == "update" and not server:
+ if app.config["ALLOW_UPDATE_WITHOUT_OLD"]:
+ action = "start"
+ else:
+ return "Server to update not found.", 404
+
+ addr_info = get_addr_info(obj["address"], obj["port"])
+
+ if addr_info is None:
+ return f"Failed to resolve server address {obj['address']!r}.", 400
+
+ if "world_uuid" not in obj:
+ valid = verify_announce(addr_info, obj["address"], obj["ip"])
+
+ if not valid:
+ return render_template("address_verification_failed.txt",
+ announce_ip=announce_ip,
+ valid_addresses=[data[4][0] for data in addr_info]), 400
+
+ obj["addr_info"] = addr_info
+
+ update_server.delay(obj)
+
+ return "Done.", 202