Add support for persistent storage using MongoDB

README.md

@@ -12,17 +12,20 @@ First install node.js, e.g.:
 	# apt-get install nodejs
 	# # OR:
 	# pacman -S nodejs
+	# # OR:
+	# emerge nodejs
 
 Then install doT.js and its dependencies:
 
+	$ cd ~
 	$ npm install dot commander mkdirp
 
 And finally compile the template:
 
 	$ cd static
-	$ ../node_modules/dot/bin/dot-packer -s .
+	$ ~/node_modules/dot/bin/dot-packer -s . -d .
 
-You can now serve the webpage by copying the files in static/ to your web root, or by [starting the server list](#setting-up-the-server).
+You can now serve the webpage by copying the files in static/ to your web root, or by [starting the master server](#setting-up-the-server).
 
 
 Embedding the server list in a page
@@ -55,78 +58,52 @@ Setting up the server
 
   1. Install Python 3 and pip:
 
-	pacman -S python python-pip
+		# pacman -S python python-pip
-	# OR:
+		# # OR:
-	apt-get install python3 python3-pip
+		# apt-get install python3 python3-pip
 
   2. Install required Python packages:
 
-	# You might have to use pip3 if your system defaults to Python 2
+		# # You might have to use pip3 if your system defaults to Python 2
-	pip install -r requirements.txt
+		# pip install -r requirements.txt
 
   3. If using in production, install uwsgi and it's python plugin:
 
-	pacman -S uwsgi uwsgi-plugin-python
+		# pacman -S uwsgi uwsgi-plugin-python
-	# OR:
+		# # OR:
-	apt-get install uwsgi uwsgi-plugin-python
+		# apt-get install uwsgi uwsgi-plugin-python
-	# OR:
+		# # OR:
-	pip install uwsgi
+		# pip install uwsgi
 
-  4. Configure the server by adding options to `config.py`.
+  4. Install, start, and enable MongoDB on boot:
-       See `config-example.py` for defaults.
 
-  5. Start the server:
+		# pacman -S mongodb && systemctl enable mongodb --now
 
-	$ ./server.py
+  5. Configure the server by adding options to `config.py`.
-	$ # Or for production:
+	See `config-example.py` for defaults.
-	$ uwsgi -s /tmp/minetest-master.sock --plugin python -w server:app --enable-threads
+
-	$ # Then configure according to http://flask.pocoo.org/docs/deploying/uwsgi/
+  6. Start the server:
+
+		$ ./server.py
+		$ # Or for production:
+		$ uwsgi -s /tmp/minetest-master.sock --plugin python -w server:app --enable-threads
+		$ # Then configure according to http://flask.pocoo.org/docs/deploying/uwsgi/
 
   7. (optional) Configure the proxy server, if any.  You should make the server
 	load static files directly from the static directory.  Also, `/list`
 	should be served from `list.json`.  Example for nginx:
 
-	root /path/to/server/static;
+		root /path/to/server/static;
-	rewrite ^/list$ /list.json;
+		rewrite ^/list$ /list.json;
-	try_files $uri @uwsgi;
+		try_files $uri @uwsgi;
-	location @uwsgi {
+		location @uwsgi {
-		uwsgi_pass ...;
+			uwsgi_pass ...;
-	}
+		}
-
-Setting up the server (Apache version)
----------------------
-
-If you wish to use Apache to host the server list, do steps 1-2, 4, above. Additionally install/enable mod_wsgi and an Apache site config like the following:
-
-		# This config assumes you have the server list at DocumentRoot.
-		# Visitors to the server list in this config would visit http://local.server/ and
-		# apache would serve up the output from server.py. Static resources would be served
-		# from http://local.server/static.
-
-		# Where are the minetest-server files located?
-		DocumentRoot /var/games/minetest/serverlist
-
-		# Serve up server.py at the root of the URL.
-		WSGIScriptAlias / /var/games/minetest/serverlist/server.py
-
-		# The name of the function that we call when we invoke server.py
-		WSGICallableObject app
-
-		# These options are necessary to enable Daemon mode. Without this, you'll have strange behavior
-		# with servers dropping off your list! You can tweak threads as needed. See mod_wsgi documentation.
-		WSGIProcessGroup minetest-serverlist
-		WSGIDaemonProcess minetest-serverlist threads=2
-
-
-		<Directory /var/games/minetest/serverlist>
-			Require all granted
-		</Directory>
-
-	</VirtualHost>
 
 License
 -------
 
-The Minetest server list code is licensed under the GNU Lesser General Public
+The Minetest master server is licensed under the GNU Lesser General Public
 License version 2.1 or later (LGPLv2.1+).  A LICENSE.txt file should have been
 supplied with your copy of this software containing a copy of the license.
+

config-example.py

@@ -1,3 +1,5 @@
+from datetime import timedelta
+
 # Enables detailed tracebacks and an interactive Python console on errors.
 # Never use in production!
 DEBUG = False
@@ -7,27 +9,23 @@ HOST = "127.0.0.1"
 # Port for development server to listen on
 PORT = 5000
 
-# Amount of time, in seconds, after which servers are removed from the list
+# Amount of time after which servers are removed from the list if they haven't
-# if they haven't updated their listings.  Note: By default Minetest servers
+# updated their listings.  Note: By default Minetest servers only announce
-# only announce once every 5 minutes, so this should be more than 300.
+# once every 5 minutes, so this should be more than that.
-PURGE_TIME = 350
+UPDATE_TIME = timedelta(minutes=6)
 
-# List of banned IP addresses for announce
+# Amount of time after which servers are removed from the database if they
-# e.g. ['2620:101::44']
+# haven't updated their listings.
-BANNED_IPS = []
+PURGE_TIME = timedelta(days=30)
-
-# List of banned servers as host/port pairs, domains must be lowercase
-# e.g. ['1.2.3.4/30000', 'server.example.net', 'server.example.net/30001']
-BANNED_SERVERS = []
-
-# List of banned domain suffixes, must be lowercase
-# e.g. ['.example.net', 'server.example.com']
-BANNED_DOMAINS = []
-
-# List of domain suffixes that should not get a point bonus (e.g. free domains), must be lowercase
-IRREPUTABLE_DOMAINS = ['.cf', '.ga', '.gq', '.ml', '.tk']
 
 # Creates server entries if a server sends an 'update' and there is no entry yet.
 # This should only be used to populate the server list after list.json was deleted.
 # This WILL cause problems such as mapgen, mods and privilege information missing from the list
 ALLOW_UPDATE_WITHOUT_OLD = False
+
+# Number of days' data to factor into popularity calculation
+POP_DAYS = 3
+
+# Address of the MongoDB server.  You can use domain sockets on unix.
+MONGO_URI = "mongodb://localhost/minetest-master"
+

requirements.txt

@@ -1,2 +1,4 @@
+APScheduler>=3
 Flask>=0.10
-maxminddb-geolite2>=2018.703
+Flask-PyMongo>=0.3
+

server.py

@@ -1,11 +1,17 @@
 #!/usr/bin/env python3
-import os, re, sys, json, time, socket, ipaddress
+import os, sys, json, time, socket
-from threading import Thread, RLock
+from datetime import date, datetime, timedelta
-from geolite2 import geolite2
+from threading import Thread
 
+from apscheduler.schedulers.background import BackgroundScheduler
 from flask import Flask, request, send_from_directory
+from flask.ext.pymongo import PyMongo, ASCENDING, DESCENDING
 
 
+# Set up scheduler
+sched = BackgroundScheduler(timezone="UTC")
+sched.start()
+
 app = Flask(__name__, static_url_path = "")
 
 # Load configuration
@@ -13,6 +19,9 @@ app.config.from_pyfile("config-example.py")  # Use example for defaults
 if os.path.isfile(os.path.join(app.root_path, "config.py")):
         app.config.from_pyfile("config.py")
 
+# Set up database
+mongo = PyMongo(app)
+
 
 # Views
 
@@ -35,12 +44,9 @@ def announce():
 	if ip.startswith("::ffff:"):
 		ip = ip[7:]
 
-	if ip in app.config["BANNED_IPS"]:
-		return "Banned (IP).", 403
-
 	data = request.values["json"]
 
-	if len(data) > 8192:
+	if len(data) > 5000:
 		return "JSON data is too big.", 413
 
 	try:
@@ -58,9 +64,6 @@ def announce():
 	if action not in ("start", "update", "delete"):
 		return "Invalid action field.", 400
 
-	if action == "start":
-		server["uptime"] = 0
-
 	server["ip"] = ip
 
 	if not "port" in server:
@@ -71,23 +74,11 @@ def announce():
 		server["port"] = int(server["port"])
 	#### End compatability code ####
 
-	if "%s/%d" % (server["ip"], server["port"]) in app.config["BANNED_SERVERS"]:
-		return "Banned (Server).", 403
-	elif "address" in server:
-		# Normalize address for ban checks
-		server["address"] = server["address"].lower().rstrip(".")
-		if f'{server["address"]}/{server["port"]}' in app.config["BANNED_SERVERS"] or \
-				server["address"] in app.config["BANNED_SERVERS"]:
-			return "Banned (Server).", 403
-		for domain in app.config["BANNED_DOMAINS"]:
-			if server["address"].endswith(domain):
-				return "Banned (Domain).", 403
-
 	old = serverList.get(ip, server["port"])
 
 	if action == "delete":
 		if not old:
-			return "Server not found."
+			return "Server not found.", 500
 		serverList.remove(old)
 		serverList.save()
 		return "Removed from server list."
@@ -96,28 +87,24 @@ def announce():
 
 	if action == "update" and not old:
 		if app.config["ALLOW_UPDATE_WITHOUT_OLD"]:
-			action = "start"
+			old = server
+			old["start"] = time.time()
+			old["clients_top"] = 0
+			old["updates"] = 0
+			old["total_clients"] = 0
 		else:
-			return "Server to update not found."
+			return "Server to update not found.", 500
 
-	server["update_time"] = int(time.time())
+	server["last_update"] = datetime.utcnow()
 
-	if action == "start":
+	del server["uptime"]
-		server["start"] = int(time.time())
+	server["start"] = time.time() if action == "start" else old["start"]
-		tracker.push("%s:%d" % (server["ip"], server["port"]), server["start"])
-	else:
-		server["start"] = old["start"]
 
 	if "clients_list" in server:
 		server["clients"] = len(server["clients_list"])
 
 	server["clients_top"] = max(server["clients"], old["clients_top"]) if old else server["clients"]
 
-	if "url" in server:
-		url = server["url"]
-		if not any(url.startswith(p) for p in ["http://", "https://", "//"]):
-			del server["url"]
-
 	# Make sure that startup options are saved
 	if action == "update":
 		for field in ("dedicated", "rollback", "mapgen", "privs",
@@ -125,20 +112,49 @@ def announce():
 			if field in old:
 				server[field] = old[field]
 
-	# Popularity
+	## Popularity
-	if old:
+	# This should only include stats from the past few days, so storing
-		server["updates"] = old["updates"] + 1
+	# the necessary data is fairly complicated.  Three pieces of
-		# This is actually a count of all the client numbers we've received,
+	# information are required: the time the server's been connected
-		# it includes clients that were on in the previous update.
+	# (number of updates), the number of clients reported, and a
-		server["total_clients"] = old["total_clients"] + server["clients"]
+	# timestamp so that this information can expire.  This is stored in
+	# the format {date: (updates, clients)}.
+	total_updates = 1
+	# This is the total of all the client numbers we've received,
+	# it includes clients that were on in previous updates.
+	total_clients = server["clients"]
+	today = date.today()
+	# Key must be a string
+	today_key = str(today.toordinal())
+	if old and type(old["updates"]) == dict:
+		server["updates"] = {}
+		updates = server["updates"]
+		# Copy over only recent updates, and set the total counter
+		pop_days = app.config["POP_DAYS"]
+		for d, data in old["updates"].items():
+			if date.fromordinal(int(d)) >= \
+					today - timedelta(days=pop_days):
+				updates[d] = data
+				total_updates += data[0]
+				total_clients += data[1]
+
+		if today_key in updates:
+			updates[today_key][0] += 1
+			updates[today_key][1] += server["clients"]
+		else:
+			updates[today_key] = (1, server["clients"])
 	else:
-		server["updates"] = 1
+		server["updates"] = {today_key: (1, server["clients"])}
-		server["total_clients"] = server["clients"]
+	server["pop_v"] = total_clients / total_updates
-	server["pop_v"] = server["total_clients"] / server["updates"]
+
+	# Keep server record ID
+	if old:
+		server["_id"] = old["_id"]
 
 	finishRequestAsync(server)
 
-	return "Request has been filed.", 202
+	return "Thanks, your request has been filed.", 202
+
 
 # Utilities
 
@@ -148,37 +164,14 @@ def serverUp(info):
 		sock = socket.socket(info[0], info[1], info[2])
 		sock.settimeout(3)
 		sock.connect(info[4])
-		# send packet of type ORIGINAL, with no data
-		#     this should prompt the server to assign us a peer id
-		# [0] u32       protocol_id (PROTOCOL_ID)
-		# [4] session_t sender_peer_id (PEER_ID_INEXISTENT)
-		# [6] u8        channel
-		# [7] u8        type (PACKET_TYPE_ORIGINAL)
 		buf = b"\x4f\x45\x74\x03\x00\x00\x00\x01"
 		sock.send(buf)
 		start = time.time()
-		# receive reliable packet of type CONTROL, subtype SET_PEER_ID,
-		#     with our assigned peer id as data
-		# [0] u32        protocol_id (PROTOCOL_ID)
-		# [4] session_t  sender_peer_id
-		# [6] u8         channel
-		# [7] u8         type (PACKET_TYPE_RELIABLE)
-		# [8] u16        seqnum
-		# [10] u8        type (PACKET_TYPE_CONTROL)
-		# [11] u8        controltype (CONTROLTYPE_SET_PEER_ID)
-		# [12] session_t peer_id_new
 		data = sock.recv(1024)
 		end = time.time()
 		if not data:
 			return False
 		peer_id = data[12:14]
-		# send packet of type CONTROL, subtype DISCO,
-		#     to cleanly close our server connection
-		# [0] u32       protocol_id (PROTOCOL_ID)
-		# [4] session_t sender_peer_id
-		# [6] u8        channel
-		# [7] u8        type (PACKET_TYPE_CONTROL)
-		# [8] u8        controltype (CONTROLTYPE_DISCO)
 		buf = b"\x4f\x45\x74\x03" + peer_id + b"\x00\x00\x03"
 		sock.send(buf)
 		sock.close()
@@ -198,7 +191,6 @@ fields = {
 
 	"clients": (True, "int"),
 	"clients_max": (True, "int"),
-	"uptime": (True, "int"),
 	"game_time": (True, "int"),
 	"lag": (False, "float"),
 
@@ -206,8 +198,8 @@ fields = {
 	"mods": (False, "list", "str"),
 
 	"version": (True, "str"),
-	"proto_min": (True, "int"),
+	"proto_min": (False, "int"),
-	"proto_max": (True, "int"),
+	"proto_max": (False, "int"),
 
 	"gameid": (True, "str"),
 	"mapgen": (False, "str"),
@@ -238,9 +230,8 @@ def checkRequest(server):
 		if data[1] == "bool" and type(server[name]).__name__ == "str":
 			server[name] = True if server[name].lower() in ("true", "1") else False
 			continue
-		# Accept strings in integer fields but convert it to an
+		# clients_max was sent as a string instead of an integer
-		# integer, for interoperability with e.g. minetest.write_json.
+		if name == "clients_max" and type(server[name]).__name__ == "str":
-		if data[1] == "int" and type(server[name]).__name__ == "str":
 			server[name] = int(server[name])
 			continue
 		#### End compatibility code ####
@@ -250,6 +241,8 @@ def checkRequest(server):
 			for item in server[name]:
 				if type(item).__name__ != data[2]:
 					return False
+	if "_id" in server:
+		return False
 	return True
 
 
@@ -283,18 +276,6 @@ def asyncFinishThread(server):
 					% (server["ip"], server["address"], addresses))
 			return
 
-	reader = geolite2.reader()
-	try:
-		geo = reader.get(server["ip"])
-	except geoip2.errors.GeoIP2Error:
-		app.logger.warning("GeoIP lookup failure for %s." % (server["ip"],))
-
-	if geo and "continent" in geo:
-		server["geo_continent"] = geo["continent"]["code"]
-	else:
-		app.logger.warning("Unable to get GeoIP Continent data for %s."
-				% (server["ip"],))
-
 	server["ping"] = serverUp(info[0])
 	if not server["ping"]:
 		app.logger.warning("Server %s:%d has no ping."
@@ -303,199 +284,145 @@ def asyncFinishThread(server):
 
 	del server["action"]
 
-	serverList.update(server)
+	with app.app_context():
+		serverList.updateServer(server)
 
 
-class UptimeTracker:
-	def  __init__(self):
-		self.d = {}
-		self.cleanTime = 0
-		self.lock = RLock()
-	def push(self, id, ts):
-		with self.lock:
-			if time.time() >= self.cleanTime: # clear once in a while
-				self.d.clear()
-				self.cleanTime = time.time() + 48*60*60
-
-			if id in self.d:
-				self.d[id] = self.d[id][-1:] + [ts]
-			else:
-				self.d[id] = [0, ts]
-	# returns the before-last start time, in bulk
-	def getStartTimes(self):
-		ret = {}
-		with self.lock:
-			for k, v in self.d.items():
-				ret[k] = v[0]
-		return ret
-
 class ServerList:
 	def __init__(self):
-		self.list = []
+		self.last_cache_update = 0
-		self.maxServers = 0
+		with app.app_context():
-		self.maxClients = 0
+			mongo.db.meta.create_index("key", unique=True, name="key")
-		self.lock = RLock()
+			self.info = mongo.db.meta.find_one({"key": "list_info"}) or {}
-		self.load()
-		self.purgeOld()
 
-	def getWithIndex(self, ip, port):
+			mongo.db.servers.create_index([("ip", ASCENDING), ("port", ASCENDING)],
-		with self.lock:
+				unique=True,
-			for i, server in enumerate(self.list):
+				name="address")
-				if server["ip"] == ip and server["port"] == port:
+			mongo.db.servers.create_index("points", name="points")
-					return (i, server)
+			mongo.db.servers.create_index("last_update",
-		return (None, None)
+				expireAfterSeconds=app.config["PURGE_TIME"].total_seconds(),
+				name="expiration")
+
+		if "max_servers" not in self.info:
+			self.info["max_servers"] = 0
+		if "max_clients" not in self.info:
+			self.info["max_clients"] = 0
+
+	def __del__(self):
+		with app.app_context():
+			self.updateMeta()
+
+	def getPoints(self, server):
+		points = 0
+
+		# 1 per client without a guest or all-numeric name.
+		if "clients_list" in server:
+			for name in server["clients_list"]:
+				if not name.startswith("Guest") and \
+						not name.isdigit():
+					points += 1
+		else:
+			# Old server
+			points = server["clients"] / 4
+
+		# Penalize highly loaded servers to improve player distribution.
+		# Note: This doesn't just make more than 16 players stop
+		# increasing your points, it can actually reduce your points
+		# if you have guests/all-numerics.
+		if server["clients"] > 16:
+			points -= server["clients"] - 16
+
+		# 1 per month of age, limited to 8
+		points += min(8, server["game_time"] / (60*60*24*30))
+
+		# 1/2 per average client, limited to 4
+		points += min(4, server["pop_v"] / 2)
+
+		# -8 for unrealistic max_clients
+		if server["clients_max"] >= 128:
+			points -= 8
+
+		# -8 per second of ping over 0.4s
+		if server["ping"] > 0.4:
+			points -= (server["ping"] - 0.4) * 8
+
+		# Up to -8 for less than an hour of uptime (penalty linearly decreasing)
+		HOUR_SECS = 60 * 60
+		uptime = server["uptime"]
+		if uptime < HOUR_SECS:
+			points -= ((HOUR_SECS - uptime) / HOUR_SECS) * 8
+
+		return points
+
+	def updateCache(self):
+		servers = mongo.db.servers.find({"last_update":
+				{"$gt": datetime.utcnow() - app.config["UPDATE_TIME"]}
+			}).sort("points", DESCENDING)
+		server_list = []
+		num_clients = 0
+		for server in servers:
+			del server["_id"]
+			del server["last_update"]
+			del server["updates"]
+			server["uptime"] = time.time() - server["start"]
+			server_list.append(server)
+			num_clients += server["clients"]
+
+		info = self.info
+		info["max_servers"] = max(len(server_list), info["max_servers"])
+		info["max_clients"] = max(num_clients, info["max_clients"])
+
+		with open(os.path.join("static", "list.json"), "w") as fd:
+			json.dump({
+					"total": {"servers": len(server_list), "clients": num_clients},
+					"total_max": {"servers": info["max_servers"], "clients": info["max_clients"]},
+					"list": server_list
+				},
+				fd,
+				indent = "\t" if app.config["DEBUG"] else None
+			)
+
+		self.updateMeta()
+
+		self.last_cache_update = time.time()
+
+	# Update if servers are likely to have expired
+	def updateCacheIfNeeded(self):
+		if time.time() - self.last_cache_update < \
+				app.config["UPDATE_TIME"].total_seconds() / 2:
+			return
+		with app.app_context():
+			self.updateCache()
+
+	def updateMeta(self):
+		if not "key" in self.info:
+			self.info["key"] = "list_info"
+		mongo.db.meta.replace_one({"key": "list_info"},
+			self.info, upsert=True)
 
 	def get(self, ip, port):
-		i, server = self.getWithIndex(ip, port)
+		return mongo.db.servers.find_one({"ip": ip, "port": port})
-		return server
 
 	def remove(self, server):
-		with self.lock:
+		mongo.db.servers.delete_one({"_id": server["_id"]})
-			try:
+		self.updateCache()
-				self.list.remove(server)
-			except:
-				pass
 
-	def sort(self):
+	def updateServer(self, server):
-		start_times = tracker.getStartTimes()
+		server["points"] = self.getPoints(server)
-
+		if "_id" in server:
-		def server_points(server):
+			mongo.db.servers.replace_one({"_id": server["_id"]},
-			points = 0
+					server)
-
+		else:
-			# 1 per client, but only 1/8 per "guest" client
+			mongo.db.servers.insert_one(server)
-			if "clients_list" in server:
+		self.updateCache()
-				for name in server["clients_list"]:
-					if re.match(r"[A-Z][a-z]{3,}[1-9][0-9]{2,3}", name):
-						points += 1/8
-					else:
-						points += 1
-			else:
-				# Old server (1/4 per client)
-				points = server["clients"] / 4
-
-			# Penalize highly loaded servers to improve player distribution.
-			# Note: This doesn't just make more than 80% of max players stop
-			# increasing your points, it can actually reduce your points
-			# if you have guests.
-			cap = int(server["clients_max"] * 0.80)
-			if server["clients"] > cap:
-				points -= server["clients"] - cap
-
-			# 8 for servers with a reputable domain name
-			try:
-				ipaddress.ip_address(server["address"])
-			except ValueError:
-				for domain in app.config["IRREPUTABLE_DOMAINS"]:
-					if server["address"].endswith(domain):
-						break
-				else:
-					points += 8
-
-			# 1 per month of age, limited to 8
-			points += min(8, server["game_time"] / (60*60*24*30))
-
-			# 1/2 per average client, limited to 4
-			points += min(4, server["pop_v"] / 2)
-
-			# -8 for unrealistic max_clients
-			if server["clients_max"] > 200:
-				points -= 8
-
-			# -8 per second of ping over 0.4s
-			if server["ping"] > 0.4:
-				points -= (server["ping"] - 0.4) * 8
-
-			# Up to -8 for less than an hour of uptime (penalty linearly decreasing)
-			# only if the server has restarted before within the last 2 hours
-			HOUR_SECS = 60 * 60
-			uptime = server["uptime"]
-			if uptime < HOUR_SECS:
-				start_time = start_times.get("%s:%d" % (server["ip"], server["port"]), 0)
-				if start_time >= time.time() - 2 * HOUR_SECS:
-					points -= ((HOUR_SECS - uptime) / HOUR_SECS) * 8
-
-			# reduction to 40% for servers that support both legacy (v4) and v5 clients
-			if server["proto_min"] <= 32 and server["proto_max"] > 36:
-				points *= 0.4
-
-			return points
-
-		with self.lock:
-			self.list.sort(key=server_points, reverse=True)
-
-	def purgeOld(self):
-		cutoff = int(time.time()) - app.config["PURGE_TIME"]
-		with self.lock:
-			count = len(self.list)
-			self.list = [server for server in self.list if cutoff <= server["update_time"]]
-			if len(self.list) < count:
-				self.save()
-
-	def load(self):
-		with self.lock:
-			try:
-				with open(os.path.join(app.static_folder, "list.json"), "r") as fd:
-					data = json.load(fd)
-			except FileNotFoundError:
-				return
-
-			if not data:
-				return
-
-			self.list = data["list"]
-			self.maxServers = data["total_max"]["servers"]
-			self.maxClients = data["total_max"]["clients"]
-
-	def save(self):
-		with self.lock:
-			servers = len(self.list)
-			clients = 0
-			for server in self.list:
-				clients += server["clients"]
-
-			self.maxServers = max(servers, self.maxServers)
-			self.maxClients = max(clients, self.maxClients)
-
-			list_path = os.path.join(app.static_folder, "list.json")
-			with open(list_path + "~", "w") as fd:
-				json.dump({
-						"total": {"servers": servers, "clients": clients},
-						"total_max": {"servers": self.maxServers, "clients": self.maxClients},
-						"list": self.list
-					},
-					fd,
-					indent = "\t" if app.config["DEBUG"] else None,
-					separators = (', ', ': ') if app.config["DEBUG"] else (',', ':')
-				)
-			os.replace(list_path + "~", list_path)
-
-	def update(self, server):
-		with self.lock:
-			i, old = self.getWithIndex(server["ip"], server["port"])
-			if i is not None:
-				self.list[i] = server
-			else:
-				self.list.append(server)
-
-			self.sort()
-			self.save()
-
-class PurgeThread(Thread):
-	def __init__(self):
-		Thread.__init__(self)
-		self.daemon = True
-	def run(self):
-		while True:
-			time.sleep(60)
-			serverList.purgeOld()
-
-# Globals / Startup
-
-tracker = UptimeTracker()
 
 serverList = ServerList()
 
-PurgeThread().start()
+sched.add_job(serverList.updateCacheIfNeeded, "interval",
+		seconds=app.config["UPDATE_TIME"].total_seconds() / 2, coalesce=True,
+		max_instances=1)
 
 if __name__ == "__main__":
+	serverList.updateCacheIfNeeded()
 	app.run(host = app.config["HOST"], port = app.config["PORT"])
+

static/index.html

@@ -3,28 +3,9 @@
 <head>
 	<meta charset="utf-8">
 	<title>Minetest server list</title>
-	<style>
-		body {
-			font-family: Roboto, Ubuntu, "Segoe UI", sans;
-		}
-		a {
-			color: #336B87;
-		}
-		a:visited {
-			color: #336BA1;
-		}
-		@media only screen and (max-width: 1024px) {
-			#server_list table .version, #server_list table .flags, #server_list table .uptime {
-				display: none;
-			}
-		}
-	</style>
-	<script>
-		var master = {show_proto_select: true};
-	</script>
 </head>
 <body>
 	<div id="server_list"></div>
-	<script src="list.js"></script>
 </body>
 </html>
+<script src="list.js"></script>

static/list.js

@@ -5,9 +5,6 @@ if (!master.output)	master.output = '#server_list';
 if (!master.list)	master.list = "list";
 if (!master.list_root)	master.list_root = master.root;
 if (!master.list_url)	master.list_url = master.list_root + master.list;
-master.cached_json = null;
-
-// Utility functions used by the templating code
 
 function humanTime(seconds) {
 	if (typeof(seconds) != "number") return '?';
@@ -16,10 +13,10 @@ function humanTime(seconds) {
 		d: 86400,
 		h: 3600,
 		m: 60
-	};
+	}
 	for (var i in conv) {
 		if (seconds >= conv[i]) {
-			return (seconds / conv[i]).toFixed(i=='y'?1:0) + i;
+			return (seconds / conv[i]).toFixed(1) + i;
 		}
 	}
 	return seconds + 's';
@@ -40,22 +37,28 @@ function addressString(server) {
 	var str = '<span'
 	if (shortStr.length > 25) {
 		shortStr = shortStr.substr(0, 23) + "&hellip;";
-		str += ' title="' + addrStr + '"'
+		str += ' class="mts_tooltip" title="' + addrStr + '"'
 	}
 	if (server.port != 30000)
 		shortStr += ':' + server.port;
 	return str + '>' + shortStr + '</span>';
 }
 
-function tooltipString(str) {
+function tooltipString(str, maxLen) {
 	str = escapeHTML(str);
-	return '<span title="' + str + '">' + str + '</div>';
+	var shortStr = str;
+	var ret = '<span';
+	if (shortStr.length > maxLen) {
+		shortStr = shortStr.substr(0, maxLen - 2) + "&hellip;";
+		ret += ' class="mts_tooltip" title="' + str + '"';
+	}
+	return ret + '>' + shortStr + '</span>';
 }
 
 function hoverList(name, list) {
 	if (!list || list.length == 0) return '';
 	var str = '<div class="mts_hover_list">'
-	str += '<b>' + name + '</b> (' + list.length + ')<br />';
+	str += name + ' (' + list.length + ')<br />';
 	for (var i in list) {
 		str += escapeHTML(list[i]) + '<br />';
 	}
@@ -64,35 +67,19 @@ function hoverList(name, list) {
 
 function hoverString(name, string) {
 	if (!string) return '';
-	return '<div class="mts_hover_list">'
+	return  '<div class="mts_hover_list">'
-		+ '<b>' + name + '</b>:<br />'
+		+ name + ':<br />'
 		+ escapeHTML(string) + '<br />'
 		+ '</div>';
 }
 
-function constantWidth(str, width) {
-	return '<span class="mts_cwidth" style="width:' + width + 'em;">' + str + '</span>';
-}
-
-// Code that fetches & displays the actual list
-
 function draw(json) {
-	if (json == null)
-		return;
-
 	var html = window.render.servers(json);
 	jQuery(master.output).html(html);
-	jQuery('.proto_select', master.output).on('change', function(e) {
-		master.proto_range = e.target.value;
-		draw(master.cached_json); // re-render
-	});
 }
 
 function get() {
-	jQuery.getJSON(master.list_url, function(json) {
+	jQuery.getJSON(master.list_url, draw);
-		master.cached_json = json;
-		draw(json);
-	});
 }
 
 function loaded(){
@@ -102,7 +89,6 @@ function loaded(){
 	get();
 }
 
-
 // https://github.com/pyrsmk/toast
 this.toast=function(){var e=document,t=e.getElementsByTagName("head")[0],n=this.setTimeout,r="createElement",i="appendChild",s="addEventListener",o="onreadystatechange",u="styleSheet",a=10,f=0,l=function(){--f},c,h=function(e,r,i,s){if(!t)n(function(){h(e)},a);else if(e.length){c=-1;while(i=e[++c]){if((s=typeof i)=="function"){r=function(){return i(),!0};break}if(s=="string")p(i);else if(i.pop){p(i[0]),r=i[1];break}}d(r,Array.prototype.slice.call(e,c+1))}},p=function(n,s){++f,/\.css$/.test(n)?(s=e[r]("link"),s.rel=u,s.href=n,t[i](s),v(s)):(s=e[r]("script"),s.src=n,t[i](s),s[o]===null?s[o]=m:s.onload=l)},d=function(e,t){if(!f)if(!e||e()){h(t);return}n(function(){d(e,t)},a)},v=function(e){if(e.sheet||e[u]){l();return}n(function(){v(e)},a)},m=function(){/ded|co/.test(this.readyState)&&l()};h(arguments)};
 
@@ -110,6 +96,6 @@ toast(master.root + 'style.css', master.root + 'servers.js', function() {
 	if (typeof(jQuery) != 'undefined')
 		return loaded();
 	else
-		toast('//ajax.googleapis.com/ajax/libs/jquery/1.12.4/jquery.min.js', loaded);
+		toast('//ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js', loaded);
 });
 

static/servers.jst

@@ -1,68 +1,59 @@
 {{? !master.no_total}}
-<div>
+<div class="total">
-	<span class="header_total">
+	Players: {{=it.total.clients}}/{{=it.total_max.clients}}&nbsp;
-		Players: {{=it.total.clients}}/{{=it.total_max.clients}}&nbsp;
+	Servers: {{=it.total.servers}}/{{=it.total_max.servers}}
-		Servers: {{=it.total.servers}}/{{=it.total_max.servers}}
-	</span>
-	{{? master.show_proto_select}}
-	, Protocol: <select class="proto_select">
-		<option value="">All</option>
-		<option value="[11,32]" {{? master.proto_range=='[11,32]'}}selected{{?}}>11-32 (Minetest 0.4)</option>
-		<option value="[37,99]" {{? master.proto_range=='[37,99]'}}selected{{?}}>37 (Minetest 5.0)</option>
-	</select>{{?}}
 </div>
 {{?}}
 <table>
 	<thead><tr>
 		{{? !master.no_address}}<th>Address[:Port]</th>{{?}}
 		{{? !master.no_clients}}<th>Players / Max{{? !master.no_avgtop}}<br/>Average / Top{{?}}</th>{{?}}
-		{{? !master.no_version}}<th class="version">Version, Subgame[, Mapgen]</th>{{?}}
+		{{? !master.no_version}}<th>Version, Subgame, Mapgenerator</th>{{?}}
 		{{? !master.no_name}}<th>Name</th>{{?}}
 		{{? !master.no_description}}<th>Description</th>{{?}}
-		{{? !master.no_flags}}<th class="flags">Flags</th>{{?}}
+		{{? !master.no_flags}}<th>Flags</th>{{?}}
-		{{? !master.no_uptime}}<th class="uptime">Uptime, Age</th>{{?}}
+		{{? !master.no_uptime}}<th>Uptime, Age</th>{{?}}
 		{{? !master.no_ping}}<th>Ping, Lag</th>{{?}}
 	</tr></thead>
 	<tbody>
-	{{ var tmp = master.proto_range ? JSON.parse(master.proto_range) : null;}}
 	{{~it.list :server:index}}
 	{{ if (master.limit && index + 1 > master.limit) break;}}
 	{{ if (master.min_clients && server.clients < master.min_clients) continue;}}
-	{{ if (tmp && (tmp[0] > server.proto_max || tmp[1] < server.proto_min)) continue;}}
 	<tr>
 		{{? !master.no_address}}
-		<td class="address">
+		<td class ="address">
 			{{=addressString(server)}}
 		</td>{{?}}
 		{{? !master.no_clients}}
 		<td class="clients{{? server.clients_list && server.clients_list.length > 0}} mts_hover_list_text{{?}}">
-			{{=constantWidth(server.clients + '/' + server.clients_max, 3.4)}}
+			{{=server.clients}}/{{=server.clients_max}}{{? !master.no_avgtop}} &nbsp;&nbsp;{{=Math.floor(server.pop_v)}}/{{=server.clients_top}}{{?}}
-			{{? !master.no_avgtop}} {{=constantWidth(Math.floor(server.pop_v) + '/' + server.clients_top, 3.4)}}{{?}}
 			{{=hoverList("Clients", server.clients_list)}}
 		</td>{{?}}
 		{{? !master.no_version}}
 		<td class="version{{? server.mods && server.mods.length > 0}} mts_hover_list_text{{?}}">
-			{{!server.version}}, {{!server.gameid}}
+			{{=escapeHTML(server.version)}}, {{=escapeHTML(server.gameid)}},&nbsp;
-			{{? server.mapgen}}, {{!server.mapgen}}{{?}}
+			{{=escapeHTML(server.mapgen || '?')}}
 			{{=hoverList("Mods", server.mods)}}
 		</td>{{?}}
 		{{? !master.no_name}}
 		<td class="name">
 			{{? server.url}}
-				<a href="{{!server.url}}">{{=tooltipString(server.name)}}</a>
+				<a href="{{=escapeHTML(server.url)}}">{{=tooltipString(server.name, 25)}}</a>
 			{{??}}
-				{{=tooltipString(server.name)}}
+				{{=tooltipString(server.name, 25)}}
 			{{?}}
 		</td>{{?}}
 		{{? !master.no_description}}
 		<td class="description">
-			{{=tooltipString(server.description)}}
+			{{=tooltipString(server.description, 50)}}
 		</td>{{?}}
 		{{? !master.no_flags}}
 		<td class="flags {{? server.privs}} mts_hover_list_text{{?}}">
-			{{=hoverString("Default privileges", server.privs)}}
+			{{=hoverString("Privs", server.privs)}}
 			{{=server.creative ? 'Cre ' : ''}}
+			{{=server.dedicated ? 'Ded ' : ''}}
 			{{=server.damage ? 'Dmg ' : ''}}
+			{{=server.liquid_finite ? 'Liq ' : ''}}
 			{{=server.pvp ? 'PvP ' : ''}}
 			{{=server.password ? 'Pwd ' : ''}}
 			{{=server.rollback ? 'Rol ' : ''}}
@@ -70,16 +61,16 @@
 		</td>{{?}}
 		{{? !master.no_uptime}}
 		<td class="uptime">
-			{{=constantWidth(humanTime(server.uptime), 3.2)}} / {{=constantWidth(humanTime(server.game_time), 3.2)}}
+			{{=humanTime(server.uptime)}}, {{=humanTime(server.game_time)}}
 		</td>{{?}}
 		{{? !master.no_ping}}
 		<td class="ping">
-			{{=constantWidth(Math.floor(server.ping * 1000), 1.8)}}{{? server.lag}} / {{=constantWidth(Math.floor(server.lag * 1000), 1.8)}}{{?}}
+			{{=Math.floor(server.ping * 1000)}}{{? server.lag}}, {{= Math.floor(server.lag * 1000)}}{{?}}
 		</td>{{?}}
 	</tr>
 	{{~}}
 	</tbody>
 </table>
 {{? master.min_clients || master.limit}}
-	<a href="javascript:delete master.min_clients; delete master.limit; get();">Show more...</a>
+	<a class="clickable" onclick="delete master.min_clients; delete master.limit; get();">More...</a>
 {{?}}

static/style.css

@@ -1,4 +1,4 @@
-#server_list .header_total {
+#server_list .total {
 	font-weight: bold;
 }
 
@@ -12,66 +12,42 @@
 }
 
 #server_list td, #server_list th {
-	border: 1px solid #2A3132;
+	border: 1px solid gray;
-	padding: 5px;
-	color: inherit;
 }
 
 #server_list thead {
-	background-color: #2A3132;
+	background-color: #FFA;
-	border-bottom: 5px solid #336B87;
-	color: white;
-	font-size: 1.1em;
 }
 
 #server_list tbody tr:nth-child(even) {
-	background-color: #E0E0E0;
+	background-color: #EEE;
 }
 
-#server_list td.clients, #server_list td.uptime, #server_list td.ping {
+#server_list tbody tr:hover {
-	text-align: center;
+	background-color: #CCC;
-}
-
-#server_list td.version, #server_list td.name, #server_list td.description {
-	text-overflow: ellipsis;
-	overflow: hidden;
-	white-space: nowrap;
-}
-
-/* Note: the column widths here are not exact as auto-layout is left enabled */
-
-#server_list td.address {
-	max-width: 24ch;
-}
-#server_list td.version {
-	max-width: 16ch;
-}
-#server_list td.name {
-	max-width: 32ch;
-}
-#server_list td.description {
-	max-width: 64ch;
 }
 
 .mts_hover_list {
 	display: none;
-	border: 1px solid #336B87;
+	border: 1px solid #88F;
-	border-radius: 10px;
+	border-radius: 4px;
-	background-color: #FFF;
+	background-color: white;
 	position: absolute;
 	z-index: 100;
 	padding: 0.5em;
-	box-shadow: 1px 1px 5px 3px rgba(0, 0, 0, 0.25);
-}
-
-.mts_hover_list b {
-	font-size: 1.1em;
 }
 
 td:hover .mts_hover_list {
 	display: block;
 }
 
-.mts_cwidth {
+.mts_hover_list_text, .mts_tooltip {
-	display: inline-block;
+	text-decoration: underline;
+	text-decoration-style: dashed;
 }
+
+.clickable {
+	text-decoration: underline;
+	cursor: pointer;
+}
+