Add funding.json

Co-authored-by: Filipino Student <hamen27545@artvara.com>

.github/FUNDING.yml

@@ -1,4 +1,5 @@
 # These are supported funding model platforms
 
+liberapay: rubenwardy
 patreon: rubenwardy
 custom: [ "https://rubenwardy.com/donate/" ]

.github/SECURITY.md

@@ -2,8 +2,8 @@
 
 ## Supported Versions
 
-We only support the latest production version, deployed to <https://content.minetest.net>.
-See the [releases page](https://github.com/minetest/contentdb/releases).
+We only support the latest production version, deployed to <https://content.luanti.org>.
+This is usually the latest `master` commit.
 
 ## Reporting a Vulnerability
 
@@ -12,8 +12,5 @@ to give us time to fix them. You can do that by using one of the methods outline
 
 * https://rubenwardy.com/contact/
 
-Depending on severity, we will either create a private issue for the vulnerability
-and release a security update, or give you permission to file the issue publicly.
-
 For more information on the justification of this policy, see
 [Responsible Disclosure](https://en.wikipedia.org/wiki/Responsible_disclosure).

.github/workflows/test.yml

@@ -6,7 +6,9 @@ jobs:
   test:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - name: Install docker-compose
+      run: sudo apt-get install -y docker-compose
+    - uses: actions/checkout@v4
     - name: Copy config
       run: cp utils/ci/* .
     - name: Build the Docker image

Dockerfile

@@ -1,16 +1,20 @@
-FROM python:3.10
+FROM python:3.10.11-alpine
 
-RUN groupadd -g 5123 cdb && \
-    useradd -r -u 5123 -g cdb cdb
+RUN addgroup --gid 5123 cdb && \
+    adduser --uid 5123 -S cdb -G cdb
 
 WORKDIR /home/cdb
 
+RUN \
+	apk add --no-cache postgresql-libs git bash unzip && \
+	apk add --no-cache --virtual .build-deps gcc musl-dev postgresql-dev g++
+
 RUN mkdir /var/cdb
 RUN chown -R cdb:cdb /var/cdb
 
 COPY requirements.lock.txt requirements.lock.txt
-RUN pip install -r requirements.lock.txt
-RUN pip install gunicorn
+RUN pip install -r requirements.lock.txt && \
+	pip install gunicorn
 
 COPY utils utils
 COPY config.cfg config.cfg

README.md

@@ -1,7 +1,7 @@
-# Content Database
-![Build Status](https://github.com/minetest/contentdb/actions/workflows/test.yml/badge.svg)
+# ContentDB
+![Build Status](https://github.com/luanti-org/contentdb/actions/workflows/test.yml/badge.svg)
 
-Content database for Minetest mods, games, and more.\
+A content database for Luanti mods, games, and more.\
 Developed by rubenwardy, license AGPLv3.0+.
 
 See [Getting Started](docs/getting_started.md) for setting up a development/prodiction environment.
@@ -29,6 +29,9 @@ See [Developer Intro](docs/dev_intro.md) for an overview of the code organisatio
 
 # Create new migration
 ./utils/create_migration.sh
+
+# Delete database
+docker-compose down && sudo rm -rf data/db
 ```
 
 
@@ -36,7 +39,7 @@ See [Developer Intro](docs/dev_intro.md) for an overview of the code organisatio
 
 * (optional) Install the [Docker extension](https://marketplace.visualstudio.com/items?itemName=ms-azuretools.vscode-docker)
 * Install the [Python extension](https://marketplace.visualstudio.com/items?itemName=ms-python.python)
-	* Click no to installing pylint (we don't want it to be installed outside of a virtual env)
+	* Click no to installing pylint (we don't want it to be installed outside a virtual env)
 * Set up a virtual env
 	* Replace `psycopg2` with `psycopg2_binary` in requirements.txt (because postgresql won't be installed on the system)
 	* `python3 -m venv env`
@@ -79,7 +82,7 @@ Package "1" --> "*" Release
 Package "1" --> "*" Dependency
 Package "1" --> "*" Tag
 Package "1" --> "*" MetaPackage : provides
-Release --> MinetestVersion
+Release --> LuantiVersion
 Package --> License
 Dependency --> Package
 Dependency --> MetaPackage

app/__init__.py

@@ -15,53 +15,98 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 import datetime
+import os
+import redis
 
-from flask import *
-from flask_gravatar import Gravatar
-from flask_mail import Mail
-from flask_github import GitHub
-from flask_wtf.csrf import CSRFProtect
-from flask_flatpages import FlatPages
+from flask import redirect, url_for, render_template, flash, request, Flask, send_from_directory, make_response, render_template_string
 from flask_babel import Babel, gettext
+from flask_flatpages import FlatPages
+from flask_github import GitHub
 from flask_login import logout_user, current_user, LoginManager
-import os, redis
-from app.markdown import init_markdown, MARKDOWN_EXTENSIONS, MARKDOWN_EXTENSION_CONFIG
+from flask_mail import Mail
+from flask_wtf.csrf import CSRFProtect
+
+from app.markdown import init_markdown, render_markdown
+
+import sentry_sdk
+from sentry_sdk.integrations.flask import FlaskIntegration
+
+
+if os.getenv("SENTRY_DSN"):
+	def before_send(event, hint):
+		from app.tasks import TaskError
+		if "exc_info" in hint:
+			exc_type, exc_value, tb = hint["exc_info"]
+			if isinstance(exc_value, TaskError):
+				return None
+		return event
+
+	environment = os.getenv("SENTRY_ENVIRONMENT")
+	assert environment is not None
+	sentry_sdk.init(
+		dsn=os.getenv("SENTRY_DSN"),
+		environment=environment,
+
+		integrations=[FlaskIntegration()],
+		# Set traces_sample_rate to 1.0 to capture 100%
+		# of transactions for performance monitoring.
+		traces_sample_rate=0.1,
+		# Set profiles_sample_rate to 1.0 to profile 100%
+		# of sampled transactions.
+		# We recommend adjusting this value in production.
+		profiles_sample_rate=0.1,
+
+		before_send=before_send,
+	)
+
 
 app = Flask(__name__, static_folder="public/static")
+
+
+def my_flatpage_renderer(text):
+	# Render with jinja first
+	prerendered_body = render_template_string(text)
+	return render_markdown(prerendered_body, clean=False)
+
+
 app.config["FLATPAGES_ROOT"] = "flatpages"
 app.config["FLATPAGES_EXTENSION"] = ".md"
-app.config["FLATPAGES_MARKDOWN_EXTENSIONS"] = MARKDOWN_EXTENSIONS
-app.config["FLATPAGES_EXTENSION_CONFIG"] = MARKDOWN_EXTENSION_CONFIG
+app.config["FLATPAGES_HTML_RENDERER"] = my_flatpage_renderer
+app.config["WTF_CSRF_TIME_LIMIT"] = None
+
 app.config["BABEL_TRANSLATION_DIRECTORIES"] = "../translations"
 app.config["LANGUAGES"] = {
 	"en": "English",
+	"cs": "čeština",
 	"de": "Deutsch",
+	"es": "Español",
 	"fr": "Français",
 	"id": "Bahasa Indonesia",
+	"it": "Italiano",
 	"ms": "Bahasa Melayu",
 	"pl": "Język Polski",
 	"ru": "русский язык",
 	"sk": "Slovenčina",
-	"zh_Hans": "汉语",
+	"sv": "Svenska",
+	"ta": "தமிழ்",
+	"tr": "Türkçe",
+	"uk": "Українська",
+	"vi": "tiếng Việt",
+	"zh_CN": "汉语",
 }
 
 app.config.from_pyfile(os.environ["FLASK_CONFIG"])
 
-r = redis.Redis.from_url(app.config["REDIS_URL"])
+if not app.config["ADMIN_CONTACT_URL"]:
+	raise Exception("Missing config property: ADMIN_CONTACT_URL")
+
+redis_client = redis.Redis.from_url(app.config["REDIS_URL"])
 
 github = GitHub(app)
 csrf = CSRFProtect(app)
 mail = Mail(app)
 pages = FlatPages(app)
-babel = Babel(app)
-gravatar = Gravatar(app,
-		size=64,
-		rating="g",
-		default="retro",
-		force_default=False,
-		force_lower=False,
-		use_ssl=True,
-		base_url=None)
+babel = Babel()
 init_markdown(app)
 
 login_manager = LoginManager()
@@ -73,11 +118,6 @@ from .sass import init_app as sass
 sass(app)
 
 
-if not app.debug and app.config["MAIL_UTILS_ERROR_SEND_TO"]:
-	from .maillogger import build_handler
-	app.logger.addHandler(build_handler(app))
-
-
 from . import models, template_filters
 
 
@@ -110,7 +150,7 @@ def check_for_ban():
 			if current_user.rank == models.UserRank.BANNED:
 				current_user.rank = models.UserRank.MEMBER
 			models.db.session.commit()
-		elif current_user.ban or current_user.rank == models.UserRank.BANNED:
+		elif current_user.is_banned:
 			if current_user.ban:
 				flash(gettext("Banned:") + " " + current_user.ban.message, "danger")
 			else:
@@ -118,17 +158,16 @@ def check_for_ban():
 			logout_user()
 			return redirect(url_for("users.login"))
 		elif current_user.rank == models.UserRank.NOT_JOINED:
-			current_user.rank = models.UserRank.MEMBER
+			current_user.rank = models.UserRank.NEW_MEMBER
 			models.db.session.commit()
 
 
-from .utils import clearNotifications, is_safe_url
+from .utils import clear_notifications, is_safe_url, create_session
 
 
 @app.before_request
 def check_for_notifications():
-	if current_user.is_authenticated:
-		clearNotifications(request.path)
+	clear_notifications(request.path)
 
 
 @app.errorhandler(404)
@@ -141,7 +180,6 @@ def server_error(e):
 	return render_template("500.html"), 500
 
 
-@babel.localeselector
 def get_locale():
 	if not request:
 		return None
@@ -156,16 +194,18 @@ def get_locale():
 		locale = request.accept_languages.best_match(locales)
 
 	if locale and current_user.is_authenticated:
-		new_session = models.db.create_session({})()
-		new_session.query(models.User) \
-			.filter(models.User.username == current_user.username) \
-			.update({ "locale": locale })
-		new_session.commit()
-		new_session.close()
+		with create_session() as new_session:
+			new_session.query(models.User) \
+				.filter(models.User.username == current_user.username) \
+				.update({"locale": locale})
+			new_session.commit()
 
 	return locale
 
 
+babel.init_app(app, locale_selector=get_locale)
+
+
 @app.route("/set-locale/", methods=["POST"])
 @csrf.exempt
 def set_locale():
@@ -183,10 +223,23 @@ def set_locale():
 	if locale:
 		expire_date = datetime.datetime.now()
 		expire_date = expire_date + datetime.timedelta(days=5*365)
-		resp.set_cookie("locale", locale, expires=expire_date)
+		resp.set_cookie("locale", locale, expires=expire_date, secure=True, samesite="Lax")
 
 		if current_user.is_authenticated:
 			current_user.locale = locale
 			models.db.session.commit()
 
 	return resp
+
+
+@app.route("/set-nonfree/", methods=["POST"])
+def set_nonfree():
+	resp = redirect(url_for("homepage.home"))
+	if request.cookies.get("hide_nonfree") == "1":
+		resp.set_cookie("hide_nonfree", "0", expires=0, secure=True, samesite="Lax")
+	else:
+		expire_date = datetime.datetime.now()
+		expire_date = expire_date + datetime.timedelta(days=5*365)
+		resp.set_cookie("hide_nonfree", "1", expires=expire_date, secure=True, samesite="Lax")
+
+	return resp

app/_translations.py

@@ -0,0 +1,252 @@
+# THIS FILE IS AUTOGENERATED: utils/extract_translations.py
+
+from flask_babel import pgettext
+
+# NOTE: tags: title for 128px
+pgettext("tags", "128px+")
+# NOTE: tags: description for 128px
+pgettext("tags", "For 128px or higher texture packs")
+# NOTE: tags: title for 16px
+pgettext("tags", "16px")
+# NOTE: tags: description for 16px
+pgettext("tags", "For 16px texture packs")
+# NOTE: tags: title for 32px
+pgettext("tags", "32px")
+# NOTE: tags: description for 32px
+pgettext("tags", "For 32px texture packs")
+# NOTE: tags: title for 64px
+pgettext("tags", "64px")
+# NOTE: tags: description for 64px
+pgettext("tags", "For 64px texture packs")
+# NOTE: tags: title for adventure__rpg
+pgettext("tags", "Adventure / RPG")
+# NOTE: tags: title for april_fools
+pgettext("tags", "Joke")
+# NOTE: tags: description for april_fools
+pgettext("tags", "For humorous content, meant as a novelty or joke, not to be taken seriously, and that is not meant to be used seriously or long-term.")
+# NOTE: tags: title for building
+pgettext("tags", "Building")
+# NOTE: tags: description for building
+pgettext("tags", "Focuses on building, such as adding new materials or nodes")
+# NOTE: tags: title for building_mechanics
+pgettext("tags", "Building Mechanics and Tools")
+# NOTE: tags: description for building_mechanics
+pgettext("tags", "Adds game mechanics or tools that change how players build.")
+# NOTE: tags: title for chat
+pgettext("tags", "Chat / Commands")
+# NOTE: tags: description for chat
+pgettext("tags", "Focus on player chat/communication or console interaction.")
+# NOTE: tags: title for commerce
+pgettext("tags", "Commerce / Economy")
+# NOTE: tags: description for commerce
+pgettext("tags", "Related to economies, money, and trading")
+# NOTE: tags: title for complex_installation
+pgettext("tags", "Complex installation")
+# NOTE: tags: description for complex_installation
+pgettext("tags", "Requires futher installation steps, such as installing LuaRocks or editing the trusted mod setting")
+# NOTE: tags: title for crafting
+pgettext("tags", "Crafting")
+# NOTE: tags: description for crafting
+pgettext("tags", "Big changes to crafting gameplay")
+# NOTE: tags: title for creative
+pgettext("tags", "Creative")
+# NOTE: tags: description for creative
+pgettext("tags", "Written specifically or exclusively for use in creative mode. Adds content only available through a creative inventory, or provides tools that facilitate ingame creation and doesn't add difficulty or scarcity")
+# NOTE: tags: title for custom_mapgen
+pgettext("tags", "Custom mapgen")
+# NOTE: tags: description for custom_mapgen
+pgettext("tags", "Contains a completely custom mapgen implemented in Lua, usually requires worlds to be set to the 'singlenode' mapgen.")
+# NOTE: tags: title for decorative
+pgettext("tags", "Decorative")
+# NOTE: tags: description for decorative
+pgettext("tags", "Adds nodes with no other purpose than for use in building")
+# NOTE: tags: title for developer_tools
+pgettext("tags", "Developer Tools")
+# NOTE: tags: description for developer_tools
+pgettext("tags", "Tools for game and mod developers")
+# NOTE: tags: title for education
+pgettext("tags", "Education")
+# NOTE: tags: description for education
+pgettext("tags", "Either has educational value, or is a tool to help teachers ")
+# NOTE: tags: title for environment
+pgettext("tags", "Environment / Weather")
+# NOTE: tags: description for environment
+pgettext("tags", "Improves the world, adding weather, ambient sounds, or other environment mechanics")
+# NOTE: tags: title for food
+pgettext("tags", "Food / Drinks")
+# NOTE: tags: title for gui
+pgettext("tags", "GUI")
+# NOTE: tags: description for gui
+pgettext("tags", "For content whose main utility or features are provided within a GUI, on-screen menu, or similar")
+# NOTE: tags: title for hud
+pgettext("tags", "HUD")
+# NOTE: tags: description for hud
+pgettext("tags", "For mods that grant the player extra information in the HUD")
+# NOTE: tags: title for inventory
+pgettext("tags", "Inventory")
+# NOTE: tags: description for inventory
+pgettext("tags", "Changes the inventory GUI")
+# NOTE: tags: title for jam_combat_mod
+pgettext("tags", "Jam / Combat 2020")
+# NOTE: tags: description for jam_combat_mod
+pgettext("tags", "For mods created for the Discord \"Combat\" modding event in 2020")
+# NOTE: tags: title for jam_game_2021
+pgettext("tags", "Jam / Game 2021")
+# NOTE: tags: description for jam_game_2021
+pgettext("tags", "Entries to the 2021 Minetest Game Jam")
+# NOTE: tags: title for jam_game_2022
+pgettext("tags", " Jam / Game 2022")
+# NOTE: tags: description for jam_game_2022
+pgettext("tags", "Entries to the 2022 Minetest Game Jam ")
+# NOTE: tags: title for jam_game_2023
+pgettext("tags", "Jam / Game 2023")
+# NOTE: tags: description for jam_game_2023
+pgettext("tags", "Entries to the 2023 Minetest Game Jam ")
+# NOTE: tags: title for jam_game_2024
+pgettext("tags", "Jam / Game 2024")
+# NOTE: tags: description for jam_game_2024
+pgettext("tags", "Entries to the 2024 Luanti Game Jam")
+# NOTE: tags: title for jam_weekly_2021
+pgettext("tags", "Jam / Weekly Challenges 2021")
+# NOTE: tags: description for jam_weekly_2021
+pgettext("tags", "For mods created for the Discord \"Weekly Challenges\" modding event in 2021")
+# NOTE: tags: title for less_than_px
+pgettext("tags", "<16px")
+# NOTE: tags: description for less_than_px
+pgettext("tags", "For less than 16px texture packs ")
+# NOTE: tags: title for library
+pgettext("tags", "API / Library")
+# NOTE: tags: description for library
+pgettext("tags", "Primarily adds an API for other mods to use")
+# NOTE: tags: title for magic
+pgettext("tags", "Magic / Enchanting")
+# NOTE: tags: title for mapgen
+pgettext("tags", "Mapgen / Biomes / Decoration")
+# NOTE: tags: description for mapgen
+pgettext("tags", "New mapgen or changes mapgen")
+# NOTE: tags: title for mini-game
+pgettext("tags", "Mini-game")
+# NOTE: tags: description for mini-game
+pgettext("tags", "Adds a mini-game to be played within Luanti")
+# NOTE: tags: title for mobs
+pgettext("tags", "Mobs / Animals / NPCs")
+# NOTE: tags: description for mobs
+pgettext("tags", "Adds mobs, animals, and non-player characters")
+# NOTE: tags: title for mtg
+pgettext("tags", "Minetest Game improved")
+# NOTE: tags: description for mtg
+pgettext("tags", "Forks of Minetest Game")
+# NOTE: tags: title for multiplayer
+pgettext("tags", "Multiplayer-focused")
+# NOTE: tags: description for multiplayer
+pgettext("tags", "Can/should only be used in multiplayer")
+# NOTE: tags: title for oneofakind__original
+pgettext("tags", "One-of-a-kind / Original")
+# NOTE: tags: description for oneofakind__original
+pgettext("tags", "For games and such that are of their own kind, distinct and original in nature to others of the same category.")
+# NOTE: tags: title for plants_and_farming
+pgettext("tags", "Plants and Farming")
+# NOTE: tags: description for plants_and_farming
+pgettext("tags", "Adds new plants or other farmable resources.")
+# NOTE: tags: title for player_effects
+pgettext("tags", "Player Effects / Power Ups")
+# NOTE: tags: description for player_effects
+pgettext("tags", "For content that changes player effects, including physics, for example: speed, jump height or gravity.")
+# NOTE: tags: title for puzzle
+pgettext("tags", "Puzzle")
+# NOTE: tags: description for puzzle
+pgettext("tags", "Focus on puzzle solving instead of combat")
+# NOTE: tags: title for pve
+pgettext("tags", "Player vs Environment (PvE)")
+# NOTE: tags: description for pve
+pgettext("tags", "For content designed for one or more players that focus on combat against the world, mobs, or NPCs.")
+# NOTE: tags: title for pvp
+pgettext("tags", "Player vs Player (PvP)")
+# NOTE: tags: description for pvp
+pgettext("tags", "Designed to be played competitively against other players")
+# NOTE: tags: title for seasonal
+pgettext("tags", "Seasonal")
+# NOTE: tags: description for seasonal
+pgettext("tags", "For content generally themed around a certain season or holiday")
+# NOTE: tags: title for server_tools
+pgettext("tags", "Server Moderation and Tools")
+# NOTE: tags: description for server_tools
+pgettext("tags", "Helps with server maintenance and moderation")
+# NOTE: tags: title for shooter
+pgettext("tags", "Shooter")
+# NOTE: tags: description for shooter
+pgettext("tags", "First person shooters (FPS) and more")
+# NOTE: tags: title for simulation
+pgettext("tags", "Sims")
+# NOTE: tags: description for simulation
+pgettext("tags", "Mods and games that aim to simulate real life activity. Similar to SimCity/The Sims/OpenTTD/etc.")
+# NOTE: tags: title for singleplayer
+pgettext("tags", "Singleplayer-focused")
+# NOTE: tags: description for singleplayer
+pgettext("tags", "Content that can be played alone")
+# NOTE: tags: title for skins
+pgettext("tags", "Player customization / Skins")
+# NOTE: tags: description for skins
+pgettext("tags", "Allows the player to customize their character by changing the texture or adding accessories.")
+# NOTE: tags: title for sound_music
+pgettext("tags", "Sounds / Music")
+# NOTE: tags: description for sound_music
+pgettext("tags", "Focuses on or adds new sounds or musical things")
+# NOTE: tags: title for sports
+pgettext("tags", "Sports")
+# NOTE: tags: title for storage
+pgettext("tags", "Storage")
+# NOTE: tags: description for storage
+pgettext("tags", "Adds or improves item storage mechanics")
+# NOTE: tags: title for strategy_rts
+pgettext("tags", "Strategy / RTS")
+# NOTE: tags: description for strategy_rts
+pgettext("tags", "Games and mods with a heavy strategy component, whether real-time or turn-based")
+# NOTE: tags: title for survival
+pgettext("tags", "Survival")
+# NOTE: tags: description for survival
+pgettext("tags", "Written specifically for survival gameplay with a focus on game-balance, difficulty level, or resources available through crafting, mining, ...")
+# NOTE: tags: title for technology
+pgettext("tags", "Machines / Electronics")
+# NOTE: tags: description for technology
+pgettext("tags", "Adds machines useful in automation, tubes, or power.")
+# NOTE: tags: title for tools
+pgettext("tags", "Tools / Weapons / Armor")
+# NOTE: tags: description for tools
+pgettext("tags", "Adds or changes tools, weapons, and armor")
+# NOTE: tags: title for transport
+pgettext("tags", "Transport")
+# NOTE: tags: description for transport
+pgettext("tags", "Adds or changes transportation methods. Includes teleportation, vehicles, ridable mobs, transport infrastructure and thematic content")
+# NOTE: tags: title for world_tools
+pgettext("tags", "World Maintenance and Tools")
+# NOTE: tags: description for world_tools
+pgettext("tags", "Tools to manage the world")
+# NOTE: content_warnings: title for alcohol_tobacco
+pgettext("content_warnings", "Alcohol / Tobacco")
+# NOTE: content_warnings: description for alcohol_tobacco
+pgettext("content_warnings", "Contains alcohol and/or tobacco")
+# NOTE: content_warnings: title for bad_language
+pgettext("content_warnings", "Bad Language")
+# NOTE: content_warnings: description for bad_language
+pgettext("content_warnings", "Contains swearing")
+# NOTE: content_warnings: title for drugs
+pgettext("content_warnings", "Drugs")
+# NOTE: content_warnings: description for drugs
+pgettext("content_warnings", "Contains recreational drugs other than alcohol or tobacco")
+# NOTE: content_warnings: title for gambling
+pgettext("content_warnings", "Gambling")
+# NOTE: content_warnings: description for gambling
+pgettext("content_warnings", "Games of chance, gambling games, etc")
+# NOTE: content_warnings: title for gore
+pgettext("content_warnings", "Gore")
+# NOTE: content_warnings: description for gore
+pgettext("content_warnings", "Blood, etc")
+# NOTE: content_warnings: title for horror
+pgettext("content_warnings", "Fear / Horror")
+# NOTE: content_warnings: description for horror
+pgettext("content_warnings", "Shocking and scary content. May scare young children")
+# NOTE: content_warnings: title for violence
+pgettext("content_warnings", "Violence")
+# NOTE: content_warnings: description for violence
+pgettext("content_warnings", "Non-cartoon violence. May be towards fantasy or  human-like characters")

app/blueprints/__init__.py

@@ -1,4 +1,22 @@
-import os, importlib
+# ContentDB
+# Copyright (C) 2018-21 rubenwardy
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+import importlib
+import os
+
 
 def create_blueprints(app):
 	dir = os.path.dirname(os.path.realpath(__file__))

app/blueprints/admin/__init__.py

@@ -19,4 +19,4 @@ from flask import Blueprint
 
 bp = Blueprint("admin", __name__)
 
-from . import admin, audit, licenseseditor, tagseditor, versioneditor, warningseditor, email
+from . import admin, audit, licenseseditor, tagseditor, versioneditor, warningseditor, languageseditor, email, approval_stats

app/blueprints/admin/actions.py

@@ -13,25 +13,24 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-
-
+import datetime
 import os
-import sys
 from typing import List
 
 import requests
-from celery import group
-from flask import redirect, url_for, flash, current_app, jsonify
-from sqlalchemy import or_, and_
+from celery import group, uuid
+from flask import redirect, url_for, flash, current_app
+from sqlalchemy import or_, and_, not_, func
 
-from app.logic.game_support import GameSupportResolver
 from app.models import PackageRelease, db, Package, PackageState, PackageScreenshot, MetaPackage, User, \
-	NotificationType, PackageUpdateConfig, License, UserRank, PackageType, PackageGameSupport
+	NotificationType, PackageUpdateConfig, License, UserRank, PackageType, Thread, AuditLogEntry, ReportAttachment
 from app.tasks.emails import send_pending_digests
-from app.tasks.forumtasks import importTopicList, checkAllForumAccounts
-from app.tasks.importtasks import importRepoScreenshot, checkZipRelease, check_for_updates
-from app.utils import addNotification, get_system_user
-from app.utils.image import get_image_size
+from app.tasks.forumtasks import import_topic_list, check_all_forum_accounts
+from app.tasks.importtasks import import_repo_screenshot, check_zip_release, check_for_updates, update_all_game_support, \
+	import_languages, check_all_zip_files
+from app.tasks.usertasks import import_github_user_ids, do_delete_likely_spammers
+from app.tasks.pkgtasks import notify_about_git_forum_links, clear_removed_packages, check_package_for_broken_links, update_file_size_bytes
+from app.utils import add_notification, get_system_user
 
 actions = {}
 
@@ -55,67 +54,7 @@ def del_stuck_releases():
 	db.session.commit()
 	return redirect(url_for("admin.admin_page"))
 
-
-@action("Check all releases (postReleaseCheckUpdate)")
-def check_releases():
-	releases = PackageRelease.query.filter(PackageRelease.url.like("/uploads/%")).all()
-
-	tasks = []
-	for release in releases:
-		tasks.append(checkZipRelease.s(release.id, release.file_path))
-
-	result = group(tasks).apply_async()
-
-	while not result.ready():
-		import time
-		time.sleep(0.1)
-
-	return redirect(url_for("todo.view_editor"))
-
-
-@action("Check latest release of all packages (postReleaseCheckUpdate)")
-def reimport_packages():
-	tasks = []
-	for package in Package.query.filter(Package.state != PackageState.DELETED).all():
-		release = package.releases.first()
-		if release:
-			tasks.append(checkZipRelease.s(release.id, release.file_path))
-
-	result = group(tasks).apply_async()
-
-	while not result.ready():
-		import time
-		time.sleep(0.1)
-
-	return redirect(url_for("todo.view_editor"))
-
-
-@action("Import forum topic list")
-def import_topic_list():
-	task = importTopicList.delay()
-	return redirect(url_for("tasks.check", id=task.id, r=url_for("todo.topics")))
-
-
-@action("Check all forum accounts")
-def check_all_forum_accounts():
-	task = checkAllForumAccounts.delay()
-	return redirect(url_for("tasks.check", id=task.id, r=url_for("admin.admin_page")))
-
-
-@action("Import screenshots from Git")
-def import_screenshots():
-	packages = Package.query \
-		.filter(Package.state != PackageState.DELETED) \
-		.outerjoin(PackageScreenshot, Package.id == PackageScreenshot.package_id) \
-		.filter(PackageScreenshot.id.is_(None)) \
-		.all()
-	for package in packages:
-		importRepoScreenshot.delay(package.id)
-
-	return redirect(url_for("admin.admin_page"))
-
-
-@action("Remove unused uploads")
+@action("Delete unused uploads")
 def clean_uploads():
 	upload_dir = current_app.config['UPLOAD_DIR']
 
@@ -129,8 +68,10 @@ def clean_uploads():
 
 		release_urls = get_filenames_from_column(PackageRelease.url)
 		screenshot_urls = get_filenames_from_column(PackageScreenshot.url)
+		attachment_urls = get_filenames_from_column(ReportAttachment.url)
+		pp_urls = get_filenames_from_column(User.profile_pic)
 
-		db_urls = release_urls.union(screenshot_urls)
+		db_urls = release_urls.union(screenshot_urls).union(pp_urls).union(attachment_urls)
 		unreachable = existing_uploads.difference(db_urls)
 
 		import sys
@@ -148,30 +89,40 @@ def clean_uploads():
 	return redirect(url_for("admin.admin_page"))
 
 
-@action("Delete unused metapackages")
-def del_meta_packages():
+@action("Delete unused mod names")
+def del_mod_names():
 	query = MetaPackage.query.filter(~MetaPackage.dependencies.any(), ~MetaPackage.packages.any())
 	count = query.count()
 	query.delete(synchronize_session=False)
 	db.session.commit()
 
-	flash("Deleted " + str(count) + " unused meta packages", "success")
+	flash("Deleted " + str(count) + " unused mod names", "success")
 	return redirect(url_for("admin.admin_page"))
 
 
-@action("Delete removed packages")
-def del_removed_packages():
-	query = Package.query.filter_by(state=PackageState.DELETED)
-	count = query.count()
-	for pkg in query.all():
-		pkg.review_thread = None
-		db.session.delete(pkg)
+@action("Recalc package scores")
+def recalc_scores():
+	for package in Package.query.all():
+		package.recalculate_score()
+
 	db.session.commit()
 
-	flash("Deleted {} soft deleted packages packages".format(count), "success")
+	flash("Recalculated package scores", "success")
 	return redirect(url_for("admin.admin_page"))
 
 
+@action("Import forum topic list")
+def do_import_topic_list():
+	task = import_topic_list.delay()
+	return redirect(url_for("tasks.check", id=task.id, r=url_for("admin.admin_page")))
+
+
+@action("Check all forum accounts")
+def check_all_forum_accounts():
+	task = check_all_forum_accounts.delay()
+	return redirect(url_for("tasks.check", id=task.id, r=url_for("admin.admin_page")))
+
+
 @action("Run update configs")
 def run_update_config():
 	check_for_updates.delay()
@@ -184,31 +135,29 @@ def _package_list(packages: List[str]):
 	# Who needs translations?
 	if len(packages) >= 3:
 		packages[len(packages) - 1] = "and " + packages[len(packages) - 1]
-		packages_list = ", ".join(packages)
+		return ", ".join(packages)
 	else:
-		packages_list = " and ".join(packages)
-	return packages_list
+		return " and ".join(packages)
 
 
 @action("Send WIP package notification")
 def remind_wip():
-	users = User.query.filter(User.packages.any(or_(Package.state == PackageState.WIP, Package.state == PackageState.CHANGES_NEEDED)))
+	users = User.query.filter(User.packages.any(or_(
+			Package.state == PackageState.WIP, Package.state == PackageState.CHANGES_NEEDED)))
 	system_user = get_system_user()
 	for user in users:
-		packages = db.session.query(Package.title).filter(
+		packages = Package.query.filter(
 				Package.author_id == user.id,
-				or_(Package.state == PackageState.WIP, Package.state==PackageState.CHANGES_NEEDED)) \
+				or_(Package.state == PackageState.WIP, Package.state == PackageState.CHANGES_NEEDED)) \
 			.all()
 
-		packages = [pkg[0] for pkg in packages]
+		packages = [pkg.title for pkg in packages]
 		packages_list = _package_list(packages)
 		havent = "haven't" if len(packages) > 1 else "hasn't"
-		if len(packages_list) + 54 > 100:
-			packages_list = packages_list[0:(100-54-1)] + "…"
 
-		addNotification(user, system_user, NotificationType.PACKAGE_APPROVAL,
+		add_notification(user, system_user, NotificationType.PACKAGE_APPROVAL,
 			f"Did you forget? {packages_list} {havent} been submitted for review yet",
-				url_for('todo.view_user', username=user.username))
+						 url_for('todo.view_user', username=user.username))
 	db.session.commit()
 
 
@@ -218,17 +167,17 @@ def remind_outdated():
 			Package.update_config.has(PackageUpdateConfig.outdated_at.isnot(None))))
 	system_user = get_system_user()
 	for user in users:
-		packages = db.session.query(Package.title).filter(
-				Package.maintainers.any(User.id==user.id),
+		packages = Package.query.filter(
+				Package.maintainers.contains(user),
 				Package.update_config.has(PackageUpdateConfig.outdated_at.isnot(None))) \
 			.all()
 
-		packages = [pkg[0] for pkg in packages]
+		packages = [pkg.title for pkg in packages]
 		packages_list = _package_list(packages)
 
-		addNotification(user, system_user, NotificationType.PACKAGE_APPROVAL,
+		add_notification(user, system_user, NotificationType.PACKAGE_APPROVAL,
 				f"The following packages may be outdated: {packages_list}",
-				url_for('todo.view_user', username=user.username))
+						 url_for('todo.view_user', username=user.username))
 
 	db.session.commit()
 
@@ -265,17 +214,16 @@ def import_licenses():
 	licenses = r.json()["licenses"]
 
 	existing_licenses = {}
-	for license in License.query.all():
-		assert license.name not in renames.keys()
-		existing_licenses[license.name.lower()] = license
+	for license_data in License.query.all():
+		assert license_data.name not in renames.keys()
+		existing_licenses[license_data.name.lower()] = license_data
 
-	for license in licenses:
-		obj = existing_licenses.get(license["licenseId"].lower())
+	for license_data in licenses:
+		obj = existing_licenses.get(license_data["licenseId"].lower())
 		if obj:
-			obj.url = license["reference"]
-		elif license.get("isOsiApproved") and license.get("isFsfLibre") and \
-				not license["isDeprecatedLicenseId"]:
-			obj = License(license["licenseId"], True, license["reference"])
+			obj.url = license_data["reference"]
+		elif license_data.get("isOsiApproved") and license_data.get("isFsfLibre") and not license_data["isDeprecatedLicenseId"]:
+			obj = License(license_data["licenseId"], True, license_data["reference"])
 			db.session.add(obj)
 
 	db.session.commit()
@@ -293,46 +241,194 @@ def delete_inactive_users():
 @action("Send Video URL notification")
 def remind_video_url():
 	users = User.query.filter(User.maintained_packages.any(
-			and_(Package.video_url.is_(None), Package.type==PackageType.GAME, Package.state==PackageState.APPROVED)))
+			and_(Package.video_url == None, Package.type == PackageType.GAME, Package.state == PackageState.APPROVED)))
 	system_user = get_system_user()
 	for user in users:
-		packages = db.session.query(Package.title).filter(
-				or_(Package.author==user, Package.maintainers.any(User.id==user.id)),
-				Package.video_url.is_(None),
+		packages = Package.query.filter(
+				or_(Package.author == user, Package.maintainers.contains(user)),
+				Package.video_url == None,
 				Package.type == PackageType.GAME,
 				Package.state == PackageState.APPROVED) \
 			.all()
 
-		packages = [pkg[0] for pkg in packages]
-		packages_list = _package_list(packages)
+		package_names = [pkg.title for pkg in packages]
+		packages_list = _package_list(package_names)
 
-		addNotification(user, system_user, NotificationType.PACKAGE_APPROVAL,
+		add_notification(user, system_user, NotificationType.PACKAGE_APPROVAL,
 				f"You should add a video to {packages_list}",
-				url_for('users.profile', username=user.username))
+						 url_for('users.profile', username=user.username))
 
 	db.session.commit()
 
 
-@action("Update screenshot sizes")
-def update_screenshot_sizes():
-	import sys
+@action("Send missing game support notifications")
+def remind_missing_game_support():
+	users = User.query.filter(
+		User.maintained_packages.any(and_(
+			Package.state != PackageState.DELETED,
+			Package.type.in_([PackageType.MOD, PackageType.TXP]),
+			~Package.supported_games.any(),
+			Package.supports_all_games == False))).all()
 
-	for screenshot in PackageScreenshot.query.all():
-		width, height = get_image_size(screenshot.file_path)
-		print(f"{screenshot.url}: {width}, {height}", file=sys.stderr)
-		screenshot.width = width
-		screenshot.height = height
+	system_user = get_system_user()
+	for user in users:
+		packages = Package.query.filter(
+			Package.maintainers.contains(user),
+			Package.state != PackageState.DELETED,
+			Package.type.in_([PackageType.MOD, PackageType.TXP]),
+			~Package.supported_games.any(),
+			Package.supports_all_games == False) \
+			.all()
+
+		packages = [pkg.title for pkg in packages]
+		packages_list = _package_list(packages)
+
+		add_notification(user, system_user, NotificationType.PACKAGE_APPROVAL,
+				f"You need to confirm whether the following packages support all games: {packages_list}",
+				url_for('todo.all_game_support', username=user.username))
 
 	db.session.commit()
 
 
 @action("Detect game support")
 def detect_game_support():
-	resolver = GameSupportResolver()
-	resolver.update_all()
-	db.session.commit()
+	task_id = uuid()
+	update_all_game_support.apply_async((), task_id=task_id)
+	return redirect(url_for("tasks.check", id=task_id, r=url_for("admin.admin_page")))
 
 
 @action("Send pending notif digests")
 def do_send_pending_digests():
 	send_pending_digests.delay()
+
+
+@action("Import user ids from GitHub")
+def do_import_github_user_ids():
+	task_id = uuid()
+	import_github_user_ids.apply_async((), task_id=task_id)
+	return redirect(url_for("tasks.check", id=task_id, r=url_for("admin.admin_page")))
+
+
+@action("Notify about links to git/forums instead of CDB")
+def do_notify_git_forums_links():
+	task_id = uuid()
+	notify_about_git_forum_links.apply_async((), task_id=task_id)
+	return redirect(url_for("tasks.check", id=task_id, r=url_for("admin.admin_page")))
+
+
+@action("Check all zip files")
+def do_check_all_zip_files():
+	task_id = uuid()
+	check_all_zip_files.apply_async((), task_id=task_id)
+	return redirect(url_for("tasks.check", id=task_id, r=url_for("admin.admin_page")))
+
+
+@action("Update file_size_bytes")
+def do_update_file_size_bytes():
+	task_id = uuid()
+	update_file_size_bytes.apply_async((), task_id=task_id)
+	return redirect(url_for("tasks.check", id=task_id, r=url_for("admin.admin_page")))
+
+
+@action("DANGER: Delete less popular removed packages")
+def del_less_popular_removed_packages():
+	task_id = uuid()
+	clear_removed_packages.apply_async((False, ), task_id=task_id)
+	return redirect(url_for("tasks.check", id=task_id, r=url_for("admin.admin_page")))
+
+
+@action("DANGER: Delete all removed packages")
+def del_removed_packages():
+	task_id = uuid()
+	clear_removed_packages.apply_async((True, ), task_id=task_id)
+	return redirect(url_for("tasks.check", id=task_id, r=url_for("admin.admin_page")))
+
+
+@action("DANGER: Check all releases (postReleaseCheckUpdate)")
+def check_releases():
+	releases = PackageRelease.query.filter(PackageRelease.url.like("/uploads/%")).all()
+
+	tasks = []
+	for release in releases:
+		tasks.append(check_zip_release.s(release.id, release.file_path))
+
+	result = group(tasks).apply_async()
+
+	while not result.ready():
+		import time
+		time.sleep(0.1)
+
+	return redirect(url_for("todo.view_editor"))
+
+
+@action("DANGER: Check latest release of all packages (postReleaseCheckUpdate)")
+def reimport_packages():
+	tasks = []
+	for package in Package.query.filter(Package.state == PackageState.APPROVED).all():
+		release = package.releases.first()
+		if release:
+			tasks.append(check_zip_release.s(release.id, release.file_path))
+
+	result = group(tasks).apply_async()
+
+	while not result.ready():
+		import time
+		time.sleep(0.1)
+
+	return redirect(url_for("todo.view_editor"))
+
+
+@action("DANGER: Import translations")
+def reimport_translations():
+	tasks = []
+	for package in Package.query.filter(Package.state == PackageState.APPROVED).all():
+		release = package.releases.first()
+		if release:
+			tasks.append(import_languages.s(release.id, release.file_path))
+
+	result = group(tasks).apply_async()
+	while not result.ready():
+		import time
+		time.sleep(0.1)
+
+	return redirect(url_for("todo.view_editor"))
+
+
+@action("DANGER: Import screenshots from Git")
+def import_screenshots():
+	packages = Package.query \
+		.filter(Package.state != PackageState.DELETED) \
+		.outerjoin(PackageScreenshot, Package.id == PackageScreenshot.package_id) \
+		.filter(PackageScreenshot.id == None) \
+		.all()
+	for package in packages:
+		import_repo_screenshot.delay(package.id)
+
+	return redirect(url_for("admin.admin_page"))
+
+
+@action("DANGER: Delete empty threads")
+def delete_empty_threads():
+	query = Thread.query.filter(~Thread.replies.any())
+	count = query.count()
+	for thread in query.all():
+		thread.watchers.clear()
+		db.session.delete(thread)
+	db.session.commit()
+
+	flash(f"Deleted {count} threads", "success")
+
+	return redirect(url_for("admin.admin_page"))
+
+
+@action("DANGER: Check for broken links in all packages")
+def check_for_broken_links():
+	for package in Package.query.filter_by(state=PackageState.APPROVED).all():
+		check_package_for_broken_links.delay(package.id)
+
+
+@action("DANGER: Delete likely spammers")
+def delete_likely_spammers():
+	task_id = uuid()
+	do_delete_likely_spammers.apply_async((), task_id=task_id)
+	return redirect(url_for("tasks.check", id=task_id, r=url_for("admin.admin_page")))

app/blueprints/admin/admin.py

@@ -17,30 +17,23 @@
 from flask import redirect, render_template, url_for, request, flash
 from flask_login import current_user, login_user
 from flask_wtf import FlaskForm
-from wtforms import StringField, SubmitField
-from wtforms.validators import InputRequired, Length
-from app.utils import rank_required, addAuditLog, addNotification, get_system_user
+from wtforms import StringField, SubmitField, BooleanField
+from wtforms.validators import InputRequired, Length, Optional
+from app.utils import rank_required, add_audit_log, add_notification, get_system_user, nonempty_or_none, \
+	get_int_or_abort
+from sqlalchemy import func
 from . import bp
 from .actions import actions
-from ...models import UserRank, Package, db, PackageState, User, AuditSeverity, NotificationType
+from app.models import UserRank, Package, db, PackageState, PackageRelease, PackageScreenshot, User, AuditSeverity, NotificationType, PackageAlias
+from ...querybuilder import QueryBuilder
 
 
 @bp.route("/admin/", methods=["GET", "POST"])
-@rank_required(UserRank.ADMIN)
+@rank_required(UserRank.EDITOR)
 def admin_page():
-	if request.method == "POST":
+	if request.method == "POST" and current_user.rank.at_least(UserRank.ADMIN):
 		action = request.form["action"]
-
-		if action == "restore":
-			package = Package.query.get(request.form["package"])
-			if package is None:
-				flash("Unknown package", "danger")
-			else:
-				package.state = PackageState.READY_FOR_REVIEW
-				db.session.commit()
-				return redirect(url_for("admin.admin_page"))
-
-		elif action in actions:
+		if action in actions:
 			ret = actions[action]["func"]()
 			if ret:
 				return ret
@@ -48,8 +41,7 @@ def admin_page():
 		else:
 			flash("Unknown action: " + action, "danger")
 
-	deleted_packages = Package.query.filter(Package.state == PackageState.DELETED).all()
-	return render_template("admin/list.html", deleted_packages=deleted_packages, actions=actions)
+	return render_template("admin/list.html", actions=actions)
 
 
 class SwitchUserForm(FlaskForm):
@@ -62,7 +54,7 @@ class SwitchUserForm(FlaskForm):
 def switch_user():
 	form = SwitchUserForm(formdata=request.form)
 	if form.validate_on_submit():
-		user = User.query.filter_by(username=form["username"].data).first()
+		user = User.query.filter_by(username=form.username.data).first()
 		if user is None:
 			flash("Unable to find user", "danger")
 		elif login_user(user):
@@ -85,11 +77,11 @@ class SendNotificationForm(FlaskForm):
 def send_bulk_notification():
 	form = SendNotificationForm(request.form)
 	if form.validate_on_submit():
-		addAuditLog(AuditSeverity.MODERATION, current_user,
+		add_audit_log(AuditSeverity.MODERATION, current_user,
 				"Sent bulk notification", url_for("admin.admin_page"), None, form.title.data)
 
 		users = User.query.filter(User.rank >= UserRank.NEW_MEMBER).all()
-		addNotification(users, get_system_user(), NotificationType.OTHER, form.title.data, form.url.data, None)
+		add_notification(users, get_system_user(), NotificationType.OTHER, form.title.data, form.url.data, None)
 		db.session.commit()
 
 		return redirect(url_for("admin.admin_page"))
@@ -115,11 +107,11 @@ def restore():
 		else:
 			package.state = target
 
-			addAuditLog(AuditSeverity.EDITOR, current_user, f"Restored package to state {target.value}",
-					package.getURL("packages.view"), package)
+			add_audit_log(AuditSeverity.EDITOR, current_user, f"Restored package to state {target.value}",
+						  package.get_url("packages.view"), package)
 
 			db.session.commit()
-			return redirect(package.getURL("packages.view"))
+			return redirect(package.get_url("packages.view"))
 
 	deleted_packages = Package.query \
 		.filter(Package.state == PackageState.DELETED) \
@@ -128,3 +120,104 @@ def restore():
 		.all()
 
 	return render_template("admin/restore.html", deleted_packages=deleted_packages)
+
+
+class TransferPackageForm(FlaskForm):
+	old_username = StringField("Old Username", [InputRequired()])
+	new_username = StringField("New Username", [InputRequired()])
+	package = StringField("Package", [Optional()])
+	remove_maintainer = BooleanField("Remove current owner from maintainers")
+	submit = SubmitField("Transfer")
+
+
+def perform_transfer(form: TransferPackageForm):
+	query = Package.query.filter(Package.author.has(username=form.old_username.data))
+	if nonempty_or_none(form.package.data):
+		query = query.filter_by(name=form.package.data)
+
+	packages = query.all()
+	if len(packages) == 0:
+		flash("Unable to find package(s)", "danger")
+		return
+
+	new_user = User.query.filter_by(username=form.new_username.data).first()
+	if new_user is None:
+		flash("Unable to find new user", "danger")
+		return
+
+	names = [x.name for x in packages]
+	already_existing = Package.query.filter(Package.author_id == new_user.id, Package.name.in_(names)).all()
+	if len(already_existing) > 0:
+		existing_names = [x.name for x in already_existing]
+		flash("Unable to transfer packages as names exist at destination: " + ", ".join(existing_names), "danger")
+		return
+
+	for package in packages:
+		if form.remove_maintainer.data:
+			package.maintainers.remove(package.author)
+		package.author = new_user
+		package.maintainers.append(new_user)
+		package.aliases.append(PackageAlias(form.old_username.data, package.name))
+
+		add_audit_log(AuditSeverity.MODERATION, current_user,
+				f"Transferred {form.old_username.data}/{package.name} to {form.new_username.data}",
+				package.get_url("packages.view"), package)
+
+	db.session.commit()
+
+	flash("Transferred " + ", ".join([x.name for x in packages]), "success")
+
+	return redirect(url_for("admin.transfer"))
+
+
+@bp.route("/admin/transfer/", methods=["GET", "POST"])
+@rank_required(UserRank.MODERATOR)
+def transfer():
+	form = TransferPackageForm(formdata=request.form)
+	if form.validate_on_submit():
+		ret = perform_transfer(form)
+		if ret is not None:
+			return ret
+
+	# Process GET or invalid POST
+	return render_template("admin/transfer.html", form=form)
+
+
+def sum_file_sizes(clazz):
+	ret = {}
+	for entry in (db.session
+			.query(clazz.package_id, func.sum(clazz.file_size_bytes))
+			.select_from(clazz)
+			.group_by(clazz.package_id)
+			.all()):
+		ret[entry[0]] = entry[1]
+	return ret
+
+
+@bp.route("/admin/storage/")
+@rank_required(UserRank.EDITOR)
+def storage():
+	qb = QueryBuilder(request.args, cookies=True)
+	qb.only_approved = False
+	packages = qb.build_package_query().all()
+
+	show_all = len(packages) < 100
+	min_size = get_int_or_abort(request.args.get("min_size"), 0 if show_all else 50)
+
+	package_size_releases = sum_file_sizes(PackageRelease)
+	package_size_screenshots = sum_file_sizes(PackageScreenshot)
+
+	data = []
+	for package in packages:
+		size_releases = package_size_releases.get(package.id, 0)
+		size_screenshots = package_size_screenshots.get(package.id, 0)
+		size_total = size_releases + size_screenshots
+		if size_total < min_size * 1024 * 1024:
+			continue
+
+		latest_release = package.releases.first()
+		size_latest = latest_release.file_size_bytes if latest_release else 0
+		data.append([package, size_total, size_releases, size_screenshots, size_latest])
+
+	data.sort(key=lambda x: x[1], reverse=True)
+	return render_template("admin/storage.html", data=data)

app/blueprints/admin/approval_stats.py

@@ -0,0 +1,77 @@
+# ContentDB
+# Copyright (C) 2024 rubenwardy
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+import datetime
+from flask import render_template, request, abort, redirect, url_for, jsonify
+
+from . import bp
+from app.logic.approval_stats import get_approval_statistics
+from app.models import UserRank
+from app.utils import rank_required
+
+
+@bp.route("/admin/approval_stats/")
+@rank_required(UserRank.APPROVER)
+def approval_stats():
+	start = request.args.get("start")
+	end = request.args.get("end")
+	if start and end:
+		try:
+			start = datetime.datetime.fromisoformat(start)
+			end = datetime.datetime.fromisoformat(end)
+		except ValueError:
+			abort(400)
+	elif start:
+		return redirect(url_for("admin.approval_stats", start=start, end=datetime.datetime.utcnow().date().isoformat()))
+	elif end:
+		return redirect(url_for("admin.approval_stats", start="2020-07-01", end=end))
+	else:
+		end = datetime.datetime.utcnow()
+		start = end - datetime.timedelta(days=365)
+
+	stats = get_approval_statistics(start, end)
+	return render_template("admin/approval_stats.html", stats=stats, start=start, end=end)
+
+
+@bp.route("/admin/approval_stats.json")
+@rank_required(UserRank.APPROVER)
+def approval_stats_json():
+	start = request.args.get("start")
+	end = request.args.get("end")
+	if start and end:
+		try:
+			start = datetime.datetime.fromisoformat(start)
+			end = datetime.datetime.fromisoformat(end)
+		except ValueError:
+			abort(400)
+	else:
+		end = datetime.datetime.utcnow()
+		start = end - datetime.timedelta(days=365)
+
+	stats = get_approval_statistics(start, end)
+	for key, value in stats.packages_info.items():
+		stats.packages_info[key] = value.__dict__()
+
+	return jsonify({
+		"start": start.isoformat(),
+		"end": end.isoformat(),
+		"editor_approvals": stats.editor_approvals,
+		"packages_info": stats.packages_info,
+		"turnaround_time": {
+			"avg": stats.avg_turnaround_time,
+			"max": stats.max_turnaround_time,
+		},
+	})

app/blueprints/admin/audit.py

@@ -15,32 +15,59 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 from flask import render_template, request, abort
-from app.models import db, AuditLogEntry, UserRank, User
+from flask_babel import lazy_gettext
+from flask_login import current_user, login_required
+from flask_wtf import FlaskForm
+from wtforms import StringField, SubmitField
+from wtforms.validators import Optional, Length
+
+from app.models import db, AuditLogEntry, UserRank, User, Permission
 from app.utils import rank_required, get_int_or_abort
 
 from . import bp
 
 
+class AuditForm(FlaskForm):
+	username = StringField(lazy_gettext("Username"), [Optional(), Length(0, 25)])
+	q = StringField(lazy_gettext("Query"), [Optional(), Length(0, 300)])
+	url = StringField(lazy_gettext("URL"), [Optional(), Length(0, 300)])
+	submit = SubmitField(lazy_gettext("Search"), name=None)
+
+
 @bp.route("/admin/audit/")
-@rank_required(UserRank.MODERATOR)
+@rank_required(UserRank.APPROVER)
 def audit():
 	page = get_int_or_abort(request.args.get("page"), 1)
 	num = min(40, get_int_or_abort(request.args.get("n"), 100))
 
 	query = AuditLogEntry.query.order_by(db.desc(AuditLogEntry.created_at))
 
-	if "username" in request.args:
-		user = User.query.filter_by(username=request.args.get("username")).first()
-		if not user:
-			abort(404)
+	form = AuditForm(request.args)
+	username = form.username.data
+	q = form.q.data
+	url = form.url.data
+	if username:
+		user = User.query.filter_by(username=username).first_or_404()
 		query = query.filter_by(causer=user)
 
-	pagination = query.paginate(page, num, True)
-	return render_template("admin/audit.html", log=pagination.items, pagination=pagination)
+	if q:
+		query = query.filter(AuditLogEntry.title.ilike(f"%{q}%"))
+
+	if url:
+		query = query.filter(AuditLogEntry.url.ilike(f"%{url}%"))
+
+	if not current_user.rank.at_least(UserRank.MODERATOR):
+		query = query.filter(AuditLogEntry.package)
+
+	pagination = query.paginate(page=page, per_page=num)
+	return render_template("admin/audit.html", log=pagination.items, pagination=pagination, form=form)
 
 
 @bp.route("/admin/audit/<int:id_>/")
-@rank_required(UserRank.MODERATOR)
+@login_required
 def audit_view(id_):
-	entry = AuditLogEntry.query.get(id_)
+	entry: AuditLogEntry = AuditLogEntry.query.get_or_404(id_)
+	if not entry.check_perm(current_user, Permission.VIEW_AUDIT_DESCRIPTION):
+		abort(403)
+
 	return render_template("admin/audit_view.html", entry=entry)

app/blueprints/admin/email.py

@@ -22,14 +22,14 @@ from wtforms.validators import InputRequired, Length
 
 from app.markdown import render_markdown
 from app.tasks.emails import send_user_email, send_bulk_email as task_send_bulk
-from app.utils import rank_required, addAuditLog
+from app.utils import rank_required, add_audit_log, normalize_line_endings
 from . import bp
-from ...models import UserRank, User, AuditSeverity
+from app.models import UserRank, User, AuditSeverity
 
 
 class SendEmailForm(FlaskForm):
 	subject = StringField("Subject", [InputRequired(), Length(1, 300)])
-	text    = TextAreaField("Message", [InputRequired()])
+	text    = TextAreaField("Message", [InputRequired()], filters=[normalize_line_endings])
 	submit  = SubmitField("Send")
 
 
@@ -49,12 +49,12 @@ def send_single_email():
 
 	form = SendEmailForm(request.form)
 	if form.validate_on_submit():
-		addAuditLog(AuditSeverity.MODERATION, current_user,
+		add_audit_log(AuditSeverity.MODERATION, current_user,
 				"Sent email to {}".format(user.display_name), url_for("users.profile", username=username))
 
 		text = form.text.data
 		html = render_markdown(text)
-		task = send_user_email.delay(user.email, user.locale or "en",form.subject.data, text, html)
+		task = send_user_email.delay(user.email, user.locale or "en", form.subject.data, text, html)
 		return redirect(url_for("tasks.check", id=task.id, r=next_url))
 
 	return render_template("admin/send_email.html", form=form, user=user)
@@ -65,7 +65,7 @@ def send_single_email():
 def send_bulk_email():
 	form = SendEmailForm(request.form)
 	if form.validate_on_submit():
-		addAuditLog(AuditSeverity.MODERATION, current_user,
+		add_audit_log(AuditSeverity.MODERATION, current_user,
 				"Sent bulk email", url_for("admin.admin_page"), None, form.text.data)
 
 		text = form.text.data

app/blueprints/admin/languageseditor.py

@@ -0,0 +1,73 @@
+# ContentDB
+# Copyright (C) 2018-24 rubenwardy
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+
+from flask import redirect, render_template, abort, url_for
+from flask_login import current_user
+from flask_wtf import FlaskForm
+from wtforms import StringField, TextAreaField, SubmitField
+from wtforms.validators import InputRequired, Length, Optional
+
+from app.models import db, AuditSeverity, UserRank, Language, Package, PackageState, PackageTranslation
+from app.utils import add_audit_log, rank_required, normalize_line_endings
+from . import bp
+
+
+@bp.route("/admin/languages/")
+@rank_required(UserRank.ADMIN)
+def language_list():
+	at_least_one_count = db.session.query(PackageTranslation.package_id).group_by(PackageTranslation.package_id).count()
+	total_package_count = Package.query.filter_by(state=PackageState.APPROVED).count()
+	return render_template("admin/languages/list.html",
+		languages=Language.query.all(), total_package_count=total_package_count,
+		at_least_one_count=at_least_one_count)
+
+
+class LanguageForm(FlaskForm):
+	id = StringField("Id", [InputRequired(), Length(2, 10)])
+	title = TextAreaField("Title", [Optional(), Length(2, 100)], filters=[normalize_line_endings])
+	submit = SubmitField("Save")
+
+
+@bp.route("/admin/languages/new/", methods=["GET", "POST"])
+@bp.route("/admin/languages/<id_>/edit/", methods=["GET", "POST"])
+@rank_required(UserRank.ADMIN)
+def create_edit_language(id_=None):
+	language = None
+	if id_ is not None:
+		language = Language.query.filter_by(id=id_).first()
+		if language is None:
+			abort(404)
+
+	form = LanguageForm(obj=language)
+	if form.validate_on_submit():
+		if language is None:
+			language = Language()
+			db.session.add(language)
+			form.populate_obj(language)
+
+			add_audit_log(AuditSeverity.EDITOR, current_user, f"Created language {language.id}",
+					url_for("admin.create_edit_language", id_=language.id))
+		else:
+			form.populate_obj(language)
+
+			add_audit_log(AuditSeverity.EDITOR, current_user, f"Edited language {language.id}",
+					url_for("admin.create_edit_language", id_=language.id))
+
+		db.session.commit()
+		return redirect(url_for("admin.create_edit_language", id_=language.id))
+
+	return render_template("admin/languages/edit.html", language=language, form=form)

app/blueprints/admin/licenseseditor.py

@@ -16,13 +16,14 @@
 
 
 from flask import redirect, render_template, abort, url_for, request, flash
+from flask_login import current_user
 from flask_wtf import FlaskForm
 from wtforms import StringField, BooleanField, SubmitField, URLField
 from wtforms.validators import InputRequired, Length, Optional
 
-from app.utils import rank_required, nonEmptyOrNone
+from app.utils import rank_required, nonempty_or_none, add_audit_log
 from . import bp
-from ...models import UserRank, License, db
+from app.models import UserRank, License, db, AuditSeverity
 
 
 @bp.route("/licenses/")
@@ -34,7 +35,7 @@ def license_list():
 class LicenseForm(FlaskForm):
 	name = StringField("Name", [InputRequired(), Length(3, 100)])
 	is_foss = BooleanField("Is FOSS")
-	url = URLField("URL", [Optional], filters=[nonEmptyOrNone])
+	url = URLField("URL", [Optional()], filters=[nonempty_or_none])
 	submit = SubmitField("Save")
 
 
@@ -56,9 +57,15 @@ def create_edit_license(name=None):
 			license = License(form.name.data)
 			db.session.add(license)
 			flash("Created license " + form.name.data, "success")
+
+			add_audit_log(AuditSeverity.MODERATION, current_user, f"Created license {license.name}",
+						  url_for("admin.license_list"))
 		else:
 			flash("Updated license " + form.name.data, "success")
 
+			add_audit_log(AuditSeverity.MODERATION, current_user, f"Edited license {license.name}",
+						  url_for("admin.license_list"))
+
 		form.populate_obj(license)
 		db.session.commit()
 		return redirect(url_for("admin.license_list"))

app/blueprints/admin/tagseditor.py

@@ -22,7 +22,8 @@ from wtforms import StringField, TextAreaField, BooleanField, SubmitField
 from wtforms.validators import InputRequired, Length, Optional, Regexp
 
 from . import bp
-from ...models import Permission, Tag, db
+from app.models import Permission, Tag, db, AuditSeverity
+from app.utils import add_audit_log, normalize_line_endings
 
 
 @bp.route("/tags/")
@@ -43,9 +44,9 @@ def tag_list():
 
 class TagForm(FlaskForm):
 	title = StringField("Title", [InputRequired(), Length(3, 100)])
-	description = TextAreaField("Description", [Optional(), Length(0, 500)])
-	name = StringField("Name", [Optional(), Length(1, 20), Regexp("^[a-z0-9_]", 0, "Lower case letters (a-z), digits (0-9), and underscores (_) only")])
-	is_protected = BooleanField("Is Protected")
+	description = TextAreaField("Description", [Optional(), Length(0, 500)], filters=[normalize_line_endings])
+	name = StringField("Name", [Optional(), Length(1, 20), Regexp("^[a-z0-9_]", 0,
+			"Lower case letters (a-z), digits (0-9), and underscores (_) only")])
 	submit = SubmitField("Save")
 
 
@@ -59,19 +60,24 @@ def create_edit_tag(name=None):
 		if tag is None:
 			abort(404)
 
-	if not Permission.checkPerm(current_user, Permission.EDIT_TAGS if tag else Permission.CREATE_TAG):
+	if not Permission.check_perm(current_user, Permission.EDIT_TAGS if tag else Permission.CREATE_TAG):
 		abort(403)
 
-	form = TagForm( obj=tag)
+	form = TagForm(obj=tag)
 	if form.validate_on_submit():
 		if tag is None:
 			tag = Tag(form.title.data)
 			tag.description = form.description.data
-			tag.is_protected = form.is_protected.data
 			db.session.add(tag)
+
+			add_audit_log(AuditSeverity.EDITOR, current_user, f"Created tag {tag.name}",
+						  url_for("admin.create_edit_tag", name=tag.name))
 		else:
 			form.populate_obj(tag)
 
+			add_audit_log(AuditSeverity.EDITOR, current_user, f"Edited tag {tag.name}",
+						  url_for("admin.create_edit_tag", name=tag.name))
+
 		db.session.commit()
 
 		if Permission.EDIT_TAGS.check(current_user):

app/blueprints/admin/versioneditor.py

@@ -16,19 +16,21 @@
 
 
 from flask import redirect, render_template, abort, url_for, request, flash
+from flask_login import current_user
 from flask_wtf import FlaskForm
 from wtforms import StringField, IntegerField, SubmitField
 from wtforms.validators import InputRequired, Length
 
-from app.utils import rank_required
+from app.utils import rank_required, add_audit_log
 from . import bp
-from ...models import UserRank, MinetestRelease, db
+from app.models import UserRank, LuantiRelease, db, AuditSeverity
 
 
 @bp.route("/versions/")
 @rank_required(UserRank.MODERATOR)
 def version_list():
-	return render_template("admin/versions/list.html", versions=MinetestRelease.query.order_by(db.asc(MinetestRelease.id)).all())
+	return render_template("admin/versions/list.html",
+			versions=LuantiRelease.query.order_by(db.asc(LuantiRelease.id)).all())
 
 
 class VersionForm(FlaskForm):
@@ -43,19 +45,25 @@ class VersionForm(FlaskForm):
 def create_edit_version(name=None):
 	version = None
 	if name is not None:
-		version = MinetestRelease.query.filter_by(name=name).first()
+		version = LuantiRelease.query.filter_by(name=name).first()
 		if version is None:
 			abort(404)
 
 	form = VersionForm(formdata=request.form, obj=version)
 	if form.validate_on_submit():
 		if version is None:
-			version = MinetestRelease(form.name.data)
+			version = LuantiRelease(form.name.data)
 			db.session.add(version)
 			flash("Created version " + form.name.data, "success")
+
+			add_audit_log(AuditSeverity.MODERATION, current_user, f"Created version {version.name}",
+						  url_for("admin.license_list"))
 		else:
 			flash("Updated version " + form.name.data, "success")
 
+			add_audit_log(AuditSeverity.MODERATION, current_user, f"Edited version {version.name}",
+						  url_for("admin.version_list"))
+
 		form.populate_obj(version)
 		db.session.commit()
 		return redirect(url_for("admin.version_list"))

app/blueprints/admin/warningseditor.py

@@ -15,14 +15,14 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 
-from flask import redirect, render_template, abort, url_for, request, flash
+from flask import redirect, render_template, abort, url_for, request
 from flask_wtf import FlaskForm
 from wtforms import StringField, TextAreaField, SubmitField
 from wtforms.validators import InputRequired, Length, Optional, Regexp
 
-from app.utils import rank_required
+from app.utils import rank_required, normalize_line_endings
 from . import bp
-from ...models import UserRank, ContentWarning, db
+from app.models import UserRank, ContentWarning, db
 
 
 @bp.route("/admin/warnings/")
@@ -33,7 +33,7 @@ def warning_list():
 
 class WarningForm(FlaskForm):
 	title = StringField("Title", [InputRequired(), Length(3, 100)])
-	description = TextAreaField("Description", [Optional(), Length(0, 500)])
+	description = TextAreaField("Description", [Optional(), Length(0, 500)], filters=[normalize_line_endings])
 	name = StringField("Name", [Optional(), Length(1, 20),
 					Regexp("^[a-z0-9_]", 0, "Lower case letters (a-z), digits (0-9), and underscores (_) only")])
 	submit = SubmitField("Save")

app/blueprints/api/__init__.py

@@ -14,8 +14,36 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
+import json
 from flask import Blueprint
 
+from .support import error
+
 bp = Blueprint("api", __name__)
 
+
 from . import tokens, endpoints
+
+
+@bp.errorhandler(400)
+@bp.errorhandler(401)
+@bp.errorhandler(403)
+@bp.errorhandler(404)
+def handle_exception(e):
+	"""Return JSON instead of HTML for HTTP errors."""
+	# start with the correct headers and status code from the error
+	response = e.get_response()
+	# replace the body with JSON
+	response.data = json.dumps({
+		"success": False,
+		"code": e.code,
+		"name": e.name,
+		"description": e.description,
+	})
+	response.content_type = "application/json"
+	return response
+
+
+@bp.route("/api/<path:path>")
+def page_not_found(path):
+	error(404, "Endpoint or method not found")

app/blueprints/api/auth.py

@@ -39,7 +39,7 @@ def is_api_authd(f):
 			if token is None:
 				error(403, "Unknown API token")
 		else:
-			abort(403, "Unsupported authentication method")
+			error(403, "Unsupported authentication method")
 
 		return f(token=token, *args, **kwargs)
 

app/blueprints/api/endpoints.py

@@ -15,58 +15,145 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 import math
+import os
 from typing import List
 
 import flask_sqlalchemy
 from flask import request, jsonify, current_app
-from flask_login import current_user, login_required
+from flask_babel import gettext
+from sqlalchemy import and_, or_
 from sqlalchemy.orm import joinedload
 from sqlalchemy.sql.expression import func
 
 from app import csrf
+from app.logic.graphs import get_package_stats, get_package_stats_for_user, get_all_package_stats
 from app.markdown import render_markdown
-from app.models import Tag, PackageState, PackageType, Package, db, PackageRelease, Permission, ForumTopic, \
-	MinetestRelease, APIToken, PackageScreenshot, License, ContentWarning, User, PackageReview, Thread
+from app.models import Tag, PackageState, PackageType, Package, db, PackageRelease, Permission, \
+	LuantiRelease, APIToken, PackageScreenshot, License, ContentWarning, User, PackageReview, Thread, Collection, \
+	PackageAlias, Language
 from app.querybuilder import QueryBuilder
-from app.utils import is_package_page, get_int_or_abort, url_set_query, abs_url, isYes
+from app.utils import is_package_page, get_int_or_abort, url_set_query, abs_url, is_yes, get_request_date, cached, \
+	cors_allowed
+from app.utils.luanti_hypertext import html_to_luanti, package_info_as_hypertext, package_reviews_as_hypertext
 from . import bp
 from .auth import is_api_authd
 from .support import error, api_create_vcs_release, api_create_zip_release, api_create_screenshot, \
 	api_order_screenshots, api_edit_package, api_set_cover_image
-from functools import wraps
-
-
-def cors_allowed(f):
-	@wraps(f)
-	def inner(*args, **kwargs):
-		res = f(*args, **kwargs)
-		res.headers["Access-Control-Allow-Origin"] = "*"
-		res.headers["Access-Control-Allow-Methods"] = "GET, POST, PUT, DELETE, OPTIONS"
-		res.headers["Access-Control-Allow-Headers"] = "Content-Type, Authorization"
-		return res
-	return inner
 
 
 @bp.route("/api/packages/")
 @cors_allowed
+@cached(300)
 def packages():
-	qb    = QueryBuilder(request.args)
-	query = qb.buildPackageQuery()
+	allowed_languages = set([x[0] for x in db.session.query(Language.id).all()])
+	lang = request.accept_languages.best_match(allowed_languages)
 
-	if request.args.get("fmt") == "keys":
-		return jsonify([package.getAsDictionaryKey() for package in query.all()])
+	qb = QueryBuilder(request.args, lang=lang)
+	query = qb.build_package_query()
 
-	pkgs = qb.convertToDictionary(query.all())
+	fmt = request.args.get("fmt")
+	if fmt == "keys":
+		return jsonify([pkg.as_key_dict() for pkg in query.all()])
+
+	include_vcs = fmt == "vcs"
+	pkgs = qb.convert_to_dictionary(query.all(), include_vcs)
 	if "engine_version" in request.args or "protocol_version" in request.args:
-		pkgs = [package for package in pkgs if package.get("release")]
-	return jsonify(pkgs)
+		pkgs = [pkg for pkg in pkgs if pkg.get("release")]
+
+	# Promote featured packages
+	if "sort" not in request.args and \
+			"order" not in request.args and \
+			"q" not in request.args and \
+			"limit" not in request.args:
+		featured_lut = set()
+		featured = qb.convert_to_dictionary(query.filter(
+			Package.collections.any(and_(Collection.name == "featured", Collection.author.has(username="ContentDB")))).all(),
+			include_vcs)
+		for pkg in featured:
+			featured_lut.add(f"{pkg['author']}/{pkg['name']}")
+			pkg["short_description"] = gettext("Featured") + ". " + pkg["short_description"]
+			pkg["featured"] = True
+
+		not_featured = [pkg for pkg in pkgs if f"{pkg['author']}/{pkg['name']}" not in featured_lut]
+		pkgs = featured + not_featured
+
+	resp = jsonify(pkgs)
+	resp.vary = "Accept-Language"
+	return resp
 
 
 @bp.route("/api/packages/<author>/<name>/")
 @is_package_page
 @cors_allowed
-def package(package):
-	return jsonify(package.getAsDictionary(current_app.config["BASE_URL"]))
+def package_view(package):
+	allowed_languages = set([x[0] for x in db.session.query(Language.id).all()])
+	lang = request.accept_languages.best_match(allowed_languages)
+
+	data = package.as_dict(current_app.config["BASE_URL"], lang=lang)
+	resp = jsonify(data)
+	resp.vary = "Accept-Language"
+	return resp
+
+
+@bp.route("/api/packages/<author>/<name>/for-client/")
+@is_package_page
+@cors_allowed
+def package_view_client(package: Package):
+	protocol_version = request.args.get("protocol_version")
+	engine_version = request.args.get("engine_version")
+	if protocol_version or engine_version:
+		version = LuantiRelease.get(engine_version, get_int_or_abort(protocol_version))
+	else:
+		version = None
+
+	allowed_languages = set([x[0] for x in db.session.query(Language.id).all()])
+	lang = request.accept_languages.best_match(allowed_languages)
+
+	data = package.as_dict(current_app.config["BASE_URL"], version, lang=lang, screenshots_dict=True)
+
+	formspec_version = get_int_or_abort(request.args["formspec_version"])
+	include_images = is_yes(request.args.get("include_images", "true"))
+	page_url = package.get_url("packages.view", absolute=True)
+	if data["long_description"] is not None:
+		html = render_markdown(data["long_description"])
+		data["long_description"] = html_to_luanti(html, page_url, formspec_version, include_images)
+
+	data["info_hypertext"] = package_info_as_hypertext(package, formspec_version)
+
+	data["download_size"] = package.get_download_release(version).file_size
+
+	data["reviews"] = {
+		"positive": package.reviews.filter(PackageReview.rating > 3).count(),
+		"neutral": package.reviews.filter(PackageReview.rating == 3).count(),
+		"negative": package.reviews.filter(PackageReview.rating < 3).count(),
+	}
+
+	resp = jsonify(data)
+	resp.vary = "Accept-Language"
+	return resp
+
+
+@bp.route("/api/packages/<author>/<name>/for-client/reviews/")
+@is_package_page
+@cors_allowed
+def package_view_client_reviews(package: Package):
+	formspec_version = get_int_or_abort(request.args["formspec_version"])
+	data = package_reviews_as_hypertext(package, formspec_version)
+
+	resp = jsonify(data)
+	resp.vary = "Accept-Language"
+	return resp
+
+
+@bp.route("/api/packages/<author>/<name>/hypertext/")
+@is_package_page
+@cors_allowed
+def package_hypertext(package):
+	formspec_version = get_int_or_abort(request.args["formspec_version"])
+	include_images = is_yes(request.args.get("include_images", "true"))
+	html = render_markdown(package.desc if package.desc else "")
+	page_url = package.get_url("packages.view", absolute=True)
+	return jsonify(html_to_luanti(html, page_url, formspec_version, include_images))
 
 
 @bp.route("/api/packages/<author>/<name>/", methods=["PUT"])
@@ -82,12 +169,12 @@ def edit_package(token, package):
 
 
 def resolve_package_deps(out, package, only_hard, depth=1):
-	id = package.getId()
-	if id in out:
+	id_ = package.get_id()
+	if id_ in out:
 		return
 
 	ret = []
-	out[id] = ret
+	out[id_] = ret
 
 	if package.type != PackageType.MOD:
 		return
@@ -98,12 +185,12 @@ def resolve_package_deps(out, package, only_hard, depth=1):
 
 		if dep.package:
 			name = dep.package.name
-			fulfilled_by = [ dep.package.getId() ]
+			fulfilled_by = [ dep.package.get_id() ]
 			resolve_package_deps(out, dep.package, only_hard, depth)
 
 		elif dep.meta_package:
 			name = dep.meta_package.name
-			fulfilled_by = [ pkg.getId() for pkg in dep.meta_package.packages if pkg.state == PackageState.APPROVED]
+			fulfilled_by = [ pkg.get_id() for pkg in dep.meta_package.packages if pkg.state == PackageState.APPROVED]
 
 			if depth == 1 and not dep.optional:
 				most_likely = next((pkg for pkg in dep.meta_package.packages \
@@ -124,6 +211,7 @@ def resolve_package_deps(out, package, only_hard, depth=1):
 @bp.route("/api/packages/<author>/<name>/dependencies/")
 @is_package_page
 @cors_allowed
+@cached(300)
 def package_dependencies(package):
 	only_hard = request.args.get("only_hard")
 
@@ -136,27 +224,9 @@ def package_dependencies(package):
 @bp.route("/api/topics/")
 @cors_allowed
 def topics():
-	qb     = QueryBuilder(request.args)
-	query  = qb.buildTopicQuery(show_added=True)
-	return jsonify([t.getAsDictionary() for t in query.all()])
-
-
-@bp.route("/api/topic_discard/", methods=["POST"])
-@login_required
-def topic_set_discard():
-	tid = request.args.get("tid")
-	discard = request.args.get("discard")
-	if tid is None or discard is None:
-		error(400, "Missing topic ID or discard bool")
-
-	topic = ForumTopic.query.get(tid)
-	if not topic.checkPerm(current_user, Permission.TOPIC_DISCARD):
-		error(403, "Permission denied, need: TOPIC_DISCARD")
-
-	topic.discarded = discard == "true"
-	db.session.commit()
-
-	return jsonify(topic.getAsDictionary())
+	qb = QueryBuilder(request.args)
+	query = qb.build_topic_query(show_added=True)
+	return jsonify([t.as_dict() for t in query.all()])
 
 
 @bp.route("/api/whoami/")
@@ -169,6 +239,20 @@ def whoami(token):
 		return jsonify({ "is_authenticated": True, "username": token.owner.username })
 
 
+@bp.route("/api/delete-token/", methods=["DELETE"])
+@csrf.exempt
+@is_api_authd
+@cors_allowed
+def api_delete_token(token):
+	if token is None:
+		error(404, "Token not found")
+
+	db.session.delete(token)
+	db.session.commit()
+
+	return jsonify({"success": True})
+
+
 @bp.route("/api/markdown/", methods=["POST"])
 @csrf.exempt
 def markdown():
@@ -180,7 +264,7 @@ def markdown():
 def list_all_releases():
 	query = PackageRelease.query.filter_by(approved=True) \
 			.filter(PackageRelease.package.has(state=PackageState.APPROVED)) \
-			.order_by(db.desc(PackageRelease.releaseDate))
+			.order_by(db.desc(PackageRelease.created_at))
 
 	if "author" in request.args:
 		author = User.query.filter_by(username=request.args["author"]).first()
@@ -193,16 +277,16 @@ def list_all_releases():
 		if maintainer is None:
 			error(404, "Maintainer not found")
 		query = query.join(Package)
-		query = query.filter(Package.maintainers.any(id=maintainer.id))
+		query = query.filter(Package.maintainers.contains(maintainer))
 
-	return jsonify([ rel.getLongAsDictionary() for rel in query.limit(30).all() ])
+	return jsonify([ rel.as_long_dict() for rel in query.limit(30).all() ])
 
 
 @bp.route("/api/packages/<author>/<name>/releases/")
 @is_package_page
 @cors_allowed
 def list_releases(package):
-	return jsonify([ rel.getAsDictionary() for rel in package.releases.all() ])
+	return jsonify([ rel.as_dict() for rel in package.releases.all() ])
 
 
 @bp.route("/api/packages/<author>/<name>/releases/new/", methods=["POST"])
@@ -214,19 +298,27 @@ def create_release(token, package):
 	if not token:
 		error(401, "Authentication needed")
 
-	if not package.checkPerm(token.owner, Permission.APPROVE_RELEASE):
+	if not package.check_perm(token.owner, Permission.APPROVE_RELEASE):
 		error(403, "You do not have the permission to approve releases")
 
-	data = request.json or request.form
-	if "title" not in data:
-		error(400, "Title is required in the POST data")
+	if request.headers.get("Content-Type") == "application/json":
+		data = request.json
+	else:
+		data = request.form
+
+	if not ("title" in data or "name" in data):
+		error(400, "name is required in the POST data")
+
+	name = data.get("name")
+	title = data.get("title") or name
+	name = name or title
 
 	if data.get("method") == "git":
 		for option in ["method", "ref"]:
 			if option not in data:
 				error(400, option + " is required in the POST data")
 
-		return api_create_vcs_release(token, package, data["title"], data["ref"])
+		return api_create_vcs_release(token, package, name, title, data.get("release_notes"), data["ref"])
 
 	elif request.files:
 		file = request.files.get("file")
@@ -235,7 +327,7 @@ def create_release(token, package):
 
 		commit_hash = data.get("commit")
 
-		return api_create_zip_release(token, package, data["title"], file, None, None, "API", commit_hash)
+		return api_create_zip_release(token, package,  name, title, data.get("release_notes"), file, None, None, "API", commit_hash)
 
 	else:
 		error(400, "Unknown release-creation method. Specify the method or provide a file.")
@@ -244,12 +336,12 @@ def create_release(token, package):
 @bp.route("/api/packages/<author>/<name>/releases/<int:id>/")
 @is_package_page
 @cors_allowed
-def release(package: Package, id: int):
+def release_view(package: Package, id: int):
 	release = PackageRelease.query.get(id)
 	if release is None or release.package != package:
 		error(404, "Release not found")
 
-	return jsonify(release.getAsDictionary())
+	return jsonify(release.as_dict())
 
 
 @bp.route("/api/packages/<author>/<name>/releases/<int:id>/", methods=["DELETE"])
@@ -265,15 +357,18 @@ def delete_release(token: APIToken, package: Package, id: int):
 	if not token:
 		error(401, "Authentication needed")
 
-	if not token.canOperateOnPackage(package):
+	if not token.can_operate_on_package(package):
 		error(403, "API token does not have access to the package")
 
-	if not release.checkPerm(token.owner, Permission.DELETE_RELEASE):
+	if not release.check_perm(token.owner, Permission.DELETE_RELEASE):
 		error(403, "Unable to delete the release, make sure there's a newer release available")
 
 	db.session.delete(release)
 	db.session.commit()
 
+	if release.file_path and os.path.isfile(release.file_path):
+		os.remove(release.file_path)
+
 	return jsonify({"success": True})
 
 
@@ -282,7 +377,7 @@ def delete_release(token: APIToken, package: Package, id: int):
 @cors_allowed
 def list_screenshots(package):
 	screenshots = package.screenshots.all()
-	return jsonify([ss.getAsDictionary(current_app.config["BASE_URL"]) for ss in screenshots])
+	return jsonify([ss.as_dict(current_app.config["BASE_URL"]) for ss in screenshots])
 
 
 @bp.route("/api/packages/<author>/<name>/screenshots/new/", methods=["POST"])
@@ -294,7 +389,7 @@ def create_screenshot(token: APIToken, package: Package):
 	if not token:
 		error(401, "Authentication needed")
 
-	if not package.checkPerm(token.owner, Permission.ADD_SCREENSHOTS):
+	if not package.check_perm(token.owner, Permission.ADD_SCREENSHOTS):
 		error(403, "You do not have the permission to create screenshots")
 
 	data = request.form
@@ -305,7 +400,7 @@ def create_screenshot(token: APIToken, package: Package):
 	if file is None:
 		error(400, "Missing 'file' in multipart body")
 
-	return api_create_screenshot(token, package, data["title"], file, isYes(data.get("is_cover_image")))
+	return api_create_screenshot(token, package, data["title"], file, is_yes(data.get("is_cover_image")))
 
 
 @bp.route("/api/packages/<author>/<name>/screenshots/<int:id>/")
@@ -316,7 +411,7 @@ def screenshot(package, id):
 	if ss is None or ss.package != package:
 		error(404, "Screenshot not found")
 
-	return jsonify(ss.getAsDictionary(current_app.config["BASE_URL"]))
+	return jsonify(ss.as_dict(current_app.config["BASE_URL"]))
 
 
 @bp.route("/api/packages/<author>/<name>/screenshots/<int:id>/", methods=["DELETE"])
@@ -332,10 +427,10 @@ def delete_screenshot(token: APIToken, package: Package, id: int):
 	if not token:
 		error(401, "Authentication needed")
 
-	if not package.checkPerm(token.owner, Permission.ADD_SCREENSHOTS):
+	if not package.check_perm(token.owner, Permission.ADD_SCREENSHOTS):
 		error(403, "You do not have the permission to delete screenshots")
 
-	if not token.canOperateOnPackage(package):
+	if not token.can_operate_on_package(package):
 		error(403, "API token does not have access to the package")
 
 	if package.cover_image == ss:
@@ -345,6 +440,8 @@ def delete_screenshot(token: APIToken, package: Package, id: int):
 	db.session.delete(ss)
 	db.session.commit()
 
+	os.remove(ss.file_path)
+
 	return jsonify({ "success": True })
 
 
@@ -357,10 +454,10 @@ def order_screenshots(token: APIToken, package: Package):
 	if not token:
 		error(401, "Authentication needed")
 
-	if not package.checkPerm(token.owner, Permission.ADD_SCREENSHOTS):
+	if not package.check_perm(token.owner, Permission.ADD_SCREENSHOTS):
 		error(403, "You do not have the permission to change screenshots")
 
-	if not token.canOperateOnPackage(package):
+	if not token.can_operate_on_package(package):
 		error(403, "API token does not have access to the package")
 
 	json = request.json
@@ -379,10 +476,10 @@ def set_cover_image(token: APIToken, package: Package):
 	if not token:
 		error(401, "Authentication needed")
 
-	if not package.checkPerm(token.owner, Permission.ADD_SCREENSHOTS):
+	if not package.check_perm(token.owner, Permission.ADD_SCREENSHOTS):
 		error(403, "You do not have the permission to change screenshots")
 
-	if not token.canOperateOnPackage(package):
+	if not token.can_operate_on_package(package):
 		error(403, "API token does not have access to the package")
 
 	json = request.json
@@ -397,29 +494,37 @@ def set_cover_image(token: APIToken, package: Package):
 @cors_allowed
 def list_reviews(package):
 	reviews = package.reviews
-	return jsonify([review.getAsDictionary() for review in reviews])
+	return jsonify([review.as_dict() for review in reviews])
 
 
 @bp.route("/api/reviews/")
 @cors_allowed
 def list_all_reviews():
 	page = get_int_or_abort(request.args.get("page"), 1)
-	num = min(get_int_or_abort(request.args.get("n"), 100), 100)
+	num = min(get_int_or_abort(request.args.get("n"), 100), 200)
 
 	query = PackageReview.query
 	query = query.options(joinedload(PackageReview.author), joinedload(PackageReview.package))
 
-	if request.args.get("author"):
+	if "for_user" in request.args:
+		query = query.filter(PackageReview.package.has(Package.author.has(username=request.args["for_user"])))
+
+	if "author" in request.args:
 		query = query.filter(PackageReview.author.has(User.username == request.args.get("author")))
 
-	if request.args.get("is_positive"):
-		query = query.filter(PackageReview.recommends == isYes(request.args.get("is_positive")))
+	if "is_positive" in request.args:
+		if is_yes(request.args.get("is_positive")):
+			query = query.filter(PackageReview.rating > 3)
+		else:
+			query = query.filter(PackageReview.rating <= 3)
 
 	q = request.args.get("q")
 	if q:
 		query = query.filter(PackageReview.thread.has(Thread.title.ilike(f"%{q}%")))
 
-	pagination: flask_sqlalchemy.Pagination = query.paginate(page, num, True)
+	query = query.order_by(db.desc(PackageReview.created_at))
+
+	pagination: flask_sqlalchemy.Pagination = query.paginate(page=page, per_page=num)
 	return jsonify({
 		"page": pagination.page,
 		"per_page": pagination.per_page,
@@ -429,48 +534,71 @@ def list_all_reviews():
 			"previous": abs_url(url_set_query(page=page - 1)) if pagination.has_prev else None,
 			"next": abs_url(url_set_query(page=page + 1)) if pagination.has_next else None,
 		},
-		"items": [review.getAsDictionary(True) for review in pagination.items],
+		"items": [review.as_dict(True) for review in pagination.items],
 	})
 
 
+@bp.route("/api/packages/<author>/<name>/stats/")
+@is_package_page
+@cors_allowed
+@cached(300)
+def package_stats(package: Package):
+	start = get_request_date("start")
+	end = get_request_date("end")
+	return jsonify(get_package_stats(package, start, end))
+
+
+@bp.route("/api/package_stats/")
+@cors_allowed
+@cached(900)
+def all_package_stats():
+	return jsonify(get_all_package_stats())
+
+
 @bp.route("/api/scores/")
 @cors_allowed
+@cached(900)
 def package_scores():
-	qb    = QueryBuilder(request.args)
-	query = qb.buildPackageQuery()
+	qb = QueryBuilder(request.args)
+	query = qb.build_package_query()
 
-	pkgs = [package.getScoreDict() for package in query.all()]
+	pkgs = [package.as_score_dict() for package in query.all()]
 	return jsonify(pkgs)
 
 
 @bp.route("/api/tags/")
 @cors_allowed
+@cached(60*60)
 def tags():
-	return jsonify([tag.getAsDictionary() for tag in Tag.query.all() ])
+	return jsonify([tag.as_dict() for tag in Tag.query.order_by(db.asc(Tag.name)).all()])
 
 
 @bp.route("/api/content_warnings/")
 @cors_allowed
+@cached(60*60)
 def content_warnings():
-	return jsonify([warning.getAsDictionary() for warning in ContentWarning.query.all() ])
+	return jsonify([warning.as_dict() for warning in ContentWarning.query.order_by(db.asc(ContentWarning.name)).all() ])
 
 
 @bp.route("/api/licenses/")
 @cors_allowed
+@cached(60*60)
 def licenses():
-	return jsonify([ { "name": license.name, "is_foss": license.is_foss } \
-		for license in License.query.order_by(db.asc(License.name)).all() ])
+	all_licenses = License.query.order_by(db.asc(License.name)).all()
+	return jsonify([{"name": license.name, "is_foss": license.is_foss} for license in all_licenses])
 
 
 @bp.route("/api/homepage/")
 @cors_allowed
+@cached(300)
 def homepage():
-	query   = Package.query.filter_by(state=PackageState.APPROVED)
-	count   = query.count()
+	query = Package.query.filter_by(state=PackageState.APPROVED)
+	count = query.count()
 
-	featured = query.filter(Package.tags.any(name="featured")).order_by(
-			func.random()).limit(6).all()
-	new     = query.order_by(db.desc(Package.approved_at)).limit(4).all()
+	spotlight = query.filter(
+			Package.collections.any(and_(Collection.name == "spotlight", Collection.author.has(username="ContentDB")))) \
+		.order_by(func.random()).limit(6).all()
+	new = query.order_by(db.desc(Package.approved_at)).limit(4).all()
 	pop_mod = query.filter_by(type=PackageType.MOD).order_by(db.desc(Package.score)).limit(8).all()
 	pop_gam = query.filter_by(type=PackageType.GAME).order_by(db.desc(Package.score)).limit(8).all()
 	pop_txp = query.filter_by(type=PackageType.TXP).order_by(db.desc(Package.score)).limit(8).all()
@@ -479,46 +607,26 @@ def homepage():
 
 	updated = db.session.query(Package).select_from(PackageRelease).join(Package) \
 			.filter_by(state=PackageState.APPROVED) \
-			.order_by(db.desc(PackageRelease.releaseDate)) \
+			.order_by(db.desc(PackageRelease.created_at)) \
 			.limit(20).all()
 	updated = updated[:4]
 
 	downloads_result = db.session.query(func.sum(Package.downloads)).one_or_none()
 	downloads = 0 if not downloads_result or not downloads_result[0] else downloads_result[0]
 
-	def mapPackages(packages: List[Package]):
-		return [pkg.getAsDictionaryShort(current_app.config["BASE_URL"]) for pkg in packages]
+	def map_packages(packages: List[Package]):
+		return [pkg.as_short_dict(current_app.config["BASE_URL"]) for pkg in packages]
 
 	return jsonify({
 		"count": count,
 		"downloads": downloads,
-		"featured": mapPackages(featured),
-		"new": mapPackages(new),
-		"updated": mapPackages(updated),
-		"pop_mod": mapPackages(pop_mod),
-		"pop_txp": mapPackages(pop_txp),
-		"pop_game": mapPackages(pop_gam),
-		"high_reviewed": mapPackages(high_reviewed)
-	})
-
-
-@bp.route("/api/welcome/v1/")
-@cors_allowed
-def welcome_v1():
-	featured = Package.query \
-		.filter(Package.type == PackageType.GAME, Package.state == PackageState.APPROVED,
-				Package.tags.any(name="featured")) \
-		.order_by(func.random()) \
-		.limit(5).all()
-
-	mtg = Package.query.filter(Package.author.has(username="Minetest"), Package.name == "minetest_game").one()
-	featured.insert(2, mtg)
-
-	def map_packages(packages: List[Package]):
-		return [pkg.getAsDictionaryShort(current_app.config["BASE_URL"]) for pkg in packages]
-
-	return jsonify({
-		"featured": map_packages(featured),
+		"spotlight": map_packages(spotlight),
+		"new": map_packages(new),
+		"updated": map_packages(updated),
+		"pop_mod": map_packages(pop_mod),
+		"pop_txp": map_packages(pop_txp),
+		"pop_game": map_packages(pop_gam),
+		"high_reviewed": map_packages(high_reviewed)
 	})
 
 
@@ -528,25 +636,31 @@ def versions():
 	protocol_version = request.args.get("protocol_version")
 	engine_version = request.args.get("engine_version")
 	if protocol_version or engine_version:
-		rel = MinetestRelease.get(engine_version, get_int_or_abort(protocol_version))
+		rel = LuantiRelease.get(engine_version, get_int_or_abort(protocol_version))
 		if rel is None:
 			error(404, "No releases found")
 
-		return jsonify(rel.getAsDictionary())
+		return jsonify(rel.as_dict())
 
-	return jsonify([rel.getAsDictionary() \
-			for rel in MinetestRelease.query.all() if rel.getActual() is not None])
+	return jsonify([rel.as_dict() \
+			for rel in LuantiRelease.query.all() if rel.get_actual() is not None])
+
+
+@bp.route("/api/languages/")
+@cors_allowed
+def languages():
+	return jsonify([x.as_dict() for x in Language.query.all()])
 
 
 @bp.route("/api/dependencies/")
 @cors_allowed
 def all_deps():
 	qb = QueryBuilder(request.args)
-	query = qb.buildPackageQuery()
+	query = qb.build_package_query()
 
 	def format_pkg(pkg: Package):
 		return {
-			"type": pkg.type.toName(),
+			"type": pkg.type.to_name(),
 			"author": pkg.author.username,
 			"name": pkg.name,
 			"provides": [x.name for x in pkg.provides],
@@ -556,7 +670,7 @@ def all_deps():
 
 	page = get_int_or_abort(request.args.get("page"), 1)
 	num = min(get_int_or_abort(request.args.get("n"), 100), 300)
-	pagination: flask_sqlalchemy.Pagination = query.paginate(page, num, True)
+	pagination: flask_sqlalchemy.Pagination = query.paginate(page=page, per_page=num)
 	return jsonify({
 		"page": pagination.page,
 		"per_page": pagination.per_page,
@@ -568,3 +682,259 @@ def all_deps():
 		},
 		"items": [format_pkg(pkg) for pkg in pagination.items],
 	})
+
+
+@bp.route("/api/users/<username>/")
+@cors_allowed
+def user_view(username: str):
+	user = User.query.filter_by(username=username).first()
+	if user is None:
+		error(404, "User not found")
+
+	return jsonify(user.get_dict())
+
+
+@bp.route("/api/users/<username>/stats/")
+@cors_allowed
+@cached(300)
+def user_stats(username: str):
+	user = User.query.filter_by(username=username).first()
+	if user is None:
+		error(404, "User not found")
+
+	start = get_request_date("start")
+	end = get_request_date("end")
+	return jsonify(get_package_stats_for_user(user, start, end))
+
+
+@bp.route("/api/cdb_schema/")
+@cors_allowed
+@cached(60*60)
+def json_schema():
+	tags = Tag.query.all()
+	warnings = ContentWarning.query.all()
+	licenses = License.query.order_by(db.asc(License.name)).all()
+	return jsonify({
+		"title": "CDB Config",
+		"description": "Package Configuration",
+		"type": "object",
+		"$defs": {
+			"license": {
+				"enum": [license.name for license in licenses],
+				"enumDescriptions": [license.is_foss and "FOSS" or "NON-FOSS" for license in licenses]
+			},
+		},
+		"properties": {
+			"type": {
+				"description": "Package Type",
+				"enum": ["MOD", "GAME", "TXP"],
+				"enumDescriptions": ["Mod", "Game", "Texture Pack"]
+			},
+			"title": {
+				"description": "Human-readable title",
+				"type": "string"
+			},
+			"name": {
+				"description": "Technical name (needs permission if already approved).",
+				"type": "string",
+				"pattern": "^[a-z_]+$"
+			},
+			"short_description": {
+				"description": "Package Short Description",
+				"type": ["string", "null"]
+			},
+			"dev_state": {
+				"description": "Development State",
+				"enum": [
+					"WIP",
+					"BETA",
+					"ACTIVELY_DEVELOPED",
+					"MAINTENANCE_ONLY",
+					"AS_IS",
+					"DEPRECATED",
+					"LOOKING_FOR_MAINTAINER"
+				]
+			},
+			"tags": {
+				"description": "Package Tags",
+				"type": "array",
+				"items": {
+					"enum": [tag.name for tag in tags],
+					"enumDescriptions": [tag.title for tag in tags]
+				},
+				"uniqueItems": True,
+			},
+			"content_warnings": {
+				"description": "Package Content Warnings",
+				"type": "array",
+				"items": {
+					"enum": [warning.name for warning in warnings],
+					"enumDescriptions": [warning.title for warning in warnings]
+				},
+				"uniqueItems": True,
+			},
+			"license": {
+				"description": "Package License",
+				"$ref": "#/$defs/license"
+			},
+			"media_license": {
+				"description": "Package Media License",
+				"$ref": "#/$defs/license"
+			},
+			"long_description": {
+				"description": "Package Long Description",
+				"type": ["string", "null"]
+			},
+			"repo": {
+				"description": "Git Repository URL",
+				"type": "string",
+				"format": "uri"
+			},
+			"website": {
+				"description": "Website URL",
+				"type": ["string", "null"],
+				"format": "uri"
+			},
+			"issue_tracker": {
+				"description": "Issue Tracker URL",
+				"type": ["string", "null"],
+				"format": "uri"
+			},
+			"forums": {
+				"description": "Forum Topic ID",
+				"type": ["integer", "null"],
+				"minimum": 0
+			},
+			"video_url": {
+				"description": "URL to a Video",
+				"type": ["string", "null"],
+				"format": "uri"
+			},
+			"donate_url": {
+				"description": "URL to a donation page",
+				"type": ["string", "null"],
+				"format": "uri"
+			},
+			"translation_url": {
+				"description": "URL to send users interested in translating your package",
+				"type": ["string", "null"],
+				"format": "uri"
+			}
+		},
+	})
+
+
+@bp.route("/api/hypertext/", methods=["POST"])
+@csrf.exempt
+@cors_allowed
+def hypertext():
+	formspec_version = get_int_or_abort(request.args["formspec_version"])
+	include_images = is_yes(request.args.get("include_images", "true"))
+
+	html = request.data.decode("utf-8")
+	if request.content_type == "text/markdown":
+		html = render_markdown(html)
+
+	return jsonify(html_to_luanti(html, "", formspec_version, include_images))
+
+
+@bp.route("/api/collections/")
+@cors_allowed
+def collection_list():
+	if "author" in request.args:
+		user = User.query.filter_by(username=request.args["author"]).one_or_404()
+		query = user.collections
+	else:
+		query = Collection.query.order_by(db.asc(Collection.title))
+
+	if "package" in request.args:
+		id_ = request.args["package"]
+		package = Package.get_by_key(id_)
+		if package is None:
+			error(404, f"Package {id_} not found")
+
+		query = query.filter(Collection.packages.contains(package))
+
+	collections = [x.as_short_dict() for x in query.all() if not x.private]
+	return jsonify(collections)
+
+
+@bp.route("/api/collections/<author>/<name>/")
+@is_api_authd
+@cors_allowed
+def collection_view(token, author, name):
+	user = token.owner if token else None
+
+	collection = Collection.query \
+		.filter(Collection.name == name, Collection.author.has(username=author)) \
+		.one_or_404()
+
+	if not collection.check_perm(user, Permission.VIEW_COLLECTION):
+		error(404, "Collection not found")
+
+	items = collection.items
+	if not collection.check_perm(user, Permission.EDIT_COLLECTION):
+		items = [x for x in items if x.package.check_perm(user, Permission.VIEW_PACKAGE)]
+
+	ret = collection.as_dict()
+	ret["items"] = [x.as_dict() for x in items]
+	return jsonify(ret)
+
+
+@bp.route("/api/updates/")
+@cors_allowed
+@cached(300)
+def updates():
+	protocol_version = get_int_or_abort(request.args.get("protocol_version"))
+	engine_version = request.args.get("engine_version")
+	if protocol_version or engine_version:
+		version = LuantiRelease.get(engine_version, protocol_version)
+	else:
+		version = None
+
+	# Subquery to get the latest release for each package
+	latest_release_query = (db.session.query(
+		PackageRelease.package_id,
+		func.max(PackageRelease.id).label('max_release_id'))
+			.select_from(PackageRelease)
+			.filter(PackageRelease.approved == True))
+
+	if version:
+		latest_release_query = (latest_release_query
+			.filter(or_(PackageRelease.min_rel_id == None,
+				PackageRelease.min_rel_id <= version.id))
+			.filter(or_(PackageRelease.max_rel_id == None,
+				PackageRelease.max_rel_id >= version.id)))
+
+	latest_release_subquery = (
+		latest_release_query
+		.group_by(PackageRelease.package_id)
+		.subquery()
+	)
+
+	# Get package id and latest release
+	query = (db.session.query(User.username, Package.name, latest_release_subquery.c.max_release_id)
+		.select_from(Package)
+		.join(User, Package.author)
+		.join(latest_release_subquery, Package.id == latest_release_subquery.c.package_id)
+		.filter(Package.state == PackageState.APPROVED)
+		.all())
+
+	ret = {}
+	for author_username, package_name, release_id in query:
+		ret[f"{author_username}/{package_name}"] = release_id
+
+	# Get aliases
+	aliases = (db.session.query(PackageAlias.author, PackageAlias.name, User.username, Package.name)
+		.select_from(PackageAlias)
+		.join(Package, PackageAlias.package)
+		.join(User, Package.author)
+		.filter(Package.state == PackageState.APPROVED)
+		.all())
+
+	for old_author, old_name, new_author, new_name in aliases:
+		new_release = ret.get(f"{new_author}/{new_name}")
+		if new_release is not None:
+			ret[f"{old_author}/{old_name}"] = new_release
+
+	return jsonify(ret)

app/blueprints/api/support.py

@@ -14,18 +14,19 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-
+from typing import Optional
 from flask import jsonify, abort, make_response, url_for, current_app
 
 from app.logic.packages import do_edit_package
 from app.logic.releases import LogicError, do_create_vcs_release, do_create_zip_release
 from app.logic.screenshots import do_create_screenshot, do_order_screenshots, do_set_cover_image
-from app.models import APIToken, Package, MinetestRelease, PackageScreenshot
+from app.models import APIToken, Package, LuantiRelease, PackageScreenshot
 
 
 def error(code: int, msg: str):
 	abort(make_response(jsonify({ "success": False, "error": msg }), code))
 
+
 # Catches LogicErrors and aborts with JSON error
 def guard(f):
 	def ret(*args, **kwargs):
@@ -37,40 +38,40 @@ def guard(f):
 	return ret
 
 
-def api_create_vcs_release(token: APIToken, package: Package, title: str, ref: str,
-		min_v: MinetestRelease = None, max_v: MinetestRelease = None, reason="API"):
-	if not token.canOperateOnPackage(package):
+def api_create_vcs_release(token: APIToken, package: Package, name: str, title: Optional[str], release_notes: Optional[str], ref: str,
+		min_v: LuantiRelease = None, max_v: LuantiRelease = None, reason="API"):
+	if not token.can_operate_on_package(package):
 		error(403, "API token does not have access to the package")
 
 	reason += ", token=" + token.name
 
-	rel = guard(do_create_vcs_release)(token.owner, package, title, ref, min_v, max_v, reason)
+	rel = guard(do_create_vcs_release)(token.owner, package, name, title, release_notes, ref, min_v, max_v, reason)
 
 	return jsonify({
 		"success": True,
 		"task": url_for("tasks.check", id=rel.task_id),
-		"release": rel.getAsDictionary()
+		"release": rel.as_dict()
 	})
 
 
-def api_create_zip_release(token: APIToken, package: Package, title: str, file,
-		min_v: MinetestRelease = None, max_v: MinetestRelease = None, reason="API", commit_hash:str=None):
-	if not token.canOperateOnPackage(package):
+def api_create_zip_release(token: APIToken, package: Package, name: str, title: Optional[str], release_notes: Optional[str], file,
+		min_v: LuantiRelease = None, max_v: LuantiRelease = None, reason="API", commit_hash: str = None):
+	if not token.can_operate_on_package(package):
 		error(403, "API token does not have access to the package")
 
 	reason += ", token=" + token.name
 
-	rel = guard(do_create_zip_release)(token.owner, package, title, file, min_v, max_v, reason, commit_hash)
+	rel = guard(do_create_zip_release)(token.owner, package, name, title, release_notes, file, min_v, max_v, reason, commit_hash)
 
 	return jsonify({
 		"success": True,
 		"task": url_for("tasks.check", id=rel.task_id),
-		"release": rel.getAsDictionary()
+		"release": rel.as_dict()
 	})
 
 
 def api_create_screenshot(token: APIToken, package: Package, title: str, file, is_cover_image: bool, reason="API"):
-	if not token.canOperateOnPackage(package):
+	if not token.can_operate_on_package(package):
 		error(403, "API token does not have access to the package")
 
 	reason += ", token=" + token.name
@@ -79,12 +80,12 @@ def api_create_screenshot(token: APIToken, package: Package, title: str, file, i
 
 	return jsonify({
 		"success": True,
-		"screenshot": ss.getAsDictionary()
+		"screenshot": ss.as_dict()
 	})
 
 
 def api_order_screenshots(token: APIToken, package: Package, order: [any]):
-	if not token.canOperateOnPackage(package):
+	if not token.can_operate_on_package(package):
 		error(403, "API token does not have access to the package")
 
 	guard(do_order_screenshots)(token.owner, package, order)
@@ -95,7 +96,7 @@ def api_order_screenshots(token: APIToken, package: Package, order: [any]):
 
 
 def api_set_cover_image(token: APIToken, package: Package, cover_image):
-	if not token.canOperateOnPackage(package):
+	if not token.can_operate_on_package(package):
 		error(403, "API token does not have access to the package")
 
 	guard(do_set_cover_image)(token.owner, package, cover_image)
@@ -106,14 +107,14 @@ def api_set_cover_image(token: APIToken, package: Package, cover_image):
 
 
 def api_edit_package(token: APIToken, package: Package, data: dict, reason: str = "API"):
-	if not token.canOperateOnPackage(package):
+	if not token.can_operate_on_package(package):
 		error(403, "API token does not have access to the package")
 
 	reason += ", token=" + token.name
 
-	package = guard(do_edit_package)(token.owner, package, False, False, data, reason)
-
+	was_modified = guard(do_edit_package)(token.owner, package, False, False, data, reason)
 	return jsonify({
 		"success": True,
-		"package": package.getAsDictionary(current_app.config["BASE_URL"])
+		"package": package.as_dict(current_app.config["BASE_URL"]),
+		"was_modified": was_modified,
 	})

app/blueprints/api/tokens.py

@@ -19,12 +19,12 @@ from flask import render_template, redirect, request, session, url_for, abort
 from flask_babel import lazy_gettext
 from flask_login import login_required, current_user
 from flask_wtf import FlaskForm
-from wtforms import *
+from wtforms import StringField, SubmitField
+from wtforms.validators import InputRequired, Length
 from wtforms_sqlalchemy.fields import QuerySelectField
-from wtforms.validators import *
 
-from app.models import db, User, APIToken, Package, Permission
-from app.utils import randomString
+from app.models import db, User, APIToken, Permission
+from app.utils import random_string
 from . import bp
 from ..users.settings import get_setting_tabs
 
@@ -49,7 +49,7 @@ def list_tokens(username):
 	if user is None:
 		abort(404)
 
-	if not user.checkPerm(current_user, Permission.CREATE_TOKEN):
+	if not user.check_perm(current_user, Permission.CREATE_TOKEN):
 		abort(403)
 
 	return render_template("api/list_tokens.html", user=user, tabs=get_setting_tabs(user), current_tab="api_tokens")
@@ -59,11 +59,8 @@ def list_tokens(username):
 @bp.route("/users/<username>/tokens/<int:id>/edit/", methods=["GET", "POST"])
 @login_required
 def create_edit_token(username, id=None):
-	user = User.query.filter_by(username=username).first()
-	if user is None:
-		abort(404)
-
-	if not user.checkPerm(current_user, Permission.CREATE_TOKEN):
+	user = User.query.filter_by(username=username).one_or_404()
+	if not user.check_perm(current_user, Permission.CREATE_TOKEN):
 		abort(403)
 
 	is_new = id is None
@@ -72,10 +69,8 @@ def create_edit_token(username, id=None):
 	access_token = None
 	if not is_new:
 		token = APIToken.query.get(id)
-		if token is None:
+		if token is None or token.owner != user:
 			abort(404)
-		elif token.owner != user:
-			abort(403)
 
 		access_token = session.pop("token_" + str(token.id), None)
 
@@ -85,12 +80,12 @@ def create_edit_token(username, id=None):
 	if form.validate_on_submit():
 		if is_new:
 			token = APIToken()
+			db.session.add(token)
 			token.owner = user
-			token.access_token = randomString(32)
+			token.access_token = random_string(32)
 
 		form.populate_obj(token)
-		db.session.add(token)
-		db.session.commit() # save
+		db.session.commit()
 
 		if is_new:
 			# Store token so it can be shown in the edit page
@@ -108,7 +103,7 @@ def reset_token(username, id):
 	if user is None:
 		abort(404)
 
-	if not user.checkPerm(current_user, Permission.CREATE_TOKEN):
+	if not user.check_perm(current_user, Permission.CREATE_TOKEN):
 		abort(403)
 
 	token = APIToken.query.get(id)
@@ -117,7 +112,7 @@ def reset_token(username, id):
 	elif token.owner != user:
 		abort(403)
 
-	token.access_token = randomString(32)
+	token.access_token = random_string(32)
 
 	db.session.commit() # save
 
@@ -134,11 +129,9 @@ def delete_token(username, id):
 	if user is None:
 		abort(404)
 
-	if not user.checkPerm(current_user, Permission.CREATE_TOKEN):
+	if not user.check_perm(current_user, Permission.CREATE_TOKEN):
 		abort(403)
 
-	is_new = id is None
-
 	token = APIToken.query.get(id)
 	if token is None:
 		abort(404)

app/blueprints/collections/__init__.py

@@ -0,0 +1,384 @@
+# ContentDB
+# Copyright (C) 2023 rubenwardy
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+import re
+import typing
+
+from flask import Blueprint, request, redirect, render_template, flash, abort, url_for, jsonify
+from flask_babel import lazy_gettext, gettext
+from flask_login import current_user, login_required
+from flask_wtf import FlaskForm
+from wtforms import StringField, BooleanField, SubmitField, FieldList, HiddenField, TextAreaField
+from wtforms.validators import InputRequired, Length, Optional, Regexp
+
+from app.models import Collection, db, Package, Permission, CollectionPackage, User, UserRank, AuditSeverity
+from app.utils import nonempty_or_none, normalize_line_endings, should_return_json
+from app.utils.models import is_package_page, add_audit_log, create_session
+
+bp = Blueprint("collections", __name__)
+
+
+regex_invalid_chars = re.compile("[^a-z0-9_]")
+
+
+@bp.route("/collections/")
+@bp.route("/collections/<author>/")
+def list_all(author=None):
+	if author:
+		user = User.query.filter_by(username=author).one_or_404()
+		query = user.collections
+	else:
+		user = None
+		query = Collection.query.filter(Collection.items.any()).order_by(db.asc(Collection.title))
+
+	if "package" in request.args:
+		package = Package.get_by_key(request.args["package"])
+		if package is None:
+			abort(404)
+
+		query = query.filter(Collection.packages.contains(package))
+
+	collections = [x for x in query.all() if x.check_perm(current_user, Permission.VIEW_COLLECTION)]
+	return render_template("collections/list.html",
+		user=user, collections=collections,
+		noindex=len(collections) == 0)
+
+
+@bp.route("/collections/<author>/<name>/")
+def view(author, name):
+	collection = Collection.query \
+		.filter(Collection.name == name, Collection.author.has(username=author)) \
+		.one_or_404()
+
+	if not collection.check_perm(current_user, Permission.VIEW_COLLECTION):
+		abort(404)
+
+	items = collection.items
+	if not collection.check_perm(current_user, Permission.EDIT_COLLECTION):
+		items = [x for x in items if x.package.check_perm(current_user, Permission.VIEW_PACKAGE)]
+
+	if should_return_json():
+		return jsonify([ item.package.as_key_dict() for item in items ])
+	else:
+		return render_template("collections/view.html", collection=collection, items=items)
+
+
+class CollectionForm(FlaskForm):
+	title = StringField(lazy_gettext("Title"), [InputRequired(), Length(3, 100)])
+	name = StringField("URL", [Optional(), Length(1, 20), Regexp("^[a-z0-9_]", 0,
+		"Lower case letters (a-z), digits (0-9), and underscores (_) only")])
+	short_description = StringField(lazy_gettext("Short Description"), [Optional(), Length(0, 200)])
+	long_description = TextAreaField(lazy_gettext("Page Content"), [Optional()], filters=[nonempty_or_none, normalize_line_endings])
+	private = BooleanField(lazy_gettext("Private"))
+	pinned = BooleanField(lazy_gettext("Pinned to my profile"))
+	descriptions = FieldList(
+		StringField(lazy_gettext("Short Description"), [Optional(), Length(0, 500)], filters=[nonempty_or_none]),
+		min_entries=0)
+	package_ids = FieldList(HiddenField(), min_entries=0)
+	package_removed = FieldList(HiddenField(), min_entries=0)
+	order = HiddenField()
+	submit = SubmitField(lazy_gettext("Save"))
+
+
+@bp.route("/collections/new/", methods=["GET", "POST"])
+@bp.route("/collections/<author>/<name>/edit/", methods=["GET", "POST"])
+@login_required
+def create_edit(author=None, name=None):
+	collection: typing.Optional[Collection] = None
+	if author is not None and name is not None:
+		collection = Collection.query \
+			.filter(Collection.name == name, Collection.author.has(username=author)) \
+			.one_or_404()
+		if not collection.check_perm(current_user, Permission.EDIT_COLLECTION):
+			abort(403)
+	elif "author" in request.args:
+		author = request.args["author"]
+		if author != current_user.username and not current_user.rank.at_least(UserRank.EDITOR):
+			abort(403)
+
+	if author is None:
+		author = current_user
+	else:
+		author = User.query.filter_by(username=author).one()
+
+	form = CollectionForm(formdata=request.form, obj=collection)
+
+	initial_packages = []
+	if "package" in request.args:
+		for package_id in request.args.getlist("package"):
+			package = Package.get_by_key(package_id)
+			if package:
+				initial_packages.append(package)
+
+	if request.method == "GET":
+		# HACK: fix bug in wtforms
+		form.private.data = collection.private if collection else False
+		form.pinned.data = collection.pinned if collection else False
+		if collection:
+			for item in collection.items:
+				form.descriptions.append_entry(item.description)
+				form.package_ids.append_entry(item.package.get_id())
+				form.package_removed.append_entry("0")
+		else:
+			form.name = None
+			form.pinned = None
+
+	if form.validate_on_submit():
+		ret = handle_create_edit(collection, form, initial_packages, author)
+		if ret:
+			return ret
+
+	return render_template("collections/create_edit.html",
+			collection=collection, form=form)
+
+
+def handle_create_edit(collection: Collection, form: CollectionForm,
+		initial_packages: typing.List[Package], author: User):
+
+	severity = AuditSeverity.NORMAL if author == current_user else AuditSeverity.EDITOR
+	name = form.name.data if collection else regex_invalid_chars.sub("", form.title.data.lower().replace(" ", "_"))
+
+	if collection is None or name != collection.name:
+		if Collection.query \
+				.filter(Collection.name == name, Collection.author == author) \
+				.count() > 0:
+			flash(gettext("A collection with a similar title already exists"), "danger")
+			return
+
+		if Package.query \
+				.filter(Package.name == name, Package.author == author) \
+				.count() > 0:
+			flash(gettext("Unable to create collection as a package with that name already exists"), "danger")
+			return
+
+	if collection is None:
+		collection = Collection()
+		collection.author = author
+		form.populate_obj(collection)
+		collection.name = name
+		db.session.add(collection)
+
+		for package in initial_packages:
+			link = CollectionPackage()
+			link.package = package
+			link.collection = collection
+			link.order = len(collection.items)
+			db.session.add(link)
+
+		add_audit_log(severity, current_user,
+				f"Created collection {collection.author.username}/{collection.name}",
+				collection.get_url("collections.view"), None)
+
+	else:
+		form.populate_obj(collection)
+		collection.name = name
+
+		link_lookup = {}
+		for link in collection.items:
+			link_lookup[link.package.get_id()] = link
+
+		for i, package_id in enumerate(form.package_ids):
+			link = link_lookup.get(package_id.data)
+			to_delete = form.package_removed[i].data == "1"
+			if link is None:
+				if to_delete:
+					continue
+
+				package = Package.get_by_key(package_id.data)
+				if package is None:
+					abort(400)
+
+				link = CollectionPackage()
+				link.package = package
+				link.collection = collection
+				link.description = form.descriptions[i].data
+				link_lookup[link.package.get_id()] = link
+				db.session.add(link)
+			elif to_delete:
+				db.session.delete(link)
+			else:
+				link.description = form.descriptions[i].data
+
+		for i, package_id in enumerate(form.order.data.split(",")):
+			if package_id != "":
+				link_lookup[package_id].order = i + 1
+
+		add_audit_log(severity, current_user,
+				f"Edited collection {collection.author.username}/{collection.name}",
+				collection.get_url("collections.view"), None)
+
+	db.session.commit()
+	return redirect(collection.get_url("collections.view"))
+
+
+@bp.route("/collections/<author>/<name>/delete/", methods=["GET", "POST"])
+@login_required
+def delete(author, name):
+	collection = Collection.query \
+		.filter(Collection.name == name, Collection.author.has(username=author)) \
+		.one_or_404()
+	if not collection.check_perm(current_user, Permission.EDIT_COLLECTION):
+		abort(403)
+
+	if request.method == "POST":
+		add_audit_log(AuditSeverity.NORMAL, current_user,
+				f"Deleted collection {collection.author.username}/{collection.name}",
+				collection.get_url("collections.view"), None)
+
+		db.session.delete(collection)
+		db.session.commit()
+		return redirect(url_for("collections.list_all", author=author))
+
+	return render_template("collections/delete.html", collection=collection)
+
+
+def toggle_package(collection: Collection, package: Package):
+	severity = AuditSeverity.NORMAL if collection.author == current_user else AuditSeverity.EDITOR
+
+	author = User.query.get(collection.author_id) if collection.author is None else collection.author
+
+	if package in collection.packages:
+		CollectionPackage.query \
+			.filter(CollectionPackage.collection == collection, CollectionPackage.package == package) \
+			.delete(synchronize_session=False)
+		add_audit_log(severity, current_user,
+				f"Removed {package.get_id()} from collection {author.username}/{collection.name}",
+				collection.get_url("collections.view"), None)
+		db.session.commit()
+		return False
+	else:
+		link = CollectionPackage()
+		link.package = package
+		link.collection = collection
+		link.order = len(collection.items)
+		db.session.add(link)
+		add_audit_log(severity, current_user,
+				f"Added {package.get_id()} to collection {author.username}/{collection.name}",
+				collection.get_url("collections.view"), None)
+		db.session.commit()
+		return True
+
+
+def get_or_create_favorites(session):
+	collection = Collection.query.filter(Collection.name == "favorites", Collection.author == current_user).first()
+	if collection is None:
+		is_new = True
+		collection = Collection()
+		collection.title = "Favorites"
+		collection.name = "favorites"
+		collection.short_description = "My favorites"
+		collection.author_id = current_user.id
+		session.add(collection)
+	else:
+		is_new = False
+
+	return collection, is_new
+
+
+@bp.route("/packages/<author>/<name>/add-to/", methods=["GET", "POST"])
+@is_package_page
+@login_required
+def package_add(package):
+	with create_session() as new_session:
+		collection, is_new = get_or_create_favorites(new_session)
+		if is_new:
+			new_session.commit()
+
+	if request.method == "POST":
+		collection_id = request.form["collection"]
+		collection = Collection.query.get(collection_id)
+		if collection is None:
+			abort(404)
+
+		if not collection.check_perm(current_user, Permission.EDIT_COLLECTION):
+			abort(403)
+
+		if toggle_package(collection, package):
+			flash(gettext("Added package to collection"), "success")
+		else:
+			flash(gettext("Removed package from collection"), "success")
+
+		return redirect(package.get_url("collections.package_add"))
+
+	collections = current_user.collections.all()
+	if current_user.rank.at_least(UserRank.EDITOR) and current_user.username != "ContentDB":
+		collections.extend(Collection.query.filter(Collection.author.has(username="ContentDB")).all())
+
+	return render_template("collections/package_add_to.html", package=package, collections=collections)
+
+
+@bp.route("/packages/<author>/<name>/favorite/", methods=["POST"])
+@is_package_page
+@login_required
+def package_toggle_favorite(package):
+	collection, _is_new = get_or_create_favorites(db.session)
+	collection.author = current_user
+
+	if toggle_package(collection, package):
+		msg = gettext("Added package to favorites collection")
+		if not collection.private:
+			msg += " " + gettext("(Public, change from Profile > My Collections)")
+
+		flash(msg, "success")
+	else:
+		flash(gettext("Removed package from favorites collection"), "success")
+
+	return redirect(package.get_url("packages.view"))
+
+
+@bp.route("/collections/<author>/<name>/clone/", methods=["POST"])
+@login_required
+def clone(author, name):
+	old_collection: typing.Optional[Collection] = Collection.query \
+		.filter(Collection.name == name, Collection.author.has(username=author)) \
+		.one_or_404()
+
+	index = 0
+	new_name = name
+	new_title = old_collection.title
+	while True:
+		if Collection.query \
+				.filter(Collection.name == new_name, Collection.author == current_user) \
+				.count() == 0:
+			break
+
+		index += 1
+		new_name = f"{name}_{index}"
+		new_title = f"{old_collection.title} ({index})"
+
+	collection = Collection()
+	collection.title = new_title
+	collection.author = current_user
+	collection.short_description = old_collection.short_description
+	collection.name = new_name
+	collection.private = True
+	db.session.add(collection)
+
+	for item in old_collection.items:
+		new_item = CollectionPackage()
+		new_item.package = item.package
+		new_item.collection = collection
+		new_item.description = item.description
+		new_item.order = item.order
+		db.session.add(new_item)
+
+	add_audit_log(AuditSeverity.NORMAL, current_user,
+			f"Created collection {collection.name} from {old_collection.author.username}/{old_collection.name} ",
+			collection.get_url("collections.view"), None)
+
+	db.session.commit()
+
+	return redirect(collection.get_url("collections.view"))

app/blueprints/donate/__init__.py

@@ -0,0 +1,49 @@
+# ContentDB
+# Copyright (C) 2023 rubenwardy
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+
+from flask import Blueprint, render_template
+from flask_login import current_user
+from sqlalchemy import or_, and_
+
+from app.models import User, Package, PackageState, db, License, PackageReview, Collection
+
+bp = Blueprint("donate", __name__)
+
+
+@bp.route("/donate/")
+def donate():
+	reviewed_packages = None
+	if current_user.is_authenticated:
+		reviewed_packages = Package.query.filter(
+			Package.state == PackageState.APPROVED,
+			or_(Package.reviews.any(and_(PackageReview.author_id == current_user.id, PackageReview.rating >= 3)),
+				Package.collections.any(and_(Collection.author_id == current_user.id, Collection.name == "favorites"))),
+			or_(Package.donate_url.isnot(None), Package.author.has(User.donate_url.isnot(None)))
+		).order_by(db.asc(Package.title)).all()
+
+	query = Package.query.filter(
+			Package.license.has(License.is_foss == True),
+			Package.media_license.has(License.is_foss == True),
+			Package.state == PackageState.APPROVED,
+			or_(Package.donate_url.isnot(None), Package.author.has(User.donate_url.isnot(None)))
+	).order_by(db.desc(Package.score))
+
+	packages_count = query.count()
+	top_packages = query.limit(40).all()
+
+	return render_template("donate/index.html",
+			reviewed_packages=reviewed_packages, top_packages=top_packages, packages_count=packages_count)

app/blueprints/feeds/__init__.py

@@ -0,0 +1,179 @@
+# ContentDB
+# Copyright (C) 2024 rubenwardy
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+
+from flask import Blueprint, jsonify, render_template, make_response
+from flask_babel import gettext
+
+from app.markdown import render_markdown
+from app.models import Package, PackageState, db, PackageRelease
+from app.utils import is_package_page, abs_url_for, cached, cors_allowed
+
+bp = Blueprint("feeds", __name__)
+
+
+def _make_feed(title: str, feed_url: str, items: list):
+	return {
+		"version": "https://jsonfeed.org/version/1",
+		"title": title,
+		"description": gettext("Welcome to the best place to find Luanti mods, games, and texture packs"),
+		"home_page_url": "https://content.luanti.org/",
+		"feed_url": feed_url,
+		"icon": "https://content.luanti.org/favicon-128.png",
+		"expired": False,
+		"items": items,
+	}
+
+
+def _render_link(url: str):
+	return f"<p><a href='{url}'>Read more</a></p>"
+
+
+def _get_new_packages_feed(feed_url: str) -> dict:
+	packages = (Package.query
+		.filter(Package.state == PackageState.APPROVED)
+		.order_by(db.desc(Package.approved_at))
+		.limit(100)
+		.all())
+
+	items = [{
+			"id": package.get_url("packages.view", absolute=True),
+			"language": "en",
+			"title": f"New: {package.title}",
+			"content_html": render_markdown(package.desc) \
+				if package.desc else _render_link(package.get_url("packages.view", absolute=True)),
+			"author": {
+				"name": package.author.display_name,
+				"avatar": package.author.get_profile_pic_url(absolute=True),
+				"url": abs_url_for("users.profile", username=package.author.username),
+			},
+			"image": package.get_thumb_url(level=4, abs=True, format="png"),
+			"url": package.get_url("packages.view", absolute=True),
+			"summary": package.short_desc,
+			"date_published": package.approved_at.isoformat(timespec="seconds") + "Z",
+			"tags": ["new_package"],
+		} for package in packages]
+
+	return _make_feed(gettext("ContentDB new packages"), feed_url, items)
+
+
+def _get_releases_feed(query, feed_url: str):
+	releases = (query
+		.filter(PackageRelease.package.has(state=PackageState.APPROVED), PackageRelease.approved==True)
+		.order_by(db.desc(PackageRelease.created_at))
+		.limit(250)
+		.all())
+
+	items = [{
+			"id": release.package.get_url("packages.view_release", id=release.id, absolute=True),
+			"language": "en",
+			"title": f"\"{release.package.title}\" updated: {release.title}",
+			"content_html": render_markdown(release.release_notes) \
+				if release.release_notes else _render_link(release.package.get_url("packages.view_release", id=release.id, absolute=True)),
+			"author": {
+				"name": release.package.author.display_name,
+				"avatar": release.package.author.get_profile_pic_url(absolute=True),
+				"url": abs_url_for("users.profile", username=release.package.author.username),
+			},
+			"url": release.package.get_url("packages.view_release", id=release.id, absolute=True),
+			"image": release.package.get_thumb_url(level=4, abs=True, format="png"),
+			"summary": release.summary,
+			"date_published": release.created_at.isoformat(timespec="seconds") + "Z",
+			"tags": ["release"],
+		} for release in releases]
+
+	return _make_feed(gettext("ContentDB package updates"), feed_url, items)
+
+
+def _get_all_feed(feed_url: str):
+	releases = _get_releases_feed(PackageRelease.query, "")["items"]
+	packages = _get_new_packages_feed("")["items"]
+	items = releases + packages
+	items.sort(reverse=True, key=lambda x: x["date_published"])
+
+	return _make_feed(gettext("ContentDB all"), feed_url, items)
+
+
+def _atomify(feed):
+	resp = make_response(render_template("feeds/json_to_atom.xml", feed=feed))
+	resp.headers["Content-type"] = "application/atom+xml; charset=utf-8"
+	return resp
+
+
+@bp.route("/feeds/all.json")
+@cors_allowed
+@cached(1800)
+def all_json():
+	feed = _get_all_feed(abs_url_for("feeds.all_json"))
+	return jsonify(feed)
+
+
+@bp.route("/feeds/all.atom")
+@cors_allowed
+@cached(1800)
+def all_atom():
+	feed = _get_all_feed(abs_url_for("feeds.all_atom"))
+	return _atomify(feed)
+
+
+@bp.route("/feeds/packages.json")
+@cors_allowed
+@cached(1800)
+def packages_all_json():
+	feed = _get_new_packages_feed(abs_url_for("feeds.packages_all_json"))
+	return jsonify(feed)
+
+
+@bp.route("/feeds/packages.atom")
+@cors_allowed
+@cached(1800)
+def packages_all_atom():
+	feed = _get_new_packages_feed(abs_url_for("feeds.packages_all_atom"))
+	return _atomify(feed)
+
+
+@bp.route("/feeds/releases.json")
+@cors_allowed
+@cached(1800)
+def releases_all_json():
+	feed = _get_releases_feed(PackageRelease.query, abs_url_for("feeds.releases_all_json"))
+	return jsonify(feed)
+
+
+@bp.route("/feeds/releases.atom")
+@cors_allowed
+@cached(1800)
+def releases_all_atom():
+	feed = _get_releases_feed(PackageRelease.query, abs_url_for("feeds.releases_all_atom"))
+	return _atomify(feed)
+
+
+@bp.route("/packages/<author>/<name>/releases_feed.json")
+@cors_allowed
+@is_package_page
+@cached(1800)
+def releases_package_json(package: Package):
+	feed = _get_releases_feed(package.releases, package.get_url("feeds.releases_package_json", absolute=True))
+	return jsonify(feed)
+
+
+@bp.route("/packages/<author>/<name>/releases_feed.atom")
+@cors_allowed
+@is_package_page
+@cached(1800)
+def releases_package_atom(package: Package):
+	feed = _get_releases_feed(package.releases, package.get_url("feeds.releases_package_atom", absolute=True))
+	return _atomify(feed)

app/blueprints/github/__init__.py

@@ -1,165 +0,0 @@
-# ContentDB
-# Copyright (C) 2018-21 rubenwardy
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-
-from flask import Blueprint
-from flask_babel import gettext
-
-bp = Blueprint("github", __name__)
-
-from flask import redirect, url_for, request, flash, jsonify, current_app
-from flask_login import current_user
-from sqlalchemy import func, or_, and_
-from app import github, csrf
-from app.models import db, User, APIToken, Package, Permission, AuditSeverity
-from app.utils import abs_url_for, addAuditLog, login_user_set_active
-from app.blueprints.api.support import error, api_create_vcs_release
-import hmac, requests
-
-@bp.route("/github/start/")
-def start():
-	return github.authorize("", redirect_uri=abs_url_for("github.callback"))
-
-@bp.route("/github/view/")
-def view_permissions():
-	url = "https://github.com/settings/connections/applications/" + \
-			current_app.config["GITHUB_CLIENT_ID"]
-	return redirect(url)
-
-@bp.route("/github/callback/")
-@github.authorized_handler
-def callback(oauth_token):
-	next_url = request.args.get("next")
-	if oauth_token is None:
-		flash(gettext("Authorization failed [err=gh-oauth-login-failed]"), "danger")
-		return redirect(url_for("users.login"))
-
-	# Get Github username
-	url = "https://api.github.com/user"
-	r = requests.get(url, headers={"Authorization": "token " + oauth_token})
-	username = r.json()["login"]
-
-	# Get user by github username
-	userByGithub = User.query.filter(func.lower(User.github_username) == func.lower(username)).first()
-
-	# If logged in, connect
-	if current_user and current_user.is_authenticated:
-		if userByGithub is None:
-			current_user.github_username = username
-			db.session.commit()
-			flash(gettext("Linked GitHub to account"), "success")
-			return redirect(url_for("homepage.home"))
-		else:
-			flash(gettext("GitHub account is already associated with another user"), "danger")
-			return redirect(url_for("homepage.home"))
-
-	# If not logged in, log in
-	else:
-		if userByGithub is None:
-			flash(gettext("Unable to find an account for that GitHub user"), "danger")
-			return redirect(url_for("users.claim_forums"))
-
-		ret = login_user_set_active(userByGithub, remember=True)
-		if ret is None:
-			flash(gettext("Authorization failed [err=gh-login-failed]"), "danger")
-			return redirect(url_for("users.login"))
-
-		addAuditLog(AuditSeverity.USER, userByGithub, "Logged in using GitHub OAuth",
-				url_for("users.profile", username=userByGithub.username))
-		db.session.commit()
-		return ret
-
-
-@bp.route("/github/webhook/", methods=["POST"])
-@csrf.exempt
-def webhook():
-	json = request.json
-
-	# Get package
-	github_url = "github.com/" + json["repository"]["full_name"]
-	package = Package.query.filter(Package.repo.ilike("%{}%".format(github_url))).first()
-	if package is None:
-		return error(400, "Could not find package, did you set the VCS repo in CDB correctly? Expected {}".format(github_url))
-
-	# Get all tokens for package
-	tokens_query = APIToken.query.filter(or_(APIToken.package==package,
-			and_(APIToken.package==None, APIToken.owner==package.author)))
-
-	possible_tokens = tokens_query.all()
-	actual_token = None
-
-	#
-	# Check signature
-	#
-
-	header_signature = request.headers.get('X-Hub-Signature')
-	if header_signature is None:
-		return error(403, "Expected payload signature")
-
-	sha_name, signature = header_signature.split('=')
-	if sha_name != 'sha1':
-		return error(403, "Expected SHA1 payload signature")
-
-	for token in possible_tokens:
-		mac = hmac.new(token.access_token.encode("utf-8"), msg=request.data, digestmod='sha1')
-
-		if hmac.compare_digest(str(mac.hexdigest()), signature):
-			actual_token = token
-			break
-
-	if actual_token is None:
-		return error(403, "Invalid authentication, couldn't validate API token")
-
-	if not package.checkPerm(actual_token.owner, Permission.APPROVE_RELEASE):
-		return error(403, "You do not have the permission to approve releases")
-
-	#
-	# Check event
-	#
-
-	event = request.headers.get("X-GitHub-Event")
-	if event == "push":
-		ref = json["after"]
-		title = json["head_commit"]["message"].partition("\n")[0]
-		branch = json["ref"].replace("refs/heads/", "")
-		if branch not in [ "master", "main" ]:
-			return jsonify({ "success": False, "message": "Webhook ignored, as it's not on the master/main branch" })
-
-	elif event == "create":
-		ref_type = json.get("ref_type")
-		if ref_type != "tag":
-			return jsonify({
-					"success": False,
-					"message": "Webhook ignored, as it's a non-tag create event. ref_type='{}'.".format(ref_type)
-			})
-
-		ref = json["ref"]
-		title = ref
-
-	elif event == "ping":
-		return jsonify({ "success": True, "message": "Ping successful" })
-
-	else:
-		return error(400, "Unsupported event: '{}'. Only 'push', 'create:tag', and 'ping' are supported."
-				.format(event or "null"))
-
-	#
-	# Perform release
-	#
-
-	if package.releases.filter_by(commit_hash=ref).count() > 0:
-		return
-
-	return api_create_vcs_release(actual_token, package, title, ref, reason="Webhook")

app/blueprints/gitlab/__init__.py

@@ -1,85 +0,0 @@
-# ContentDB
-# Copyright (C) 2020  rubenwardy
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-
-from flask import Blueprint, request, jsonify
-
-bp = Blueprint("gitlab", __name__)
-
-from app import csrf
-from app.models import Package, APIToken, Permission
-from app.blueprints.api.support import error, api_create_vcs_release
-
-
-def webhook_impl():
-	json = request.json
-
-	# Get package
-	gitlab_url = json["project"]["web_url"].replace("https://", "").replace("http://", "")
-	package = Package.query.filter(Package.repo.ilike("%{}%".format(gitlab_url))).first()
-	if package is None:
-		return error(400,
-				"Could not find package, did you set the VCS repo in CDB correctly? Expected {}".format(gitlab_url))
-
-	# Get all tokens for package
-	secret = request.headers.get("X-Gitlab-Token")
-	if secret is None:
-		return error(403, "Token required")
-
-	token = APIToken.query.filter_by(access_token=secret).first()
-	if token is None:
-		return error(403, "Invalid authentication")
-
-	if not package.checkPerm(token.owner, Permission.APPROVE_RELEASE):
-		return error(403, "You do not have the permission to approve releases")
-
-	#
-	# Check event
-	#
-
-	event = json["event_name"]
-	if event == "push":
-		ref = json["after"]
-		title = ref[:5]
-
-		branch = json["ref"].replace("refs/heads/", "")
-		if branch not in ["master", "main"]:
-			return jsonify({"success": False,
-				"message": "Webhook ignored, as it's not on the master/main branch"})
-
-	elif event == "tag_push":
-		ref = json["ref"]
-		title = ref.replace("refs/tags/", "")
-	else:
-		return error(400, "Unsupported event: '{}'. Only 'push', 'create:tag', and 'ping' are supported."
-					 .format(event or "null"))
-
-	#
-	# Perform release
-	#
-
-	if package.releases.filter_by(commit_hash=ref).count() > 0:
-		return
-
-	return api_create_vcs_release(token, package, title, ref, reason="Webhook")
-
-
-@bp.route("/gitlab/webhook/", methods=["POST"])
-@csrf.exempt
-def webhook():
-	try:
-		return webhook_impl()
-	except KeyError as err:
-		return error(400, "Missing field: {}".format(err.args[0]))

app/blueprints/homepage/__init__.py

@@ -1,44 +1,136 @@
+# ContentDB
+# Copyright (C) 2018-23 rubenwardy
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
 from flask import Blueprint, render_template, redirect
+from sqlalchemy import and_
+
+from app.models import Package, PackageReview, Thread, User, PackageState, db, PackageType, PackageRelease, Tags, Tag, \
+	Collection, License, Language
 
 bp = Blueprint("homepage", __name__)
 
-from app.models import *
-from sqlalchemy.orm import joinedload
+from sqlalchemy.orm import joinedload, subqueryload, load_only, noload
 from sqlalchemy.sql.expression import func
 
 
+PKGS_PER_ROW = 4
+
+# GAMEJAM_BANNER = "https://jam.luanti.org/img/banner.png"
+#
+# class GameJam:
+# 	cover_image = type("", (), dict(url=GAMEJAM_BANNER))()
+# 	tags = []
+#
+# 	def get_cover_image_url(self):
+# 		return GAMEJAM_BANNER
+#
+# 	def get_url(self, _name):
+# 		return "/gamejam/"
+#
+# 	title = "Luanti Game Jam 2023: \"Unexpected\""
+# 	author = None
+#
+# 	short_desc = "The game jam has finished! It's now up to the community to play and rate the games."
+# 	type = type("", (), dict(value="Competition"))()
+# 	content_warnings = []
+# 	reviews = []
+
+
+@bp.route("/gamejam/")
+def gamejam():
+	return redirect("https://jam.luanti.org/")
+
+
 @bp.route("/")
 def home():
-	def join(query):
+	def package_load(query):
 		return query.options(
-				joinedload(Package.license),
-				joinedload(Package.media_license))
+				load_only(Package.name, Package.title, Package.short_desc, Package.state, raiseload=True),
+				subqueryload(Package.main_screenshot),
+				joinedload(Package.author).load_only(User.username, User.display_name, raiseload=True),
+				joinedload(Package.license).load_only(License.name, License.is_foss, raiseload=True),
+				joinedload(Package.media_license).load_only(License.name, License.is_foss, raiseload=True))
 
-	query   = Package.query.filter_by(state=PackageState.APPROVED)
-	count   = query.count()
+	def package_spotlight_load(query):
+		return query.options(
+				load_only(Package.name, Package.title, Package.type, Package.short_desc, Package.state, Package.cover_image_id, raiseload=True),
+				subqueryload(Package.main_screenshot),
+				joinedload(Package.tags),
+				joinedload(Package.content_warnings),
+				joinedload(Package.author).load_only(User.username, User.display_name, raiseload=True),
+				subqueryload(Package.cover_image),
+				joinedload(Package.license).load_only(License.name, License.is_foss, raiseload=True),
+				joinedload(Package.media_license).load_only(License.name, License.is_foss, raiseload=True))
 
-	featured = query.filter(Package.tags.any(name="featured")).order_by(func.random()).limit(6).all()
+	def review_load(query):
+		return query.options(
+			load_only(PackageReview.id, PackageReview.rating, PackageReview.created_at, PackageReview.language_id, raiseload=True),
+			joinedload(PackageReview.author).load_only(User.username, User.rank, User.email, User.display_name, User.profile_pic, User.is_active, raiseload=True),
+			joinedload(PackageReview.votes),
+			joinedload(PackageReview.language).load_only(Language.title, raiseload=True),
+			joinedload(PackageReview.thread).load_only(Thread.title, Thread.replies_count, raiseload=True).subqueryload(Thread.first_reply),
+			joinedload(PackageReview.package)
+				.load_only(Package.title, Package.name, raiseload=True)
+				.joinedload(Package.author).load_only(User.username, User.display_name, raiseload=True))
 
-	new     = join(query.order_by(db.desc(Package.approved_at))).limit(4).all()
-	pop_mod = join(query.filter_by(type=PackageType.MOD).order_by(db.desc(Package.score))).limit(8).all()
-	pop_gam = join(query.filter_by(type=PackageType.GAME).order_by(db.desc(Package.score))).limit(8).all()
-	pop_txp = join(query.filter_by(type=PackageType.TXP).order_by(db.desc(Package.score))).limit(8).all()
-	high_reviewed = join(query.order_by(db.desc(Package.score - Package.score_downloads))) \
-			.filter(Package.reviews.any()).limit(4).all()
+	query = Package.query.filter_by(state=PackageState.APPROVED)
+	count = db.session.query(Package.id).filter(Package.state == PackageState.APPROVED).count()
 
-	updated = db.session.query(Package).select_from(PackageRelease).join(Package) \
-			.filter_by(state=PackageState.APPROVED) \
-			.order_by(db.desc(PackageRelease.releaseDate)) \
-			.limit(20).all()
-	updated = updated[:4]
+	spotlight_pkgs = package_spotlight_load(query.filter(
+			Package.collections.any(and_(Collection.name == "spotlight", Collection.author.has(username="ContentDB"))))
+		.order_by(func.random())).limit(6).all()
+	# spotlight_pkgs.insert(0, GameJam())
 
-	reviews = PackageReview.query.filter_by(recommends=True).order_by(db.desc(PackageReview.created_at)).limit(5).all()
+	new = package_load(query).order_by(db.desc(Package.approved_at)).limit(PKGS_PER_ROW).all() # 0.06
+	pop_mod = package_load(query).filter_by(type=PackageType.MOD).order_by(db.desc(Package.score)).limit(2*PKGS_PER_ROW).all()
+	pop_gam = package_load(query).filter_by(type=PackageType.GAME).order_by(db.desc(Package.score)).limit(2*PKGS_PER_ROW).all()
+	pop_txp = package_load(query).filter_by(type=PackageType.TXP).order_by(db.desc(Package.score)).limit(2*PKGS_PER_ROW).all()
+
+	high_reviewed = package_load(query.order_by(db.desc(Package.score - Package.score_downloads))
+			.filter(Package.reviews.any()).limit(PKGS_PER_ROW)).all()
+
+	recent_releases_query = (
+		db.session.query(
+			Package.id,
+			func.max(PackageRelease.created_at).label("max_created_at")
+		)
+		.join(PackageRelease, Package.releases)
+		.group_by(Package.id)
+		.order_by(db.desc("max_created_at"))
+		.limit(3*PKGS_PER_ROW)
+		.subquery())
+
+	updated = (
+		package_load(db.session.query(Package)
+			.select_from(recent_releases_query)
+			.join(Package, Package.id == recent_releases_query.c.id)
+			.filter(Package.state == PackageState.APPROVED)
+			.limit(PKGS_PER_ROW))
+			.all())
+
+	reviews = review_load(PackageReview.query.filter(PackageReview.rating > 3)
+			.order_by(db.desc(PackageReview.created_at))).limit(5).all()
 
 	downloads_result = db.session.query(func.sum(Package.downloads)).one_or_none()
 	downloads = 0 if not downloads_result or not downloads_result[0] else downloads_result[0]
 
 	tags = db.session.query(func.count(Tags.c.tag_id), Tag) \
-		.select_from(Tag).outerjoin(Tags).group_by(Tag.id).order_by(db.asc(Tag.title)).all()
+		.select_from(Tag).outerjoin(Tags).join(Package).filter(Package.state == PackageState.APPROVED)\
+		.group_by(Tag.id).order_by(db.asc(Tag.title)).all()
 
-	return render_template("index.html", count=count, downloads=downloads, tags=tags, featured=featured,
-			new=new, updated=updated, pop_mod=pop_mod, pop_txp=pop_txp, pop_gam=pop_gam, high_reviewed=high_reviewed, reviews=reviews)
+	return render_template("index.html", count=count, downloads=downloads, tags=tags, spotlight_pkgs=spotlight_pkgs,
+			new=new, updated=updated, pop_mod=pop_mod, pop_txp=pop_txp, pop_gam=pop_gam, high_reviewed=high_reviewed,
+			reviews=reviews)

app/blueprints/metrics/__init__.py

@@ -14,59 +14,111 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
+import datetime
+
 from flask import Blueprint, make_response
+from sqlalchemy import or_, and_
 from sqlalchemy.sql.expression import func
 
-from app.models import Package, db, User, UserRank, PackageState
+from app.models import Package, db, User, UserRank, PackageState, PackageReview, ThreadReply, Collection, AuditLogEntry, \
+	PackageTranslation, Language
+from app.rediscache import get_key
 
 bp = Blueprint("metrics", __name__)
 
-def generate_metrics(full=False):
+
+def generate_metrics():
 	def write_single_stat(name, help, type, value):
 		fmt = "# HELP {name} {help}\n# TYPE {name} {type}\n{name} {value}\n\n"
 
 		return fmt.format(name=name, help=help, type=type, value=value)
 
 	def gen_labels(labels):
-		pieces = [key + "=" + str(val) for key, val in labels.items()]
+		pieces = [f"{key}=\"{val}\"" for key, val in labels.items()]
 		return ",".join(pieces)
 
-
 	def write_array_stat(name, help, type, data):
-		ret = "# HELP {name} {help}\n# TYPE {name} {type}\n" \
+		result = "# HELP {name} {help}\n# TYPE {name} {type}\n" \
 			.format(name=name, help=help, type=type)
 
 		for entry in data:
 			assert(len(entry) == 2)
-			ret += "{name}{{{labels}}} {value}\n" \
+			result += "{name}{{{labels}}} {value}\n" \
 				.format(name=name, labels=gen_labels(entry[0]), value=entry[1])
 
-		return ret + "\n"
+		return result + "\n"
 
 	downloads_result = db.session.query(func.sum(Package.downloads)).one_or_none()
 	downloads = 0 if not downloads_result or not downloads_result[0] else downloads_result[0]
 
 	packages = Package.query.filter_by(state=PackageState.APPROVED).count()
-	users = User.query.filter(User.rank != UserRank.NOT_JOINED).count()
+	users = User.query.filter(User.rank > UserRank.NOT_JOINED, User.rank != UserRank.BOT, User.is_active).count()
+	authors = User.query.filter(User.packages.any(state=PackageState.APPROVED)).count()
+
+	one_day_ago = datetime.datetime.now() - datetime.timedelta(days=1)
+	one_week_ago = datetime.datetime.now() - datetime.timedelta(weeks=1)
+	one_month_ago = datetime.datetime.now() - datetime.timedelta(weeks=4)
+
+	active_users_day = User.query.filter(and_(User.rank != UserRank.BOT, or_(
+		User.audit_log_entries.any(AuditLogEntry.created_at > one_day_ago),
+		User.replies.any(ThreadReply.created_at > one_day_ago)))).count()
+	active_users_week = User.query.filter(and_(User.rank != UserRank.BOT, or_(
+		User.audit_log_entries.any(AuditLogEntry.created_at > one_week_ago),
+		User.replies.any(ThreadReply.created_at > one_week_ago)))).count()
+	active_users_month = User.query.filter(and_(User.rank != UserRank.BOT, or_(
+		User.audit_log_entries.any(AuditLogEntry.created_at > one_month_ago),
+		User.replies.any(ThreadReply.created_at > one_month_ago)))).count()
+
+	reviews = PackageReview.query.count()
+	comments = ThreadReply.query.count()
+	collections = Collection.query.count()
+
+	score_result = db.session.query(func.sum(Package.score)).one_or_none()
+	score = 0 if not score_result or not score_result[0] else score_result[0]
+
+	packages_with_translations = (db.session.query(PackageTranslation.package_id)
+			.filter(PackageTranslation.language_id != "en")
+			.group_by(PackageTranslation.package_id).count())
+	packages_with_translations_meta = (db.session.query(PackageTranslation.package_id)
+			.filter(PackageTranslation.short_desc.is_not(None), PackageTranslation.language_id != "en")
+			.group_by(PackageTranslation.package_id).count())
+	languages_packages = (db.session.query(PackageTranslation.language_id, func.count(Package.id))
+			.select_from(PackageTranslation).outerjoin(Package)
+			.order_by(db.asc(PackageTranslation.language_id))
+			.group_by(PackageTranslation.language_id).all())
+	languages_packages_meta = (db.session.query(PackageTranslation.language_id, func.count(Package.id))
+			.select_from(PackageTranslation).outerjoin(Package)
+			.filter(PackageTranslation.short_desc.is_not(None))
+			.order_by(db.asc(PackageTranslation.language_id))
+			.group_by(PackageTranslation.language_id).all())
 
 	ret = ""
 	ret += write_single_stat("contentdb_packages", "Total packages", "gauge", packages)
 	ret += write_single_stat("contentdb_users", "Number of registered users", "gauge", users)
+	ret += write_single_stat("contentdb_authors", "Number of users with packages", "gauge", authors)
+	ret += write_single_stat("contentdb_users_active_1d", "Number of daily active registered users", "gauge", active_users_day)
+	ret += write_single_stat("contentdb_users_active_1w", "Number of weekly active registered users", "gauge", active_users_week)
+	ret += write_single_stat("contentdb_users_active_1m", "Number of monthly active registered users", "gauge", active_users_month)
 	ret += write_single_stat("contentdb_downloads", "Total downloads", "gauge", downloads)
-
-	if full:
-		scores = Package.query.join(User).with_entities(User.username, Package.name, Package.score) \
-			.filter(Package.state==PackageState.APPROVED).all()
-
-		ret += write_array_stat("contentdb_package_score", "Package score", "gauge",
-				[({ "author": score[0], "name": score[1] }, score[2])  for score in scores])
-	else:
-		score_result = db.session.query(func.sum(Package.score)).one_or_none()
-		score = 0 if not score_result or not score_result[0] else score_result[0]
-		ret += write_single_stat("contentdb_score", "Total package score", "gauge", score)
+	ret += write_single_stat("contentdb_emails", "Number of emails sent", "counter", int(get_key("emails_sent", "0")))
+	ret += write_single_stat("contentdb_reviews", "Number of reviews", "gauge", reviews)
+	ret += write_single_stat("contentdb_comments", "Number of comments", "gauge", comments)
+	ret += write_single_stat("contentdb_collections", "Number of collections", "gauge", collections)
+	ret += write_single_stat("contentdb_score", "Total package score", "gauge", score)
+	ret += write_single_stat("contentdb_packages_with_translations", "Number of packages with translations", "gauge",
+			packages_with_translations)
+	ret += write_single_stat("contentdb_packages_with_translations_meta", "Number of packages with translated meta",
+			"gauge", packages_with_translations_meta)
+	ret += write_array_stat("contentdb_languages_translated",
+			"Number of packages per language", "gauge",
+			[({"language": x[0]}, x[1]) for x in languages_packages])
+	ret += write_array_stat("contentdb_languages_translated_meta",
+			"Number of packages with translated short desc per language", "gauge",
+			[({"language": x[0]}, x[1]) for x in languages_packages_meta])
 
 	return ret
 
+
 @bp.route("/metrics")
 def metrics():
 	response = make_response(generate_metrics(), 200)

app/blueprints/modnames/__init__.py

@@ -14,27 +14,31 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-
-from flask import *
+from flask import Blueprint, redirect, render_template, abort
 from sqlalchemy import func
 from app.models import MetaPackage, Package, db, Dependency, PackageState, ForumTopic
 
-bp = Blueprint("metapackages", __name__)
+bp = Blueprint("modnames", __name__)
 
 
-@bp.route("/metapackages/")
+@bp.route("/metapackages/<path:path>")
+def mp_redirect(path):
+	return redirect("/modnames/" + path)
+
+
+@bp.route("/modnames/")
 def list_all():
-	mpackages = db.session.query(MetaPackage, func.count(Package.id)) \
+	modnames = db.session.query(MetaPackage, func.count(Package.id)) \
 			.select_from(MetaPackage).outerjoin(MetaPackage.packages) \
 			.order_by(db.asc(MetaPackage.name)) \
 			.group_by(MetaPackage.id).all()
-	return render_template("metapackages/list.html", mpackages=mpackages)
+	return render_template("modnames/list.html", modnames=modnames)
 
 
-@bp.route("/metapackages/<name>/")
+@bp.route("/modnames/<name>/")
 def view(name):
-	mpackage = MetaPackage.query.filter_by(name=name).first()
-	if mpackage is None:
+	modname = MetaPackage.query.filter_by(name=name).first()
+	if modname is None:
 		abort(404)
 
 	dependers = db.session.query(Package) \
@@ -59,6 +63,6 @@ def view(name):
 		.order_by(db.asc(ForumTopic.name), db.asc(ForumTopic.title)) \
 		.all()
 
-	return render_template("metapackages/view.html", mpackage=mpackage,
+	return render_template("modnames/view.html", modname=modname,
 			dependers=dependers, optional_dependers=optional_dependers,
 			similar_topics=similar_topics)

app/blueprints/notifications/__init__.py

@@ -14,7 +14,6 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-
 from flask import Blueprint, render_template, redirect, url_for
 from flask_login import current_user, login_required
 from sqlalchemy import or_, desc

app/blueprints/oauth/__init__.py

@@ -0,0 +1,264 @@
+# ContentDB
+# Copyright (C) 2023 rubenwardy
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+import urllib.parse as urlparse
+from urllib.parse import urlencode
+
+import typing
+from flask import Blueprint, render_template, redirect, url_for, request, jsonify, abort, make_response, flash
+from flask_babel import lazy_gettext, gettext
+from flask_login import current_user, login_required
+from flask_wtf import FlaskForm
+from wtforms import StringField, SubmitField, URLField, SelectField
+from wtforms.validators import InputRequired, Length, Optional
+
+from app import csrf
+from app.blueprints.users.settings import get_setting_tabs
+from app.models import db, OAuthClient, User, Permission, APIToken, AuditSeverity, UserRank
+from app.utils import random_string, add_audit_log
+
+bp = Blueprint("oauth", __name__)
+
+
+def build_redirect_url(url: str, code: str, state: typing.Optional[str]):
+	params = {"code": code}
+	if state is not None:
+		params["state"] = state
+	url_parts = list(urlparse.urlparse(url))
+	query = dict(urlparse.parse_qsl(url_parts[4]))
+	query.update(params)
+	url_parts[4] = urlencode(query)
+	return urlparse.urlunparse(url_parts)
+
+
+@bp.route("/oauth/authorize/", methods=["GET", "POST"])
+@login_required
+def oauth_start():
+	response_type = request.args.get("response_type", "code")
+	if response_type != "code":
+		return "Unsupported response_type, only code is supported", 400
+
+	client_id = request.args.get("client_id", "")
+	if client_id == "":
+		return "Missing client_id", 400
+
+	redirect_uri = request.args.get("redirect_uri", "")
+	if redirect_uri == "":
+		return "Missing redirect_uri", 400
+
+	client = OAuthClient.query.get_or_404(client_id)
+	if client.redirect_url != redirect_uri:
+		return "redirect_uri does not match client", 400
+
+	if not client.approved and client.owner != current_user:
+		abort(404)
+
+	scope = request.args.get("scope", "public")
+	if scope != "public":
+		return "Unsupported scope, only public is supported", 400
+
+	state = request.args.get("state")
+
+	token = APIToken.query.filter(APIToken.client == client, APIToken.owner == current_user).first()
+	if token:
+		token.access_token = random_string(32)
+		token.auth_code = random_string(32)
+		db.session.commit()
+		return redirect(build_redirect_url(client.redirect_url, token.auth_code, state))
+
+	if request.method == "POST":
+		action = request.form["action"]
+		if action == "cancel":
+			return redirect(client.redirect_url)
+
+		elif action == "authorize":
+			token = APIToken()
+			token.access_token = random_string(32)
+			token.name = f"Token for {client.title} by {client.owner.username}"
+			token.owner = current_user
+			token.client = client
+			assert client is not None
+			token.auth_code = random_string(32)
+			db.session.add(token)
+
+			add_audit_log(AuditSeverity.USER, current_user,
+					f"Granted \"{scope}\" to OAuth2 application \"{client.title}\" by {client.owner.username} [{client_id}] ",
+					url_for("users.profile", username=current_user.username))
+
+			db.session.commit()
+
+			return redirect(build_redirect_url(client.redirect_url, token.auth_code, state))
+
+	return render_template("oauth/authorize.html", client=client)
+
+
+def error(code: int, msg: str):
+	abort(make_response(jsonify({"success": False, "error": msg}), code))
+
+
+@bp.route("/oauth/token/", methods=["POST"])
+@csrf.exempt
+def oauth_grant():
+	form = request.form
+
+	grant_type = request.args.get("grant_type", "authorization_code")
+	if grant_type != "authorization_code":
+		error(400, "Unsupported grant_type, only authorization_code is supported")
+
+	client_id = form.get("client_id", "")
+	if client_id == "":
+		error(400, "Missing client_id")
+
+	client_secret = form.get("client_secret", "")
+	if client_secret == "":
+		error(400, "Missing client_secret")
+
+	code = form.get("code", "")
+	if code == "":
+		error(400, "Missing code")
+
+	client = OAuthClient.query.filter_by(id=client_id, secret=client_secret).first()
+	if client is None:
+		error(400, "client_id and/or client_secret is incorrect")
+
+	token = APIToken.query.filter_by(auth_code=code).first()
+	if token is None or token.client != client:
+		error(400, "Incorrect code. It may have already been redeemed")
+
+	token.auth_code = None
+	db.session.commit()
+
+	return jsonify({
+		"success": True,
+		"access_token": token.access_token,
+		"token_type": "Bearer",
+	})
+
+
+@bp.route("/user/apps/")
+@login_required
+def list_clients_redirect():
+	return redirect(url_for("oauth.list_clients", username=current_user.username))
+
+
+@bp.route("/users/<username>/apps/")
+@login_required
+def list_clients(username):
+	user = User.query.filter_by(username=username).first_or_404()
+	if not user.check_perm(current_user, Permission.CREATE_OAUTH_CLIENT):
+		abort(403)
+
+	return render_template("oauth/list_clients.html", user=user, tabs=get_setting_tabs(user), current_tab="oauth_clients")
+
+
+class OAuthClientForm(FlaskForm):
+	title = StringField(lazy_gettext("Title"), [InputRequired(), Length(5, 30)])
+	description = StringField(lazy_gettext("Description"), [Optional()])
+	redirect_url = URLField(lazy_gettext("Redirect URL"), [InputRequired(), Length(5, 123)])
+	app_type = SelectField(lazy_gettext("App Type"), [InputRequired()], choices=[
+		("server", "Server-side (client_secret is kept safe)"),
+		("client", "Client-side (client_secret is visible to all users)"),
+	], coerce=lambda x: x)
+	submit = SubmitField(lazy_gettext("Save"))
+
+
+@bp.route("/users/<username>/apps/new/", methods=["GET", "POST"])
+@bp.route("/users/<username>/apps/<id_>/edit/", methods=["GET", "POST"])
+@login_required
+def create_edit_client(username, id_=None):
+	user = User.query.filter_by(username=username).first_or_404()
+	if not user.check_perm(current_user, Permission.CREATE_OAUTH_CLIENT):
+		abort(403)
+
+	is_new = id_ is None
+	client = None
+	if id_ is not None:
+		client = OAuthClient.query.get_or_404(id_)
+		if client.owner != user:
+			abort(404)
+
+	form = OAuthClientForm(formdata=request.form, obj=client)
+
+	if form.validate_on_submit():
+		if is_new:
+			if OAuthClient.query.filter(OAuthClient.title.ilike(form.title.data.strip())).count() > 0:
+				flash(gettext("An OAuth client with that title already exists. Please choose a new title."), "danger")
+				return render_template("oauth/create_edit.html", user=user, form=form, client=client)
+
+			client = OAuthClient()
+			db.session.add(client)
+			client.owner = user
+			client.id = random_string(24)
+			client.secret = random_string(32)
+			client.approved = current_user.rank.at_least(UserRank.EDITOR)
+
+
+		form.populate_obj(client)
+
+		verb = "Created" if is_new else "Edited"
+		add_audit_log(AuditSeverity.NORMAL, current_user,
+				f"{verb} OAuth2 application {client.title} by {client.owner.username} [{client.id}]",
+				url_for("oauth.create_edit_client", username=client.owner.username, id_=client.id))
+
+		db.session.commit()
+
+		return redirect(url_for("oauth.create_edit_client", username=username, id_=client.id))
+
+	return render_template("oauth/create_edit.html", user=user, form=form, client=client)
+
+
+@bp.route("/users/<username>/apps/<id_>/delete/", methods=["POST"])
+@login_required
+def delete_client(username, id_):
+	user = User.query.filter_by(username=username).first_or_404()
+	if not user.check_perm(current_user, Permission.CREATE_OAUTH_CLIENT):
+		abort(403)
+
+	client = OAuthClient.query.get(id_)
+	if client is None or client.owner != user:
+		abort(404)
+
+	add_audit_log(AuditSeverity.NORMAL, current_user,
+			f"Deleted OAuth2 application {client.title} by {client.owner.username} [{client.id}]",
+			url_for("users.profile", username=current_user.username))
+
+	db.session.delete(client)
+	db.session.commit()
+
+	return redirect(url_for("oauth.list_clients", username=username))
+
+
+@bp.route("/users/<username>/apps/<id_>/revoke-all/", methods=["POST"])
+@login_required
+def revoke_all(username, id_):
+	user = User.query.filter_by(username=username).first_or_404()
+	if not user.check_perm(current_user, Permission.CREATE_OAUTH_CLIENT):
+		abort(403)
+
+	client = OAuthClient.query.get(id_)
+	if client is None or client.owner != user:
+		abort(404)
+
+	add_audit_log(AuditSeverity.NORMAL, current_user,
+			f"Revoked all user tokens for OAuth2 application {client.title} by {client.owner.username} [{client.id}]",
+			url_for("oauth.create_edit_client", username=client.owner.username, id_=client.id))
+
+	client.tokens = []
+	db.session.commit()
+
+	flash(gettext("Revoked all user tokens"), "success")
+
+	return redirect(url_for("oauth.create_edit_client", username=client.owner.username, id_=client.id))

app/blueprints/packages/__init__.py

@@ -17,52 +17,71 @@
 from flask import Blueprint
 from flask_babel import gettext
 
-from app.models import User, Package, Permission
+from app.models import User, Package, Permission, PackageType
 
 bp = Blueprint("packages", __name__)
 
 
 def get_package_tabs(user: User, package: Package):
-	if package is None or not package.checkPerm(user, Permission.EDIT_PACKAGE):
+	if package is None or not package.check_perm(user, Permission.EDIT_PACKAGE):
 		return []
 
-	return [
+	retval = [
 		{
 			"id": "edit",
 			"title": gettext("Edit Details"),
-			"url": package.getURL("packages.create_edit")
+			"url": package.get_url("packages.create_edit")
+		},
+		{
+			"id": "translation",
+			"title": gettext("Translation"),
+			"url": package.get_url("packages.translation")
 		},
 		{
 			"id": "releases",
 			"title": gettext("Releases"),
-			"url": package.getURL("packages.list_releases")
+			"url": package.get_url("packages.list_releases")
 		},
 		{
 			"id": "screenshots",
 			"title": gettext("Screenshots"),
-			"url": package.getURL("packages.screenshots")
+			"url": package.get_url("packages.screenshots")
 		},
 		{
 			"id": "maintainers",
 			"title": gettext("Maintainers"),
-			"url": package.getURL("packages.edit_maintainers")
+			"url": package.get_url("packages.edit_maintainers")
 		},
 		{
 			"id": "audit",
 			"title": gettext("Audit Log"),
-			"url": package.getURL("packages.audit")
+			"url": package.get_url("packages.audit")
+		},
+		{
+			"id": "stats",
+			"title": gettext("Statistics"),
+			"url": package.get_url("packages.statistics")
 		},
 		{
 			"id": "share",
 			"title": gettext("Share and Badges"),
-			"url": package.getURL("packages.share")
+			"url": package.get_url("packages.share")
 		},
 		{
 			"id": "remove",
-			"title": gettext("Remove"),
-			"url": package.getURL("packages.remove")
+			"title": gettext("Remove / Unpublish"),
+			"url": package.get_url("packages.remove")
 		}
 	]
 
+	if package.type == PackageType.MOD or package.type == PackageType.TXP:
+		retval.insert(2, {
+			"id": "game_support",
+			"title": gettext("Supported Games"),
+			"url": package.get_url("packages.game_support")
+		})
 
-from . import packages, screenshots, releases, reviews, game_hub
+	return retval
+
+
+from . import packages, advanced_search, screenshots, releases, reviews, game_hub

app/blueprints/packages/advanced_search.py

@@ -0,0 +1,103 @@
+# ContentDB
+# Copyright (C) 2024 rubenwardy
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+from flask import render_template
+from flask_babel import lazy_gettext, gettext
+from flask_wtf import FlaskForm
+from wtforms.fields.choices import SelectField, SelectMultipleField
+from wtforms.fields.simple import StringField, BooleanField
+from wtforms.validators import Optional
+from wtforms_sqlalchemy.fields import QuerySelectMultipleField, QuerySelectField
+
+from . import bp
+from ...models import PackageType, Tag, db, ContentWarning, License, Language, LuantiRelease, Package, PackageState
+
+
+def make_label(obj: Tag | ContentWarning):
+	translated = obj.get_translated()
+	if translated["description"]:
+		return "{}: {}".format(translated["title"], translated["description"])
+	else:
+		return translated["title"]
+
+
+def get_hide_choices():
+	ret = [
+		("android_default", gettext("Android Default")),
+		("desktop_default", gettext("Desktop Default")),
+		("nonfree", gettext("Non-free")),
+		("wip", gettext("Work in Progress")),
+		("deprecated", gettext("Deprecated")),
+		("*", gettext("All content warnings")),
+	]
+	content_warnings = ContentWarning.query.order_by(db.asc(ContentWarning.name)).all()
+	tags = Tag.query.order_by(db.asc(Tag.name)).all()
+	ret += [(x.name, make_label(x)) for x in content_warnings + tags]
+	return ret
+
+
+class AdvancedSearchForm(FlaskForm):
+	q = StringField(lazy_gettext("Query"), [Optional()])
+	type = SelectMultipleField(lazy_gettext("Type"), [Optional()],
+			choices=PackageType.choices(), coerce=PackageType.coerce)
+	author = StringField(lazy_gettext("Author"), [Optional()])
+	tag = QuerySelectMultipleField(lazy_gettext('Tags'),
+			query_factory=lambda: Tag.query.order_by(db.asc(Tag.name)),
+			get_pk=lambda a: a.name, get_label=make_label)
+	flag = QuerySelectMultipleField(lazy_gettext('Content Warnings'),
+			query_factory=lambda: ContentWarning.query.order_by(db.asc(ContentWarning.name)),
+			get_pk=lambda a: a.name, get_label=make_label)
+	license = QuerySelectMultipleField(lazy_gettext("License"), [Optional()],
+			query_factory=lambda: License.query.order_by(db.asc(License.name)),
+			allow_blank=True, blank_value="",
+			get_pk=lambda a: a.name, get_label=lambda a: a.name)
+	game = QuerySelectField(lazy_gettext("Supports Game"), [Optional()],
+			query_factory=lambda: Package.query.filter(Package.type == PackageType.GAME, Package.state == PackageState.APPROVED).order_by(db.asc(Package.name)),
+			allow_blank=True, blank_value="",
+			get_pk=lambda a: f"{a.author.username}/{a.name}",
+			get_label=lambda a: lazy_gettext("%(title)s by %(author)s", title=a.title, author=a.author.display_name))
+	lang = QuerySelectField(lazy_gettext("Language"),
+			query_factory=lambda: Language.query.order_by(db.asc(Language.title)),
+			allow_blank=True, blank_value="",
+			get_pk=lambda a: a.id, get_label=lambda a: a.title)
+	hide = SelectMultipleField(lazy_gettext("Hide Tags and Content Warnings"), [Optional()])
+	engine_version = QuerySelectField(lazy_gettext("Luanti Version"),
+			query_factory=lambda: LuantiRelease.query.order_by(db.asc(LuantiRelease.id)),
+			allow_blank=True, blank_value="",
+			get_pk=lambda a: a.value, get_label=lambda a: a.name)
+	sort = SelectField(lazy_gettext("Sort by"), [Optional()], choices=[
+		("", ""),
+		("name", lazy_gettext("Name")),
+		("title", lazy_gettext("Title")),
+		("score", lazy_gettext("Package score")),
+		("reviews", lazy_gettext("Reviews")),
+		("downloads", lazy_gettext("Downloads")),
+		("created_at", lazy_gettext("Created At")),
+		("approved_at", lazy_gettext("Approved At")),
+		("last_release", lazy_gettext("Last Release")),
+	])
+	order = SelectField(lazy_gettext("Order"), [Optional()], choices=[
+		("desc", lazy_gettext("Descending")),
+		("asc", lazy_gettext("Ascending")),
+	])
+	random = BooleanField(lazy_gettext("Random order"))
+
+
+@bp.route("/packages/advanced-search/")
+def advanced_search():
+	form = AdvancedSearchForm()
+	form.hide.choices = get_hide_choices()
+	return render_template("packages/advanced_search.html", form=form)

app/blueprints/packages/game_hub.py

@@ -19,7 +19,7 @@ from sqlalchemy.orm import joinedload
 
 from . import bp
 from app.utils import is_package_page
-from ...models import Package, PackageType, PackageState, db, PackageRelease
+from app.models import Package, PackageType, PackageState, db, PackageRelease
 
 
 @bp.route("/packages/<author>/<name>/hub/")
@@ -33,22 +33,21 @@ def game_hub(package: Package):
 			joinedload(Package.license),
 			joinedload(Package.media_license))
 
-	query = Package.query.filter(Package.supported_games.any(game=package), Package.state==PackageState.APPROVED)
+	query = Package.query.filter(Package.supported_games.any(game=package, supports=True), Package.state==PackageState.APPROVED)
 	count = query.count()
 
 	new = join(query.order_by(db.desc(Package.approved_at))).limit(4).all()
 	pop_mod = join(query.filter_by(type=PackageType.MOD).order_by(db.desc(Package.score))).limit(8).all()
-	pop_gam = join(query.filter_by(type=PackageType.GAME).order_by(db.desc(Package.score))).limit(8).all()
 	pop_txp = join(query.filter_by(type=PackageType.TXP).order_by(db.desc(Package.score))).limit(8).all()
 	high_reviewed = join(query.order_by(db.desc(Package.score - Package.score_downloads))) \
 		.filter(Package.reviews.any()).limit(4).all()
 
 	updated = db.session.query(Package).select_from(PackageRelease).join(Package) \
-		.filter(Package.supported_games.any(game=package), Package.state==PackageState.APPROVED) \
-		.order_by(db.desc(PackageRelease.releaseDate)) \
+		.filter(Package.supported_games.any(game=package, supports=True), Package.state==PackageState.APPROVED) \
+		.order_by(db.desc(PackageRelease.created_at)) \
 		.limit(20).all()
 	updated = updated[:4]
 
 	return render_template("packages/game_hub.html", package=package, count=count,
-			new=new, updated=updated, pop_mod=pop_mod, pop_txp=pop_txp, pop_gam=pop_gam,
+			new=new, updated=updated, pop_mod=pop_mod, pop_txp=pop_txp,
 			high_reviewed=high_reviewed)

app/blueprints/packages/packages.py

@@ -13,35 +13,48 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+import datetime
 import typing
 from urllib.parse import quote as urlescape
 
-from flask import render_template
-from flask_babel import lazy_gettext, gettext
+from celery import uuid
+from flask import render_template, make_response, request, redirect, flash, url_for, abort
+from flask_babel import gettext, lazy_gettext
+from flask_login import login_required, current_user
 from flask_wtf import FlaskForm
-from flask_login import login_required
-from jinja2 import Markup
-from sqlalchemy import or_, func, and_
+from jinja2.utils import markupsafe
+from sqlalchemy import func, or_, and_
 from sqlalchemy.orm import joinedload, subqueryload
-from wtforms import *
+from wtforms import SelectField, StringField, TextAreaField, IntegerField, SubmitField, BooleanField
+from wtforms.validators import InputRequired, Length, Regexp, DataRequired, Optional, URL, NumberRange, ValidationError
 from wtforms_sqlalchemy.fields import QuerySelectField, QuerySelectMultipleField
-from wtforms.validators import *
 
-from app.querybuilder import QueryBuilder
-from app.rediscache import has_key, set_key
-from app.tasks.importtasks import importRepoScreenshot
-from app.utils import *
-from . import bp, get_package_tabs
 from app.logic.LogicError import LogicError
 from app.logic.packages import do_edit_package
-from app.models.packages import PackageProvides
+from app.querybuilder import QueryBuilder
+from app.rediscache import has_key, set_temp_key
+from app.tasks.importtasks import import_repo_screenshot, check_zip_release, remove_package_game_support, \
+	update_package_game_support
+from app.tasks.pkgtasks import check_package_on_submit
 from app.tasks.webhooktasks import post_discord_webhook
 
+from . import bp, get_package_tabs
+from app.models import Package, Tag, db, User, Tags, PackageState, Permission, PackageType, MetaPackage, ForumTopic, \
+	Dependency, Thread, UserRank, PackageReview, PackageDevState, ContentWarning, License, AuditSeverity, \
+	PackageScreenshot, NotificationType, AuditLogEntry, PackageAlias, PackageProvides, PackageGameSupport, \
+	PackageDailyStats, Collection
+from app.utils import is_user_bot, get_int_or_abort, is_package_page, abs_url_for, add_audit_log, get_package_by_info, \
+	add_notification, get_system_user, rank_required, get_games_from_csv, get_daterange_options, \
+	post_to_approval_thread, normalize_line_endings
+from app.logic.package_approval import validate_package_for_approval, can_move_to_state
+from app.logic.game_support import game_support_set
+
 
 @bp.route("/packages/")
 def list_all():
-	qb    = QueryBuilder(request.args)
-	query = qb.buildPackageQuery()
+	qb    = QueryBuilder(request.args, cookies=True)
+	query = qb.build_package_query()
 	title = qb.title
 
 	query = query.options(
@@ -56,7 +69,7 @@ def list_all():
 			edited = True
 			key = "tag/{}/{}".format(ip, tag.name)
 			if not has_key(key):
-				set_key(key, "true")
+				set_temp_key(key, "true")
 				Tag.query.filter_by(id=tag.id).update({
 						"views": Tag.views + 1
 					})
@@ -67,15 +80,15 @@ def list_all():
 	if qb.lucky:
 		package = query.first()
 		if package:
-			return redirect(package.getURL("packages.view"))
+			return redirect(package.get_url("packages.view"))
 
-		topic = qb.buildTopicQuery().first()
+		topic = qb.build_topic_query().first()
 		if qb.search and topic:
-			return redirect("https://forum.minetest.net/viewtopic.php?t=" + str(topic.topic_id))
+			return redirect(topic.url)
 
 	page  = get_int_or_abort(request.args.get("page"), 1)
 	num   = min(40, get_int_or_abort(request.args.get("n"), 100))
-	query = query.paginate(page, num, True)
+	query = query.paginate(page=page, per_page=num)
 
 	search = request.args.get("q")
 	type_name = request.args.get("type")
@@ -90,50 +103,38 @@ def list_all():
 
 	topics = None
 	if qb.search and not query.has_next:
-		qb.show_discarded = True
-		topics = qb.buildTopicQuery().all()
+		topics = qb.build_topic_query().all()
 
 	tags_query = db.session.query(func.count(Tags.c.tag_id), Tag) \
-  		.select_from(Tag).join(Tags).join(Package).group_by(Tag.id).order_by(db.asc(Tag.title))
-	tags = qb.filterPackageQuery(tags_query).all()
+		.select_from(Tag).join(Tags).join(Package).filter(Package.state==PackageState.APPROVED) \
+		.group_by(Tag.id).order_by(db.asc(Tag.title))
+	tags = qb.filter_package_query(tags_query).all()
 
 	selected_tags = set(qb.tags)
 
 	return render_template("packages/list.html",
-			query_hint=title, packages=query.items, pagination=query,
+			query_hint=qb.query_hint, packages=query.items, pagination=query,
 			query=search, tags=tags, selected_tags=selected_tags, type=type_name,
-			authors=authors, packages_count=query.total, topics=topics)
+			authors=authors, packages_count=query.total, topics=topics, noindex=qb.noindex)
 
 
-def getReleases(package):
-	if package.checkPerm(current_user, Permission.MAKE_RELEASE):
+def get_releases(package):
+	if package.check_perm(current_user, Permission.MAKE_RELEASE):
 		return package.releases.limit(5)
 	else:
 		return package.releases.filter_by(approved=True).limit(5)
 
 
+@bp.route("/packages/<author>/")
+def user_redirect(author):
+	return redirect(url_for("users.profile", username=author))
+
+
 @bp.route("/packages/<author>/<name>/")
 @is_package_page
 def view(package):
-	show_similar = not package.approved and (
-			current_user in package.maintainers or
-				package.checkPerm(current_user, Permission.APPROVE_NEW))
-
-	conflicting_modnames = None
-	if show_similar and package.type != PackageType.TXP:
-		conflicting_modnames = db.session.query(MetaPackage.name) \
-				.filter(MetaPackage.id.in_([ mp.id for mp in package.provides ])) \
-				.filter(MetaPackage.packages.any(and_(Package.id != package.id, Package.state == PackageState.APPROVED))) \
-				.all()
-
-		conflicting_modnames += db.session.query(ForumTopic.name) \
-				.filter(ForumTopic.name.in_([ mp.name for mp in package.provides ])) \
-				.filter(ForumTopic.topic_id != package.forums) \
-				.filter(~ db.exists().where(Package.forums==ForumTopic.topic_id)) \
-				.order_by(db.asc(ForumTopic.name), db.asc(ForumTopic.title)) \
-				.all()
-
-		conflicting_modnames = set([x[0] for x in conflicting_modnames])
+	if not package.check_perm(current_user, Permission.VIEW_PACKAGE):
+		return render_template("packages/gone.html", package=package), 403
 
 	packages_uses = None
 	if package.type == PackageType.MOD:
@@ -145,44 +146,44 @@ def view(package):
 						Dependency.meta_package_id.in_([p.id for p in package.provides]))) \
 			.order_by(db.desc(Package.score)).limit(6).all()
 
-	releases = getReleases(package)
+	releases = get_releases(package)
 
 	review_thread = package.review_thread
-	if review_thread is not None and not review_thread.checkPerm(current_user, Permission.SEE_THREAD):
+	if review_thread is not None and not review_thread.check_perm(current_user, Permission.SEE_THREAD):
 		review_thread = None
 
-	topic_error = None
-	topic_error_lvl = "warning"
-	if package.state != PackageState.APPROVED and package.forums is not None:
-		errors = []
-		if Package.query.filter(Package.forums==package.forums, Package.state!=PackageState.DELETED).count() > 1:
-			errors.append("<b>" + gettext("Error: Another package already uses this forum topic!") + "</b>")
-			topic_error_lvl = "danger"
-
-		topic = ForumTopic.query.get(package.forums)
-		if topic is not None:
-			if topic.author != package.author:
-				errors.append("<b>" + gettext("Error: Forum topic author doesn't match package author.") + "</b>")
-				topic_error_lvl = "danger"
-		elif package.type != PackageType.TXP:
-			errors.append(gettext("Warning: Forum topic not found. This may happen if the topic has only just been created."))
-
-		topic_error = "<br />".join(errors)
-
-
 	threads = Thread.query.filter_by(package_id=package.id, review_id=None)
 	if not current_user.is_authenticated:
 		threads = threads.filter_by(private=False)
-	elif not current_user.rank.atLeast(UserRank.APPROVER) and not current_user == package.author:
+	elif not current_user.rank.at_least(UserRank.APPROVER) and not current_user == package.author:
 		threads = threads.filter(or_(Thread.private == False, Thread.author == current_user))
 
-	has_review = current_user.is_authenticated and PackageReview.query.filter_by(package=package, author=current_user).count() > 0
+	has_review = current_user.is_authenticated and \
+		PackageReview.query.filter_by(package=package, author=current_user).count() > 0
+
+	validation = None
+	if package.state != PackageState.APPROVED:
+		validation = validate_package_for_approval(package)
+
+	favorites_count = Collection.query.filter(
+			Collection.packages.contains(package),
+			Collection.name == "favorites").count()
+
+	public_collection_count = Collection.query.filter(
+		Collection.packages.contains(package),
+		Collection.private == False).count()
+
+	is_favorited = current_user.is_authenticated and \
+		Collection.query.filter(
+			Collection.author == current_user,
+			Collection.packages.contains(package),
+			Collection.name == "favorites").count() > 0
 
 	return render_template("packages/view.html",
 			package=package, releases=releases, packages_uses=packages_uses,
-			conflicting_modnames=conflicting_modnames,
-			review_thread=review_thread, topic_error=topic_error, topic_error_lvl=topic_error_lvl,
-			threads=threads.all(), has_review=has_review)
+			review_thread=review_thread, threads=threads.all(), validation=validation,
+			has_review=has_review, favorites_count=favorites_count, is_favorited=is_favorited,
+			public_collection_count=public_collection_count)
 
 
 @bp.route("/packages/<author>/<name>/shields/<type>/")
@@ -192,11 +193,11 @@ def shield(package, type):
 		url = "https://img.shields.io/static/v1?label=ContentDB&message={}&color={}" \
 			.format(urlescape(package.title), urlescape("#375a7f"))
 	elif type == "downloads":
-		#api_url = abs_url_for("api.package", author=package.author.username, name=package.name)
-		api_url = "https://content.minetest.net" + url_for("api.package", author=package.author.username, name=package.name)
+		api_url = abs_url_for("api.package_view", author=package.author.username, name=package.name)
 		url = "https://img.shields.io/badge/dynamic/json?color={}&label=ContentDB&query=downloads&suffix=+downloads&url={}" \
 			.format(urlescape("#375a7f"), urlescape(api_url))
 	else:
+		from flask import abort
 		abort(404)
 
 	return redirect(url)
@@ -205,24 +206,25 @@ def shield(package, type):
 @bp.route("/packages/<author>/<name>/download/")
 @is_package_page
 def download(package):
-	release = package.getDownloadRelease()
+	release = package.get_download_release()
 
 	if release is None:
 		if "application/zip" in request.accept_mimetypes and \
-				not "text/html" in request.accept_mimetypes:
+				"text/html" not in request.accept_mimetypes:
 			return "", 204
 		else:
 			flash(gettext("No download available."), "danger")
-			return redirect(package.getURL("packages.view"))
+			return redirect(package.get_url("packages.view"))
 	else:
-		return redirect(release.getDownloadURL())
+		return redirect(release.get_download_url())
 
 
-def makeLabel(obj):
-	if obj.description:
-		return "{}: {}".format(obj.title, obj.description)
+def make_label(obj: Tag | ContentWarning):
+	translated = obj.get_translated()
+	if translated["description"]:
+		return "{}: {}".format(translated["title"], translated["description"])
 	else:
-		return obj.title
+		return translated["title"]
 
 
 class PackageForm(FlaskForm):
@@ -231,39 +233,54 @@ class PackageForm(FlaskForm):
 	name             = StringField(lazy_gettext("Name (Technical)"), [InputRequired(), Length(1, 100), Regexp("^[a-z0-9_]+$", 0, lazy_gettext("Lower case letters (a-z), digits (0-9), and underscores (_) only"))])
 	short_desc       = StringField(lazy_gettext("Short Description (Plaintext)"), [InputRequired(), Length(1,200)])
 
-	dev_state        = SelectField(lazy_gettext("Maintenance State"), [InputRequired()], choices=PackageDevState.choices(with_none=True), coerce=PackageDevState.coerce)
+	dev_state        = SelectField(lazy_gettext("Maintenance State"), [DataRequired()], choices=PackageDevState.choices(with_none=True), coerce=PackageDevState.coerce)
 
-	tags             = QuerySelectMultipleField(lazy_gettext('Tags'), query_factory=lambda: Tag.query.order_by(db.asc(Tag.name)), get_pk=lambda a: a.id, get_label=makeLabel)
-	content_warnings = QuerySelectMultipleField(lazy_gettext('Content Warnings'), query_factory=lambda: ContentWarning.query.order_by(db.asc(ContentWarning.name)), get_pk=lambda a: a.id, get_label=makeLabel)
+	tags             = QuerySelectMultipleField(lazy_gettext('Tags'), query_factory=lambda: Tag.query.order_by(db.asc(Tag.name)), get_pk=lambda a: a.id, get_label=make_label)
+	content_warnings = QuerySelectMultipleField(lazy_gettext('Content Warnings'), query_factory=lambda: ContentWarning.query.order_by(db.asc(ContentWarning.name)), get_pk=lambda a: a.id, get_label=make_label)
 	license          = QuerySelectField(lazy_gettext("License"), [DataRequired()], allow_blank=True, query_factory=lambda: License.query.order_by(db.asc(License.name)), get_pk=lambda a: a.id, get_label=lambda a: a.name)
 	media_license    = QuerySelectField(lazy_gettext("Media License"), [DataRequired()], allow_blank=True, query_factory=lambda: License.query.order_by(db.asc(License.name)), get_pk=lambda a: a.id, get_label=lambda a: a.name)
 
-	desc             = TextAreaField(lazy_gettext("Long Description (Markdown)"), [Optional(), Length(0,10000)])
+	desc             = TextAreaField(lazy_gettext("Long Description (Markdown)"), [Optional(), Length(0,10000)], filters=[normalize_line_endings])
 
 	repo             = StringField(lazy_gettext("VCS Repository URL"), [Optional(), URL()], filters = [lambda x: x or None])
 	website          = StringField(lazy_gettext("Website URL"), [Optional(), URL()], filters = [lambda x: x or None])
 	issueTracker     = StringField(lazy_gettext("Issue Tracker URL"), [Optional(), URL()], filters = [lambda x: x or None])
-	forums           = IntegerField(lazy_gettext("Forum Topic ID"), [Optional(), NumberRange(0,999999)])
-	video_url        = StringField(lazy_gettext("Video URL"), [Optional(), URL()], filters = [lambda x: x or None])
+	forums           = IntegerField(lazy_gettext("Forum Topic ID"), [Optional(), NumberRange(0, 999999)])
+	video_url        = StringField(lazy_gettext("Video URL"), [Optional(), URL()], filters=[lambda x: x or None])
+	donate_url       = StringField(lazy_gettext("Donate URL"), [Optional(), URL()], filters=[lambda x: x or None])
+	translation_url  = StringField(lazy_gettext("Translation URL"), [Optional(), URL()], filters=[lambda x: x or None])
 
 	submit           = SubmitField(lazy_gettext("Save"))
 
+	def validate_name(self, field):
+		if field.data == "_game":
+			raise ValidationError(lazy_gettext("_game is not an allowed name"))
+
 
 def handle_create_edit(package: typing.Optional[Package], form: PackageForm, author: User):
 	wasNew = False
 	if package is None:
-		package = Package.query.filter_by(name=form["name"].data, author_id=author.id).first()
+		package = Package.query.filter_by(name=form.name.data, author_id=author.id).first()
 		if package is not None:
 			if package.state == PackageState.DELETED:
-				package.review_thread_id = None
-				db.session.delete(package)
+				flash(
+					gettext("Package already exists, but is removed. Please contact ContentDB staff to restore the package"),
+					"danger")
+				return redirect(url_for("report.report", url=package.get_url("packages.view")))
 			else:
-				flash(Markup(
-					f"<a class='btn btn-sm btn-danger float-right' href='{package.getURL('packages.view')}'>View</a>" +
+				flash(markupsafe.Markup(
+					f"<a class='btn btn-sm btn-danger float-end' href='{package.get_url('packages.view')}'>View</a>" +
 					gettext("Package already exists")), "danger")
-				return None
+			return None
+
+		if Collection.query \
+				.filter(Collection.name == form.name.data, Collection.author == author) \
+				.count() > 0:
+			flash(gettext("A collection with a similar name already exists"), "danger")
+			return
 
 		package = Package()
+		db.session.add(package)
 		package.author = author
 		package.maintainers.append(author)
 		wasNew = True
@@ -285,33 +302,38 @@ def handle_create_edit(package: typing.Optional[Package], form: PackageForm, aut
 			"issueTracker": form.issueTracker.data,
 			"forums": form.forums.data,
 			"video_url": form.video_url.data,
+			"donate_url": form.donate_url.data,
+			"translation_url": form.translation_url.data,
 		})
 
-		if wasNew:
-			msg = f"Created package {author.username}/{form.name.data}"
-			addAuditLog(AuditSeverity.NORMAL, current_user, msg, package.getURL("packages.view"), package)
-
 		if wasNew and package.repo is not None:
-			importRepoScreenshot.delay(package.id)
+			import_repo_screenshot.delay(package.id)
 
-		next_url = package.getURL("packages.view")
+		next_url = package.get_url("packages.view")
 		if wasNew and ("WTFPL" in package.license.name or "WTFPL" in package.media_license.name):
 			next_url = url_for("flatpage", path="help/wtfpl", r=next_url)
 		elif wasNew:
-			next_url = package.getURL("packages.setup_releases")
+			next_url = package.get_url("packages.setup_releases")
 
 		return redirect(next_url)
 	except LogicError as e:
 		flash(e.message, "danger")
+		db.session.rollback()
 
 
 @bp.route("/packages/new/", methods=["GET", "POST"])
 @bp.route("/packages/<author>/<name>/edit/", methods=["GET", "POST"])
 @login_required
 def create_edit(author=None, name=None):
+	if current_user.email is None and not current_user.rank.at_least(UserRank.ADMIN):
+		flash(gettext("You must add an email address to your account and confirm it before you can manage packages"), "danger")
+		return redirect(url_for("users.email_notifications"))
+
 	package = None
 	if author is None:
 		form = PackageForm(formdata=request.form)
+		form.submit.label.text = lazy_gettext("Save draft")
+
 		author = request.args.get("author")
 		if author is None or author == current_user.username:
 			author = current_user
@@ -321,16 +343,16 @@ def create_edit(author=None, name=None):
 				flash(gettext("Unable to find that user"), "danger")
 				return redirect(url_for("packages.create_edit"))
 
-			if not author.checkPerm(current_user, Permission.CHANGE_AUTHOR):
+			if not author.check_perm(current_user, Permission.CHANGE_AUTHOR):
 				flash(gettext("Permission denied"), "danger")
 				return redirect(url_for("packages.create_edit"))
 
 	else:
-		package = getPackageByInfo(author, name)
+		package = get_package_by_info(author, name)
 		if package is None:
 			abort(404)
-		if not package.checkPerm(current_user, Permission.EDIT_PACKAGE):
-			return redirect(package.getURL("packages.view"))
+		if not package.check_perm(current_user, Permission.EDIT_PACKAGE):
+			abort(403)
 
 		author = package.author
 
@@ -365,7 +387,7 @@ def create_edit(author=None, name=None):
 	return render_template("packages/create_edit.html", package=package,
 			form=form, author=author, enable_wizard=enableWizard,
 			packages=package_query.all(),
-			mpackages=MetaPackage.query.order_by(db.asc(MetaPackage.name)).all(),
+			modnames=MetaPackage.query.order_by(db.asc(MetaPackage.name)).all(),
 			tabs=get_package_tabs(current_user, package), current_tab="edit")
 
 
@@ -377,17 +399,22 @@ def move_to_state(package):
 	if state is None:
 		abort(400)
 
-	if not package.canMoveToState(current_user, state):
-		flash(gettext("You don't have permission to do that"), "danger")
-		return redirect(package.getURL("packages.view"))
+	if package.state == state:
+		return redirect(package.get_url("packages.view"))
 
+	if not can_move_to_state(package, current_user, state):
+		flash(gettext("You don't have permission to do that"), "danger")
+		return redirect(package.get_url("packages.view"))
+
+	old_state = package.state
 	package.state = state
 	msg = "Marked {} as {}".format(package.title, state.value)
 
 	if state == PackageState.APPROVED:
 		if not package.approved_at:
-			post_discord_webhook.delay(package.author.username,
-					"New package {}".format(package.getURL("packages.view", absolute=True)), False)
+			post_discord_webhook.delay(package.author.display_name,
+					"New package {}".format(package.get_url("packages.view", absolute=True)), False,
+					package.title, package.short_desc, package.get_thumb_url(2, True, "png"))
 			package.approved_at = datetime.datetime.now()
 
 		screenshots = PackageScreenshot.query.filter_by(package=package, approved=False).all()
@@ -395,68 +422,123 @@ def move_to_state(package):
 			s.approved = True
 
 		msg = "Approved {}".format(package.title)
+		update_package_game_support.delay(package.id)
 	elif state == PackageState.READY_FOR_REVIEW:
-		post_discord_webhook.delay(package.author.username,
-				"Ready for Review: {}".format(package.getURL("packages.view", absolute=True)), True)
+		post_discord_webhook.delay(package.author.display_name,
+				"Ready for Review: {}".format(package.get_url("packages.view", absolute=True)), True,
+				package.title, package.short_desc, package.get_thumb_url(2, True, "png"))
 
-	addNotification(package.maintainers, current_user, NotificationType.PACKAGE_APPROVAL, msg, package.getURL("packages.view"), package)
+	add_notification(package.maintainers, current_user, NotificationType.PACKAGE_APPROVAL, msg, package.get_url("packages.view"), package)
 	severity = AuditSeverity.NORMAL if current_user in package.maintainers else AuditSeverity.EDITOR
-	addAuditLog(severity, current_user, msg, package.getURL("packages.view"), package)
+	add_audit_log(severity, current_user, msg, package.get_url("packages.view"), package)
+	post_to_approval_thread(package, current_user, msg, True)
 
 	db.session.commit()
 
+	check_package_on_submit.delay(package.id)
+
 	if package.state == PackageState.CHANGES_NEEDED:
 		flash(gettext("Please comment what changes are needed in the approval thread"), "warning")
 		if package.review_thread:
-			return redirect(package.review_thread.getViewURL())
+			return redirect(package.review_thread.get_view_url())
 		else:
 			return redirect(url_for('threads.new', pid=package.id, title='Package approval comments'))
+	elif (package.review_thread and
+			old_state == PackageState.CHANGES_NEEDED and package.state == PackageState.READY_FOR_REVIEW):
+		flash(gettext("Please comment in the approval thread so editors know what you have changed"), "warning")
+		return redirect(package.review_thread.get_view_url())
 
-	return redirect(package.getURL("packages.view"))
+	return redirect(package.get_url("packages.view"))
+
+
+@bp.route("/packages/<author>/<name>/translation/")
+@login_required
+@is_package_page
+def translation(package):
+	return render_template("packages/translation.html", package=package,
+			has_content_translations=any([x.title or x.short_desc for x in package.translations.all()]),
+			tabs=get_package_tabs(current_user, package), current_tab="translation")
 
 
 @bp.route("/packages/<author>/<name>/remove/", methods=["GET", "POST"])
 @login_required
 @is_package_page
 def remove(package):
+	if not package.check_perm(current_user, Permission.EDIT_PACKAGE):
+		abort(403)
+
+	states = [PackageDevState.AS_IS, PackageDevState.DEPRECATED, PackageDevState.LOOKING_FOR_MAINTAINER]
+
 	if request.method == "GET":
-		return render_template("packages/remove.html", package=package,
+		# Find packages that will having missing hard deps after this action
+		broken_meta = MetaPackage.query.filter(MetaPackage.packages.contains(package),
+				~MetaPackage.packages.any(and_(Package.id != package.id, Package.state == PackageState.APPROVED)))
+		hard_deps = Package.query.filter(
+			Package.state == PackageState.APPROVED,
+			Package.dependencies.any(
+				and_(Dependency.meta_package_id.in_([x.id for x in broken_meta]), Dependency.optional == False))).all()
+
+		return render_template("packages/remove.html",
+				package=package, hard_deps=hard_deps, states=states,
 				tabs=get_package_tabs(current_user, package), current_tab="remove")
 
+	for state in states:
+		if state.name in request.form:
+			flash(gettext("Set state to %(state)s", state=state.title), "success")
+			package.dev_state = state
+			msg = "Set dev state of {} to {}".format(package.title, state.title)
+			add_audit_log(AuditSeverity.NORMAL, current_user, msg, package.get_url("packages.view"), package)
+			db.session.commit()
+			return redirect(package.get_url("packages.view"))
+
 	reason = request.form.get("reason") or "?"
+	if len(reason) > 500:
+		abort(400)
 
 	if "delete" in request.form:
-		if not package.checkPerm(current_user, Permission.DELETE_PACKAGE):
+		if not package.check_perm(current_user, Permission.DELETE_PACKAGE):
 			flash(gettext("You don't have permission to do that"), "danger")
-			return redirect(package.getURL("packages.view"))
+			return redirect(package.get_url("packages.view"))
 
 		package.state = PackageState.DELETED
 
 		url = url_for("users.profile", username=package.author.username)
 		msg = "Deleted {}, reason={}".format(package.title, reason)
-		addNotification(package.maintainers, current_user, NotificationType.PACKAGE_EDIT, msg, url, package)
-		addAuditLog(AuditSeverity.EDITOR, current_user, msg, url, package)
+		add_notification(package.maintainers, current_user, NotificationType.PACKAGE_EDIT, msg, url, package)
+		add_audit_log(AuditSeverity.EDITOR, current_user, msg, url, package)
 		db.session.commit()
 
+		post_discord_webhook.delay(current_user.username,
+			f"Deleted package {package.author.username}/{package.name} with reason '{reason}'",
+			True, package.title, package.short_desc, package.get_thumb_url(2, True, "png"))
+
+		remove_package_game_support.delay(package.id)
+
 		flash(gettext("Deleted package"), "success")
 
 		return redirect(url)
 	elif "unapprove" in request.form:
-		if not package.checkPerm(current_user, Permission.UNAPPROVE_PACKAGE):
+		if not package.check_perm(current_user, Permission.UNAPPROVE_PACKAGE):
 			flash(gettext("You don't have permission to do that"), "danger")
-			return redirect(package.getURL("packages.view"))
+			return redirect(package.get_url("packages.view"))
 
 		package.state = PackageState.WIP
 
 		msg = "Unapproved {}, reason={}".format(package.title, reason)
-		addNotification(package.maintainers, current_user, NotificationType.PACKAGE_APPROVAL, msg, package.getURL("packages.view"), package)
-		addAuditLog(AuditSeverity.EDITOR, current_user, msg, package.getURL("packages.view"), package)
+		add_notification(package.maintainers, current_user, NotificationType.PACKAGE_APPROVAL, msg, package.get_url("packages.view"), package)
+		add_audit_log(AuditSeverity.EDITOR, current_user, msg, package.get_url("packages.view"), package)
 
 		db.session.commit()
 
+		post_discord_webhook.delay(current_user.username,
+			"Unapproved package with reason {}\n\n{}".format(reason, package.get_url("packages.view", absolute=True)), True,
+			package.title, package.short_desc, package.get_thumb_url(2, True, "png"))
+
+		remove_package_game_support.delay(package.id)
+
 		flash(gettext("Unapproved package"), "success")
 
-		return redirect(package.getURL("packages.view"))
+		return redirect(package.get_url("packages.view"))
 	else:
 		abort(400)
 
@@ -471,9 +553,9 @@ class PackageMaintainersForm(FlaskForm):
 @login_required
 @is_package_page
 def edit_maintainers(package):
-	if not package.checkPerm(current_user, Permission.EDIT_MAINTAINERS):
+	if not package.check_perm(current_user, Permission.EDIT_MAINTAINERS):
 		flash(gettext("You don't have permission to edit maintainers"), "danger")
-		return redirect(package.getURL("packages.view"))
+		return redirect(package.get_url("packages.view"))
 
 	form = PackageMaintainersForm(formdata=request.form)
 	if request.method == "GET":
@@ -487,15 +569,15 @@ def edit_maintainers(package):
 
 		for user in users:
 			if not user in package.maintainers:
-				if thread:
+				if thread and user not in thread.watchers:
 					thread.watchers.append(user)
-				addNotification(user, current_user, NotificationType.MAINTAINER,
-						"Added you as a maintainer of {}".format(package.title), package.getURL("packages.view"), package)
+				add_notification(user, current_user, NotificationType.MAINTAINER,
+						"Added you as a maintainer of {}".format(package.title), package.get_url("packages.view"), package)
 
 		for user in package.maintainers:
 			if user != package.author and not user in users:
-				addNotification(user, current_user, NotificationType.MAINTAINER,
-						"Removed you as a maintainer of {}".format(package.title), package.getURL("packages.view"), package)
+				add_notification(user, current_user, NotificationType.MAINTAINER,
+						"Removed you as a maintainer of {}".format(package.title), package.get_url("packages.view"), package)
 
 		package.maintainers.clear()
 		package.maintainers.extend(users)
@@ -503,13 +585,13 @@ def edit_maintainers(package):
 			package.maintainers.append(package.author)
 
 		msg = "Edited {} maintainers".format(package.title)
-		addNotification(package.author, current_user, NotificationType.MAINTAINER, msg, package.getURL("packages.view"), package)
+		add_notification(package.author, current_user, NotificationType.MAINTAINER, msg, package.get_url("packages.view"), package)
 		severity = AuditSeverity.NORMAL if current_user == package.author else AuditSeverity.MODERATION
-		addAuditLog(severity, current_user, msg, package.getURL("packages.view"), package)
+		add_audit_log(severity, current_user, msg, package.get_url("packages.view"), package)
 
 		db.session.commit()
 
-		return redirect(package.getURL("packages.view"))
+		return redirect(package.get_url("packages.view"))
 
 	users = User.query.filter(User.rank >= UserRank.NEW_MEMBER).order_by(db.asc(User.username)).all()
 
@@ -530,20 +612,20 @@ def remove_self_maintainers(package):
 	else:
 		package.maintainers.remove(current_user)
 
-		addNotification(package.author, current_user, NotificationType.MAINTAINER,
-				"Removed themself as a maintainer of {}".format(package.title), package.getURL("packages.view"), package)
+		add_notification(package.author, current_user, NotificationType.MAINTAINER,
+				"Removed themself as a maintainer of {}".format(package.title), package.get_url("packages.view"), package)
 
 		db.session.commit()
 
-	return redirect(package.getURL("packages.view"))
+	return redirect(package.get_url("packages.view"))
 
 
 @bp.route("/packages/<author>/<name>/audit/")
 @login_required
 @is_package_page
 def audit(package):
-	if not (package.checkPerm(current_user, Permission.EDIT_PACKAGE) or
-			package.checkPerm(current_user, Permission.APPROVE_NEW)):
+	if not (package.check_perm(current_user, Permission.EDIT_PACKAGE) or
+			package.check_perm(current_user, Permission.APPROVE_NEW)):
 		abort(403)
 
 	page = get_int_or_abort(request.args.get("page"), 1)
@@ -551,7 +633,7 @@ def audit(package):
 
 	query = package.audit_log_entries.order_by(db.desc(AuditLogEntry.created_at))
 
-	pagination = query.paginate(page, num, True)
+	pagination = query.paginate(page=page, per_page=num)
 	return render_template("packages/audit.html", log=pagination.items, pagination=pagination,
 		package=package, tabs=get_package_tabs(current_user, package), current_tab="audit")
 
@@ -564,7 +646,7 @@ class PackageAliasForm(FlaskForm):
 
 
 @bp.route("/packages/<author>/<name>/aliases/")
-@rank_required(UserRank.EDITOR)
+@rank_required(UserRank.ADMIN)
 @is_package_page
 def alias_list(package: Package):
 	return render_template("packages/alias_list.html", package=package)
@@ -572,7 +654,7 @@ def alias_list(package: Package):
 
 @bp.route("/packages/<author>/<name>/aliases/new/", methods=["GET", "POST"])
 @bp.route("/packages/<author>/<name>/aliases/<int:alias_id>/", methods=["GET", "POST"])
-@rank_required(UserRank.EDITOR)
+@rank_required(UserRank.ADMIN)
 @is_package_page
 def alias_create_edit(package: Package, alias_id: int = None):
 	alias = None
@@ -591,7 +673,7 @@ def alias_create_edit(package: Package, alias_id: int = None):
 		form.populate_obj(alias)
 		db.session.commit()
 
-		return redirect(package.getURL("packages.alias_list"))
+		return redirect(package.get_url("packages.alias_list"))
 
 	return render_template("packages/alias_create_edit.html", package=package, form=form)
 
@@ -608,10 +690,10 @@ def share(package):
 @is_package_page
 def similar(package):
 	packages_modnames = {}
-	for metapackage in package.provides:
-		packages_modnames[metapackage] = Package.query.filter(Package.id != package.id,
+	for mname in package.provides:
+		packages_modnames[mname] = Package.query.filter(Package.id != package.id,
 				Package.state != PackageState.DELETED) \
-			.filter(Package.provides.any(PackageProvides.c.metapackage_id == metapackage.id)) \
+			.filter(Package.provides.any(PackageProvides.c.metapackage_id == mname.id)) \
 			.order_by(db.desc(Package.score)) \
 			.all()
 
@@ -624,3 +706,166 @@ def similar(package):
 
 	return render_template("packages/similar.html", package=package,
 			packages_modnames=packages_modnames, similar_topics=similar_topics)
+
+
+def csv_games_check(_form, field):
+	game_names = [name.strip() for name in field.data.split(",")]
+	if len(game_names) == 0 or (len(game_names) == 1 and game_names[0] == ""):
+		return
+
+	missing = set()
+	for game_name in game_names:
+		if game_name.endswith("_game"):
+			game_name = game_name[:-5]
+		if Package.query.filter(and_(Package.state==PackageState.APPROVED, Package.type==PackageType.GAME,
+				or_(Package.name==game_name, Package.name==game_name + "_game"))).count() == 0:
+			missing.add(game_name)
+
+	if len(missing) > 0:
+		raise ValidationError(f"Unable to find game {','.join(missing)}")
+
+
+class GameSupportForm(FlaskForm):
+	enable_support_detection = BooleanField(lazy_gettext("Enable support detection based on dependencies (recommended)"), [Optional()])
+	supported = StringField(lazy_gettext("Supported games"), [Optional(), csv_games_check])
+	unsupported = StringField(lazy_gettext("Unsupported games"), [Optional(), csv_games_check])
+	supports_all_games = BooleanField(lazy_gettext("Supports all games (unless stated) / is game independent"), [Optional()])
+	submit = SubmitField(lazy_gettext("Save"))
+
+
+@bp.route("/packages/<author>/<name>/support/", methods=["GET", "POST"])
+@login_required
+@is_package_page
+def game_support(package):
+	if package.type != PackageType.MOD and package.type != PackageType.TXP:
+		abort(404)
+
+	can_edit = package.check_perm(current_user, Permission.EDIT_PACKAGE)
+	if not (can_edit or package.check_perm(current_user, Permission.APPROVE_NEW)):
+		abort(403)
+
+	if package.releases.count() == 0:
+		flash(gettext("You need at least one release before you can edit game support"), "danger")
+		return redirect(package.get_url('packages.create_release' if package.update_config else 'packages.setup_releases'))
+
+	if package.type == PackageType.MOD and len(package.provides) == 0:
+		flash(gettext("Mod(pack) needs to contain at least one mod. Please create a new release"), "danger")
+		return redirect(package.get_url('packages.list_releases'))
+
+	force_game_detection = package.supported_games.filter(and_(
+		PackageGameSupport.confidence > 1, PackageGameSupport.supports == True)).count() == 0
+
+	can_support_all_games = package.type != PackageType.TXP and \
+		package.supported_games.filter(and_(
+			PackageGameSupport.confidence == 1, PackageGameSupport.supports == True)).count() == 0
+
+	can_override = can_edit
+
+	form = GameSupportForm() if can_edit else None
+	if form and request.method == "GET":
+		form.enable_support_detection.data = package.enable_game_support_detection
+		form.supports_all_games.data = package.supports_all_games and can_support_all_games
+
+		if can_override:
+			manual_supported_games = package.supported_games.filter_by(confidence=11).all()
+			form.supported.data = ", ".join([x.game.name for x in manual_supported_games if x.supports])
+			form.unsupported.data = ", ".join([x.game.name for x in manual_supported_games if not x.supports])
+		else:
+			form.supported = None
+			form.unsupported = None
+
+	if form and form.validate_on_submit():
+		detect_update_needed = False
+
+		if can_override:
+			try:
+				game_is_supported = {}
+				for game in get_games_from_csv(db.session, form.supported.data or ""):
+					game_is_supported[game.id] = True
+				for game in get_games_from_csv(db.session, form.unsupported.data or ""):
+					game_is_supported[game.id] = False
+				game_support_set(db.session, package, game_is_supported, 11)
+				detect_update_needed = True
+			except LogicError as e:
+				flash(e.message, "danger")
+
+		next_url = package.get_url("packages.game_support")
+
+		enable_support_detection = form.enable_support_detection.data or force_game_detection
+		if enable_support_detection != package.enable_game_support_detection:
+			package.enable_game_support_detection = enable_support_detection
+			if package.enable_game_support_detection:
+				detect_update_needed = True
+			else:
+				package.supported_games.filter_by(confidence=1).delete()
+
+		if can_support_all_games:
+			package.supports_all_games = form.supports_all_games.data
+
+		add_audit_log(AuditSeverity.NORMAL, current_user, "Edited game support", package.get_url("packages.game_support"), package)
+
+		db.session.commit()
+
+		if detect_update_needed:
+			release = package.releases.first()
+			if release:
+				task_id = uuid()
+				check_zip_release.apply_async((release.id, release.file_path), task_id=task_id)
+				next_url = url_for("tasks.check", id=task_id, r=next_url)
+
+		return redirect(next_url)
+
+	all_game_support = package.supported_games.all()
+	all_game_support.sort(key=lambda x: -x.game.score)
+	supported_games_list: typing.List[str] = [x.game.name for x in all_game_support if x.supports]
+	if package.supports_all_games:
+		supported_games_list.insert(0, "*")
+	supported_games = ", ".join(supported_games_list)
+	unsupported_games = ", ".join([x.game.name for x in all_game_support if not x.supports])
+
+	mod_conf_lines = ""
+	if supported_games:
+		mod_conf_lines += f"supported_games = {supported_games}"
+	if unsupported_games:
+		mod_conf_lines += f"\nunsupported_games = {unsupported_games}"
+
+	return render_template("packages/game_support.html", package=package, form=form,
+			mod_conf_lines=mod_conf_lines, force_game_detection=force_game_detection,
+			can_support_all_games=can_support_all_games, tabs=get_package_tabs(current_user, package),
+			current_tab="game_support")
+
+
+@bp.route("/packages/<author>/<name>/stats/")
+@is_package_page
+def statistics(package):
+	start = request.args.get("start")
+	end = request.args.get("end")
+
+	return render_template("packages/stats.html",
+		package=package, tabs=get_package_tabs(current_user, package), current_tab="stats",
+		start=start, end=end, options=get_daterange_options(), noindex=start or end)
+
+
+@bp.route("/packages/<author>/<name>/stats.csv")
+@is_package_page
+def stats_csv(package):
+	stats: typing.List[PackageDailyStats] = package.daily_stats.order_by(db.asc(PackageDailyStats.date)).all()
+
+	columns = ["platform_minetest", "platform_other", "reason_new",
+				"reason_dependency", "reason_update"]
+
+	result = "Date, " + ", ".join(columns) + "\n"
+
+	for stat in stats:
+		stat: PackageDailyStats
+		result += stat.date.isoformat()
+		for i, key in enumerate(columns):
+			result += ", " + str(getattr(stat, key))
+		result += "\n"
+
+	date = datetime.datetime.utcnow().date()
+
+	res = make_response(result, 200)
+	res.headers["Content-Disposition"] = f"attachment; filename={package.author.username}_{package.name}_stats_{date.isoformat()}.csv"
+	res.headers["Content-type"] = "text/csv"
+	return res

app/blueprints/packages/releases.py

@@ -13,20 +13,23 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
+import os
 
-
-from flask import *
-from flask_babel import gettext, lazy_gettext
-from flask_login import login_required
+from flask import render_template, request, redirect, flash, url_for, abort
+from flask_babel import lazy_gettext, gettext
+from flask_login import login_required, current_user
 from flask_wtf import FlaskForm
-from wtforms import *
+from wtforms import StringField, SubmitField, BooleanField, RadioField, FileField
+from wtforms.fields.simple import TextAreaField
+from wtforms.validators import InputRequired, Length, Optional
 from wtforms_sqlalchemy.fields import QuerySelectField
-from wtforms.validators import *
 
 from app.logic.releases import do_create_vcs_release, LogicError, do_create_zip_release
-from app.rediscache import has_key, set_key, make_download_key
+from app.models import Package, db, User, PackageState, Permission, UserRank, PackageDailyStats, LuantiRelease, \
+	PackageRelease, PackageUpdateTrigger, PackageUpdateConfig
+from app.rediscache import has_key, set_temp_key, make_download_key
 from app.tasks.importtasks import check_update_config
-from app.utils import *
+from app.utils import is_user_bot, is_package_page, nonempty_or_none, normalize_line_endings
 from . import bp, get_package_tabs
 
 
@@ -39,35 +42,41 @@ def list_releases(package):
 
 
 def get_mt_releases(is_max):
-	query = MinetestRelease.query.order_by(db.asc(MinetestRelease.id))
+	query = LuantiRelease.query.order_by(db.asc(LuantiRelease.id))
 	if is_max:
 		query = query.limit(query.count() - 1)
 	else:
-		query = query.filter(MinetestRelease.name != "0.4.17")
+		query = query.filter(LuantiRelease.name != "0.4.17")
 
 	return query
 
 
 class CreatePackageReleaseForm(FlaskForm):
-	title      = StringField(lazy_gettext("Title"), [InputRequired(), Length(1, 30)])
-	uploadOpt  = RadioField(lazy_gettext("Method"), choices=[("upload", lazy_gettext("File Upload"))], default="upload")
-	vcsLabel   = StringField(lazy_gettext("Git reference (ie: commit hash, branch, or tag)"), default=None)
-	fileUpload = FileField(lazy_gettext("File Upload"))
-	min_rel    = QuerySelectField(lazy_gettext("Minimum Minetest Version"), [InputRequired()],
+	name        = StringField(lazy_gettext("Name"), [InputRequired(), Length(1, 30)])
+	title       = StringField(lazy_gettext("Title"), [Optional(), Length(1, 100)], filters=[nonempty_or_none])
+	release_notes = TextAreaField(lazy_gettext("Release Notes"), [Optional(), Length(1, 5000)],
+			filters=[nonempty_or_none, normalize_line_endings])
+	upload_mode = RadioField(lazy_gettext("Method"), choices=[("upload", lazy_gettext("File Upload"))], default="upload")
+	vcs_label   = StringField(lazy_gettext("Git reference (ie: commit hash, branch, or tag)"), default=None)
+	file_upload = FileField(lazy_gettext("File Upload"))
+	min_rel     = QuerySelectField(lazy_gettext("Minimum Luanti Version"), [InputRequired()],
 			query_factory=lambda: get_mt_releases(False), get_pk=lambda a: a.id, get_label=lambda a: a.name)
-	max_rel    = QuerySelectField(lazy_gettext("Maximum Minetest Version"), [InputRequired()],
+	max_rel     = QuerySelectField(lazy_gettext("Maximum Luanti Version"), [InputRequired()],
 			query_factory=lambda: get_mt_releases(True), get_pk=lambda a: a.id, get_label=lambda a: a.name)
-	submit     = SubmitField(lazy_gettext("Save"))
+	submit      = SubmitField(lazy_gettext("Save"))
 
 
 class EditPackageReleaseForm(FlaskForm):
-	title    = StringField(lazy_gettext("Title"), [InputRequired(), Length(1, 30)])
+	name     = StringField(lazy_gettext("Name"), [InputRequired(), Length(1, 30)])
+	title    = StringField(lazy_gettext("Title"), [InputRequired(), Length(1, 30)], filters=[nonempty_or_none])
+	release_notes = TextAreaField(lazy_gettext("Release Notes"), [Optional(), Length(1, 5000)],
+			filters=[nonempty_or_none, normalize_line_endings])
 	url      = StringField(lazy_gettext("URL"), [Optional()])
 	task_id  = StringField(lazy_gettext("Task ID"), filters = [lambda x: x or None])
 	approved = BooleanField(lazy_gettext("Is Approved"))
-	min_rel  = QuerySelectField(lazy_gettext("Minimum Minetest Version"), [InputRequired()],
+	min_rel  = QuerySelectField(lazy_gettext("Minimum Luanti Version"), [InputRequired()],
 			query_factory=lambda: get_mt_releases(False), get_pk=lambda a: a.id, get_label=lambda a: a.name)
-	max_rel  = QuerySelectField(lazy_gettext("Maximum Minetest Version"), [InputRequired()],
+	max_rel  = QuerySelectField(lazy_gettext("Maximum Luanti Version"), [InputRequired()],
 			query_factory=lambda: get_mt_releases(True), get_pk=lambda a: a.id, get_label=lambda a: a.name)
 	submit   = SubmitField(lazy_gettext("Save"))
 
@@ -76,36 +85,40 @@ class EditPackageReleaseForm(FlaskForm):
 @login_required
 @is_package_page
 def create_release(package):
-	if not package.checkPerm(current_user, Permission.MAKE_RELEASE):
-		return redirect(package.getURL("packages.view"))
+	if current_user.email is None and not current_user.rank.at_least(UserRank.ADMIN):
+		flash(gettext("You must add an email address to your account and confirm it before you can manage packages"), "danger")
+		return redirect(url_for("users.email_notifications"))
+
+	if not package.check_perm(current_user, Permission.MAKE_RELEASE):
+		return redirect(package.get_url("packages.view"))
 
 	# Initial form class from post data and default data
 	form = CreatePackageReleaseForm()
 	if package.repo is not None:
-		form["uploadOpt"].choices = [("vcs", gettext("Import from Git")), ("upload", gettext("Upload .zip file"))]
+		form.upload_mode.choices = [("vcs", gettext("Import from Git")), ("upload", gettext("Upload .zip file"))]
 		if request.method == "GET":
-			form["uploadOpt"].data = "vcs"
-			form.vcsLabel.data = request.args.get("ref")
+			form.upload_mode.data = "vcs"
+			form.vcs_label.data = request.args.get("ref")
 
 	if request.method == "GET":
 		form.title.data = request.args.get("title")
 
 	if form.validate_on_submit():
 		try:
-			if form["uploadOpt"].data == "vcs":
-				rel = do_create_vcs_release(current_user, package, form.title.data,
-						form.vcsLabel.data, form.min_rel.data.getActual(), form.max_rel.data.getActual())
+			if form.upload_mode.data == "vcs":
+				rel = do_create_vcs_release(current_user, package, form.name.data, form.title.data, form.release_notes.data,
+						form.vcs_label.data, form.min_rel.data.get_actual(), form.max_rel.data.get_actual())
 			else:
-				rel = do_create_zip_release(current_user, package, form.title.data,
-						form.fileUpload.data, form.min_rel.data.getActual(), form.max_rel.data.getActual())
-			return redirect(url_for("tasks.check", id=rel.task_id, r=rel.getEditURL()))
+				rel = do_create_zip_release(current_user, package, form.name.data, form.title.data, form.release_notes.data,
+						form.file_upload.data, form.min_rel.data.get_actual(), form.max_rel.data.get_actual())
+			return redirect(url_for("tasks.check", id=rel.task_id, r=rel.get_edit_url()))
 		except LogicError as e:
 			flash(e.message, "danger")
 
 	return render_template("packages/release_new.html", package=package, form=form)
 
 
-@bp.route("/packages/<author>/<name>/releases/<id>/download/")
+@bp.route("/packages/<author>/<name>/releases/<int:id>/download/")
 @is_package_page
 def download_release(package, id):
 	release = PackageRelease.query.get(id)
@@ -114,11 +127,20 @@ def download_release(package, id):
 
 	ip = request.headers.get("X-Forwarded-For") or request.remote_addr
 	if ip is not None and not is_user_bot():
+		user_agent = request.headers.get("User-Agent") or ""
+		is_luanti = user_agent.startswith("Luanti") or user_agent.startswith("Minetest")
+		reason = request.args.get("reason")
+		PackageDailyStats.update(package, is_luanti, reason)
+
 		key = make_download_key(ip, release.package)
 		if not has_key(key):
-			set_key(key, "true")
+			set_temp_key(key, "true")
 
-			bonus = 1
+			bonus = 0
+			if reason == "new":
+				bonus = 1
+			elif reason == "dependency" or reason == "update":
+				bonus = 0.5
 
 			PackageRelease.query.filter_by(id=release.id).update({
 					"downloads": PackageRelease.downloads + 1
@@ -130,23 +152,33 @@ def download_release(package, id):
 					"score": Package.score + bonus
 				})
 
-			db.session.commit()
+		db.session.commit()
 
 	return redirect(release.url)
 
 
-@bp.route("/packages/<author>/<name>/releases/<id>/", methods=["GET", "POST"])
-@login_required
+@bp.route("/packages/<author>/<name>/releases/<int:id>/")
 @is_package_page
-def edit_release(package, id):
-	release : PackageRelease = PackageRelease.query.get(id)
+def view_release(package, id):
+	release: PackageRelease = PackageRelease.query.get(id)
 	if release is None or release.package != package:
 		abort(404)
 
-	canEdit	= package.checkPerm(current_user, Permission.MAKE_RELEASE)
-	canApprove = release.checkPerm(current_user, Permission.APPROVE_RELEASE)
+	return render_template("packages/release_view.html", package=package, release=release)
+
+
+@bp.route("/packages/<author>/<name>/releases/<int:id>/edit/", methods=["GET", "POST"])
+@login_required
+@is_package_page
+def edit_release(package, id):
+	release: PackageRelease = PackageRelease.query.get(id)
+	if release is None or release.package != package:
+		abort(404)
+
+	canEdit	= package.check_perm(current_user, Permission.MAKE_RELEASE)
+	canApprove = release.check_perm(current_user, Permission.APPROVE_RELEASE)
 	if not (canEdit or canApprove):
-		return redirect(package.getURL("packages.view"))
+		return redirect(package.get_url("packages.view"))
 
 	# Initial form class from post data and default data
 	form = EditPackageReleaseForm(formdata=request.form, obj=release)
@@ -157,13 +189,15 @@ def edit_release(package, id):
 
 	if form.validate_on_submit():
 		if canEdit:
-			release.title = form["title"].data
-			release.min_rel = form["min_rel"].data.getActual()
-			release.max_rel = form["max_rel"].data.getActual()
+			release.name = form.name.data
+			release.title = form.title.data
+			release.release_notes = form.release_notes.data
+			release.min_rel = form.min_rel.data.get_actual()
+			release.max_rel = form.max_rel.data.get_actual()
 
-		if package.checkPerm(current_user, Permission.CHANGE_RELEASE_URL):
-			release.url = form["url"].data
-			release.task_id = form["task_id"].data
+		if package.check_perm(current_user, Permission.CHANGE_RELEASE_URL):
+			release.url = form.url.data
+			release.task_id = form.task_id.data
 			if release.task_id is not None:
 				release.task_id = None
 
@@ -173,7 +207,7 @@ def edit_release(package, id):
 			release.approved = False
 
 		db.session.commit()
-		return redirect(package.getURL("packages.list_releases"))
+		return redirect(package.get_url("packages.list_releases"))
 
 	return render_template("packages/release_edit.html", package=package, release=release, form=form)
 
@@ -181,10 +215,10 @@ def edit_release(package, id):
 
 class BulkReleaseForm(FlaskForm):
 	set_min = BooleanField(lazy_gettext("Set Min"))
-	min_rel  = QuerySelectField(lazy_gettext("Minimum Minetest Version"), [InputRequired()],
+	min_rel  = QuerySelectField(lazy_gettext("Minimum Luanti Version"), [InputRequired()],
 			query_factory=lambda: get_mt_releases(False), get_pk=lambda a: a.id, get_label=lambda a: a.name)
 	set_max = BooleanField(lazy_gettext("Set Max"))
-	max_rel  = QuerySelectField(lazy_gettext("Maximum Minetest Version"), [InputRequired()],
+	max_rel  = QuerySelectField(lazy_gettext("Maximum Luanti Version"), [InputRequired()],
 			query_factory=lambda: get_mt_releases(True), get_pk=lambda a: a.id, get_label=lambda a: a.name)
 	only_change_none = BooleanField(lazy_gettext("Only change values previously set as none"))
 	submit   = SubmitField(lazy_gettext("Update"))
@@ -194,8 +228,8 @@ class BulkReleaseForm(FlaskForm):
 @login_required
 @is_package_page
 def bulk_change_release(package):
-	if not package.checkPerm(current_user, Permission.MAKE_RELEASE):
-		return redirect(package.getURL("packages.view"))
+	if not package.check_perm(current_user, Permission.MAKE_RELEASE):
+		return redirect(package.get_url("packages.view"))
 
 	# Initial form class from post data and default data
 	form = BulkReleaseForm()
@@ -206,19 +240,19 @@ def bulk_change_release(package):
 		only_change_none = form.only_change_none.data
 
 		for release in package.releases.all():
-			if form["set_min"].data and (not only_change_none or release.min_rel is None):
-				release.min_rel = form["min_rel"].data.getActual()
-			if form["set_max"].data and (not only_change_none or release.max_rel is None):
-				release.max_rel = form["max_rel"].data.getActual()
+			if form.set_min.data and (not only_change_none or release.min_rel is None):
+				release.min_rel = form.min_rel.data.get_actual()
+			if form.set_max.data and (not only_change_none or release.max_rel is None):
+				release.max_rel = form.max_rel.data.get_actual()
 
 		db.session.commit()
 
-		return redirect(package.getURL("packages.list_releases"))
+		return redirect(package.get_url("packages.list_releases"))
 
 	return render_template("packages/release_bulk_change.html", package=package, form=form)
 
 
-@bp.route("/packages/<author>/<name>/releases/<id>/delete/", methods=["POST"])
+@bp.route("/packages/<author>/<name>/releases/<int:id>/delete/", methods=["POST"])
 @login_required
 @is_package_page
 def delete_release(package, id):
@@ -226,13 +260,16 @@ def delete_release(package, id):
 	if release is None or release.package != package:
 		abort(404)
 
-	if not release.checkPerm(current_user, Permission.DELETE_RELEASE):
-		return redirect(package.getURL("packages.list_releases"))
+	if not release.check_perm(current_user, Permission.DELETE_RELEASE):
+		return redirect(package.get_url("packages.list_releases"))
 
 	db.session.delete(release)
 	db.session.commit()
 
-	return redirect(package.getURL("packages.view"))
+	if release.file_path and os.path.isfile(release.file_path):
+		os.remove(release.file_path)
+
+	return redirect(package.get_url("packages.view"))
 
 
 class PackageUpdateConfigFrom(FlaskForm):
@@ -254,7 +291,7 @@ def set_update_config(package, form):
 		db.session.add(package.update_config)
 
 	form.populate_obj(package.update_config)
-	package.update_config.ref = nonEmptyOrNone(form.ref.data)
+	package.update_config.ref = nonempty_or_none(form.ref.data)
 	package.update_config.make_release = form.action.data == "make_release"
 
 	if package.update_config.trigger == PackageUpdateTrigger.COMMIT:
@@ -280,12 +317,12 @@ def set_update_config(package, form):
 @login_required
 @is_package_page
 def update_config(package):
-	if not package.checkPerm(current_user, Permission.MAKE_RELEASE):
+	if not package.check_perm(current_user, Permission.MAKE_RELEASE):
 		abort(403)
 
 	if not package.repo:
 		flash(gettext("Please add a Git repository URL in order to set up automatic releases"), "danger")
-		return redirect(package.getURL("packages.create_edit"))
+		return redirect(package.get_url("packages.create_edit"))
 
 	form = PackageUpdateConfigFrom(obj=package.update_config)
 	if request.method == "GET":
@@ -309,9 +346,9 @@ def update_config(package):
 
 		if not form.disable.data and package.releases.count() == 0:
 			flash(gettext("Now, please create an initial release"), "success")
-			return redirect(package.getURL("packages.create_release"))
+			return redirect(package.get_url("packages.create_release"))
 
-		return redirect(package.getURL("packages.list_releases"))
+		return redirect(package.get_url("packages.list_releases"))
 
 	return render_template("packages/update_config.html", package=package, form=form)
 
@@ -320,11 +357,11 @@ def update_config(package):
 @login_required
 @is_package_page
 def setup_releases(package):
-	if not package.checkPerm(current_user, Permission.MAKE_RELEASE):
+	if not package.check_perm(current_user, Permission.MAKE_RELEASE):
 		abort(403)
 
 	if package.update_config:
-		return redirect(package.getURL("packages.update_config"))
+		return redirect(package.get_url("packages.update_config"))
 
 	return render_template("packages/release_wizard.html", package=package)
 
@@ -340,7 +377,7 @@ def bulk_update_config(username=None):
 	if not user:
 		abort(404)
 
-	if current_user != user and not current_user.rank.atLeast(UserRank.EDITOR):
+	if current_user != user and not current_user.rank.at_least(UserRank.EDITOR):
 		abort(403)
 
 	form = PackageUpdateConfigFrom()

app/blueprints/packages/reviews.py

@@ -13,21 +13,24 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
 from collections import namedtuple
 
-from flask_babel import gettext, lazy_gettext
-
-from . import bp
-
-from flask import *
+import typing
+from flask import render_template, request, redirect, flash, url_for, abort, jsonify
+from flask_babel import gettext, lazy_gettext, get_locale
 from flask_login import current_user, login_required
 from flask_wtf import FlaskForm
-from wtforms import *
-from wtforms.validators import *
+from wtforms import StringField, TextAreaField, SubmitField, RadioField
+from wtforms.validators import InputRequired, Length, DataRequired
+from wtforms_sqlalchemy.fields import QuerySelectField
+
 from app.models import db, PackageReview, Thread, ThreadReply, NotificationType, PackageReviewVote, Package, UserRank, \
-	Permission, AuditSeverity
-from app.utils import is_package_page, addNotification, get_int_or_abort, isYes, is_safe_url, rank_required, addAuditLog
+	Permission, AuditSeverity, PackageState, Language
 from app.tasks.webhooktasks import post_discord_webhook
+from app.utils import is_package_page, add_notification, get_int_or_abort, is_yes, is_safe_url, rank_required, \
+	add_audit_log, has_blocked_domains, should_return_json, normalize_line_endings
+from . import bp
 
 
 @bp.route("/reviews/")
@@ -35,16 +38,30 @@ def list_reviews():
 	page = get_int_or_abort(request.args.get("page"), 1)
 	num = min(40, get_int_or_abort(request.args.get("n"), 100))
 
-	pagination = PackageReview.query.order_by(db.desc(PackageReview.created_at)).paginate(page, num, True)
+	pagination = PackageReview.query.order_by(db.desc(PackageReview.created_at)).paginate(page=page, per_page=num)
 	return render_template("packages/reviews_list.html", pagination=pagination, reviews=pagination.items)
 
 
+def get_default_language():
+	locale = get_locale()
+	if locale:
+		return Language.query.filter_by(id=locale.language).first()
+
+	return None
+
 class ReviewForm(FlaskForm):
-	title	= StringField(lazy_gettext("Title"), [InputRequired(), Length(3,100)])
-	comment = TextAreaField(lazy_gettext("Comment"), [InputRequired(), Length(10, 2000)])
-	recommends = RadioField(lazy_gettext("Private"), [InputRequired()],
-			choices=[("yes", lazy_gettext("Yes")), ("no", lazy_gettext("No"))])
-	submit  = SubmitField(lazy_gettext("Save"))
+	title = StringField(lazy_gettext("Title"), [InputRequired(), Length(3, 100)])
+	language = QuerySelectField(lazy_gettext("Language"), [DataRequired()],
+			allow_blank=True,
+			query_factory=lambda: Language.query.order_by(db.asc(Language.title)),
+			get_pk=lambda a: a.id,
+			get_label=lambda a: a.title,
+			default=get_default_language)
+	comment = TextAreaField(lazy_gettext("Comment"), [InputRequired(), Length(10, 2000)], filters=[normalize_line_endings])
+	rating = RadioField(lazy_gettext("Rating"), [InputRequired()],
+			choices=[("5", lazy_gettext("Yes")), ("3", lazy_gettext("Neutral")), ("1", lazy_gettext("No"))])
+	btn_submit = SubmitField(lazy_gettext("Save"))
+
 
 @bp.route("/packages/<author>/<name>/review/", methods=["GET", "POST"])
 @login_required
@@ -52,10 +69,13 @@ class ReviewForm(FlaskForm):
 def review(package):
 	if current_user in package.maintainers:
 		flash(gettext("You can't review your own package!"), "danger")
-		return redirect(package.getURL("packages.view"))
+		return redirect(package.get_url("packages.view"))
+
+	if package.state != PackageState.APPROVED:
+		abort(404)
 
 	review = PackageReview.query.filter_by(package=package, author=current_user).first()
-	can_review = review is not None or current_user.canReviewRL()
+	can_review = review is not None or current_user.can_review_ratelimit()
 
 	if not can_review:
 		flash(gettext("You've reviewed too many packages recently. Please wait before trying again, and consider making your reviews more detailed"), "danger")
@@ -65,66 +85,70 @@ def review(package):
 	# Set default values
 	if request.method == "GET" and review:
 		form.title.data = review.thread.title
-		form.recommends.data = "yes" if review.recommends else "no"
-		form.comment.data = review.thread.replies[0].comment
+		form.rating.data = str(review.rating)
+		form.comment.data = review.thread.first_reply.comment
 
 	# Validate and submit
 	elif can_review and form.validate_on_submit():
-		was_new = False
-		if not review:
-			was_new = True
-			review = PackageReview()
-			review.package = package
-			review.author  = current_user
-			db.session.add(review)
-
-		review.recommends = form.recommends.data == "yes"
-
-		thread = review.thread
-		if not thread:
-			thread = Thread()
-			thread.author  = current_user
-			thread.private = False
-			thread.package = package
-			thread.review = review
-			db.session.add(thread)
-
-			thread.watchers.append(current_user)
-
-			reply = ThreadReply()
-			reply.thread  = thread
-			reply.author  = current_user
-			reply.comment = form.comment.data
-			db.session.add(reply)
-
-			thread.replies.append(reply)
+		if has_blocked_domains(form.comment.data, current_user.username, f"review of {package.get_id()}"):
+			flash(gettext("Linking to blocked sites is not allowed"), "danger")
 		else:
-			reply = thread.replies[0]
-			reply.comment = form.comment.data
+			was_new = False
+			if not review:
+				was_new = True
+				review = PackageReview()
+				review.package = package
+				review.author  = current_user
+				db.session.add(review)
 
-		thread.title   = form.title.data
+			review.rating = int(form.rating.data)
+			review.language = form.language.data
 
-		db.session.commit()
+			thread = review.thread
+			if not thread:
+				thread = Thread()
+				thread.author  = current_user
+				thread.private = False
+				thread.package = package
+				thread.review = review
+				db.session.add(thread)
 
-		package.recalcScore()
+				thread.watchers.append(current_user)
 
-		if was_new:
-			notif_msg = "New review '{}'".format(form.title.data)
-			type = NotificationType.NEW_REVIEW
-		else:
-			notif_msg = "Updated review '{}'".format(form.title.data)
-			type = NotificationType.OTHER
+				reply = ThreadReply()
+				reply.thread  = thread
+				reply.author  = current_user
+				reply.comment = form.comment.data
+				db.session.add(reply)
 
-		addNotification(package.maintainers, current_user, type, notif_msg,
-				url_for("threads.view", id=thread.id), package)
+				thread.replies.append(reply)
+			else:
+				reply = thread.first_reply
+				reply.comment = form.comment.data
 
-		if was_new:
-			post_discord_webhook.delay(thread.author.username,
-					"Reviewed {}: {}".format(package.title, thread.getViewURL(absolute=True)), False)
+			thread.title   = form.title.data
 
-		db.session.commit()
+			db.session.commit()
 
-		return redirect(package.getURL("packages.view"))
+			package.recalculate_score()
+
+			if was_new:
+				notif_msg = "New review '{}'".format(form.title.data)
+				type = NotificationType.NEW_REVIEW
+			else:
+				notif_msg = "Updated review '{}'".format(form.title.data)
+				type = NotificationType.OTHER
+
+			add_notification(package.maintainers, current_user, type, notif_msg,
+							 url_for("threads.view", id=thread.id), package)
+
+			if was_new:
+				msg = f"Reviewed {package.title} ({review.language.title}): {thread.get_view_url(absolute=True)}"
+				post_discord_webhook.delay(thread.author.display_name, msg, False)
+
+			db.session.commit()
+
+			return redirect(package.get_url("packages.view"))
 
 	return render_template("packages/review_create_edit.html",
 			form=form, package=package, review=review)
@@ -140,7 +164,7 @@ def delete_review(package, reviewer):
 	if review is None or review.package != package:
 		abort(404)
 
-	if not review.checkPerm(current_user, Permission.DELETE_REVIEW):
+	if not review.check_perm(current_user, Permission.DELETE_REVIEW):
 		abort(403)
 
 	thread = review.thread
@@ -155,35 +179,33 @@ def delete_review(package, reviewer):
 	thread.review = None
 
 	msg = "Converted review by {} to thread".format(review.author.display_name)
-	addAuditLog(AuditSeverity.MODERATION if current_user.username != reviewer else AuditSeverity.NORMAL,
-			current_user, msg, thread.getViewURL(), thread.package)
+	add_audit_log(AuditSeverity.MODERATION if current_user.username != reviewer else AuditSeverity.NORMAL,
+				  current_user, msg, thread.get_view_url(), thread.package)
 
 	notif_msg = "Deleted review '{}', comments were kept as a thread".format(thread.title)
-	addNotification(package.maintainers, current_user, NotificationType.OTHER, notif_msg, url_for("threads.view", id=thread.id), package)
+	add_notification(package.maintainers, current_user, NotificationType.OTHER, notif_msg, url_for("threads.view", id=thread.id), package)
 
 	db.session.delete(review)
 
-	package.recalcScore()
+	package.recalculate_score()
 
 	db.session.commit()
 
-	return redirect(thread.getViewURL())
+	return redirect(thread.get_view_url())
 
 
-def handle_review_vote(package: Package, review_id: int):
+def handle_review_vote(package: Package, review_id: int) -> typing.Optional[str]:
 	if current_user in package.maintainers:
-		flash(gettext("You can't vote on the reviews on your own package!"), "danger")
-		return
+		return gettext("You can't vote on the reviews on your own package!")
 
 	review: PackageReview = PackageReview.query.get(review_id)
 	if review is None or review.package != package:
 		abort(404)
 
 	if review.author == current_user:
-		flash(gettext("You can't vote on your own reviews!"), "danger")
-		return
+		return gettext("You can't vote on your own reviews!")
 
-	is_positive = isYes(request.form["is_positive"])
+	is_positive = is_yes(request.form["is_positive"])
 
 	vote = PackageReviewVote.query.filter_by(review=review, user=current_user).first()
 	if vote is None:
@@ -205,14 +227,21 @@ def handle_review_vote(package: Package, review_id: int):
 @login_required
 @is_package_page
 def review_vote(package, review_id):
-	handle_review_vote(package, review_id)
+	msg = handle_review_vote(package, review_id)
+	if should_return_json():
+		if msg:
+			return jsonify({"success": False, "error": msg}), 403
+		else:
+			return jsonify({"success": True})
+
+	if msg:
+		flash(msg, "danger")
 
 	next_url = request.args.get("r")
 	if next_url and is_safe_url(next_url):
 		return redirect(next_url)
 	else:
-		return redirect(review.thread.getViewURL())
-
+		return redirect(review.thread.get_view_url())
 
 
 @bp.route("/packages/<author>/<name>/review-votes/")
@@ -221,7 +250,7 @@ def review_vote(package, review_id):
 def review_votes(package):
 	user_biases = {}
 	for review in package.reviews:
-		review_sign = 1 if review.recommends else -1
+		review_sign = review.as_weight()
 		for vote in review.votes:
 			user_biases[vote.user.username] = user_biases.get(vote.user.username, [0, 0])
 			vote_sign = 1 if vote.is_positive else -1
@@ -231,15 +260,17 @@ def review_votes(package):
 			else:
 				user_biases[vote.user.username][1] += 1
 
+	reviews = package.reviews.all()
+
 	BiasInfo = namedtuple("BiasInfo", "username balance with_ against no_vote perc_with")
 	user_biases_info = []
 	for username, bias in user_biases.items():
 		total_votes = bias[0] + bias[1]
 		balance = bias[0] - bias[1]
 		perc_with = round((100 * bias[0]) / total_votes)
-		user_biases_info.append(BiasInfo(username, balance, bias[0], bias[1], len(package.reviews) - total_votes, perc_with))
+		user_biases_info.append(BiasInfo(username, balance, bias[0], bias[1], len(reviews) - total_votes, perc_with))
 
 	user_biases_info.sort(key=lambda x: -abs(x.balance))
 
-	return render_template("packages/review_votes.html", form=form, package=package, reviews=package.reviews,
+	return render_template("packages/review_votes.html", package=package, reviews=reviews,
 			user_biases=user_biases_info)

app/blueprints/packages/screenshots.py

@@ -13,25 +13,27 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
+import os
 
-
-from flask import *
-from flask_babel import gettext, lazy_gettext
+from flask import render_template, request, redirect, flash, url_for, abort
+from flask_babel import lazy_gettext, gettext
+from flask_login import login_required, current_user
 from flask_wtf import FlaskForm
-from flask_login import login_required
-from wtforms import *
+from flask_wtf.file import FileRequired
+from wtforms import StringField, SubmitField, BooleanField, FileField
+from wtforms.validators import Length, DataRequired, Optional
 from wtforms_sqlalchemy.fields import QuerySelectField
-from wtforms.validators import *
 
-from app.utils import *
-from . import bp, get_package_tabs
 from app.logic.LogicError import LogicError
 from app.logic.screenshots import do_create_screenshot, do_order_screenshots
+from . import bp, get_package_tabs
+from app.models import Permission, db, PackageScreenshot
+from app.utils import is_package_page
 
 
 class CreateScreenshotForm(FlaskForm):
 	title	   = StringField(lazy_gettext("Title/Caption"), [Optional(), Length(-1, 100)])
-	fileUpload = FileField(lazy_gettext("File Upload"), [InputRequired()])
+	file_upload = FileField(lazy_gettext("File Upload"), [FileRequired()])
 	submit	   = SubmitField(lazy_gettext("Save"))
 
 
@@ -50,11 +52,8 @@ class EditPackageScreenshotsForm(FlaskForm):
 @login_required
 @is_package_page
 def screenshots(package):
-	if not package.checkPerm(current_user, Permission.ADD_SCREENSHOTS):
-		return redirect(package.getURL("packages.view"))
-
-	if package.screenshots.count() == 0:
-		return redirect(package.getURL("packages.create_screenshot"))
+	if not package.check_perm(current_user, Permission.ADD_SCREENSHOTS):
+		return redirect(package.get_url("packages.view"))
 
 	form = EditPackageScreenshotsForm(obj=package)
 	form.cover_image.query = package.screenshots
@@ -64,7 +63,7 @@ def screenshots(package):
 		if order:
 			try:
 				do_order_screenshots(current_user, package, order.split(","))
-				return redirect(package.getURL("packages.view"))
+				return redirect(package.get_url("packages.view"))
 			except LogicError as e:
 				flash(e.message, "danger")
 
@@ -80,22 +79,22 @@ def screenshots(package):
 @login_required
 @is_package_page
 def create_screenshot(package):
-	if not package.checkPerm(current_user, Permission.ADD_SCREENSHOTS):
-		return redirect(package.getURL("packages.view"))
+	if not package.check_perm(current_user, Permission.ADD_SCREENSHOTS):
+		return redirect(package.get_url("packages.view"))
 
 	# Initial form class from post data and default data
 	form = CreateScreenshotForm()
 	if form.validate_on_submit():
 		try:
-			do_create_screenshot(current_user, package, form.title.data, form.fileUpload.data, False)
-			return redirect(package.getURL("packages.screenshots"))
+			do_create_screenshot(current_user, package, form.title.data, form.file_upload.data, False)
+			return redirect(package.get_url("packages.screenshots"))
 		except LogicError as e:
 			flash(e.message, "danger")
 
 	return render_template("packages/screenshot_new.html", package=package, form=form)
 
 
-@bp.route("/packages/<author>/<name>/screenshots/<id>/edit/", methods=["GET", "POST"])
+@bp.route("/packages/<author>/<name>/screenshots/<int:id>/edit/", methods=["GET", "POST"])
 @login_required
 @is_package_page
 def edit_screenshot(package, id):
@@ -103,31 +102,31 @@ def edit_screenshot(package, id):
 	if screenshot is None or screenshot.package != package:
 		abort(404)
 
-	canEdit	= package.checkPerm(current_user, Permission.ADD_SCREENSHOTS)
-	canApprove = package.checkPerm(current_user, Permission.APPROVE_SCREENSHOT)
-	if not (canEdit or canApprove):
-		return redirect(package.getURL("packages.screenshots"))
+	can_edit = package.check_perm(current_user, Permission.ADD_SCREENSHOTS)
+	can_approve = package.check_perm(current_user, Permission.APPROVE_SCREENSHOT)
+	if not (can_edit or can_approve):
+		return redirect(package.get_url("packages.screenshots"))
 
 	# Initial form class from post data and default data
 	form = EditScreenshotForm(obj=screenshot)
 	if form.validate_on_submit():
-		wasApproved = screenshot.approved
+		was_approved = screenshot.approved
 
-		if canEdit:
-			screenshot.title = form["title"].data or "Untitled"
+		if can_edit:
+			screenshot.title = form.title.data or "Untitled"
 
-		if canApprove:
-			screenshot.approved = form["approved"].data
+		if can_approve:
+			screenshot.approved = form.approved.data
 		else:
-			screenshot.approved = wasApproved
+			screenshot.approved = was_approved
 
 		db.session.commit()
-		return redirect(package.getURL("packages.screenshots"))
+		return redirect(package.get_url("packages.screenshots"))
 
 	return render_template("packages/screenshot_edit.html", package=package, screenshot=screenshot, form=form)
 
 
-@bp.route("/packages/<author>/<name>/screenshots/<id>/delete/", methods=["POST"])
+@bp.route("/packages/<author>/<name>/screenshots/<int:id>/delete/", methods=["POST"])
 @login_required
 @is_package_page
 def delete_screenshot(package, id):
@@ -135,7 +134,7 @@ def delete_screenshot(package, id):
 	if screenshot is None or screenshot.package != package:
 		abort(404)
 
-	if not package.checkPerm(current_user, Permission.ADD_SCREENSHOTS):
+	if not package.check_perm(current_user, Permission.ADD_SCREENSHOTS):
 		flash(gettext("Permission denied"), "danger")
 		return redirect(url_for("homepage.home"))
 
@@ -146,4 +145,6 @@ def delete_screenshot(package, id):
 	db.session.delete(screenshot)
 	db.session.commit()
 
-	return redirect(package.getURL("packages.screenshots"))
+	os.remove(screenshot.file_path)
+
+	return redirect(package.get_url("packages.screenshots"))

app/blueprints/report/__init__.py

@@ -14,51 +14,169 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-from flask import Blueprint, request, render_template, url_for
+from flask import Blueprint, request, render_template, url_for, abort, flash
 from flask_babel import lazy_gettext
 from flask_login import current_user
 from flask_wtf import FlaskForm
 from werkzeug.utils import redirect
-from wtforms import TextAreaField, SubmitField
-from wtforms.validators import InputRequired, Length
+from wtforms import TextAreaField, SubmitField, URLField, StringField, SelectField, FileField
+from wtforms.validators import InputRequired, Length, Optional, DataRequired
 
-from app.models import User, UserRank
-from app.tasks.emails import send_user_email
+from app.logic.uploads import upload_file
+from app.models import User, UserRank, Report, db, AuditSeverity, ReportCategory, Thread, Permission, ReportAttachment
 from app.tasks.webhooktasks import post_discord_webhook
-from app.utils import isNo, abs_url_samesite
+from app.utils import (is_no, abs_url_samesite, normalize_line_endings, rank_required, add_audit_log, abs_url_for,
+					   random_string, add_replies)
 
 bp = Blueprint("report", __name__)
 
 
 class ReportForm(FlaskForm):
-	message = TextAreaField(lazy_gettext("Message"), [InputRequired(), Length(10, 10000)])
+	category = SelectField(lazy_gettext("Category"), [DataRequired()], choices=ReportCategory.choices(with_none=True), coerce=ReportCategory.coerce)
+
+	url = URLField(lazy_gettext("URL"), [Optional()])
+	title = StringField(lazy_gettext("Subject / Title"), [InputRequired(), Length(10, 300)])
+	message = TextAreaField(lazy_gettext("Message"), [Optional(), Length(0, 10000)], filters=[normalize_line_endings])
+	file_upload = FileField(lazy_gettext("Image Upload"), [Optional()])
 	submit = SubmitField(lazy_gettext("Report"))
 
 
 @bp.route("/report/", methods=["GET", "POST"])
 def report():
-	is_anon = not current_user.is_authenticated or not isNo(request.args.get("anon"))
+	is_anon = not current_user.is_authenticated or not is_no(request.args.get("anon"))
 
 	url = request.args.get("url")
 	if url:
+		if url.startswith("/report/"):
+			abort(404)
+
 		url = abs_url_samesite(url)
 
-	form = ReportForm(formdata=request.form)
-	if form.validate_on_submit():
+	form = ReportForm() if current_user.is_authenticated else None
+	if form and request.method == "GET":
+		try:
+			form.category.data = ReportCategory.coerce(request.args.get("category"))
+		except KeyError:
+			pass
+		form.url.data = url
+		form.title.data = request.args.get("title", "")
+
+	if form and form.validate_on_submit():
+		report = Report()
+		report.id = random_string(8)
+		report.user = current_user if current_user.is_authenticated else None
+		form.populate_obj(report)
+
 		if current_user.is_authenticated:
-			user_info = f"{current_user.username}"
+			thread = Thread()
+			thread.title = f"Report: {form.title.data}"
+			thread.author = current_user
+			thread.private = True
+			thread.watchers.extend(User.query.filter(User.rank >= UserRank.MODERATOR).all())
+			db.session.add(thread)
+			db.session.flush()
+
+			report.thread = thread
+
+			add_replies(thread, current_user, f"**{report.category.title} report created**\n\n{form.message.data}")
 		else:
-			user_info = request.headers.get("X-Forwarded-For") or request.remote_addr
+			ip_addr = request.headers.get("X-Forwarded-For") or request.remote_addr
+			report.message = ip_addr + "\n\n" + report.message
 
-		text = f"{url}\n\n{form.message.data}"
+		db.session.add(report)
+		db.session.flush()
 
-		task = None
-		for admin in User.query.filter_by(rank=UserRank.ADMIN).all():
-			task = send_user_email.delay(admin.email, admin.locale or "en",
-					f"User report from {user_info}", text)
+		if form.file_upload.data:
+			atmt = ReportAttachment()
+			report.attachments.add(atmt)
+			uploaded_url, _ = upload_file(form.file_upload.data, "image", lazy_gettext("a PNG, JPEG, or WebP image file"))
+			atmt.url = uploaded_url
+			db.session.add(atmt)
 
-		post_discord_webhook.delay(None if is_anon else current_user.username, f"**New Report**\n{url}\n\n{form.message.data}", True)
+		if current_user.is_authenticated:
+			add_audit_log(AuditSeverity.USER, current_user, f"New report: {report.title}",
+					url_for("report.view", rid=report.id))
 
-		return redirect(url_for("tasks.check", id=task.id, r=url_for("homepage.home")))
+		db.session.commit()
 
-	return render_template("report/index.html", form=form, url=url, is_anon=is_anon)
+		abs_url = abs_url_for("report.view", rid=report.id)
+		msg = f"**New Report**\nReport on `{report.url}`\n\n{report.title}\n\nView: {abs_url}"
+		post_discord_webhook.delay(None if is_anon else current_user.username, msg, True)
+
+		return redirect(url_for("report.report_received", rid=report.id))
+
+	return render_template("report/report.html", form=form, url=url, is_anon=is_anon, noindex=url is not None)
+
+
+@bp.route("/report/received/")
+def report_received():
+	rid = request.args.get("rid")
+	report = Report.query.get_or_404(rid)
+	return render_template("report/report_received.html", report=report)
+
+
+@bp.route("/admin/reports/")
+@rank_required(UserRank.EDITOR)
+def list_all():
+	reports = Report.query.order_by(db.asc(Report.is_resolved), db.asc(Report.created_at)).all()
+	return render_template("report/list.html", reports=reports)
+
+
+@bp.route("/admin/reports/<rid>/", methods=["GET", "POST"])
+def view(rid: str):
+	report = Report.query.get_or_404(rid)
+	if not report.check_perm(current_user, Permission.SEE_REPORT):
+		abort(404)
+
+	if request.method == "POST":
+		if report.is_resolved:
+			if "reopen" in request.form:
+				report.is_resolved = False
+				url = url_for("report.view", rid=report.id)
+				add_audit_log(AuditSeverity.MODERATION, current_user, f"Reopened report \"{report.title}\"", url)
+
+				if report.thread:
+					add_replies(report.thread, current_user, f"Reopened report", is_status_update=True)
+
+				db.session.commit()
+		else:
+			if "completed" in request.form:
+				outcome = "as completed"
+			elif "removed" in request.form:
+				outcome = "as content removed"
+			elif "invalid" in request.form:
+				outcome = "without action"
+				if report.thread:
+					flash("Make sure to comment why the report is invalid in the thread", "warning")
+			else:
+				abort(400)
+
+			report.is_resolved = True
+			url = url_for("report.view", rid=report.id)
+			add_audit_log(AuditSeverity.MODERATION, current_user, f"Report closed {outcome} \"{report.title}\"", url)
+
+			if report.thread:
+				add_replies(report.thread, current_user, f"Closed report {outcome}", is_status_update=True)
+
+			db.session.commit()
+
+	return render_template("report/view.html", report=report)
+
+
+@bp.route("/admin/reports/<rid>/edit/", methods=["GET", "POST"])
+def edit(rid: str):
+	report = Report.query.get_or_404(rid)
+	if not report.check_perm(current_user, Permission.SEE_REPORT):
+		abort(404)
+
+	form = ReportForm(request.form, obj=report)
+	form.submit.label.text = lazy_gettext("Save")
+	if form.validate_on_submit():
+		form.populate_obj(report)
+		url = url_for("report.view", rid=report.id)
+		add_audit_log(AuditSeverity.MODERATION, current_user, f"Edited report \"{report.title}\"", url)
+		db.session.commit()
+
+		return redirect(url_for("report.view", rid=report.id))
+
+	return render_template("report/edit.html", report=report, form=form)

app/blueprints/tasks/__init__.py

@@ -14,14 +14,14 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-
-from flask import *
-from flask_login import login_required
+from flask import Blueprint, jsonify, url_for, request, redirect, render_template
+from flask_login import login_required, current_user
 
 from app import csrf
+from app.models import UserRank
 from app.tasks import celery
-from app.tasks.importtasks import getMeta
-from app.utils import *
+from app.tasks.importtasks import get_meta
+from app.utils import should_return_json
 
 bp = Blueprint("tasks", __name__)
 
@@ -30,9 +30,10 @@ bp = Blueprint("tasks", __name__)
 @bp.route("/tasks/getmeta/new/", methods=["POST"])
 @login_required
 def start_getmeta():
+	from flask import request
 	author = request.args.get("author")
 	author = current_user.forums_username if author is None else author
-	aresult = getMeta.delay(request.args.get("url"), author)
+	aresult = get_meta.delay(request.args.get("url"), author)
 	return jsonify({
 		"poll_url": url_for("tasks.check", id=aresult.id),
 	})
@@ -51,10 +52,13 @@ def check(id):
 				'status': status,
 			}
 
-		if current_user.is_authenticated and current_user.rank.atLeast(UserRank.ADMIN):
+		if current_user.is_authenticated and current_user.rank.at_least(UserRank.ADMIN):
 			info["error"] = str(traceback)
 		elif str(result)[1:12] == "TaskError: ":
-			info["error"] = str(result)[12:-1]
+			if hasattr(result, "value"):
+				info["error"] = result.value
+			else:
+				info["error"] = str(result)
 		else:
 			info["error"] = "Unknown server error"
 	else:
@@ -64,7 +68,7 @@ def check(id):
 				'result': result,
 			}
 
-	if shouldReturnJson():
+	if should_return_json():
 		return jsonify(info)
 	else:
 		r = request.args.get("r")

app/blueprints/threads/__init__.py

@@ -13,8 +13,10 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from flask import *
+
+from flask import Blueprint, request, render_template, abort, flash, redirect, url_for
 from flask_babel import gettext, lazy_gettext
+from sqlalchemy.orm import selectinload
 
 from app.markdown import get_user_mentions, render_markdown
 from app.tasks.webhooktasks import post_discord_webhook
@@ -22,11 +24,13 @@ from app.tasks.webhooktasks import post_discord_webhook
 bp = Blueprint("threads", __name__)
 
 from flask_login import current_user, login_required
-from app.models import *
-from app.utils import addNotification, isYes, addAuditLog, get_system_user
+from app.models import Package, db, User, Permission, Thread, UserRank, AuditSeverity, \
+	NotificationType, ThreadReply
+from app.utils import add_notification, is_yes, add_audit_log, get_system_user, has_blocked_domains, \
+	normalize_line_endings
 from flask_wtf import FlaskForm
-from wtforms import *
-from wtforms.validators import *
+from wtforms import StringField, TextAreaField, SubmitField
+from wtforms.validators import InputRequired, Length
 from app.utils import get_int_or_abort
 
 
@@ -40,7 +44,7 @@ def list_all():
 	pid = request.args.get("pid")
 	if pid:
 		pid = get_int_or_abort(pid)
-		package = Package.query.get(pid)
+		package = Package.query.get_or_404(pid)
 		query = query.filter_by(package=package)
 
 	query = query.filter_by(review_id=None)
@@ -50,16 +54,17 @@ def list_all():
 	page = get_int_or_abort(request.args.get("page"), 1)
 	num = min(40, get_int_or_abort(request.args.get("n"), 100))
 
-	pagination = query.paginate(page, num, True)
+	pagination = query.paginate(page=page, per_page=num)
 
-	return render_template("threads/list.html", pagination=pagination, threads=pagination.items, package=package)
+	return render_template("threads/list.html", pagination=pagination, threads=pagination.items,
+			package=package, noindex=pid)
 
 
 @bp.route("/threads/<int:id>/subscribe/", methods=["POST"])
 @login_required
 def subscribe(id):
 	thread = Thread.query.get(id)
-	if thread is None or not thread.checkPerm(current_user, Permission.SEE_THREAD):
+	if thread is None or not thread.check_perm(current_user, Permission.SEE_THREAD):
 		abort(404)
 
 	if current_user in thread.watchers:
@@ -69,14 +74,14 @@ def subscribe(id):
 		thread.watchers.append(current_user)
 		db.session.commit()
 
-	return redirect(thread.getViewURL())
+	return redirect(thread.get_view_url())
 
 
 @bp.route("/threads/<int:id>/unsubscribe/", methods=["POST"])
 @login_required
 def unsubscribe(id):
 	thread = Thread.query.get(id)
-	if thread is None or not thread.checkPerm(current_user, Permission.SEE_THREAD):
+	if thread is None or not thread.check_perm(current_user, Permission.SEE_THREAD):
 		abort(404)
 
 	if current_user in thread.watchers:
@@ -86,21 +91,20 @@ def unsubscribe(id):
 	else:
 		flash(gettext("Already not subscribed!"), "success")
 
-	return redirect(thread.getViewURL())
+	return redirect(thread.get_view_url())
 
 
 @bp.route("/threads/<int:id>/set-lock/", methods=["POST"])
 @login_required
 def set_lock(id):
 	thread = Thread.query.get(id)
-	if thread is None or not thread.checkPerm(current_user, Permission.LOCK_THREAD):
+	if thread is None or not thread.check_perm(current_user, Permission.LOCK_THREAD):
 		abort(404)
 
-	thread.locked = isYes(request.args.get("lock"))
+	thread.locked = is_yes(request.args.get("lock"))
 	if thread.locked is None:
 		abort(400)
 
-	msg = None
 	if thread.locked:
 		msg = "Locked thread '{}'".format(thread.title)
 		flash(gettext("Locked thread"), "success")
@@ -108,19 +112,19 @@ def set_lock(id):
 		msg = "Unlocked thread '{}'".format(thread.title)
 		flash(gettext("Unlocked thread"), "success")
 
-	addNotification(thread.watchers, current_user, NotificationType.OTHER, msg, thread.getViewURL(), thread.package)
-	addAuditLog(AuditSeverity.MODERATION, current_user, msg, thread.getViewURL(), thread.package)
+	add_notification(thread.watchers, current_user, NotificationType.OTHER, msg, thread.get_view_url(), thread.package)
+	add_audit_log(AuditSeverity.MODERATION, current_user, msg, thread.get_view_url(), thread.package)
 
 	db.session.commit()
 
-	return redirect(thread.getViewURL())
+	return redirect(thread.get_view_url())
 
 
 @bp.route("/threads/<int:id>/delete/", methods=["GET", "POST"])
 @login_required
 def delete_thread(id):
 	thread = Thread.query.get(id)
-	if thread is None or not thread.checkPerm(current_user, Permission.DELETE_THREAD):
+	if thread is None or not thread.check_perm(current_user, Permission.DELETE_THREAD):
 		abort(404)
 
 	if request.method == "GET":
@@ -132,7 +136,7 @@ def delete_thread(id):
 
 	db.session.delete(thread)
 
-	addAuditLog(AuditSeverity.MODERATION, current_user, msg, None, thread.package, summary)
+	add_audit_log(AuditSeverity.MODERATION, current_user, msg, None, thread.package, summary)
 
 	db.session.commit()
 
@@ -154,28 +158,28 @@ def delete_reply(id):
 	if reply is None or reply.thread != thread:
 		abort(404)
 
-	if thread.replies[0] == reply:
+	if thread.first_reply == reply:
 		flash(gettext("Cannot delete thread opening post!"), "danger")
-		return redirect(thread.getViewURL())
+		return redirect(thread.get_view_url())
 
-	if not reply.checkPerm(current_user, Permission.DELETE_REPLY):
+	if not reply.check_perm(current_user, Permission.DELETE_REPLY):
 		abort(403)
 
 	if request.method == "GET":
 		return render_template("threads/delete_reply.html", thread=thread, reply=reply)
 
 	msg = "Deleted reply by {}".format(reply.author.display_name)
-	addAuditLog(AuditSeverity.MODERATION, current_user, msg, thread.getViewURL(), thread.package, reply.comment)
+	add_audit_log(AuditSeverity.MODERATION, current_user, msg, thread.get_view_url(), thread.package, reply.comment)
 
 	db.session.delete(reply)
 	db.session.commit()
 
-	return redirect(thread.getViewURL())
+	return redirect(thread.get_view_url())
 
 
 class CommentForm(FlaskForm):
-	comment = TextAreaField(lazy_gettext("Comment"), [InputRequired(), Length(10, 2000)])
-	submit  = SubmitField(lazy_gettext("Comment"))
+	comment = TextAreaField(lazy_gettext("Comment"), [InputRequired(), Length(2, 2000)], filters=[normalize_line_endings])
+	btn_submit = SubmitField(lazy_gettext("Comment"))
 
 
 @bp.route("/threads/<int:id>/edit/", methods=["GET", "POST"])
@@ -189,27 +193,29 @@ def edit_reply(id):
 	if reply_id is None:
 		abort(404)
 
-	reply = ThreadReply.query.get(reply_id)
+	reply: ThreadReply = ThreadReply.query.get(reply_id)
 	if reply is None or reply.thread != thread:
 		abort(404)
 
-	if not reply.checkPerm(current_user, Permission.EDIT_REPLY):
+	if not reply.check_perm(current_user, Permission.EDIT_REPLY):
 		abort(403)
 
 	form = CommentForm(formdata=request.form, obj=reply)
 	if form.validate_on_submit():
 		comment = form.comment.data
+		if has_blocked_domains(comment, current_user.username, f"edit to reply {reply.get_url(True)}"):
+			flash(gettext("Linking to blocked sites is not allowed"), "danger")
+		else:
+			msg = "Edited reply by {}".format(reply.author.display_name)
+			severity = AuditSeverity.NORMAL if current_user == reply.author else AuditSeverity.MODERATION
+			add_notification(reply.author, current_user, NotificationType.OTHER, msg, thread.get_view_url(), thread.package)
+			add_audit_log(severity, current_user, msg, thread.get_view_url(), thread.package, reply.comment)
 
-		msg = "Edited reply by {}".format(reply.author.display_name)
-		severity = AuditSeverity.NORMAL if current_user == reply.author else AuditSeverity.MODERATION
-		addNotification(reply.author, current_user, NotificationType.OTHER, msg, thread.getViewURL(), thread.package)
-		addAuditLog(severity, current_user, msg, thread.getViewURL(), thread.package, reply.comment)
+			reply.comment = comment
 
-		reply.comment = comment
+			db.session.commit()
 
-		db.session.commit()
-
-		return redirect(thread.getViewURL())
+			return redirect(thread.get_view_url())
 
 	return render_template("threads/edit_reply.html", thread=thread, reply=reply, form=form)
 
@@ -217,18 +223,22 @@ def edit_reply(id):
 @bp.route("/threads/<int:id>/", methods=["GET", "POST"])
 def view(id):
 	thread: Thread = Thread.query.get(id)
-	if thread is None or not thread.checkPerm(current_user, Permission.SEE_THREAD):
+	if thread is None or not thread.check_perm(current_user, Permission.SEE_THREAD):
 		abort(404)
 
-	form = CommentForm(formdata=request.form) if thread.checkPerm(current_user, Permission.COMMENT_THREAD) else None
+	form = CommentForm(formdata=request.form) if thread.check_perm(current_user, Permission.COMMENT_THREAD) else None
 
 	# Check that title is none to load comments into textarea if redirected from new thread page
 	if form and form.validate_on_submit() and request.form.get("title") is None:
 		comment = form.comment.data
 
-		if not current_user.canCommentRL():
+		if not current_user.can_comment_ratelimit():
 			flash(gettext("Please wait before commenting again"), "danger")
-			return redirect(thread.getViewURL())
+			return redirect(thread.get_view_url())
+
+		if has_blocked_domains(comment, current_user.username, f"reply to {thread.get_view_url(True)}"):
+			flash(gettext("Linking to blocked sites is not allowed"), "danger")
+			return render_template("threads/view.html", thread=thread, form=form)
 
 		reply = ThreadReply()
 		reply.author = current_user
@@ -244,34 +254,37 @@ def view(id):
 			if mentioned is None:
 				continue
 
-			msg = "Mentioned by {} in '{}'".format(current_user.display_name, thread.title)
-			addNotification(mentioned, current_user, NotificationType.THREAD_REPLY,
-					msg, thread.getViewURL(), thread.package)
+			if not thread.check_perm(mentioned, Permission.SEE_THREAD):
+				continue
 
-			thread.watchers.append(mentioned)
+			msg = "Mentioned by {} in '{}'".format(current_user.display_name, thread.title)
+			add_notification(mentioned, current_user, NotificationType.THREAD_REPLY,
+							 msg, thread.get_view_url(), thread.package)
+
+			if mentioned not in thread.watchers:
+				thread.watchers.append(mentioned)
 
 		msg = "New comment on '{}'".format(thread.title)
-		addNotification(thread.watchers, current_user, NotificationType.THREAD_REPLY, msg, thread.getViewURL(), thread.package)
+		add_notification(thread.watchers, current_user, NotificationType.THREAD_REPLY, msg, thread.get_view_url(), thread.package)
 
 		if thread.author == get_system_user():
 			approvers = User.query.filter(User.rank >= UserRank.APPROVER).all()
-			addNotification(approvers, current_user, NotificationType.EDITOR_MISC, msg,
-					thread.getViewURL(), thread.package)
-			post_discord_webhook.delay(current_user.username,
-					"Replied to bot messages: {}".format(thread.getViewURL(absolute=True)), True)
+			add_notification(approvers, current_user, NotificationType.EDITOR_MISC, msg,
+							 thread.get_view_url(), thread.package)
+			post_discord_webhook.delay(current_user.display_name,
+					"Replied to bot messages: {}".format(thread.get_view_url(absolute=True)), True)
 
 		db.session.commit()
 
-		return redirect(thread.getViewURL())
+		return redirect(thread.get_view_url())
 
 	return render_template("threads/view.html", thread=thread, form=form)
 
 
 class ThreadForm(FlaskForm):
 	title	= StringField(lazy_gettext("Title"), [InputRequired(), Length(3,100)])
-	comment = TextAreaField(lazy_gettext("Comment"), [InputRequired(), Length(10, 2000)])
-	private = BooleanField(lazy_gettext("Private"))
-	submit  = SubmitField(lazy_gettext("Open Thread"))
+	comment = TextAreaField(lazy_gettext("Comment"), [InputRequired(), Length(10, 2000)], filters=[normalize_line_endings])
+	btn_submit  = SubmitField(lazy_gettext("Open Thread"))
 
 
 @bp.route("/threads/new/", methods=["GET", "POST"])
@@ -285,15 +298,14 @@ def new():
 		if package is None:
 			abort(404)
 
-	def_is_private = request.args.get("private") or False
-	if package is None and not current_user.rank.atLeast(UserRank.APPROVER):
+	if package is None and not current_user.rank.at_least(UserRank.APPROVER):
 		abort(404)
 
-	allow_private_change = not package or package.approved
 	is_review_thread = package and not package.approved
+	is_private_thread = is_review_thread
 
 	# Check that user can make the thread
-	if package and not package.checkPerm(current_user, Permission.CREATE_THREAD):
+	if package and not package.check_perm(current_user, Permission.CREATE_THREAD):
 		flash(gettext("Unable to create thread!"), "danger")
 		return redirect(url_for("homepage.home"))
 
@@ -301,75 +313,77 @@ def new():
 	elif is_review_thread and package.review_thread is not None:
 		# Redirect submit to `view` page, which checks for `title` in the form data and so won't commit the reply
 		flash(gettext("An approval thread already exists! Consider replying there instead"), "danger")
-		return redirect(package.review_thread.getViewURL(), code=307)
+		return redirect(package.review_thread.get_view_url(), code=307)
 
-	elif not current_user.canOpenThreadRL():
+	elif not current_user.can_open_thread_ratelimit():
 		flash(gettext("Please wait before opening another thread"), "danger")
 
 		if package:
-			return redirect(package.getURL("packages.view"))
+			return redirect(package.get_url("packages.view"))
 		else:
 			return redirect(url_for("homepage.home"))
 
 	# Set default values
 	elif request.method == "GET":
-		form.private.data = def_is_private
 		form.title.data   = request.args.get("title") or ""
 
 	# Validate and submit
 	elif form.validate_on_submit():
-		thread = Thread()
-		thread.author  = current_user
-		thread.title   = form.title.data
-		thread.private = form.private.data if allow_private_change else def_is_private
-		thread.package = package
-		db.session.add(thread)
+		if has_blocked_domains(form.comment.data, current_user.username, f"new thread"):
+			flash(gettext("Linking to blocked sites is not allowed"), "danger")
+		else:
+			thread = Thread()
+			thread.author  = current_user
+			thread.title   = form.title.data
+			thread.private = is_private_thread
+			thread.package = package
+			db.session.add(thread)
 
-		thread.watchers.append(current_user)
-		if package and package.author != current_user:
-			thread.watchers.append(package.author)
+			thread.watchers.append(current_user)
+			if package and package.author != current_user:
+				thread.watchers.append(package.author)
 
-		reply = ThreadReply()
-		reply.thread  = thread
-		reply.author  = current_user
-		reply.comment = form.comment.data
-		db.session.add(reply)
+			reply = ThreadReply()
+			reply.thread  = thread
+			reply.author  = current_user
+			reply.comment = form.comment.data
+			db.session.add(reply)
 
-		thread.replies.append(reply)
+			thread.replies.append(reply)
 
-		db.session.commit()
+			db.session.commit()
 
-		if is_review_thread:
-			package.review_thread = thread
+			if is_review_thread:
+				package.review_thread = thread
 
-		for mentioned_username in get_user_mentions(render_markdown(form.comment.data)):
-			mentioned = User.query.filter_by(username=mentioned_username).first()
-			if mentioned is None:
-				continue
+			for mentioned_username in get_user_mentions(render_markdown(form.comment.data)):
+				mentioned = User.query.filter_by(username=mentioned_username).first()
+				if mentioned is None:
+					continue
 
-			msg = "Mentioned by {} in new thread '{}'".format(current_user.display_name, thread.title)
-			addNotification(mentioned, current_user, NotificationType.NEW_THREAD,
-							msg, thread.getViewURL(), thread.package)
+				msg = "Mentioned by {} in new thread '{}'".format(current_user.display_name, thread.title)
+				add_notification(mentioned, current_user, NotificationType.NEW_THREAD,
+								 msg, thread.get_view_url(), thread.package)
 
-			thread.watchers.append(mentioned)
+				if mentioned not in thread.watchers:
+					thread.watchers.append(mentioned)
 
-		notif_msg = "New thread '{}'".format(thread.title)
-		if package is not None:
-			addNotification(package.maintainers, current_user, NotificationType.NEW_THREAD, notif_msg, thread.getViewURL(), package)
+			notif_msg = "New thread '{}'".format(thread.title)
+			if package is not None:
+				add_notification(package.maintainers, current_user, NotificationType.NEW_THREAD, notif_msg, thread.get_view_url(), package)
 
-		approvers = User.query.filter(User.rank >= UserRank.APPROVER).all()
-		addNotification(approvers, current_user, NotificationType.EDITOR_MISC, notif_msg, thread.getViewURL(), package)
+			approvers = User.query.filter(User.rank >= UserRank.APPROVER).all()
+			add_notification(approvers, current_user, NotificationType.EDITOR_MISC, notif_msg, thread.get_view_url(), package)
 
-		if is_review_thread:
-			post_discord_webhook.delay(current_user.username,
-					"Opened approval thread: {}".format(thread.getViewURL(absolute=True)), True)
+			if is_review_thread:
+				post_discord_webhook.delay(current_user.display_name,
+						"Opened approval thread: {}".format(thread.get_view_url(absolute=True)), True)
 
-		db.session.commit()
+			db.session.commit()
 
-		return redirect(thread.getViewURL())
+			return redirect(thread.get_view_url())
 
-
-	return render_template("threads/new.html", form=form, allow_private_change=allow_private_change, package=package)
+	return render_template("threads/new.html", form=form, package=package)
 
 
 @bp.route("/users/<username>/comments/")
@@ -378,4 +392,16 @@ def user_comments(username):
 	if user is None:
 		abort(404)
 
-	return render_template("threads/user_comments.html", user=user, replies=user.replies)
+	page = get_int_or_abort(request.args.get("page"), 1)
+	num = min(40, get_int_or_abort(request.args.get("n"), 40))
+
+	# Filter replies the current user can see
+	query = ThreadReply.query.options(selectinload(ThreadReply.thread)).filter_by(author=user)
+	only_public = False
+	if current_user != user and not (current_user.is_authenticated and current_user.rank.at_least(UserRank.APPROVER)):
+		query = query.filter(ThreadReply.thread.has(private=False))
+		only_public = True
+
+	pagination = query.order_by(db.desc(ThreadReply.created_at)).paginate(page=page, per_page=num)
+
+	return render_template("threads/user_comments.html", user=user, pagination=pagination, only_public=only_public)

app/blueprints/thumbnails/__init__.py

@@ -14,15 +14,23 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-
-from flask import abort, send_file, Blueprint, current_app
-
-bp = Blueprint("thumbnails", __name__)
-
+import re
+import requests
+from flask import abort, send_file, Blueprint, current_app, request
 import os
 from PIL import Image
 
-ALLOWED_RESOLUTIONS=[(100,67), (270,180), (350,233)]
+
+bp = Blueprint("thumbnails", __name__)
+
+
+ALLOWED_RESOLUTIONS = [(100, 67), (270, 180), (350, 233), (1100, 520)]
+ALLOWED_MIMETYPES = {
+	"png": "image/png",
+	"webp": "image/webp",
+	"jpg": "image/jpeg",
+}
+
 
 def mkdir(path):
 	assert path != "" and path is not None
@@ -34,52 +42,104 @@ def mkdir(path):
 
 
 def resize_and_crop(img_path, modified_path, size):
-	try:
-		img = Image.open(img_path)
-	except FileNotFoundError:
+	with Image.open(img_path) as img:
+		# Get current and desired ratio for the images
+		img_ratio = img.size[0] / float(img.size[1])
+		desired_ratio = size[0] / float(size[1])
+
+		# Is more portrait than target, scale and crop
+		if desired_ratio > img_ratio:
+			img = img.resize((int(size[0]), int(size[0] * img.size[1] / img.size[0])),
+					Image.BICUBIC)
+			box = (0, (img.size[1] - size[1]) / 2, img.size[0], (img.size[1] + size[1]) / 2)
+			img = img.crop(box)
+
+		# Is more landscape than target, scale and crop
+		elif desired_ratio < img_ratio:
+			img = img.resize((int(size[1] * img.size[0] / img.size[1]), int(size[1])),
+					Image.BICUBIC)
+			box = ((img.size[0] - size[0]) / 2, 0, (img.size[0] + size[0]) / 2, img.size[1])
+			img = img.crop(box)
+
+		# Is exactly the same ratio as target
+		else:
+			img = img.resize(size, Image.BICUBIC)
+
+		if modified_path.endswith(".jpg") and img.mode != "RGB":
+			img = img.convert("RGB")
+
+		img.save(modified_path, lossless=True)
+
+
+def find_source_file(img):
+	upload_dir = current_app.config["UPLOAD_DIR"]
+	source_filepath = os.path.join(upload_dir, img)
+	if os.path.isfile(source_filepath):
+		return source_filepath
+
+	period = source_filepath.rfind(".")
+	start = source_filepath[:period]
+	ext = source_filepath[period + 1:]
+	if ext not in ALLOWED_MIMETYPES:
 		abort(404)
 
-	# Get current and desired ratio for the images
-	img_ratio = img.size[0] / float(img.size[1])
-	ratio = size[0] / float(size[1])
+	for other_ext in ALLOWED_MIMETYPES.keys():
+		other_path = f"{start}.{other_ext}"
+		if ext != other_ext and os.path.isfile(other_path):
+			return other_path
 
-	# Is more portrait than target, scale and crop
-	if ratio > img_ratio:
-		img = img.resize((int(size[0]), int(size[0] * img.size[1] / img.size[0])),
-				Image.BICUBIC)
-		box = (0, (img.size[1] - size[1]) / 2, img.size[0], (img.size[1] + size[1]) / 2)
-		img = img.crop(box)
+	abort(404)
 
-	# Is more landscape than target, scale and crop
-	elif ratio < img_ratio:
-		img = img.resize((int(size[1] * img.size[0] / img.size[1]), int(size[1])),
-				Image.BICUBIC)
-		box = ((img.size[0] - size[0]) / 2, 0, (img.size[0] + size[0]) / 2, img.size[1])
-		img = img.crop(box)
 
-	# Is exactly the same ratio as target
-	else:
-		img = img.resize(size, Image.BICUBIC)
-
-	img.save(modified_path)
+def get_mimetype(cache_filepath: str) -> str:
+	period = cache_filepath.rfind(".")
+	ext = cache_filepath[period + 1:]
+	mimetype = ALLOWED_MIMETYPES.get(ext)
+	if mimetype is None:
+		abort(404)
+	return mimetype
 
 
 @bp.route("/thumbnails/<int:level>/<img>")
 def make_thumbnail(img, level):
 	if level > len(ALLOWED_RESOLUTIONS) or level <= 0:
 		abort(403)
-
 	w, h = ALLOWED_RESOLUTIONS[level - 1]
 
-	upload_dir = current_app.config["UPLOAD_DIR"]
 	thumbnail_dir = current_app.config["THUMBNAIL_DIR"]
 	mkdir(thumbnail_dir)
 
 	output_dir = os.path.join(thumbnail_dir, str(level))
 	mkdir(output_dir)
 
-	cache_filepath  = os.path.join(output_dir, img)
-	source_filepath = os.path.join(upload_dir, img)
+	cache_filepath = os.path.join(output_dir, img)
+	if not os.path.isfile(cache_filepath):
+		source_filepath = find_source_file(img)
+		resize_and_crop(source_filepath, cache_filepath, (w, h))
 
-	resize_and_crop(source_filepath, cache_filepath, (w, h))
-	return send_file(cache_filepath)
+	res = send_file(cache_filepath, mimetype=get_mimetype(cache_filepath))
+	res.headers["Cache-Control"] = "max-age=604800" # 1 week
+	return res
+
+
+@bp.route("/thumbnails/youtube/<id_>.jpg")
+def youtube(id_: str):
+	if not re.match(r"^[A-Za-z0-9\-_]+$", id_):
+		abort(400)
+
+	cache_dir = os.path.join(current_app.config["THUMBNAIL_DIR"], "youtube")
+	os.makedirs(cache_dir, exist_ok=True)
+	cache_filepath = os.path.join(cache_dir, id_ + ".jpg")
+
+	url = f"https://img.youtube.com/vi/{id_}/default.jpg"
+
+	response = requests.get(url, stream=True)
+	if response.status_code != 200:
+		abort(response.status_code)
+
+	with open(cache_filepath, "wb") as file:
+		file.write(response.content)
+
+	res = send_file(cache_filepath)
+	res.headers["Cache-Control"] = "max-age=604800"  # 1 week
+	return res

app/blueprints/todo/__init__.py

@@ -14,257 +14,9 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-from celery import uuid
-from flask import *
-from flask_login import current_user, login_required
-from sqlalchemy import or_, and_
 
-from app.models import *
-from app.querybuilder import QueryBuilder
-from app.utils import get_int_or_abort, addNotification, addAuditLog, isYes
-from app.tasks.importtasks import makeVCSRelease
+from flask import Blueprint
 
 bp = Blueprint("todo", __name__)
 
-@bp.route("/todo/", methods=["GET", "POST"])
-@login_required
-def view_editor():
-	canApproveNew = Permission.APPROVE_NEW.check(current_user)
-	canApproveRel = Permission.APPROVE_RELEASE.check(current_user)
-	canApproveScn = Permission.APPROVE_SCREENSHOT.check(current_user)
-
-	packages = None
-	wip_packages = None
-	if canApproveNew:
-		packages = Package.query.filter_by(state=PackageState.READY_FOR_REVIEW) \
-			.order_by(db.desc(Package.created_at)).all()
-		wip_packages = Package.query.filter(or_(Package.state==PackageState.WIP, Package.state==PackageState.CHANGES_NEEDED)) \
-			.order_by(db.desc(Package.created_at)).all()
-
-	releases = None
-	if canApproveRel:
-		releases = PackageRelease.query.filter_by(approved=False).all()
-
-	screenshots = None
-	if canApproveScn:
-		screenshots = PackageScreenshot.query.filter_by(approved=False).all()
-
-	if not canApproveNew and not canApproveRel and not canApproveScn:
-		abort(403)
-
-	if request.method == "POST":
-		if request.form["action"] == "screenshots_approve_all":
-			if not canApproveScn:
-				abort(403)
-
-			PackageScreenshot.query.update({ "approved": True })
-			db.session.commit()
-			return redirect(url_for("todo.view_editor"))
-		else:
-			abort(400)
-
-	license_needed = Package.query \
-		.filter(Package.state.in_([PackageState.READY_FOR_REVIEW, PackageState.APPROVED])) \
-		.filter(or_(Package.license.has(License.name.like("Other %")),
-			Package.media_license.has(License.name.like("Other %")))) \
-		.all()
-
-	total_packages = Package.query.filter_by(state=PackageState.APPROVED).count()
-	total_to_tag = Package.query.filter_by(state=PackageState.APPROVED, tags=None).count()
-
-	unfulfilled_meta_packages = MetaPackage.query \
-			.filter(~ MetaPackage.packages.any(state=PackageState.APPROVED)) \
-			.filter(MetaPackage.dependencies.any(Dependency.depender.has(state=PackageState.APPROVED), optional=False)) \
-			.order_by(db.asc(MetaPackage.name)).count()
-
-	audit_log = AuditLogEntry.query \
-		.filter(AuditLogEntry.package.has()) \
-		.order_by(db.desc(AuditLogEntry.created_at)) \
-		.limit(20).all()
-
-	return render_template("todo/editor.html", current_tab="editor",
-			packages=packages, wip_packages=wip_packages, releases=releases, screenshots=screenshots,
-			canApproveNew=canApproveNew, canApproveRel=canApproveRel, canApproveScn=canApproveScn,
-			license_needed=license_needed, total_packages=total_packages, total_to_tag=total_to_tag,
-			unfulfilled_meta_packages=unfulfilled_meta_packages, audit_log=audit_log)
-
-
-@bp.route("/todo/topics/")
-@login_required
-def topics():
-	qb    = QueryBuilder(request.args)
-	qb.setSortIfNone("date")
-	query = qb.buildTopicQuery()
-
-	tmp_q = ForumTopic.query
-	if not qb.show_discarded:
-		tmp_q = tmp_q.filter_by(discarded=False)
-	total = tmp_q.count()
-	topic_count = query.count()
-
-	page  = get_int_or_abort(request.args.get("page"), 1)
-	num   = get_int_or_abort(request.args.get("n"), 100)
-	if num > 100 and not current_user.rank.atLeast(UserRank.APPROVER):
-		num = 100
-
-	query = query.paginate(page, num, True)
-	next_url = url_for("todo.topics", page=query.next_num, query=qb.search,
-			show_discarded=qb.show_discarded, n=num, sort=qb.order_by) \
-			if query.has_next else None
-	prev_url = url_for("todo.topics", page=query.prev_num, query=qb.search,
-			show_discarded=qb.show_discarded, n=num, sort=qb.order_by) \
-			if query.has_prev else None
-
-	return render_template("todo/topics.html", current_tab="topics", topics=query.items, total=total,
-			topic_count=topic_count, query=qb.search, show_discarded=qb.show_discarded,
-			next_url=next_url, prev_url=prev_url, page=page, page_max=query.pages,
-			n=num, sort_by=qb.order_by)
-
-
-@bp.route("/todo/tags/")
-@login_required
-def tags():
-	qb    = QueryBuilder(request.args)
-	qb.setSortIfNone("score", "desc")
-	query = qb.buildPackageQuery()
-
-	only_no_tags = isYes(request.args.get("no_tags"))
-	if only_no_tags:
-		query = query.filter(Package.tags==None)
-
-	tags = Tag.query.order_by(db.asc(Tag.title)).all()
-
-	return render_template("todo/tags.html", current_tab="tags", packages=query.all(), \
-			tags=tags, only_no_tags=only_no_tags)
-
-
-@bp.route("/user/tags/")
-def tags_user():
-	return redirect(url_for('todo.tags', author=current_user.username))
-
-
-@bp.route("/todo/metapackages/")
-@login_required
-def metapackages():
-	mpackages = MetaPackage.query \
-			.filter(~ MetaPackage.packages.any(state=PackageState.APPROVED)) \
-			.filter(MetaPackage.dependencies.any(Dependency.depender.has(state=PackageState.APPROVED), optional=False)) \
-			.order_by(db.asc(MetaPackage.name)).all()
-
-	return render_template("todo/metapackages.html", mpackages=mpackages)
-
-
-@bp.route("/user/todo/")
-@bp.route("/users/<username>/todo/")
-@login_required
-def view_user(username=None):
-	if username is None:
-		return redirect(url_for("todo.view_user", username=current_user.username))
-
-	user : User = User.query.filter_by(username=username).first()
-	if not user:
-		abort(404)
-
-	if current_user != user and not current_user.rank.atLeast(UserRank.APPROVER):
-		abort(403)
-
-	unapproved_packages = user.packages \
-		.filter(or_(Package.state == PackageState.WIP,
-			Package.state == PackageState.CHANGES_NEEDED)) \
-		.order_by(db.asc(Package.created_at)).all()
-
-	packages_with_small_screenshots = user.maintained_packages \
-		.filter(Package.screenshots.any(and_(PackageScreenshot.width < PackageScreenshot.SOFT_MIN_SIZE[0],
-				PackageScreenshot.height < PackageScreenshot.SOFT_MIN_SIZE[1]))) \
-		.all()
-
-	outdated_packages = user.maintained_packages \
-			.filter(Package.state != PackageState.DELETED,
-					Package.update_config.has(PackageUpdateConfig.outdated_at.isnot(None))) \
-			.order_by(db.asc(Package.title)).all()
-
-	topics_to_add = ForumTopic.query \
-			.filter_by(author_id=user.id) \
-			.filter(~ db.exists().where(Package.forums == ForumTopic.topic_id)) \
-			.order_by(db.asc(ForumTopic.name), db.asc(ForumTopic.title)) \
-			.all()
-
-	needs_tags = user.maintained_packages \
-		.filter(Package.state != PackageState.DELETED, Package.tags==None) \
-		.order_by(db.asc(Package.title)).all()
-
-	return render_template("todo/user.html", current_tab="user", user=user,
-			unapproved_packages=unapproved_packages, outdated_packages=outdated_packages,
-			needs_tags=needs_tags, topics_to_add=topics_to_add,
-			packages_with_small_screenshots=packages_with_small_screenshots,
-			screenshot_min_size=PackageScreenshot.HARD_MIN_SIZE, screenshot_rec_size=PackageScreenshot.SOFT_MIN_SIZE)
-
-
-@bp.route("/users/<username>/update-configs/apply-all/", methods=["POST"])
-@login_required
-def apply_all_updates(username):
-	user: User = User.query.filter_by(username=username).first()
-	if not user:
-		abort(404)
-
-	if current_user != user and not current_user.rank.atLeast(UserRank.EDITOR):
-		abort(403)
-
-	outdated_packages = user.maintained_packages \
-		.filter(Package.state != PackageState.DELETED,
-			Package.update_config.has(PackageUpdateConfig.outdated_at.isnot(None))) \
-		.order_by(db.asc(Package.title)).all()
-
-	for package in outdated_packages:
-		if not package.checkPerm(current_user, Permission.MAKE_RELEASE):
-			continue
-
-		if package.releases.filter(or_(PackageRelease.task_id.isnot(None),
-				PackageRelease.commit_hash==package.update_config.last_commit)).count() > 0:
-			continue
-
-		title = package.update_config.get_title()
-		ref = package.update_config.get_ref()
-
-		rel = PackageRelease()
-		rel.package = package
-		rel.title = title
-		rel.url = ""
-		rel.task_id = uuid()
-		db.session.add(rel)
-		db.session.commit()
-
-		makeVCSRelease.apply_async((rel.id, ref),
-				task_id=rel.task_id)
-
-		msg = "Created release {} (Applied all Git Update Detection)".format(rel.title)
-		addNotification(package.maintainers, current_user, NotificationType.PACKAGE_EDIT, msg,
-				rel.getURL("packages.create_edit"), package)
-		addAuditLog(AuditSeverity.NORMAL, current_user, msg, package.getURL("packages.view"), package)
-		db.session.commit()
-
-	return redirect(url_for("todo.view_user", username=username))
-
-
-@bp.route("/todo/outdated/")
-@login_required
-def outdated():
-	is_mtm_only = isYes(request.args.get("mtm"))
-
-	query = db.session.query(Package).select_from(PackageUpdateConfig) \
-			.filter(PackageUpdateConfig.outdated_at.isnot(None)) \
-			.join(PackageUpdateConfig.package) \
-			.filter(Package.state == PackageState.APPROVED)
-
-	if is_mtm_only:
-		query = query.filter(Package.repo.ilike("%github.com/minetest-mods/%"))
-
-	sort_by = request.args.get("sort")
-	if sort_by == "date":
-		query = query.order_by(db.desc(PackageUpdateConfig.outdated_at))
-	else:
-		sort_by = "score"
-		query = query.order_by(db.desc(Package.score))
-
-	return render_template("todo/outdated.html", current_tab="outdated",
-			outdated_packages=query.all(), sort_by=sort_by, is_mtm_only=is_mtm_only)
+from . import editor, user

app/blueprints/todo/editor.py

@@ -0,0 +1,217 @@
+# ContentDB
+# Copyright (C) 2018-23 rubenwardy
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+
+from flask import redirect, url_for, abort, render_template, request
+from flask_login import current_user, login_required
+from sqlalchemy import or_, and_
+
+from app.models import Package, PackageState, PackageScreenshot, PackageUpdateConfig, ForumTopic, db, \
+	PackageRelease, Permission, UserRank, License, MetaPackage, Dependency, AuditLogEntry, Tag, LuantiRelease, Report
+from app.querybuilder import QueryBuilder
+from app.utils import get_int_or_abort, is_yes, rank_required
+from . import bp
+
+
+@bp.route("/todo/", methods=["GET", "POST"])
+@login_required
+def view_editor():
+	can_approve_new = Permission.APPROVE_NEW.check(current_user)
+	can_approve_rel = Permission.APPROVE_RELEASE.check(current_user)
+	can_approve_scn = Permission.APPROVE_SCREENSHOT.check(current_user)
+
+	packages = None
+	wip_packages = None
+	if can_approve_new:
+		packages = Package.query.filter_by(state=PackageState.READY_FOR_REVIEW) \
+			.order_by(db.desc(Package.created_at)).all()
+		wip_packages = Package.query \
+			.filter(or_(Package.state == PackageState.WIP, Package.state == PackageState.CHANGES_NEEDED)) \
+			.order_by(db.desc(Package.created_at)).all()
+
+	releases = None
+	if can_approve_rel:
+		releases = PackageRelease.query.filter_by(approved=False, task_id=None).all()
+
+	screenshots = None
+	if can_approve_scn:
+		screenshots = PackageScreenshot.query.filter_by(approved=False).all()
+
+	if not can_approve_new and not can_approve_rel and not can_approve_scn:
+		abort(403)
+
+	if request.method == "POST":
+		if request.form["action"] == "screenshots_approve_all":
+			if not can_approve_scn:
+				abort(403)
+
+			PackageScreenshot.query.update({"approved": True})
+			db.session.commit()
+			return redirect(url_for("todo.view_editor"))
+		else:
+			abort(400)
+
+	license_needed = Package.query \
+		.filter(Package.state.in_([PackageState.READY_FOR_REVIEW, PackageState.APPROVED])) \
+		.filter(or_(Package.license.has(License.name.like("Other %")),
+			Package.media_license.has(License.name.like("Other %")))) \
+		.all()
+
+	total_packages = Package.query.filter_by(state=PackageState.APPROVED).count()
+	total_to_tag = Package.query.filter_by(state=PackageState.APPROVED, tags=None).count()
+
+	unfulfilled_meta_packages = MetaPackage.query \
+		.filter(~ MetaPackage.packages.any(state=PackageState.APPROVED)) \
+		.filter(MetaPackage.dependencies.any(Dependency.depender.has(state=PackageState.APPROVED), optional=False)) \
+		.order_by(db.asc(MetaPackage.name)).count()
+
+	audit_log = AuditLogEntry.query \
+		.filter(AuditLogEntry.package.has()) \
+		.order_by(db.desc(AuditLogEntry.created_at)) \
+		.limit(20).all()
+
+	reports = Report.query.filter_by(is_resolved=False).order_by(db.asc(Report.created_at)).all() if current_user.rank.at_least(UserRank.EDITOR) else None
+
+	return render_template("todo/editor.html", current_tab="editor",
+			packages=packages, wip_packages=wip_packages, releases=releases, screenshots=screenshots,
+			can_approve_new=can_approve_new, can_approve_rel=can_approve_rel, can_approve_scn=can_approve_scn,
+			license_needed=license_needed, total_packages=total_packages, total_to_tag=total_to_tag,
+			unfulfilled_meta_packages=unfulfilled_meta_packages, audit_log=audit_log, reports=reports)
+
+
+@bp.route("/todo/tags/")
+@login_required
+def tags():
+	qb    = QueryBuilder(request.args, cookies=True)
+	qb.set_sort_if_none("score", "desc")
+	query = qb.build_package_query()
+
+	only_no_tags = is_yes(request.args.get("no_tags"))
+	if only_no_tags:
+		query = query.filter(Package.tags == None)
+
+	tags = Tag.query.order_by(db.asc(Tag.title)).all()
+
+	return render_template("todo/tags.html", current_tab="tags", packages=query.all(),
+			tags=tags, only_no_tags=only_no_tags)
+
+
+@bp.route("/todo/modnames/")
+@login_required
+def modnames():
+	mnames = MetaPackage.query \
+			.filter(~ MetaPackage.packages.any(state=PackageState.APPROVED)) \
+			.filter(MetaPackage.dependencies.any(Dependency.depender.has(state=PackageState.APPROVED), optional=False)) \
+			.order_by(db.asc(MetaPackage.name)).all()
+
+	return render_template("todo/modnames.html", modnames=mnames)
+
+
+@bp.route("/todo/outdated/")
+@login_required
+def outdated():
+	is_mtm_only = is_yes(request.args.get("mtm"))
+
+	query = db.session.query(Package).select_from(PackageUpdateConfig) \
+			.filter(PackageUpdateConfig.outdated_at.isnot(None)) \
+			.join(PackageUpdateConfig.package) \
+			.filter(Package.state == PackageState.APPROVED)
+
+	if is_mtm_only:
+		query = query.filter(Package.repo.ilike("%github.com/minetest-mods/%"))
+
+	sort_by = request.args.get("sort")
+	if sort_by == "date":
+		query = query.order_by(db.desc(PackageUpdateConfig.outdated_at))
+	else:
+		sort_by = "score"
+		query = query.order_by(db.desc(Package.score))
+
+	return render_template("todo/outdated.html", current_tab="outdated",
+			outdated_packages=query.all(), sort_by=sort_by, is_mtm_only=is_mtm_only)
+
+
+@bp.route("/todo/screenshots/")
+@login_required
+def screenshots():
+	is_mtm_only = is_yes(request.args.get("mtm"))
+
+	query = db.session.query(Package) \
+			.filter(~Package.screenshots.any()) \
+			.filter(Package.state == PackageState.APPROVED)
+
+	if is_mtm_only:
+		query = query.filter(Package.repo.ilike("%github.com/minetest-mods/%"))
+
+	sort_by = request.args.get("sort")
+	if sort_by == "date":
+		query = query.order_by(db.desc(Package.approved_at))
+	else:
+		sort_by = "score"
+		query = query.order_by(db.desc(Package.score))
+
+	return render_template("todo/screenshots.html", current_tab="screenshots",
+			packages=query.all(), sort_by=sort_by, is_mtm_only=is_mtm_only)
+
+
+@bp.route("/todo/mtver_support/")
+@login_required
+def mtver_support():
+	is_mtm_only = is_yes(request.args.get("mtm"))
+
+	current_stable = LuantiRelease.query.filter(~LuantiRelease.name.like("%-dev")).order_by(db.desc(LuantiRelease.id)).first()
+
+	query = db.session.query(Package) \
+			.filter(~Package.releases.any(or_(PackageRelease.max_rel==None, PackageRelease.max_rel == current_stable))) \
+			.filter(Package.state == PackageState.APPROVED)
+
+	if is_mtm_only:
+		query = query.filter(Package.repo.ilike("%github.com/minetest-mods/%"))
+
+	sort_by = request.args.get("sort")
+	if sort_by == "date":
+		query = query.order_by(db.desc(Package.approved_at))
+	else:
+		sort_by = "score"
+		query = query.order_by(db.desc(Package.score))
+
+	return render_template("todo/mtver_support.html", current_tab="screenshots",
+			packages=query.all(), sort_by=sort_by, is_mtm_only=is_mtm_only, current_stable=current_stable)
+
+
+@bp.route("/todo/topics/mismatch/")
+@rank_required(UserRank.EDITOR)
+def topics_mismatch():
+	missing_topics = Package.query.filter(Package.forums.is_not(None)) .filter(~ForumTopic.query.filter(ForumTopic.topic_id == Package.forums).exists()).all()
+
+	packages_bad_author = (
+		db.session.query(Package, ForumTopic)
+		.select_from(Package)
+		.join(ForumTopic, Package.forums == ForumTopic.topic_id)
+		.filter(Package.author_id != ForumTopic.author_id)
+		.all())
+
+	packages_bad_title = (
+		db.session.query(Package, ForumTopic)
+		.select_from(Package)
+		.join(ForumTopic, Package.forums == ForumTopic.topic_id)
+		.filter(and_(ForumTopic.name != Package.name, ~ForumTopic.title.ilike("%" + Package.title + "%"), ~ForumTopic.title.ilike("%" + Package.name + "%")))
+		.all())
+
+	return render_template("todo/topics_mismatch.html",
+			missing_topics=missing_topics,
+			packages_bad_author=packages_bad_author,
+			packages_bad_title=packages_bad_title)

app/blueprints/todo/user.py

@@ -0,0 +1,194 @@
+# ContentDB
+# Copyright (C) 2018-23 rubenwardy
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+from celery import uuid
+from flask import redirect, url_for, abort, render_template, flash
+from flask_babel import gettext
+from flask_login import current_user, login_required
+from sqlalchemy import or_, and_
+
+from app.models import User, Package, PackageState, PackageScreenshot, PackageUpdateConfig, ForumTopic, db, \
+	PackageRelease, Permission, NotificationType, AuditSeverity, UserRank, PackageType
+from app.tasks.importtasks import make_vcs_release
+from app.utils import add_notification, add_audit_log
+from . import bp
+
+
+@bp.route("/user/tags/")
+def tags_user():
+	return redirect(url_for('todo.tags', author=current_user.username))
+
+
+@bp.route("/user/todo/")
+@bp.route("/users/<username>/todo/")
+@login_required
+def view_user(username=None):
+	if username is None:
+		return redirect(url_for("todo.view_user", username=current_user.username))
+
+	user: User = User.query.filter_by(username=username).first()
+	if not user:
+		abort(404)
+
+	if current_user != user and not current_user.rank.at_least(UserRank.APPROVER):
+		abort(403)
+
+	unapproved_packages = user.packages \
+			.filter(or_(Package.state == PackageState.WIP,
+				Package.state == PackageState.CHANGES_NEEDED)) \
+			.order_by(db.asc(Package.created_at)).all()
+
+	outdated_packages = user.maintained_packages \
+			.filter(Package.state != PackageState.DELETED,
+					Package.update_config.has(PackageUpdateConfig.outdated_at.isnot(None))) \
+			.order_by(db.asc(Package.title)).all()
+
+	missing_game_support = user.maintained_packages.filter(
+			Package.state != PackageState.DELETED,
+			Package.type.in_([PackageType.MOD, PackageType.TXP]),
+			~Package.supported_games.any(),
+			Package.supports_all_games == False) \
+			.order_by(db.asc(Package.title)).all()
+
+	packages_with_no_screenshots = user.maintained_packages.filter(
+			~Package.screenshots.any(), Package.state == PackageState.APPROVED).all()
+
+	packages_with_small_screenshots = user.maintained_packages \
+			.filter(Package.state != PackageState.DELETED,
+					Package.screenshots.any(and_(PackageScreenshot.width < PackageScreenshot.SOFT_MIN_SIZE[0],
+					PackageScreenshot.height < PackageScreenshot.SOFT_MIN_SIZE[1]))) \
+			.all()
+
+	topics_to_add = ForumTopic.query \
+			.filter_by(author_id=user.id) \
+			.filter(~ db.exists().where(Package.forums == ForumTopic.topic_id)) \
+			.order_by(db.asc(ForumTopic.name), db.asc(ForumTopic.title)) \
+			.all()
+
+	needs_tags = user.maintained_packages \
+		.filter(Package.state != PackageState.DELETED, ~Package.tags.any()) \
+		.order_by(db.asc(Package.title)).all()
+
+	return render_template("todo/user.html", current_tab="user", user=user,
+			unapproved_packages=unapproved_packages, outdated_packages=outdated_packages,
+			missing_game_support=missing_game_support, needs_tags=needs_tags, topics_to_add=topics_to_add,
+			packages_with_no_screenshots=packages_with_no_screenshots,
+			packages_with_small_screenshots=packages_with_small_screenshots,
+			screenshot_min_size=PackageScreenshot.HARD_MIN_SIZE, screenshot_rec_size=PackageScreenshot.SOFT_MIN_SIZE)
+
+
+@bp.route("/users/<username>/update-configs/apply-all/", methods=["POST"])
+@login_required
+def apply_all_updates(username):
+	user: User = User.query.filter_by(username=username).first()
+	if not user:
+		abort(404)
+
+	if current_user != user and not current_user.rank.at_least(UserRank.EDITOR):
+		abort(403)
+
+	outdated_packages = user.maintained_packages \
+		.filter(Package.state != PackageState.DELETED,
+			Package.update_config.has(PackageUpdateConfig.outdated_at.isnot(None))) \
+		.order_by(db.asc(Package.title)).all()
+
+	for package in outdated_packages:
+		if not package.check_perm(current_user, Permission.MAKE_RELEASE):
+			continue
+
+		if package.releases.filter(or_(PackageRelease.task_id.isnot(None),
+				PackageRelease.commit_hash == package.update_config.last_commit)).count() > 0:
+			continue
+
+		title = package.update_config.title
+		ref = package.update_config.get_ref()
+
+		rel = PackageRelease()
+		rel.package = package
+		rel.name = title
+		rel.title = title
+		rel.url = ""
+		rel.task_id = uuid()
+		db.session.add(rel)
+		db.session.commit()
+
+		make_vcs_release.apply_async((rel.id, ref),
+									 task_id=rel.task_id)
+
+		msg = "Created release {} (Applied all Git Update Detection)".format(rel.title)
+		add_notification(package.maintainers, current_user, NotificationType.PACKAGE_EDIT, msg,
+				package.get_url("packages.create_edit"), package)
+		add_audit_log(AuditSeverity.NORMAL, current_user, msg, package.get_url("packages.view"), package)
+		db.session.commit()
+
+	return redirect(url_for("todo.view_user", username=username))
+
+
+@bp.route("/user/game_support/")
+@bp.route("/users/<username>/game_support/")
+@login_required
+def all_game_support(username=None):
+	if username is None:
+		return redirect(url_for("todo.all_game_support", username=current_user.username))
+
+	user: User = User.query.filter_by(username=username).one_or_404()
+	if current_user != user and not current_user.rank.at_least(UserRank.EDITOR):
+		abort(403)
+
+	packages = user.maintained_packages.filter(
+				Package.state != PackageState.DELETED,
+				Package.type.in_([PackageType.MOD, PackageType.TXP])) \
+			.order_by(db.asc(Package.title)).all()
+
+	bulk_support_names = db.session.query(Package.title) \
+		.select_from(Package).filter(
+			Package.maintainers.contains(user),
+			Package.state != PackageState.DELETED,
+			Package.type.in_([PackageType.MOD, PackageType.TXP]),
+			~Package.supported_games.any(),
+			Package.supports_all_games == False) \
+		.order_by(db.asc(Package.title)).all()
+
+	bulk_support_names = ", ".join([x[0] for x in bulk_support_names])
+
+	return render_template("todo/game_support.html", user=user, packages=packages, bulk_support_names=bulk_support_names)
+
+
+@bp.route("/users/<username>/confirm_supports_all_games/", methods=["POST"])
+@login_required
+def confirm_supports_all_games(username=None):
+	user: User = User.query.filter_by(username=username).one_or_404()
+	if current_user != user and not current_user.rank.at_least(UserRank.EDITOR):
+		abort(403)
+
+	packages = user.maintained_packages.filter(
+		Package.state != PackageState.DELETED,
+		Package.type.in_([PackageType.MOD, PackageType.TXP]),
+		~Package.supported_games.any(),
+		Package.supports_all_games == False) \
+		.all()
+
+	for package in packages:
+		package.supports_all_games = True
+		db.session.merge(package)
+
+		add_audit_log(AuditSeverity.NORMAL, current_user, "Enabled 'Supports all games' (bulk)",
+					  package.get_url("packages.game_support"), package)
+
+	db.session.commit()
+
+	flash(gettext("Done"), "success")
+	return redirect(url_for("todo.all_game_support", username=current_user.username))

app/blueprints/translate/__init__.py

@@ -0,0 +1,48 @@
+# ContentDB
+# Copyright (C) 2024 rubenwardy
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+
+from flask import Blueprint, render_template, request
+from sqlalchemy import or_
+
+from app.models import Package, PackageState, db, PackageTranslation
+
+bp = Blueprint("translate", __name__)
+
+
+@bp.route("/translate/")
+def translate():
+	query = Package.query.filter(
+			Package.state == PackageState.APPROVED,
+			or_(
+				Package.translation_url.is_not(None),
+				Package.translations.any(PackageTranslation.language_id != "en")
+			))
+
+	has_langs = request.args.getlist("has_lang")
+	for lang in has_langs:
+		query = query.filter(Package.translations.any(PackageTranslation.language_id == lang))
+
+	not_langs = request.args.getlist("not_lang")
+	for lang in not_langs:
+		query = query.filter(~Package.translations.any(PackageTranslation.language_id == lang))
+
+	supports_translation = (query
+			.order_by(Package.translation_url.is_(None), db.desc(Package.score))
+			.all())
+
+	return render_template("translate/index.html",
+			supports_translation=supports_translation, has_langs=has_langs, not_langs=not_langs)

app/blueprints/users/account.py

@@ -14,23 +14,22 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
+import datetime
 
-
-from flask import *
-from flask_babel import gettext, lazy_gettext, get_locale
+from flask import redirect, abort, render_template, flash, request, url_for, Response
+from flask_babel import gettext, get_locale, lazy_gettext
 from flask_login import current_user, login_required, logout_user, login_user
 from flask_wtf import FlaskForm
 from sqlalchemy import or_
-from wtforms import *
-from wtforms.validators import *
+from wtforms import StringField, SubmitField, BooleanField, PasswordField, validators
+from wtforms.validators import InputRequired, Length, Regexp, DataRequired, Optional, Email, EqualTo
 
-from app.models import *
 from app.tasks.emails import send_verify_email, send_anon_email, send_unsubscribe_verify, send_user_email
-from app.utils import randomString, make_flask_login_password, is_safe_url, check_password_hash, addAuditLog, \
-	nonEmptyOrNone, post_login, is_username_valid
-from passlib.pwd import genphrase
-
+from app.utils import random_string, make_flask_login_password, is_safe_url, check_password_hash, add_audit_log, \
+	nonempty_or_none, post_login
 from . import bp
+from app.models import User, AuditSeverity, db, EmailSubscription, UserEmailVerification
+from app.logic.users import create_user
 
 
 class LoginForm(FlaskForm):
@@ -47,7 +46,6 @@ def handle_login(form):
 		else:
 			flash(err, "danger")
 
-
 	username = form.username.data.strip()
 	user = User.query.filter(or_(User.username == username, User.email == username)).first()
 	if user is None:
@@ -60,8 +58,8 @@ def handle_login(form):
 		flash(gettext("You need to confirm the registration email"), "danger")
 		return
 
-	addAuditLog(AuditSeverity.USER, user, "Logged in using password",
-			url_for("users.profile", username=user.username))
+	add_audit_log(AuditSeverity.USER, user, "Logged in using password",
+				  url_for("users.profile", username=user.username))
 	db.session.commit()
 
 	if not login_user(user, remember=form.remember_me.data):
@@ -73,11 +71,11 @@ def handle_login(form):
 
 @bp.route("/user/login/", methods=["GET", "POST"])
 def login():
-	if current_user.is_authenticated:
-		next = request.args.get("next")
-		if next and not is_safe_url(next):
-			abort(400)
+	next = request.args.get("next")
+	if next and not is_safe_url(next):
+		abort(400)
 
+	if current_user.is_authenticated:
 		return redirect(next or url_for("homepage.home"))
 
 	form = LoginForm(request.form)
@@ -89,8 +87,7 @@ def login():
 	if request.method == "GET":
 		form.remember_me.data = True
 
-
-	return render_template("users/login.html", form=form)
+	return render_template("users/login.html", form=form, next=next)
 
 
 @bp.route("/user/logout/", methods=["GET", "POST"])
@@ -100,14 +97,14 @@ def logout():
 
 
 class RegisterForm(FlaskForm):
-	display_name = StringField(lazy_gettext("Display Name"), [Optional(), Length(1, 20)], filters=[nonEmptyOrNone])
+	display_name = StringField(lazy_gettext("Display Name"), [Optional(), Length(1, 20)], filters=[nonempty_or_none])
 	username = StringField(lazy_gettext("Username"), [InputRequired(),
 			Regexp("^[a-zA-Z0-9._-]+$", message=lazy_gettext(
 				"Only alphabetic letters (A-Za-z), numbers (0-9), underscores (_), minuses (-), and periods (.) allowed"))])
 	email    = StringField(lazy_gettext("Email"), [InputRequired(), Email()])
-	password = PasswordField(lazy_gettext("Password"), [InputRequired(), Length(6, 100)])
+	password = PasswordField(lazy_gettext("Password"), [InputRequired(), Length(12, 100)])
 	question  = StringField(lazy_gettext("What is the result of the above calculation?"), [InputRequired()])
-	agree    = BooleanField(lazy_gettext("I agree"), [DataRequired()])
+	first_name = StringField("First name", [])
 	submit   = SubmitField(lazy_gettext("Register"))
 
 
@@ -116,51 +113,20 @@ def handle_register(form):
 		flash(gettext("Incorrect captcha answer"), "danger")
 		return
 
-	if not is_username_valid(form.username.data):
-		flash(gettext("Username is invalid"))
+	user = create_user(form.username.data, form.display_name.data, form.email.data)
+	if isinstance(user, Response):
+		return user
+	elif user is None:
 		return
+	elif form.first_name.data != "":
+		abort(500)
 
-	user_by_name = User.query.filter(or_(
-			User.username == form.username.data,
-			User.username == form.display_name.data,
-			User.display_name == form.display_name.data,
-			User.forums_username == form.username.data,
-			User.github_username == form.username.data)).first()
-	if user_by_name:
-		if user_by_name.rank == UserRank.NOT_JOINED and user_by_name.forums_username:
-			flash(gettext("An account already exists for that username but hasn't been claimed yet."), "danger")
-			return redirect(url_for("users.claim_forums", username=user_by_name.forums_username))
-		else:
-			flash(gettext("That username/display name is already in use, please choose another."), "danger")
-			return
+	user.password = make_flask_login_password(form.password.data)
 
-	alias_by_name = PackageAlias.query.filter(or_(
-			PackageAlias.author==form.username.data,
-			PackageAlias.author==form.display_name.data)).first()
-	if alias_by_name:
-		flash(gettext("That username/display name is already in use, please choose another."), "danger")
-		return
+	add_audit_log(AuditSeverity.USER, user, "Registered with email, display name=" + user.display_name,
+				  url_for("users.profile", username=user.username))
 
-	user_by_email = User.query.filter_by(email=form.email.data).first()
-	if user_by_email:
-		send_anon_email.delay(form.email.data, get_locale().language, gettext("Email already in use"),
-			gettext("We were unable to create the account as the email is already in use by %(display_name)s. Try a different email address.",
-					display_name=user_by_email.display_name))
-		return redirect(url_for("users.email_sent"))
-	elif EmailSubscription.query.filter_by(email=form.email.data, blacklisted=True).count() > 0:
-		flash(gettext("That email address has been unsubscribed/blacklisted, and cannot be used"), "danger")
-		return
-
-	user = User(form.username.data, False, form.email.data, make_flask_login_password(form.password.data))
-	user.notification_preferences = UserNotificationPreferences(user)
-	if form.display_name.data:
-		user.display_name = form.display_name.data
-	db.session.add(user)
-
-	addAuditLog(AuditSeverity.USER, user, "Registered with email, display name=" + user.display_name,
-			url_for("users.profile", username=user.username))
-
-	token = randomString(32)
+	token = random_string(32)
 
 	ver = UserEmailVerification()
 	ver.user = user
@@ -182,14 +148,14 @@ def register():
 		if ret:
 			return ret
 
-	return render_template("users/register.html", form=form,
-			suggested_password=genphrase(entropy=52, wordset="bip39"))
+	return render_template("users/register.html", form=form)
 
 
 class ForgotPasswordForm(FlaskForm):
 	email = StringField(lazy_gettext("Email"), [InputRequired(), Email()])
 	submit = SubmitField(lazy_gettext("Reset Password"))
 
+
 @bp.route("/user/forgot-password/", methods=["GET", "POST"])
 def forgot_password():
 	form = ForgotPasswordForm(request.form)
@@ -197,10 +163,10 @@ def forgot_password():
 		email = form.email.data
 		user = User.query.filter_by(email=email).first()
 		if user:
-			token = randomString(32)
+			token = random_string(32)
 
-			addAuditLog(AuditSeverity.USER, user, "(Anonymous) requested a password reset",
-					url_for("users.profile", username=user.username), None)
+			add_audit_log(AuditSeverity.USER, user, "(Anonymous) requested a password reset",
+						  url_for("users.profile", username=user.username), None)
 
 			ver = UserEmailVerification()
 			ver.user = user
@@ -223,15 +189,16 @@ def forgot_password():
 
 class SetPasswordForm(FlaskForm):
 	email = StringField(lazy_gettext("Email"), [Optional(), Email()])
-	password = PasswordField(lazy_gettext("New password"), [InputRequired(), Length(8, 100)])
-	password2 = PasswordField(lazy_gettext("Verify password"), [InputRequired(), Length(8, 100),
-			validators.EqualTo('password', message=lazy_gettext('Passwords must match'))])
+	password = PasswordField(lazy_gettext("New password"), [InputRequired(), Length(12, 100)])
+	password2 = PasswordField(lazy_gettext("Verify password"), [InputRequired(), Length(12, 100),
+			EqualTo('password', message=lazy_gettext('Passwords must match'))])
 	submit = SubmitField(lazy_gettext("Save"))
 
+
 class ChangePasswordForm(FlaskForm):
-	old_password = PasswordField(lazy_gettext("Old password"), [InputRequired(), Length(8, 100)])
-	password = PasswordField(lazy_gettext("New password"), [InputRequired(), Length(8, 100)])
-	password2 = PasswordField(lazy_gettext("Verify password"), [InputRequired(), Length(8, 100),
+	old_password = PasswordField(lazy_gettext("Old password"), [InputRequired(), Length(6, 100)])
+	password = PasswordField(lazy_gettext("New password"), [InputRequired(), Length(12, 100)])
+	password2 = PasswordField(lazy_gettext("Verify password"), [InputRequired(), Length(12, 100),
 			validators.EqualTo('password', message=lazy_gettext('Passwords must match'))])
 	submit = SubmitField(lazy_gettext("Save"))
 
@@ -243,13 +210,13 @@ def handle_set_password(form):
 		flash(gettext("Passwords do not match"), "danger")
 		return
 
-	addAuditLog(AuditSeverity.USER, current_user, "Changed their password", url_for("users.profile", username=current_user.username))
+	add_audit_log(AuditSeverity.USER, current_user, "Changed their password", url_for("users.profile", username=current_user.username))
 
 	current_user.password = make_flask_login_password(form.password.data)
 
 	if hasattr(form, "email"):
-		newEmail = nonEmptyOrNone(form.email.data)
-		if newEmail and newEmail != current_user.email:
+		new_email = nonempty_or_none(form.email.data)
+		if new_email and new_email != current_user.email:
 			if EmailSubscription.query.filter_by(email=form.email.data, blacklisted=True).count() > 0:
 				flash(gettext(u"That email address has been unsubscribed/blacklisted, and cannot be used"), "danger")
 				return
@@ -260,12 +227,12 @@ def handle_set_password(form):
 					gettext(u"We were unable to create the account as the email is already in use by %(display_name)s. Try a different email address.",
 							display_name=user_by_email.display_name))
 			else:
-				token = randomString(32)
+				token = random_string(32)
 
 				ver = UserEmailVerification()
 				ver.user = current_user
 				ver.token = token
-				ver.email = newEmail
+				ver.email = new_email
 				db.session.add(ver)
 				db.session.commit()
 
@@ -292,8 +259,7 @@ def change_password():
 		else:
 			flash(gettext("Old password is incorrect"), "danger")
 
-	return render_template("users/change_set_password.html", form=form,
-			suggested_password=genphrase(entropy=52, wordset="bip39"))
+	return render_template("users/change_set_password.html", form=form)
 
 
 @bp.route("/user/set-password/", methods=["GET", "POST"])
@@ -311,8 +277,7 @@ def set_password():
 		if ret:
 			return ret
 
-	return render_template("users/change_set_password.html", form=form, optional=request.args.get("optional"),
-			suggested_password=genphrase(entropy=52, wordset="bip39"))
+	return render_template("users/change_set_password.html", form=form)
 
 
 @bp.route("/user/verify/")
@@ -323,9 +288,7 @@ def verify_email():
 		flash(gettext("Unknown verification token!"), "danger")
 		return redirect(url_for("homepage.home"))
 
-	delta = (datetime.datetime.now() - ver.created_at)
-	delta: datetime.timedelta
-	if delta.total_seconds() > 12*60*60:
+	if ver.is_expired:
 		flash(gettext("Token has expired"), "danger")
 		db.session.delete(ver)
 		db.session.commit()
@@ -333,8 +296,8 @@ def verify_email():
 
 	user = ver.user
 
-	addAuditLog(AuditSeverity.USER, user, "Confirmed their email",
-			url_for("users.profile", username=user.username))
+	add_audit_log(AuditSeverity.USER, user, "Confirmed their email",
+				  url_for("users.profile", username=user.username))
 
 	was_activating = not user.is_active
 
@@ -387,7 +350,7 @@ def unsubscribe_verify():
 			sub = EmailSubscription(email)
 			db.session.add(sub)
 
-		sub.token = randomString(32)
+		sub.token = random_string(32)
 		db.session.commit()
 		send_unsubscribe_verify.delay(form.email.data, get_locale().language)
 

app/blueprints/users/claim.py

@@ -13,14 +13,16 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
 from flask_babel import gettext
+from flask_login import current_user
 
 from . import bp
 from flask import redirect, render_template, session, request, flash, url_for
 from app.models import db, User, UserRank
-from app.utils import randomString, login_user_set_active, is_username_valid
-from app.tasks.forumtasks import checkForumAccount
-from app.utils.phpbbparser import getProfile
+from app.utils import random_string, login_user_set_active
+from app.tasks.forumtasks import check_forum_account
+from app.utils.phpbbparser import get_profile
 
 
 @bp.route("/user/claim/", methods=["GET", "POST"])
@@ -30,52 +32,52 @@ def claim():
 
 @bp.route("/user/claim-forums/", methods=["GET", "POST"])
 def claim_forums():
+	if current_user.is_authenticated:
+		return redirect(url_for("homepage.home"))
+
 	username = request.args.get("username")
 	if username is None:
 		username = ""
 	else:
 		method = request.args.get("method")
 
-		if not is_username_valid(username):
-			flash(gettext("Invalid username, Only alphabetic letters (A-Za-z), numbers (0-9), underscores (_), minuses (-), and periods (.) allowed. Consider contacting an admin"), "danger")
-			return redirect(url_for("users.claim_forums"))
-
 		user = User.query.filter_by(forums_username=username).first()
-		if user and user.rank.atLeast(UserRank.NEW_MEMBER):
+		if user and user.rank.at_least(UserRank.NEW_MEMBER):
 			flash(gettext("User has already been claimed"), "danger")
 			return redirect(url_for("users.claim_forums"))
 		elif method == "github":
 			if user is None or user.github_username is None:
-				flash(gettext("Unable to get GitHub username for user"), "danger")
+				flash(gettext("Unable to get GitHub username for user. Make sure the forum account exists."), "danger")
 				return redirect(url_for("users.claim_forums", username=username))
 			else:
-				return redirect(url_for("github.start"))
+				return redirect(url_for("vcs.github_start"))
 
 	if "forum_token" in session:
 		token = session["forum_token"]
 	else:
-		token = randomString(12)
+		token = random_string(12)
 		session["forum_token"] = token
 
 	if request.method == "POST":
-		ctype	= request.form.get("claim_type")
+		ctype = request.form.get("claim_type")
 		username = request.form.get("username")
 
-		if not is_username_valid(username):
-			flash(gettext("Invalid username, Only alphabetic letters (A-Za-z), numbers (0-9), underscores (_), minuses (-), and periods (.) allowed. Consider contacting an admin"), "danger")
-		elif ctype == "github":
-			task = checkForumAccount.delay(username)
+		if User.query.filter(User.username == username, User.forums_username.is_(None)).first():
+			flash(gettext("A ContentDB user with that name already exists. Please contact an admin to link to your forum account"), "danger")
+			return redirect(url_for("users.claim_forums"))
+
+		if ctype == "github":
+			task = check_forum_account.delay(username)
 			return redirect(url_for("tasks.check", id=task.id, r=url_for("users.claim_forums", username=username, method="github")))
 		elif ctype == "forum":
 			user = User.query.filter_by(forums_username=username).first()
-			if user is not None and user.rank.atLeast(UserRank.NEW_MEMBER):
+			if user is not None and user.rank.at_least(UserRank.NEW_MEMBER):
 				flash(gettext("That user has already been claimed!"), "danger")
 				return redirect(url_for("users.claim_forums"))
 
 			# Get signature
-			sig = None
 			try:
-				profile = getProfile("https://forum.minetest.net", username)
+				profile = get_profile("https://forum.luanti.org", username)
 				sig = profile.signature if profile else None
 			except IOError as e:
 				if hasattr(e, 'message'):

app/blueprints/users/profile.py

@@ -15,15 +15,17 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 import math
-from typing import Optional
+from typing import Optional, Tuple, List
 
-from flask import *
+from flask import redirect, url_for, abort, render_template, request
 from flask_babel import gettext
 from flask_login import current_user, login_required
-from sqlalchemy import func
+from sqlalchemy import func, text
+
+from app.models import User, db, Package, PackageReview, PackageState, PackageType, UserRank, Collection
+from app.utils import get_daterange_options
+from app.tasks.forumtasks import check_forum_account
 
-from app.models import *
-from app.tasks.forumtasks import checkForumAccount
 from . import bp
 
 
@@ -160,10 +162,7 @@ def get_user_medals(user: User) -> Tuple[List[Medal], List[Medal]]:
 	if user_package_ranks:
 		top_rank = user_package_ranks[2]
 		top_type = PackageType.coerce(user_package_ranks[0])
-		if top_rank == 1:
-			title = gettext(u"Top %(type)s", type=top_type.text.lower())
-		else:
-			title = gettext(u"Top %(group)d %(type)s", group=top_rank, type=top_type.text.lower())
+		title = top_type.get_top_ordinal(top_rank)
 		if top_type == PackageType.MOD:
 			icon = "fa-box"
 		elif top_type == PackageType.GAME:
@@ -171,8 +170,7 @@ def get_user_medals(user: User) -> Tuple[List[Medal], List[Medal]]:
 		else:
 			icon = "fa-paint-brush"
 
-		description = gettext(u"%(display_name)s has a %(type)s placed at #%(place)d.",
-				display_name=user.display_name, type=top_type.text.lower(), place=top_rank)
+		description = top_type.get_top_ordinal_description(user.display_name, top_rank)
 		unlocked.append(
 				Medal.make_unlocked(place_to_color(top_rank), icon, title, description))
 
@@ -216,7 +214,7 @@ def profile(username):
 	if not user:
 		abort(404)
 
-	if not current_user.is_authenticated or (user != current_user and not current_user.canAccessTodoList()):
+	if not current_user.is_authenticated or (user != current_user and not current_user.can_access_todo_list()):
 		packages = user.packages.filter_by(state=PackageState.APPROVED)
 		maintained_packages = user.maintained_packages.filter_by(state=PackageState.APPROVED)
 	else:
@@ -228,27 +226,62 @@ def profile(username):
 		.filter(Package.author != user) \
 		.order_by(db.asc(Package.title)).all()
 
+	pinned_collections = user.collections.filter(Collection.private == False,
+			Collection.pinned == True, Collection.packages.any()).all()
+
 	unlocked, locked = get_user_medals(user)
 	# Process GET or invalid POST
 	return render_template("users/profile.html", user=user,
 			packages=packages, maintained_packages=maintained_packages,
-			medals_unlocked=unlocked, medals_locked=locked)
+			medals_unlocked=unlocked, medals_locked=locked, pinned_collections=pinned_collections)
 
 
-@bp.route("/users/<username>/check/", methods=["POST"])
+@bp.route("/users/<username>/check-forums/", methods=["POST"])
 @login_required
-def user_check(username):
+def user_check_forums(username):
 	user = User.query.filter_by(username=username).first()
 	if user is None:
 		abort(404)
 
-	if current_user != user and not current_user.rank.atLeast(UserRank.MODERATOR):
+	if current_user != user and not current_user.rank.at_least(UserRank.MODERATOR):
 		abort(403)
 
 	if user.forums_username is None:
 		abort(404)
 
-	task = checkForumAccount.delay(user.forums_username)
+	task = check_forum_account.delay(user.forums_username, force_replace_pic=True)
 	next_url = url_for("users.profile", username=username)
 
 	return redirect(url_for("tasks.check", id=task.id, r=next_url))
+
+
+@bp.route("/users/<username>/remove-profile-pic/", methods=["POST"])
+@login_required
+def user_remove_profile_pic(username):
+	user = User.query.filter_by(username=username).one_or_404()
+	if current_user != user and not current_user.rank.at_least(UserRank.MODERATOR):
+		abort(403)
+
+	user.profile_pic = None
+	db.session.commit()
+	return redirect(url_for("users.profile_edit", username=username))
+
+
+@bp.route("/user/stats/")
+@login_required
+def statistics_redirect():
+	return redirect(url_for("users.statistics", username=current_user.username))
+
+
+@bp.route("/users/<username>/stats/")
+def statistics(username):
+	user = User.query.filter_by(username=username).first()
+	if user is None:
+		abort(404)
+
+	downloads = db.session.query(func.sum(Package.downloads)).filter(Package.author==user).one()
+
+	start = request.args.get("start")
+	end = request.args.get("end")
+	return render_template("users/stats.html", user=user, downloads=downloads[0],
+			start=start, end=end, options=get_daterange_options(), noindex=start or end)

app/blueprints/users/settings.py

@@ -1,14 +1,33 @@
-from flask import *
-from flask_babel import gettext, lazy_gettext, get_locale
+# ContentDB
+# Copyright (C) 2023 rubenwardy
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+from flask import redirect, abort, render_template, request, flash, url_for
+from flask_babel import gettext, get_locale, lazy_gettext
 from flask_login import current_user, login_required, logout_user
 from flask_wtf import FlaskForm
+from kombu import uuid
 from sqlalchemy import or_
-from wtforms import *
-from wtforms.validators import *
+from wtforms import StringField, SubmitField, BooleanField, SelectField
+from wtforms.validators import Length, Optional, Email, URL
 
-from app.models import *
-from app.utils import nonEmptyOrNone, addAuditLog, randomString, rank_required
+from app.models import User, AuditSeverity, db, UserRank, PackageAlias, EmailSubscription, UserNotificationPreferences, \
+	UserEmailVerification, Permission, NotificationType, UserBan
 from app.tasks.emails import send_verify_email
+from app.tasks.usertasks import update_github_user_id
+from app.utils import nonempty_or_none, add_audit_log, random_string, rank_required, has_blocked_domains
 from . import bp
 
 
@@ -36,7 +55,14 @@ def get_setting_tabs(user):
 		},
 	]
 
-	if current_user.rank.atLeast(UserRank.MODERATOR):
+	if user.check_perm(current_user, Permission.CREATE_OAUTH_CLIENT):
+		ret.append({
+			"id": "oauth_clients",
+			"title": gettext("OAuth2 Applications"),
+			"url": url_for("oauth.list_clients", username=user.username)
+		})
+
+	if current_user.rank.at_least(UserRank.MODERATOR):
 		ret.append({
 			"id": "modtools",
 			"title": gettext("Moderator Tools"),
@@ -47,41 +73,49 @@ def get_setting_tabs(user):
 
 
 class UserProfileForm(FlaskForm):
-	display_name = StringField(lazy_gettext("Display Name"), [Optional(), Length(1, 20)], filters=[lambda x: nonEmptyOrNone(x)])
+	display_name = StringField(lazy_gettext("Display Name"), [Optional(), Length(1, 20)], filters=[lambda x: nonempty_or_none(x.strip())])
 	website_url = StringField(lazy_gettext("Website URL"), [Optional(), URL()], filters = [lambda x: x or None])
 	donate_url = StringField(lazy_gettext("Donation URL"), [Optional(), URL()], filters = [lambda x: x or None])
 	submit = SubmitField(lazy_gettext("Save"))
 
 
-def handle_profile_edit(form, user, username):
+def handle_profile_edit(form: UserProfileForm, user: User, username: str):
 	severity = AuditSeverity.NORMAL if current_user == user else AuditSeverity.MODERATION
-	addAuditLog(severity, current_user, "Edited {}'s profile".format(user.display_name),
-			url_for("users.profile", username=username))
+	add_audit_log(severity, current_user, "Edited {}'s profile".format(user.display_name),
+				  url_for("users.profile", username=username))
+
+	display_name = form.display_name.data or user.username
+	if user.check_perm(current_user, Permission.CHANGE_DISPLAY_NAME) and \
+			user.display_name != display_name:
 
-	if user.checkPerm(current_user, Permission.CHANGE_DISPLAY_NAME) and \
-			user.display_name != form.display_name.data:
 		if User.query.filter(User.id != user.id,
-				or_(User.username == form.display_name.data,
-						User.display_name.ilike(form.display_name.data))).count() > 0:
+				or_(User.username == display_name,
+						User.display_name.ilike(display_name))).count() > 0:
 			flash(gettext("A user already has that name"), "danger")
 			return None
 
+
 		alias_by_name = PackageAlias.query.filter(or_(
-				PackageAlias.author == form.display_name.data)).first()
+				PackageAlias.author == display_name)).first()
 		if alias_by_name:
 			flash(gettext("A user already has that name"), "danger")
 			return
 
-		user.display_name = form.display_name.data
+		user.display_name = display_name
 
 		severity = AuditSeverity.USER if current_user == user else AuditSeverity.MODERATION
-		addAuditLog(severity, current_user, "Changed display name of {} to {}"
-			.format(user.username, user.display_name),
-				url_for("users.profile", username=username))
+		add_audit_log(severity, current_user, "Changed display name of {} to {}"
+					  .format(user.username, user.display_name),
+					  url_for("users.profile", username=username))
 
-	if user.checkPerm(current_user, Permission.CHANGE_PROFILE_URLS):
-		user.website_url = form["website_url"].data
-		user.donate_url = form["donate_url"].data
+	if user.check_perm(current_user, Permission.CHANGE_PROFILE_URLS):
+		if has_blocked_domains(form.website_url.data, current_user.username, f"{user.username}'s website_url") or \
+				has_blocked_domains(form.donate_url.data, current_user.username, f"{user.username}'s donate_url"):
+			flash(gettext("Linking to blocked sites is not allowed"), "danger")
+			return
+
+		user.website_url = form.website_url.data
+		user.donate_url = form.donate_url.data
 
 	db.session.commit()
 
@@ -96,8 +130,7 @@ def profile_edit(username):
 		abort(404)
 
 	if not user.can_see_edit_profile(current_user):
-		flash(gettext("Permission denied"), "danger")
-		return redirect(url_for("users.profile", username=username))
+		abort(403)
 
 	form = UserProfileForm(obj=user)
 	if form.validate_on_submit():
@@ -116,7 +149,7 @@ def make_settings_form():
 	}
 
 	for notificationType in NotificationType:
-		key = "pref_" + notificationType.toName()
+		key = "pref_" + notificationType.to_name()
 		attrs[key] = BooleanField("")
 		attrs[key + "_digest"] = BooleanField("")
 
@@ -127,27 +160,27 @@ SettingsForm = make_settings_form()
 
 def handle_email_notifications(user, prefs: UserNotificationPreferences, is_new, form):
 	for notificationType in NotificationType:
-		field_email = getattr(form, "pref_" + notificationType.toName()).data
-		field_digest = getattr(form, "pref_" + notificationType.toName() + "_digest").data or field_email
+		field_email = getattr(form, "pref_" + notificationType.to_name()).data
+		field_digest = getattr(form, "pref_" + notificationType.to_name() + "_digest").data or field_email
 		prefs.set_can_email(notificationType, field_email)
 		prefs.set_can_digest(notificationType, field_digest)
 
 	if is_new:
 		db.session.add(prefs)
 
-	if user.checkPerm(current_user, Permission.CHANGE_EMAIL):
+	if user.check_perm(current_user, Permission.CHANGE_EMAIL):
 		newEmail = form.email.data
 		if newEmail and newEmail != user.email and newEmail.strip() != "":
 			if EmailSubscription.query.filter_by(email=form.email.data, blacklisted=True).count() > 0:
 				flash(gettext("That email address has been unsubscribed/blacklisted, and cannot be used"), "danger")
 				return
 
-			token = randomString(32)
+			token = random_string(32)
 
 			severity = AuditSeverity.NORMAL if current_user == user else AuditSeverity.MODERATION
 
 			msg = "Changed email of {}".format(user.display_name)
-			addAuditLog(severity, current_user, msg, url_for("users.profile", username=user.username))
+			add_audit_log(severity, current_user, msg, url_for("users.profile", username=user.username))
 
 			ver = UserEmailVerification()
 			ver.user = user
@@ -174,7 +207,7 @@ def email_notifications(username=None):
 	if not user:
 		abort(404)
 
-	if not user.checkPerm(current_user, Permission.CHANGE_EMAIL):
+	if not user.check_perm(current_user, Permission.CHANGE_EMAIL):
 		abort(403)
 
 	is_new = False
@@ -187,8 +220,8 @@ def email_notifications(username=None):
 	types = []
 	for notificationType in NotificationType:
 		types.append(notificationType)
-		data["pref_" + notificationType.toName()] = prefs.get_can_email(notificationType)
-		data["pref_" + notificationType.toName() + "_digest"] = prefs.get_can_digest(notificationType)
+		data["pref_" + notificationType.to_name()] = prefs.get_can_email(notificationType)
+		data["pref_" + notificationType.to_name() + "_digest"] = prefs.get_can_digest(notificationType)
 
 	data["email"] = user.email
 
@@ -210,9 +243,31 @@ def account(username):
 	if not user:
 		abort(404)
 
+	if not user.can_see_edit_profile(current_user):
+		abort(403)
+
 	return render_template("users/account.html", user=user, tabs=get_setting_tabs(user), current_tab="account")
 
 
+@bp.route("/users/<username>/settings/account/disconnect-github/", methods=["POST"])
+def disconnect_github(username: str):
+	user: User = User.query.filter_by(username=username).one_or_404()
+
+	if not user.can_see_edit_profile(current_user):
+		abort(403)
+
+	if user.password and user.email:
+		user.github_user_id = None
+		user.github_username = None
+		db.session.commit()
+
+		flash(gettext("Removed GitHub account"), "success")
+	else:
+		flash(gettext("You need to add an email address and password before you can remove your GitHub account"), "danger")
+
+	return redirect(url_for("users.account", username=username))
+
+
 @bp.route("/users/<username>/delete/", methods=["GET", "POST"])
 @rank_required(UserRank.ADMIN)
 def delete(username):
@@ -220,34 +275,40 @@ def delete(username):
 	if not user:
 		abort(404)
 
-	if user.rank.atLeast(UserRank.MODERATOR):
+	if user.rank.at_least(UserRank.MODERATOR):
 		flash(gettext("Users with moderator rank or above cannot be deleted"), "danger")
 		return redirect(url_for("users.account", username=username))
 
 	if request.method == "GET":
 		return render_template("users/delete.html", user=user, can_delete=user.can_delete())
 
-	if "delete" in request.form and (user.can_delete() or current_user.rank.atLeast(UserRank.ADMIN)):
+	if "delete" in request.form and (user.can_delete() or current_user.rank.at_least(UserRank.ADMIN)):
 		msg = "Deleted user {}".format(user.username)
 		flash(msg, "success")
-		addAuditLog(AuditSeverity.MODERATION, current_user, msg, None)
+		add_audit_log(AuditSeverity.MODERATION, current_user, msg, None)
 
-		if current_user.rank.atLeast(UserRank.ADMIN):
+		if current_user.rank.at_least(UserRank.ADMIN):
 			for pkg in user.packages.all():
 				pkg.review_thread = None
 				db.session.delete(pkg)
 
 		db.session.delete(user)
 	elif "deactivate" in request.form:
-		user.replies.delete()
+		for reply in user.replies.all():
+			db.session.delete(reply)
 		for thread in user.threads.all():
 			db.session.delete(thread)
+		for token in user.tokens.all():
+			db.session.delete(token)
+		user.profile_pic = None
 		user.email = None
-		user.rank = UserRank.NOT_JOINED
+
+		if user.rank != UserRank.BANNED:
+			user.rank = UserRank.NOT_JOINED
 
 		msg = "Deactivated user {}".format(user.username)
 		flash(msg, "success")
-		addAuditLog(AuditSeverity.MODERATION, current_user, msg, None)
+		add_audit_log(AuditSeverity.MODERATION, current_user, msg, None)
 	else:
 		assert False
 
@@ -276,17 +337,19 @@ def modtools(username):
 	if not user:
 		abort(404)
 
-	if not user.checkPerm(current_user, Permission.CHANGE_EMAIL):
+	if not user.check_perm(current_user, Permission.CHANGE_EMAIL):
 		abort(403)
 
 	form = ModToolsForm(obj=user)
 	if form.validate_on_submit():
 		severity = AuditSeverity.NORMAL if current_user == user else AuditSeverity.MODERATION
-		addAuditLog(severity, current_user, "Edited {}'s account".format(user.display_name),
-				url_for("users.profile", username=username))
+		add_audit_log(severity, current_user, "Edited {}'s account".format(user.display_name),
+					  url_for("users.profile", username=username))
+
+		redirect_target = url_for("users.modtools", username=username)
 
 		# Copy form fields to user_profile fields
-		if user.checkPerm(current_user, Permission.CHANGE_USERNAMES):
+		if user.check_perm(current_user, Permission.CHANGE_USERNAMES):
 			if user.username != form.username.data:
 				for package in user.packages:
 					alias = PackageAlias(user.username, package.name)
@@ -296,23 +359,30 @@ def modtools(username):
 				user.username = form.username.data
 
 			user.display_name = form.display_name.data
-			user.forums_username = nonEmptyOrNone(form.forums_username.data)
-			user.github_username = nonEmptyOrNone(form.github_username.data)
+			user.forums_username = nonempty_or_none(form.forums_username.data)
+			github_username = nonempty_or_none(form.github_username.data)
+			if github_username is None:
+				user.github_username = None
+				user.github_user_id = None
+			else:
+				task_id = uuid()
+				update_github_user_id.apply_async((user.id, github_username), task_id=task_id)
+				redirect_target = url_for("tasks.check", id=task_id, r=redirect_target)
 
-		if user.checkPerm(current_user, Permission.CHANGE_RANK):
-			newRank = form["rank"].data
-			if current_user.rank.atLeast(newRank):
-				if newRank != user.rank:
-					user.rank = form["rank"].data
-					msg = "Set rank of {} to {}".format(user.display_name, user.rank.getTitle())
-					addAuditLog(AuditSeverity.MODERATION, current_user, msg,
-								url_for("users.profile", username=username))
+		if user.check_perm(current_user, Permission.CHANGE_RANK):
+			new_rank = form.rank.data
+			if current_user.rank.at_least(new_rank):
+				if new_rank != user.rank:
+					user.rank = form.rank.data
+					msg = "Set rank of {} to {}".format(user.display_name, user.rank.title)
+					add_audit_log(AuditSeverity.MODERATION, current_user, msg,
+								  url_for("users.profile", username=username))
 			else:
 				flash(gettext("Can't promote a user to a rank higher than yourself!"), "danger")
 
 		db.session.commit()
 
-		return redirect(url_for("users.modtools", username=username))
+		return redirect(redirect_target)
 
 	return render_template("users/modtools.html", user=user, form=form, tabs=get_setting_tabs(user), current_tab="modtools")
 
@@ -324,15 +394,15 @@ def modtools_set_email(username):
 	if not user:
 		abort(404)
 
-	if not user.checkPerm(current_user, Permission.CHANGE_EMAIL):
+	if not user.check_perm(current_user, Permission.CHANGE_EMAIL):
 		abort(403)
 
 	user.email = request.form["email"]
 	user.is_active = False
 
-	token = randomString(32)
-	addAuditLog(AuditSeverity.MODERATION, current_user, f"Set email and sent a password reset on {user.username}",
-			url_for("users.profile", username=user.username), None)
+	token = random_string(32)
+	add_audit_log(AuditSeverity.MODERATION, current_user, f"Set email and sent a password reset on {user.username}",
+				  url_for("users.profile", username=user.username), None)
 
 	ver = UserEmailVerification()
 	ver.user = user
@@ -355,7 +425,7 @@ def modtools_ban(username):
 	if not user:
 		abort(404)
 
-	if not user.checkPerm(current_user, Permission.CHANGE_RANK):
+	if not user.check_perm(current_user, Permission.CHANGE_RANK):
 		abort(403)
 
 	message = request.form["message"]
@@ -370,8 +440,8 @@ def modtools_ban(username):
 	else:
 		user.rank = UserRank.BANNED
 
-	addAuditLog(AuditSeverity.MODERATION, current_user, f"Banned {user.username}, expires {user.ban.expires_at or '-'}, message: {message}",
-			url_for("users.profile", username=user.username), None)
+	add_audit_log(AuditSeverity.MODERATION, current_user, f"Banned {user.username}, expires {user.ban.expires_at or '-'}, message: {message}",
+				  url_for("users.profile", username=user.username), None)
 	db.session.commit()
 
 	flash(f"Banned {user.username}", "success")
@@ -385,7 +455,7 @@ def modtools_unban(username):
 	if not user:
 		abort(404)
 
-	if not user.checkPerm(current_user, Permission.CHANGE_RANK):
+	if not user.check_perm(current_user, Permission.CHANGE_RANK):
 		abort(403)
 
 	if user.ban:
@@ -394,8 +464,8 @@ def modtools_unban(username):
 	if user.rank == UserRank.BANNED:
 		user.rank = UserRank.MEMBER
 
-	addAuditLog(AuditSeverity.MODERATION, current_user, f"Unbanned {user.username}",
-			url_for("users.profile", username=user.username), None)
+	add_audit_log(AuditSeverity.MODERATION, current_user, f"Unbanned {user.username}",
+				  url_for("users.profile", username=user.username), None)
 	db.session.commit()
 
 	flash(f"Unbanned {user.username}", "success")

app/blueprints/vcs/__init__.py

@@ -0,0 +1,22 @@
+# ContentDB
+# Copyright (C) 2024 rubenwardy
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+
+from flask import Blueprint
+
+bp = Blueprint("vcs", __name__)
+
+from . import github, gitlab

app/blueprints/vcs/common.py

@@ -0,0 +1,43 @@
+# ContentDB
+# Copyright (C) 2024  rubenwardy
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+
+from app.blueprints.api.support import error
+from app.models import Package, APIToken, Permission, PackageState
+
+
+def get_packages_for_vcs_and_token(token: APIToken, repo_url: str) -> list[Package]:
+	repo_url = repo_url.replace("https://", "").replace("http://", "").lower()
+	if token.package:
+		packages = [token.package]
+		if not token.package.check_perm(token.owner, Permission.APPROVE_RELEASE):
+			return error(403, "You do not have the permission to approve releases")
+
+		actual_repo_url: str = token.package.repo or ""
+		if repo_url not in actual_repo_url.lower():
+			return error(400, "Repo URL does not match the API token's package")
+	else:
+		# Get package
+		packages = Package.query.filter(
+			Package.repo.ilike("%{}%".format(repo_url)), Package.state != PackageState.DELETED).all()
+		if len(packages) == 0:
+			return error(400,
+					"Could not find package, did you set the VCS repo in CDB correctly? Expected {}".format(repo_url))
+		packages = [x for x in packages if x.check_perm(token.owner, Permission.APPROVE_RELEASE)]
+		if len(packages) == 0:
+			return error(403, "You do not have the permission to approve releases")
+
+	return packages

app/blueprints/vcs/github.py

@@ -0,0 +1,200 @@
+# ContentDB
+# Copyright (C) 2018-24 rubenwardy
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+import datetime
+import hmac
+
+import requests
+from flask import abort, Response
+from flask import redirect, url_for, request, flash, jsonify, current_app
+from flask_babel import gettext
+from flask_login import current_user
+
+from app import github, csrf
+from app.blueprints.api.support import error, api_create_vcs_release
+from app.logic.users import create_user
+from app.models import db, User, APIToken, AuditSeverity
+from app.utils import abs_url_for, add_audit_log, login_user_set_active, is_safe_url
+from . import bp
+from .common import get_packages_for_vcs_and_token
+
+
+@bp.route("/github/start/")
+def github_start():
+	next = request.args.get("next")
+	if next and not is_safe_url(next):
+		abort(400)
+
+	return github.authorize("", redirect_uri=abs_url_for("vcs.github_callback", next=next))
+
+
+@bp.route("/github/view/")
+def github_view_permissions():
+	url = "https://github.com/settings/connections/applications/" + \
+			current_app.config["GITHUB_CLIENT_ID"]
+	return redirect(url)
+
+
+@bp.route("/github/callback/")
+@github.authorized_handler
+def github_callback(oauth_token):
+	if oauth_token is None:
+		flash(gettext("Authorization failed [err=gh-oauth-login-failed]"), "danger")
+		return redirect(url_for("users.login"))
+
+	next = request.args.get("next")
+	if next and not is_safe_url(next):
+		abort(400)
+
+	redirect_to = next
+	if redirect_to is None:
+		redirect_to = url_for("homepage.home")
+
+	# Get GitGub username
+	url = "https://api.github.com/user"
+	r = requests.get(url, headers={"Authorization": "token " + oauth_token})
+	json = r.json()
+	user_id = json["id"]
+	github_username = json["login"]
+	if type(user_id) is not int:
+		abort(400)
+
+	# Get user by GitHub user ID
+	user_by_github = User.query.filter(User.github_user_id == user_id).one_or_none()
+
+	# If logged in, connect
+	if current_user and current_user.is_authenticated:
+		if user_by_github is None:
+			current_user.github_username = github_username
+			current_user.github_user_id = user_id
+			db.session.commit()
+			flash(gettext("Linked GitHub to account"), "success")
+			return redirect(redirect_to)
+		elif user_by_github == current_user:
+			return redirect(redirect_to)
+		else:
+			flash(gettext("GitHub account is already associated with another user: %(username)s",
+					username=user_by_github.username), "danger")
+			return redirect(redirect_to)
+
+	# Log in to existing account
+	elif user_by_github:
+		ret = login_user_set_active(user_by_github, next, remember=True)
+		if ret is None:
+			flash(gettext("Authorization failed [err=gh-login-failed]"), "danger")
+			return redirect(url_for("users.login"))
+
+		user_by_github.github_username = github_username
+		add_audit_log(AuditSeverity.USER, user_by_github, "Logged in using GitHub OAuth",
+				url_for("users.profile", username=user_by_github.username))
+		db.session.commit()
+		return ret
+
+	# Sign up
+	else:
+		user = create_user(github_username, github_username, None, "GitHub")
+		if isinstance(user, Response):
+			return user
+		elif user is None:
+			return redirect(url_for("users.login"))
+
+		user.github_username = github_username
+		user.github_user_id = user_id
+
+		add_audit_log(AuditSeverity.USER, user, "Registered with GitHub, display name=" + user.display_name,
+				url_for("users.profile", username=user.username))
+
+		db.session.commit()
+
+		ret = login_user_set_active(user, next, remember=True)
+		if ret is None:
+			flash(gettext("Authorization failed [err=gh-login-failed]"), "danger")
+			return redirect(url_for("users.login"))
+
+		return ret
+
+
+def _find_api_token(header_signature: str) -> APIToken:
+	sha_name, signature = header_signature.split('=')
+	if sha_name != 'sha1':
+		error(403, "Expected SHA1 payload signature")
+
+	for token in APIToken.query.all():
+		mac = hmac.new(token.access_token.encode("utf-8"), msg=request.data, digestmod='sha1')
+
+		if hmac.compare_digest(str(mac.hexdigest()), signature):
+			return token
+
+	error(401, "Invalid authentication, couldn't validate API token")
+
+
+@bp.route("/github/webhook/", methods=["POST"])
+@csrf.exempt
+def github_webhook():
+	json = request.json
+
+	header_signature = request.headers.get('X-Hub-Signature')
+	if header_signature is None:
+		return error(403, "Expected payload signature")
+
+	token = _find_api_token(header_signature)
+	packages = get_packages_for_vcs_and_token(token, "github.com/" + json["repository"]["full_name"])
+
+	for package in packages:
+		#
+		# Check event
+		#
+		event = request.headers.get("X-GitHub-Event")
+		if event == "push":
+			ref = json["after"]
+			title = datetime.datetime.utcnow().strftime("%Y-%m-%d") + " " + ref[:5]
+			branch = json["ref"].replace("refs/heads/", "")
+			if package.update_config and package.update_config.ref:
+				if branch != package.update_config.ref:
+					continue
+			elif branch not in ["master", "main"]:
+				continue
+
+		elif event == "create":
+			ref_type = json.get("ref_type")
+			if ref_type != "tag":
+				return jsonify({
+					"success": False,
+					"message": "Webhook ignored, as it's a non-tag create event. ref_type='{}'.".format(ref_type)
+				})
+
+			ref = json["ref"]
+			title = ref
+
+		elif event == "ping":
+			return jsonify({"success": True, "message": "Ping successful"})
+
+		else:
+			return error(400, "Unsupported event: '{}'. Only 'push', 'create:tag', and 'ping' are supported."
+					.format(event or "null"))
+
+		#
+		# Perform release
+		#
+		if package.releases.filter_by(commit_hash=ref).count() > 0:
+			return
+
+		return api_create_vcs_release(token, package, title, title, None, ref, reason="Webhook")
+
+	return jsonify({
+		"success": False,
+		"message": "No release made. Either the release already exists or the event was filtered based on the branch"
+	})

app/blueprints/vcs/gitlab.py

@@ -0,0 +1,86 @@
+# ContentDB
+# Copyright (C) 2020-24  rubenwardy
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+import datetime
+
+from flask import request, jsonify
+
+from app import csrf
+from app.blueprints.api.support import error, api_create_vcs_release
+from app.models import APIToken
+
+from . import bp
+from .common import get_packages_for_vcs_and_token
+
+
+def webhook_impl():
+	json = request.json
+
+	# Get all tokens for package
+	secret = request.headers.get("X-Gitlab-Token")
+	if secret is None:
+		return error(403, "Token required")
+
+	token: APIToken = APIToken.query.filter_by(access_token=secret).first()
+	if token is None:
+		return error(403, "Invalid authentication")
+
+	packages = get_packages_for_vcs_and_token(token, json["project"]["web_url"])
+	for package in packages:
+		#
+		# Check event
+		#
+		event = json["event_name"]
+		if event == "push":
+			ref = json["after"]
+			title = datetime.datetime.utcnow().strftime("%Y-%m-%d") + " " + ref[:5]
+			branch = json["ref"].replace("refs/heads/", "")
+			if package.update_config and package.update_config.ref:
+				if branch != package.update_config.ref:
+					continue
+			elif branch not in ["master", "main"]:
+				continue
+
+		elif event == "tag_push":
+			ref = json["ref"]
+			title = ref.replace("refs/tags/", "")
+
+		else:
+			return error(400, "Unsupported event: '{}'. Only 'push', 'create:tag', and 'ping' are supported."
+					.format(event or "null"))
+
+		#
+		# Perform release
+		#
+		if package.releases.filter_by(commit_hash=ref).count() > 0:
+			continue
+
+		return api_create_vcs_release(token, package, title, title, None, ref, reason="Webhook")
+
+	return jsonify({
+		"success": False,
+		"message": "No release made. Either the release already exists or the event was filtered based on the branch"
+	})
+
+
+
+@bp.route("/gitlab/webhook/", methods=["POST"])
+@csrf.exempt
+def gitlab_webhook():
+	try:
+		return webhook_impl()
+	except KeyError as err:
+		return error(400, "Missing field: {}".format(err.args[0]))

app/blueprints/zipgrep/__init__.py

@@ -14,36 +14,37 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-
 from celery import uuid
-from flask import Blueprint, render_template, redirect, request, abort
+from flask import Blueprint, render_template, redirect, request, abort, url_for
+from flask_babel import lazy_gettext
 from flask_wtf import FlaskForm
-from wtforms import StringField, BooleanField, SubmitField
-from wtforms.validators import InputRequired, Length
+from wtforms import StringField, BooleanField, SubmitField, SelectMultipleField
+from wtforms.validators import InputRequired, Length, Optional
 
 from app.tasks import celery
 from app.utils import rank_required
 
 bp = Blueprint("zipgrep", __name__)
 
-from app.models import *
+from app.models import UserRank, Package, PackageType
 from app.tasks.zipgrep import search_in_releases
 
 
 class SearchForm(FlaskForm):
-	query = StringField(lazy_gettext("Text to find (regex)"), [InputRequired(), Length(6, 100)])
+	query = StringField(lazy_gettext("Text to find (regex)"), [InputRequired(), Length(1, 100)])
 	file_filter = StringField(lazy_gettext("File filter"), [InputRequired(), Length(1, 100)], default="*.lua")
-	remember_me = BooleanField(lazy_gettext("Remember me"), default=True)
+	type = SelectMultipleField(lazy_gettext("Type"), [Optional()],
+			choices=PackageType.choices(), coerce=PackageType.coerce)
 	submit = SubmitField(lazy_gettext("Search"))
 
 
 @bp.route("/zipgrep/", methods=["GET", "POST"])
-@rank_required(UserRank.ADMIN)
+@rank_required(UserRank.EDITOR)
 def zipgrep_search():
 	form = SearchForm(request.form)
 	if form.validate_on_submit():
 		task_id = uuid()
-		search_in_releases.apply_async((form.query.data, form.file_filter.data), task_id=task_id)
+		search_in_releases.apply_async((form.query.data, form.file_filter.data, [x.name for x in form.type.data]), task_id=task_id)
 		result_url = url_for("zipgrep.view_results", id=task_id)
 		return redirect(url_for("tasks.check", id=task_id, r=result_url))
 

app/default_data.py

@@ -1,4 +1,23 @@
-from .models import *
+# ContentDB
+# Copyright (C) rubenwardy
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+import datetime
+
+from .models import User, UserRank, LuantiRelease, Tag, License, Notification, NotificationType, Package, \
+	PackageState, PackageType, PackageRelease, MetaPackage, Dependency
 from .utils import make_flask_login_password
 
 
@@ -16,26 +35,26 @@ def populate(session):
 	system_user.rank = UserRank.BOT
 	session.add(system_user)
 
-	session.add(MinetestRelease("None", 0))
-	session.add(MinetestRelease("0.4.16/17", 32))
-	session.add(MinetestRelease("5.0", 37))
-	session.add(MinetestRelease("5.1", 38))
-	session.add(MinetestRelease("5.2", 39))
-	session.add(MinetestRelease("5.3", 39))
+	session.add(LuantiRelease("None", 0))
+	session.add(LuantiRelease("0.4.16/17", 32))
+	session.add(LuantiRelease("5.0", 37))
+	session.add(LuantiRelease("5.1", 38))
+	session.add(LuantiRelease("5.2", 39))
+	session.add(LuantiRelease("5.3", 39))
 
 	tags = {}
 	for tag in ["Inventory", "Mapgen", "Building",
-		"Mobs and NPCs", "Tools", "Player effects",
-		"Environment", "Transport", "Maintenance", "Plants and farming",
-		"PvP", "PvE", "Survival", "Creative", "Puzzle", "Multiplayer", "Singleplayer", "Featured"]:
+			"Mobs and NPCs", "Tools", "Player effects",
+			"Environment", "Transport", "Maintenance", "Plants and farming",
+			"PvP", "PvE", "Survival", "Creative", "Puzzle", "Multiplayer", "Singleplayer"]:
 		row = Tag(tag)
 		tags[row.name] = row
 		session.add(row)
 
 	licenses = {}
 	for license in ["GPLv2.1", "GPLv3", "LGPLv2.1", "LGPLv3", "AGPLv2.1", "AGPLv3",
-					"Apache", "BSD 3-Clause", "BSD 2-Clause", "CC0", "CC-BY-SA",
-					"CC-BY", "MIT", "ZLib", "Other (Free)"]:
+			"Apache", "BSD 3-Clause", "BSD 2-Clause", "CC0", "CC-BY-SA",
+			"CC-BY", "MIT", "ZLib", "Other (Free)"]:
 		row = License(license)
 		licenses[row.name] = row
 		session.add(row)
@@ -50,9 +69,8 @@ def populate_test_data(session):
 	licenses = { x.name : x for x in License.query.all() }
 	tags = { x.name : x for x in Tag.query.all() }
 	admin_user = User.query.filter_by(rank=UserRank.ADMIN).first()
-	v4 = MinetestRelease.query.filter_by(protocol=32).first()
-	v50 = MinetestRelease.query.filter_by(protocol=37).first()
-	v51 = MinetestRelease.query.filter_by(protocol=38).first()
+	v4 = LuantiRelease.query.filter_by(protocol=32).first()
+	v51 = LuantiRelease.query.filter_by(protocol=38).first()
 
 	ez = User("Shara")
 	ez.github_username = "Ezhh"
@@ -68,7 +86,6 @@ def populate_test_data(session):
 	jeija.forums_username = "Jeija"
 	session.add(jeija)
 
-
 	mod = Package()
 	mod.state = PackageState.APPROVED
 	mod.name = "alpha"
@@ -88,6 +105,7 @@ def populate_test_data(session):
 
 	rel = PackageRelease()
 	rel.package = mod
+	rel.name = "v1.0.0"
 	rel.title = "v1.0.0"
 	rel.url = "https://github.com/ezhh/handholds/archive/master.zip"
 	rel.approved = True
@@ -125,6 +143,7 @@ awards.register_achievement("award_mesefind",{
 	rel = PackageRelease()
 	rel.package = mod1
 	rel.min_rel = v51
+	rel.name = "v1.0.0"
 	rel.title = "v1.0.0"
 	rel.url = "https://github.com/rubenwardy/awards/archive/master.zip"
 	rel.approved = True
@@ -237,6 +256,7 @@ No warranty is provided, express or implied, for any part of the project.
 
 	rel = PackageRelease()
 	rel.package = mod
+	rel.name = "v1.0.0"
 	rel.title = "v1.0.0"
 	rel.max_rel = v4
 	rel.url = "https://github.com/ezhh/handholds/archive/master.zip"
@@ -350,6 +370,7 @@ Uses the CTF PvP Engine.
 
 	rel = PackageRelease()
 	rel.package = game1
+	rel.name = "v1.0.0"
 	rel.title = "v1.0.0"
 	rel.url = "https://github.com/rubenwardy/capturetheflag/archive/master.zip"
 	rel.approved = True
@@ -361,7 +382,7 @@ Uses the CTF PvP Engine.
 	mod.name = "pixelbox"
 	mod.title = "PixelBOX Reloaded"
 	mod.license = licenses["CC0"]
-	mod.media_license = licenses["MIT"]
+	mod.media_license = licenses["CC0"]
 	mod.type = PackageType.TXP
 	mod.author = admin_user
 	mod.forums = 14132
@@ -371,6 +392,7 @@ Uses the CTF PvP Engine.
 
 	rel = PackageRelease()
 	rel.package = mod
+	rel.name = "v1.0.0"
 	rel.title = "v1.0.0"
 	rel.url = "http://mamadou3.free.fr/Minetest/PixelBOX.zip"
 	rel.approved = True
@@ -380,7 +402,6 @@ Uses the CTF PvP Engine.
 
 	metas = {}
 	for package in Package.query.filter_by(type=PackageType.MOD).all():
-		meta = None
 		try:
 			meta = metas[package.name]
 		except KeyError:

app/flatpages/about.md

@@ -0,0 +1,49 @@
+title: About ContentDB
+description: Information about ContentDB's development, history, and more
+toc: False
+
+## Development
+
+ContentDB was created by [rubenwardy](https://rubenwardy.com/) in 2018, he was lucky enough to have the time available
+as it was submitted as university coursework. To learn about the history and development of ContentDB,
+[see the blog post](https://blog.rubenwardy.com/2022/03/24/contentdb/).
+
+ContentDB is open source software, licensed under AGPLv3.0.
+
+<a href="https://github.com/luanti-org/contentdb/" class="btn btn-primary me-1">Source code</a>
+<a href="https://github.com/luanti-org/contentdb/issues/" class="btn btn-secondary me-1">Issue tracker</a>
+<a href="{{ admin_contact_url }}" class="btn btn-secondary me-1">Contact admin</a>
+{% if monitoring_url -%}
+<a href="{{ monitoring_url }}" class="btn btn-secondary">Stats / monitoring</a>
+{%- endif %}
+
+## Why was ContentDB created?
+
+Before ContentDB, users had to manually install mods and games by unzipping their files into a directory. This is
+poor user experience, especially for first-time users.
+
+ContentDB isn't just about supporting the in-game content downloader; it's common for technical users to find
+and review packages using the ContentDB website, but install using Git rather than the in-game installer.
+**ContentDB's purpose is to be a well-formatted source of information about mods, games,
+and texture packs for Luanti**.
+
+## How do I learn how to make mods and games for Luanti?
+
+You should read
+[the official Luanti Modding Book](https://rubenwardy.com/minetest_modding_book/)
+for a guide to making mods and games using Luanti.
+
+
+<h2 id="donate">How can I support / donate to ContentDB?</h2>
+
+You can donate to rubenwardy to cover ContentDB's costs and support future development.
+
+For more information about the cost of ContentDB and what rubenwardy does, see his donation page:
+
+<a href="https://rubenwardy.com/donate/" class="btn btn-primary me-1">Donate</a>
+<a href="/donate/" class="btn btn-secondary">Support Creators</a>
+
+## Sponsorships
+
+Luanti and ContentDB are sponsored by <a href="https://sentry.io/" rel="nofollow">sentry.io</a>.
+This provides us with improved error logging and performance insights.

app/flatpages/help.md

@@ -2,29 +2,41 @@ title: Help
 toc: False
 
 
+## Rules
+
+* [Terms of Service](/terms/)
+* [Package Inclusion Policy and Guidance](/policy_and_guidance/)
+
 ## General Help
 
-* [Frequently Asked Questions](faq)
-* [Content Ratings and Flags](content_flags)
-* [Non-free Licenses](non_free)
-* [Why WTFPL is a terrible license](wtfpl)
-* [Ranks and Permissions](ranks_permissions)
-* [Contact Us](contact_us)
-* [Top Packages Algorithm](top_packages)
-* [Featured Packages](featured)
+* [Frequently Asked Questions](faq/)
+* [Installing content](installing/)
+* [Content Ratings and Flags](content_flags/)
+* [Non-free Licenses](non_free/)
+* [Why WTFPL is a terrible license](wtfpl/)
+* [Ranks and Permissions](ranks_permissions/)
+* [Contact Us](contact_us/)
+* [Top Packages Algorithm](top_packages/)
+* [Featured Packages](featured/)
+* [Feeds](feeds/)
 
 ## Help for Package Authors
 
 * [Package Inclusion Policy and Guidance](/policy_and_guidance/)
-* [Git Update Detection](update_config)
-* [Creating Releases using Webhooks](release_webhooks)
-* [Package Configuration and Releases Guide](package_config)
+* [Copyright Guide](copyright/)
+* [Git Update Detection](update_config/)
+* [Creating Releases using Webhooks](release_webhooks/)
+* [Package Configuration and Releases Guide](package_config/)
+* [Supported Games](game_support/)
+* [Creating an appealing ContentDB page](appealing_page/)
+
 
 ## Help for Specific User Ranks
 
-* [Editors](editors)
+* [Editors](editors/)
 
 ## APIs
 
-* [API](api)
-* [Prometheus Metrics](metrics)
+* [API](api/)
+* [OAuth2 Applications](oauth/)
+* [Prometheus Metrics](metrics/)

app/flatpages/help/api.md

@@ -3,12 +3,12 @@ title: API
 
 ## Resources
 
-* [How the Minetest client uses the API](https://github.com/minetest/contentdb/blob/master/docs/minetest_client.md)
+* [How the Luanti client uses the API](https://github.com/luanti-org/contentdb/blob/master/docs/luanti_client.md)
 
 
 ## Responses and Error Handling
 
-If there is an error, the response will be JSON similar to the following with a non-200 status code: 
+If there is an error, the response will be JSON similar to the following with a non-200 status code:
 
 ```json
 {
@@ -26,7 +26,7 @@ often other keys with information. For example:
 {
     "success": true,
     "release": {
-        /* same as returned by a GET */ 
+        /* same as returned by a GET */
     }
 }
 ```
@@ -39,7 +39,7 @@ the number of items is specified using `num`
 
 The response will be a dictionary with the following keys:
 
-* `page`: page number, integer from 1 to max 
+* `page`: page number, integer from 1 to max
 * `per_page`: number of items per page, same as `n`
 * `page_count`: number of pages
 * `total`: total number of results
@@ -54,8 +54,8 @@ The response will be a dictionary with the following keys:
 Not all endpoints require authentication, but it is done using Bearer tokens:
 
 ```bash
-curl https://content.minetest.net/api/whoami/ \
-    -H "Authorization: Bearer YOURTOKEN" 
+curl https://content.luanti.org/api/whoami/ \
+    -H "Authorization: Bearer YOURTOKEN"
 ```
 
 Tokens can be attained by visiting [Settings > API Tokens](/user/tokens/).
@@ -64,6 +64,13 @@ Tokens can be attained by visiting [Settings > API Tokens](/user/tokens/).
     * `is_authenticated`:  True on successful API authentication
     * `username`:  Username of the user authenticated as, null otherwise.
     * 4xx status codes will be thrown on unsupported authentication type, invalid access token, or other errors.
+* DELETE `/api/delete-token/`: Deletes the currently used token.
+
+```bash
+# Logout
+curl -X DELETE https://content.luanti.org/api/delete-token/ \
+    -H "Authorization: Bearer YOURTOKEN"
+```
 
 
 ## Packages
@@ -71,9 +78,12 @@ Tokens can be attained by visiting [Settings > API Tokens](/user/tokens/).
 * GET `/api/packages/` (List)
     * See [Package Queries](#package-queries)
 * GET `/api/packages/<username>/<name>/` (Read)
+    * Redirects a JSON object with the keys documented by the PUT endpoint, below.
+    * Plus:
+        * `forum_url`: String or null. 
 * PUT `/api/packages/<author>/<name>/` (Update)
     * Requires authentication.
-    * JSON dictionary with any of these keys (all are optional, null to delete Nullables):
+    * JSON object with any of these keys (all are optional, null to delete Nullables):
         * `type`: One of `GAME`, `MOD`, `TXP`.
         * `title`: Human-readable title.
         * `name`: Technical name (needs permission if already approved).
@@ -83,49 +93,111 @@ Tokens can be attained by visiting [Settings > API Tokens](/user/tokens/).
         * `tags`: List of [tag](#tags) names.
         * `content_warnings`: List of [content warning](#content-warnings) names.
         * `license`: A [license](#licenses) name.
-        * `media_license`: A [license](#licenses) name.   
+        * `media_license`: A [license](#licenses) name.
         * `long_description`: Long markdown description.
-        * `repo`: Git repo URL.
+        * `repo`: Source repository (eg: Git)
         * `website`: Website URL.
         * `issue_tracker`: Issue tracker URL.
         * `forums`: forum topic ID.
         * `video_url`: URL to a video.
-        * `game_support`: Array of game support information objects. Not currently documented, as subject to change.
+        * `donate_url`: URL to a donation page.
+        * `translation_url`: URL to send users interested in translating your package.
+        * `game_support`: Array of game support information objects. Not currently documented,
+    * Returns a JSON object with:
+        * `success`
+        * `package`: updated package
+        * `was_modified`: bool, whether anything changed
+* GET `/api/packages/<username>/<name>/for-client/`
+    * Similar to the read endpoint, but optimised for the Luanti client
+    * `long_description` is given as a hypertext object, see `/hypertext/` below.
+    * `info_hypertext` is the info sidebar as a hypertext object.
+    * Query arguments
+        * `formspec_version`: Required. See /hypertext/ below.
+        * `include_images`: Optional, defaults to true. If true, images use `<img>`. If false, they're linked.
+        * `protocol_version`: Optional, used to get the correct release.
+        * `engine_version`: Optional, used to get the correct release. Ex: `5.3.0`.
+* GET `/api/packages/<username>/<name>/for-client/reviews/`
+    * Returns hypertext representing the package's reviews
+    * Query arguments
+        * `formspec_version`: Required. See /hypertext/ below.
+    * Returns JSON dictionary with following keys:
+        * `head`: markup for suggested styling and custom tags, prepend to the body before displaying.
+        * `body`: markup for long description.
+        * `links`: dictionary of anchor name to link URL.
+        * `images`: dictionary of img name to image URL.
+        * `image_tooltips`: dictionary of img name to tooltip text.
+    * The hypertext body contains some placeholders that should be replaced client-side:
+        * `<thumbsup>` with a thumbs up icon. 
+        * `<neutral>` with a thumbs up icon. 
+        * `<thumbsdown>` with a thumbs up icon. 
+* GET `/api/packages/<author>/<name>/hypertext/`
+    * Converts the long description to [Luanti Markup Language](https://github.com/luanti-org/luanti/blob/master/doc/lua_api.md#markup-language)
+      to be used in a `hypertext` formspec element.
+    * Query arguments:
+        * `formspec_version`: Required, maximum supported formspec version.
+        * `include_images`: Optional, defaults to true. If true, images use `<img>`. If false, they're linked.
+    * Returns JSON dictionary with following keys:
+        * `head`: markup for suggested styling and custom tags, prepend to the body before displaying.
+        * `body`: markup for long description.
+        * `links`: dictionary of anchor name to link URL.
+        * `images`: dictionary of img name to image URL.
+        * `image_tooltips`: dictionary of img name to tooltip text.
 * GET `/api/packages/<username>/<name>/dependencies/`
-    * Returns dependencies, with suggested candidates 
+    * Returns dependencies, with suggested candidates
     * If query argument `only_hard` is present, only hard deps will be returned.
 * GET `/api/dependencies/`
     * Returns `provides` and raw dependencies for all packages.
     * Supports [Package Queries](#package-queries)
     * [Paginated result](#paginated-results), max 300 results per page
     * Each item in `items` will be a dictionary with the following keys:
-        * `type`: One of `GAME`, `MOD`, `TXP`. 
+        * `type`: One of `GAME`, `MOD`, `TXP`.
         * `author`: Username of the package author.
         * `name`: Package name.
         * `provides`: List of technical mod names inside the package.
         * `depends`: List of hard dependencies.
-            * Each dep will either be a metapackage dependency (`name`), or a
+            * Each dep will either be a modname dependency (`name`), or a
                 package dependency (`author/name`).
         * `optional_depends`: list of optional dependencies
             * Same as above.
+* GET `/api/packages/<username>/<name>/stats/`
+    * Returns daily stats for package, or null if there is no data.
+    * Daily date is done based on the UTC timezone.
+    * EXPERIMENTAL. This API may change without warning.
+    * Query args:
+        * `start`: start date, inclusive. Optional. Default: 2022-10-01. UTC.
+        * `end`: end date, inclusive. Optional. Default: today. UTC.
+    * An object with the following keys:
+        * `start`: start date, inclusive. Ex: 2022-10-22. M
+        * `end`: end date, inclusive. Ex: 2022-11-05.
+        * `platform_minetest`: list of integers per day.
+        * `platform_other`: list of integers per day.
+        * `reason_new`: list of integers per day.
+        * `reason_dependency`: list of integers per day.
+        * `reason_update`: list of integers per day.
+* GET `/api/package_stats/`
+    * Returns last 30 days of daily stats for _all_ packages.
+    * An object with the following keys:
+        * `start`: start date, inclusive. Ex: 2022-10-22.
+        * `end`: end date, inclusive. Ex: 2022-11-05.
+        * `package_downloads`: map from package key to list of download integers.
 
 You can download a package by building one of the two URLs:
 
 ```
-https://content.minetest.net/packages/${author}/${name}/download/`
-https://content.minetest.net/packages/${author}/${name}/releases/${release}/download/`
+https://content.luanti.org/packages/${author}/${name}/download/`
+https://content.luanti.org/packages/${author}/${name}/releases/${release}/download/`
 ```
 
 Examples:
 
 ```bash
 # Edit package
-curl -X PUT https://content.minetest.net/api/packages/username/name/ \
+curl -X PUT https://content.luanti.org/api/packages/username/name/ \
     -H "Authorization: Bearer YOURTOKEN" -H "Content-Type: application/json" \
     -d '{ "title": "Foo bar", "tags": ["pvp", "survival"], "license": "MIT" }'
-    
+
 # Remove website URL
-curl -X PUT https://content.minetest.net/api/packages/username/name/ \
+curl -X PUT https://content.luanti.org/api/packages/username/name/ \
     -H "Authorization: Bearer YOURTOKEN" -H "Content-Type: application/json" \
     -d '{ "website": null }'
 ```
@@ -136,44 +208,63 @@ Example:
 
     /api/packages/?type=mod&type=game&q=mobs+fun&hide=nonfree&hide=gore
 
-Supported query parameters:
+Filter query parameters:
 
-* `type`:  Package types (`mod`, `game`, `txp`).
+* `type`: Filter by package type (`mod`, `game`, `txp`). Multiple types are OR-ed together.
 * `q`:  Query string.
 * `author`:  Filter by author.
-* `tag`:  Filter by tags.
-* `random`:  When present, enable random ordering and ignore `sort`.
-* `limit`:  Return at most `limit` packages.
-* `hide`:  Hide content based on [Content Flags](/help/content_flags/).
+* `tag`:  Filter by tags. Multiple tags are AND-ed together.
+* `flag`: Filter to show packages with [Content Flags](/help/content_flags/).
+* `hide`:  Hide content based on tags or [Content Flags](/help/content_flags/).
+* `license`: Filter by [license name](#licenses). Multiple licenses are OR-ed together, ie: `&license=MIT&license=LGPL-2.1-only`
+* `game`: Filter by [Game Support](/help/game_support/), ex: `Warr1024/nodecore`. (experimental, doesn't show items that support every game currently).
+* `lang`: Filter by translation support, eg: `en`/`de`/`ja`/`zh_TW`.
+* `protocol_version`:  Only show packages supported by this Luanti protocol version.
+* `engine_version`:  Only show packages supported by this Luanti engine version, eg: `5.3.0`.
+
+Sorting query parameters:
+
 * `sort`:  Sort by (`name`, `title`, `score`, `reviews`, `downloads`, `created_at`, `approved_at`, `last_release`).
 * `order`:  Sort ascending (`asc`) or descending (`desc`).
-* `protocol_version`:  Only show packages supported by this Minetest protocol version.
-* `engine_version`:  Only show packages supported by this Minetest engine version, eg: `5.3.0`.
-* `fmt`:  How the response is formated.
+* `random`:  When present, enable random ordering and ignore `sort`.
+
+Format query parameters:
+
+* `limit`:  Return at most `limit` packages.
+* `fmt`:  How the response is formatted.
     * `keys`:  author/name only.
-    * `short`:  stuff needed for the Minetest client. 
+    * `short`:  stuff needed for the Luanti client.
+    * `vcs`: `short` but with `repo`.
 
 
-## Releases
+### Releases
 
-* GET `/api/releases/` (List)   
+* GET `/api/releases/` (List)
     * Limited to 30 most recent releases.
     * Optional arguments:
         * `author`: Filter by author
         * `maintainer`: Filter by maintainer
     * Returns array of release dictionaries with keys:
         * `id`: release ID
+        * `name`: short release name
         * `title`: human-readable title
+        * `release_notes`: string or null, what's new in this release. Markdown.
         * `release_date`: Date released
         * `url`: download URL
         * `commit`: commit hash or null
         * `downloads`: number of downloads
-        * `min_minetest_version`: dict or null, minimum supported minetest version (inclusive).
-        * `max_minetest_version`: dict or null, minimum supported minetest version (inclusive).
+        * `min_minetest_version`: dict or null, minimum supported Luanti version (inclusive).
+        * `max_minetest_version`: dict or null, minimum supported Luanti version (inclusive).
+        * `size`: size of zip file, in bytes.
         * `package`
             * `author`: author username
             * `name`: technical name
             * `type`: `mod`, `game`, or `txp`
+* GET `/api/updates/` (Look-up table)
+    * Returns a look-up table from package key (`author/name`) to latest release id
+    * Query arguments
+        * `protocol_version`:  Only show packages supported by this Luanti protocol version.
+        * `engine_version`:  Only show packages supported by this Luanti engine version, eg: `5.3.0`.
 * GET `/api/packages/<username>/<name>/releases/` (List)
     * Returns array of release dictionaries, see above, but without package info.
 * GET `/api/packages/<username>/<name>/releases/<id>/` (Read)
@@ -181,42 +272,49 @@ Supported query parameters:
     * Requires authentication.
     * Body can be JSON or multipart form data. Zip uploads must be multipart form data.
     * `title`: human-readable name of the release.
+    * `release_notes`: string or null, what's new in this release.
     * For Git release creation:
         * `method`: must be `git`.
         * `ref`: (Optional) git reference, eg: `master`.
-    * For zip upload release creation: 
+    * For zip upload release creation:
         * `file`: multipart file to upload, like `<input type="file" name="file">`.
         * `commit`: (Optional) Source Git commit hash, for informational purposes.
-    * You can set min and max Minetest Versions [using the content's .conf file](/help/package_config/).
+    * You can set min and max Luanti Versions [using the content's .conf file](/help/package_config/).
 * DELETE `/api/packages/<username>/<name>/releases/<id>/` (Delete)
     * Requires authentication.
     * Deletes release.
-  
+
 Examples:
 
 ```bash
 # Create release from Git
-curl -X POST https://content.minetest.net/api/packages/username/name/releases/new/ \
+curl -X POST https://content.luanti.org/api/packages/username/name/releases/new/ \
     -H "Authorization: Bearer YOURTOKEN" -H "Content-Type: application/json" \
-    -d '{ "method": "git", "title": "My Release", "ref": "master" }'
+    -d '{
+        "method": "git",
+        "name": "1.2.3",
+        "title": "My Release",
+        "ref": "master",
+        "release_notes": "some\nrelease\nnotes\n"
+    }'
 
 # Create release from zip upload
-curl -X POST https://content.minetest.net/api/packages/username/name/releases/new/ \
+curl -X POST https://content.luanti.org/api/packages/username/name/releases/new/ \
     -H "Authorization: Bearer YOURTOKEN" \
     -F title="My Release" -F file=@path/to/file.zip
 
 # Create release from zip upload with commit hash
-curl -X POST https://content.minetest.net/api/packages/username/name/releases/new/ \
+curl -X POST https://content.luanti.org/api/packages/username/name/releases/new/ \
     -H "Authorization: Bearer YOURTOKEN" \
     -F title="My Release" -F commit="8ef74deec170a8ce789f6055a59d43876d16a7ea" -F file=@path/to/file.zip
 
 # Delete release
-curl -X DELETE https://content.minetest.net/api/packages/username/name/releases/3/ \
-    -H "Authorization: Bearer YOURTOKEN" 
+curl -X DELETE https://content.luanti.org/api/packages/username/name/releases/3/ \
+    -H "Authorization: Bearer YOURTOKEN"
 ```
 
 
-## Screenshots
+### Screenshots
 
 * GET `/api/packages/<username>/<name>/screenshots/` (List)
     * Returns array of screenshot dictionaries with keys:
@@ -252,38 +350,39 @@ Examples:
 
 ```bash
 # Create screenshot
-curl -X POST https://content.minetest.net/api/packages/username/name/screenshots/new/ \
+curl -X POST https://content.luanti.org/api/packages/username/name/screenshots/new/ \
     -H "Authorization: Bearer YOURTOKEN" \
     -F title="My Release" -F file=@path/to/screnshot.png
-    
+
 # Create screenshot and set it as the cover image
-curl -X POST https://content.minetest.net/api/packages/username/name/screenshots/new/ \
+curl -X POST https://content.luanti.org/api/packages/username/name/screenshots/new/ \
     -H "Authorization: Bearer YOURTOKEN" \
     -F title="My Release" -F file=@path/to/screnshot.png -F is_cover_image="true"
 
 # Delete screenshot
-curl -X DELETE https://content.minetest.net/api/packages/username/name/screenshots/3/ \
-    -H "Authorization: Bearer YOURTOKEN" 
-    
+curl -X DELETE https://content.luanti.org/api/packages/username/name/screenshots/3/ \
+    -H "Authorization: Bearer YOURTOKEN"
+
 # Reorder screenshots
-curl -X POST https://content.minetest.net/api/packages/username/name/screenshots/order/ \
+curl -X POST https://content.luanti.org/api/packages/username/name/screenshots/order/ \
     -H "Authorization: Bearer YOURTOKEN" -H "Content-Type: application/json" \
     -d "[13, 2, 5, 7]"
-    
+
 # Set cover image
-curl -X POST https://content.minetest.net/api/packages/username/name/screenshots/cover-image/ \
+curl -X POST https://content.luanti.org/api/packages/username/name/screenshots/cover-image/ \
     -H "Authorization: Bearer YOURTOKEN" -H "Content-Type: application/json" \
     -d "{ 'cover_image': 123 }"
 ```
 
 
-## Reviews
+### Reviews
 
 * GET `/api/packages/<username>/<name>/reviews/` (List)
     * Returns array of review dictionaries with keys:
         * `user`: dictionary with `display_name` and `username`.
-        * `title`: review title 
+        * `title`: review title
         * `comment`: the text
+        * `rating`: 1 for negative, 3 for neutral, 5 for positive
         * `is_positive`: boolean
         * `created_at`: iso timestamp
         * `votes`: dictionary with `helpful` and `unhelpful`,
@@ -292,28 +391,31 @@ curl -X POST https://content.minetest.net/api/packages/username/name/screenshots
         * [Paginated result](#paginated-results)
         * `items`: array of review dictionaries, like above
             * Each review also has a `package` dictionary with `type`, `author` and `name`
+        * Ordered by created at, newest to oldest.
     * Query arguments:
         * `page`: page number, integer from 1 to max
-        * `n`: number of results per page, max 100
+        * `n`: number of results per page, max 200
         * `author`: filter by review author username
+        * `for_user`: filter by package author
+        * `rating`: 1 for negative, 3 for neutral, 5 for positive
         * `is_positive`: true or false. Default: null
-        * `q`: filter by title (case insensitive, no fulltext search)
+        * `q`: filter by title (case-insensitive, no fulltext search)
 
 Example:
 
 ```json
 [
   {
-    "comment": "This is a really good mod!", 
-    "created_at": "2021-11-24T16:18:33.764084", 
-    "is_positive": true, 
-    "title": "Really good", 
+    "comment": "This is a really good mod!",
+    "created_at": "2021-11-24T16:18:33.764084",
+    "is_positive": true,
+    "title": "Really good",
     "user": {
-      "display_name": "rubenwardy", 
+      "display_name": "rubenwardy",
       "username": "rubenwardy"
-    }, 
+    },
     "votes": {
-      "helpful": 0, 
+      "helpful": 0,
       "unhelpful": 0
     }
   }
@@ -321,6 +423,39 @@ Example:
 ```
 
 
+## Users
+
+* GET `/api/users/<username>/`
+    * `username`
+    * `display_name`: human-readable name to be displayed in GUIs.
+    * `rank`: ContentDB [rank](/help/ranks_permissions/).
+    * `profile_pic_url`: URL to profile picture, or null.
+    * `website_url`: URL to website, or null.
+    * `donate_url`: URL to donate page, or null.
+    * `connections`: object
+        * `github`: GitHub username, or null.
+        * `forums`: forums username, or null.
+    * `links`: object
+        * `api_packages`: URL to API to list this user's packages.
+        * `profile`: URL to the HTML profile page.
+* GET `/api/users/<username>/stats/`
+    * Returns daily stats for the user's packages, or null if there is no data.
+    * Daily date is done based on the UTC timezone.
+    * EXPERIMENTAL. This API may change without warning.
+    * Query args:
+        * `start`: start date, inclusive. Optional. Default: 2022-10-01. UTC.
+        * `end`: end date, inclusive. Optional. Default: today. UTC.
+    * A table with the following keys:
+        * `from`: start date, inclusive. Ex: 2022-10-22.
+        * `end`: end date, inclusive. Ex: 2022-11-05.
+        * `package_downloads`: map of package title to list of integers per day.
+        * `platform_minetest`: list of integers per day.
+        * `platform_other`: list of integers per day.
+        * `reason_new`: list of integers per day.
+        * `reason_dependency`: list of integers per day.
+        * `reason_update`: list of integers per day.
+
+
 ## Topics
 
 * GET `/api/topics/` ([View](/api/topics/))
@@ -338,39 +473,79 @@ Supported query parameters:
 * `type`:  Package types (`mod`, `game`, `txp`).
 * `sort`:  Sort by (`name`, `views`, `created_at`).
 * `show_added`:  Show topics that have an existing package.
-* `show_discarded`:  Show topics marked as discarded.
 * `limit`:  Return at most `limit` topics.
 
+
+## Collections
+
+* GET `/api/collections/`
+    * Query args:
+        * `author`: collection author username.
+        * `package`: collections that contain the package.
+    * Returns JSON array of collection entries:
+        * `author`: author username.
+        * `name`: collection name.
+        * `title`
+        * `short_description`
+        * `created_at`: creation time in iso format.
+        * `private`: whether collection is private, boolean.
+        * `package_count`: number of packages, integer.
+* GET `/api/collections/<username>/<name>/`
+    * Returns JSON object for collection:
+        * `author`: author username.
+        * `name`: collection name.
+        * `title`
+        * `short_description`
+        * `long_description`
+        * `created_at`: creation time in iso format.
+        * `private`: whether collection is private, boolean.
+        * `items`: array of item objects:
+            * `package`: short info about the package.
+            * `description`: custom short description.
+            * `created_at`: when the package was added to the collection.
+            * `order`: integer.
+
 ## Types
 
 ### Tags
 
-* GET `/api/tags/` ([View](/api/tags/)):  List of:
-    * `name`:  technical name.
-    * `title`:  human-readable title.
-    * `description`:  tag description or null.
-    * `is_protected`: boolean, whether the tag is protected (can only be set by Editors in the web interface).
-    * `views`: number of views of this tag.
-    
+* GET `/api/tags/` ([View](/api/tags/))
+    * List of objects with:
+        * `name`:  technical name.
+        * `title`:  human-readable title.
+        * `description`:  tag description or null.
+        * `views`: number of views of this tag.
+
 ### Content Warnings
 
-* GET `/api/content_warnings/` ([View](/api/content_warnings/)):  List of:
-    * `name`:  technical name
-    * `title`:  human-readable title
-    * `description`:  tag description or null
+* GET `/api/content_warnings/` ([View](/api/content_warnings/))
+    * List of objects with
+        * `name`:  technical name
+        * `title`:  human-readable title
+        * `description`:  tag description or null
 
 ### Licenses
 
-* GET `/api/licenses/` ([View](/api/licenses/)):  List of:
-    * `name`
-    * `is_foss`: whether the license is foss
+* GET `/api/licenses/` ([View](/api/licenses/))
+    * List of objects with:
+        * `name`
+        * `is_foss`: whether the license is foss
 
-### Minetest Versions
+### Luanti Versions
 
 * GET `/api/minetest_versions/` ([View](/api/minetest_versions/))
-    * `name`: Version name.
-    * `is_dev`: boolean, is dev version.
-    * `protocol_version`: protocol version umber.
+    * List of objects with:
+        * `name`: Version name.
+        * `is_dev`: boolean, is dev version.
+        * `protocol_version`: protocol version number.
+
+### Languages
+
+* GET `/api/languages/` ([View](/api/languages/))
+    * List of objects with:
+        * `id`: language code.
+        * `title`: native language name.
+        * `has_contentdb_translation`: whether ContentDB has been translated into this language.
 
 
 ## Misc
@@ -385,6 +560,10 @@ Supported query parameters:
         * `score`: total package score.
         * `score_reviews`: score from reviews.
         * `score_downloads`: score from downloads.
+        * `reviews`: a dictionary of
+          * `positive`: int, number of positive reviews.
+          * `neutral`: int, number of neutral reviews.
+          * `negative`: int, number of negative reviews.
 * GET `/api/homepage/` ([View](/api/homepage/)) - get contents of homepage.
     * `count`:  number of packages
     * `downloads`:  get number of downloads
@@ -394,5 +573,20 @@ Supported query parameters:
     * `pop_txp`:  popular textures
     * `pop_game`:  popular games
     * `high_reviewed`:  highest reviewed
-* GET `/api/welcome/v1/` ([View](/api/welcome/v1/)) - in-menu welcome dialog. Experimental (may change without warning)
-    * `featured`: featured games
+* GET `/api/cdb_schema/` ([View](/api/cdb_schema/))
+    * Get JSON Schema of `.cdb.json`, including licenses, tags and content warnings.
+    * See [JSON Schema Reference](https://json-schema.org/).
+* POST `/api/hypertext/`
+    * Converts HTML or Markdown to [Luanti Markup Language](https://github.com/luanti-org/luanti/blob/master/doc/lua_api.md#markup-language)
+      to be used in a `hypertext` formspec element.
+    * Post data: HTML or Markdown as plain text.
+    * Content-Type: `text/html` or `text/markdown`.
+    * Query arguments:
+        * `formspec_version`: Required, maximum supported formspec version. Ie: 6
+        * `include_images`: Optional, defaults to true. If true, images use `<img>`. If false, they're linked.
+    * Returns JSON dictionary with following key:
+        * `head`: markup for suggested styling and custom tags, prepend to the body before displaying.
+        * `body`: markup for long description.
+        * `links`: dictionary of anchor name to link URL.
+        * `images`: dictionary of img name to image URL
+        * `image_tooltips`: dictionary of img name to tooltip text.

app/flatpages/help/appealing_page.md

@@ -0,0 +1,74 @@
+title: Creating an appealing ContentDB page
+
+## Title and short description
+
+Make sure that your package's title is unique, short, and descriptive.
+
+Expand on the title with the short description. You have a limited number
+of characters, use them wisely!
+
+```ini
+# Bad, we know this is a mod for Luanti. Doesn't give much information other than "food"
+description = The food mod for Luanti
+# Much better, says what is actually in this mod!
+description = Adds soup, cakes, bakes and juices
+```
+
+## Thumbnail
+
+A good thumbnail goes a long way to making a package more appealing. It's one of the few things
+a user sees before clicking on your package. Make sure it's possible to tell what a
+thumbnail is when it's small.
+
+For a preview of what your package will look like inside Luanti, see
+Edit Package > Screenshots.
+
+## Screenshots
+
+Upload a good selection of screenshots that show what is possible with your packages.
+You may wish to focus on a different key feature in each of your screenshots.
+
+A lot of users won't bother reading text, and will just look at screenshots.
+
+## Long description
+
+The target audience of your package page is end users.
+The long description should explain what your package is about,
+why the user should choose it, and how to use it if they download it.
+
+[NodeCore](https://content.luanti.org/packages/Warr1024/nodecore/) is a good
+example of what to do. For inspiration, you might want to look at how games on
+Steam write their descriptions.
+
+Your long description might contain:
+
+* What does the package contain/have? ie: list of high-level features.
+* What makes it special? Why should users choose this over another package?
+* How can you use it?
+
+The following are redundant and should probably not be included:
+
+* A heading with the title of the package
+* The short description
+* Links to a Git repository, the forum topic, the package's ContentDB page (ContentDB has fields for this)
+* License (unless you need to give more information than ContentDB's license fields)
+* API reference (unless your mod is a library only)
+* Development instructions for your package (this should be in the repo's README)
+* Screenshots that are already uploaded (unless you want to embed a recipe image in a specific place)
+    * Note: you should avoid images in the long description as they won't be visible inside Luanti,
+      when support for showing the long description is added.
+
+## Localize / Translate your package
+
+According to Google Play, 64% of Luanti Android users don't have English as their main language.
+Adding translation support to your package increases accessibility. Using content translation, you
+can also translate your ContentDB page. See Edit Package > Translation for more information.
+
+<p>
+    <a class="btn btn-primary me-2" href="https://rubenwardy.com/minetest_modding_book/en/quality/translations.html">
+        {{ _("Translation - Luanti Modding Book") }}
+    </a>
+    <a class="btn btn-primary" href="https://api.luanti.org/translations/#translating-content-meta">
+        {{ _("Translating content meta - lua_api.md") }}
+    </a>
+</p>

app/flatpages/help/contact_us.md

@@ -11,4 +11,4 @@ We take copyright violation and other offenses very seriously.
 
 ## Other
 
-<a href="https://rubenwardy.com/contact/" class="btn btn-primary">Contact the admin</a>
+<a href="{{ admin_contact_url }}" class="btn btn-primary">Contact the admin</a>

app/flatpages/help/content_flags.md

@@ -6,7 +6,7 @@ your client to use new flags.
 
 ## Flags
 
-Minetest allows you to specify a comma-separated list of flags to hide in the
+Luanti allows you to specify a comma-separated list of flags to hide in the
 client:
 
 ```
@@ -17,7 +17,7 @@ A flag can be:
 
 * `nonfree`: can be used to hide packages which do not qualify as
     'free software', as defined by the Free Software Foundation.
-* `wip`: packages marked as Work in Progress 
+* `wip`: packages marked as Work in Progress
 * `deprecated`: packages marked as Deprecated
 * A content warning, given below.
 * `*`: hides all content warnings.
@@ -33,8 +33,8 @@ without making a release.
 Packages with mature content will be tagged with a content warning based
 on the content type.
 
+* `alcohol_tobacco`: alcohol or tobacco.
 * `bad_language`: swearing.
-* `drugs`: drugs or alcohol.
 * `gambling`
 * `gore`: blood, etc.
 * `horror`: shocking and scary content.

app/flatpages/help/copyright.md

@@ -0,0 +1,147 @@
+title: Copyright Guide
+
+## Why should I care?
+
+Falling foul of copyright law can put you and ContentDB into legal trouble. Receiving a Cease and Desist, DMCA notice,
+or a Court Summons isn't pleasant for anyone, and can turn out to be very expensive. This page contains some
+guidance on how to ensure your content is clearly licensed and attributed to avoid these issues.
+
+Additionally, ContentDB and the forums both have some
+[requirements on the licenses](/policy_and_guidance/#41-allowed-licenses) you are allowed to use. Both require
+[free distribution and modification](/help/non_free/), allowing us to remain an open community where people can fork
+and remix each other's content. To this end, you need to make sure your content is clearly licensed.
+
+**As always, we are not lawyers and this does not constitute legal advice.**
+
+
+## What do I need to do?
+
+### Follow the licenses
+
+Make sure you understand the licenses for anything you copy into your content.
+[TL;DR Legal](https://tldrlegal.com/license/mit-license) is a good resource for quickly understanding
+licenses, although you should actually read the text as well.
+
+If you use code from other sources (such as mods or games), you'll need to make sure you follow
+their license. A common one is attribution, you should do this by adding a comment next to the
+code and crediting the author in your LICENSE file.
+
+It's sometimes fine to copy trivial/small amounts of code under fair use, but this
+is a bit of a grey area. It's better to understand the solution and rewrite it yourself.
+
+### List the sources of your media
+
+It's a good idea to create a list of all the media you used in your package, as it allows
+you to keep track of where the media came from. Media includes textures, 3d models,
+sounds, and more.
+
+You should have the following information:
+
+* File name (as found in your package)
+* Author name
+* License
+* Source (URL to the webpage, mod name, website name)
+
+It's common to do this in README.md or LICENSE.md like so:
+
+```md
+* conquer_arrow_*.png from [Simple Shooter](https://github.com/stujones11/shooter) by Stuart Jones, CC0 1.0.
+* conquer_arrow.b3d from [Simple Shooter](https://github.com/stujones11/shooter) by Stuart Jones, CC-BY-SA 3.0.
+* conquer_arrow_head.png from MTG, CC-BY-SA 3.0.
+* health_*.png from [Gauges](https://content.luanti.org/packages/Calinou/gauges/) by Calinou, CC0.
+```
+
+if you have a lot of media, then you can split it up by author like so:
+
+```md
+[Kenney](https://www.kenney.nl/assets/voxel-pack), CC0:
+
+* mymod_fence.png
+
+John Green, CC BY-SA 4.0 from [OpenGameArt](https://opengameart.org/content/tiny-16-basic):
+
+* mymod_texture.png
+* mymod_another.png
+
+Your Name, CC BY-SA 4.0:
+
+* mymod_texture_i_made.png
+```
+
+
+## Where can I get freely licensed media?
+
+* [OpenGameArt](https://opengameart.org/) - everything
+* [Kenney game assets](https://www.kenney.nl/assets) - everything
+* [Free Sound](https://freesound.org/) - sounds
+* [PolyHaven](https://polyhaven.com/) - 3d models and textures.
+* Other Luanti mods/games
+
+    Don't assume the author has correctly licensed their work.
+    Make sure they have clearly indicated the source in a list [like above](#list-the-sources-of-your-media).
+    If they didn't make it, then go to the actual source to check the license.
+
+
+## Common Situations
+
+### I made it myself, using X as a guide
+
+Copying by hand is still copying, the law doesn't distinguish this from copy+paste.
+Make your own art without copying colors or patterns from existing games/art.
+
+If you need a good set of colors, see [LOSPEC](https://lospec.com/palette-list).
+
+### I got it from Google Images / Search / the Internet
+
+You do not have permission to use things unless you are given permission to do so by the author.
+No license is exactly the same as "Copyright © All Rights Reserved".
+To use on ContentDB or the forums, you must also be given a clear license.
+
+Try searching with "creative commons" in the search term, and then clicking through to the page
+and looking for a license. Make sure the source looks trustworthy, as there are a lot of websites
+that rip off art and give an incorrect license. But it might be better to use a trusted source directly, see
+[the section above](#where-can-i-get-freely-licensed-media) for a list.
+
+### I have permission from the author
+
+You'll also need to make sure that the author gives you an explicit license for it, such as CC BY-SA 4.0.
+Permission for *you* to use it doesn't mean that *everyone* has permission to use it. A license outlines the terms of
+the permission, making things clearer and less vague.
+
+### The author said it's free for anyone to use, is that enough?
+
+No, you need an explicit license like CC0 or CC BY-SA 4.0. ContentDB does not allow custom licenses
+or public domain.
+
+### I used an AI
+
+Errrr. This is a legally untested area, we highly recommend that **you don't use AI art/code** in packages
+for that reason.
+
+For now, we haven't banned AI art/code from ContentDB. Make sure to clearly include it in your package's
+credit list (include the name of the AI tool used).
+
+Check the tools terms and conditions to see if there are any constraints on use. It looks
+like AI-generated art and code isn't copyrightable by itself, but the tool's T&Cs may still
+impose conditions.
+
+AI art/code may regurgitate copyrighted things. Make sure that you don't include the
+names of any copyrighted materials in your AI prompts, such as names of games or artists.
+
+## What does ContentDB do?
+
+The package authors and maintainers are responsible for the licenses and copyright of packages on ContentDB.
+ContentDB editors will check packages to make sure the package page's license matches up with the list of licenses
+inside the package download, but do not investigate each piece of media or line of code.
+
+If a copyright violation is reported to us, we will unlist the package and contact the author/maintainers.
+Once the problem has been fixed, the package can be restored. Repeated copyright infringement may lead to
+permanent bans.
+
+
+## Where can I get help?
+
+[Join](https://www.luanti.org/get-involved/) IRC, Matrix, or Discord to ask for help.
+In Discord, there are the #assets or #contentdb channels. In IRC or Matrix, you can just ask in the main channels.
+
+If your package is already on ContentDB, you can open a thread.

app/flatpages/help/editors.md

@@ -15,8 +15,9 @@ Editors should make sure they are familiar with the
 
 ## ContentDB is not a curated platform
 
-It's important to note that ContentDB isn't a curated platform, but it also does have some
-requirements on minimum usefulness. See 2.2 in the [Policy and Guidance](/policy_and_guidance/).
+It's important to note that ContentDB isn't a curated platform - a mod doesn't need to be good
+to be accepted, but there are some minimum requirements when it comes to usefulness and other things.
+See 2.2 in the [Policy and Guidance](/policy_and_guidance/).
 
 ## Editor Work Queue
 
@@ -25,10 +26,31 @@ The [Editor Work Queue](/todo/) and related pages contain useful information for
 * The package, release, and screenshot approval queues.
 * Packages which are outdated or are missing tags.
 * A list of forum topics without packages.
-  Editors can create the packages or "discard" them if they don't think it's worth adding them.
 
 ## Editor Notifications
 
 Editors currently receive notifications for any new thread opened on a package, so that they
 know when a user is asking for help. These notifications are shown separately in the notifications
 interface, and can be configured separately in Emails and Notifications.
+
+## Crash Course to being an Editor
+
+The [Package Inclusion Policy and Guidance](/policy_and_guidance/) is our go-to resource for making decisions in 
+changes needed, similar to how lua_api.txt is the doc for modders to consult.
+
+In the [Editor console](/todo/), the two most important tabs are the Editor Work Queue and the Forum 
+Topics tab. Primarily you will be focusing on the Editor Work Queue tab, where a list of packages to review is.
+
+When you have some free time, feel free to scroll through the Forum Topics page and import mods into ContentDB.
+But don't import a mod if it's broken, outdated, not that useful, or not worth importing - click Discard instead.
+
+A simplified process for reviewing a package is as follows:
+
+1. scan the package image if present for any obvious closed source assets.
+2. if right to a name warning is present, check its validity and if the package meets 
+   the exceptions.
+3. if the forums topic missing warning is present, feel free to check it, but it's 
+   usually incorrect.
+4. check source, etc links to make sure they work and are correct.
+5. verify that the package has license file that matches what is on the contentdb fields
+6. if the above steps pass, approve the package, else request changes needed from the author

app/flatpages/help/faq.md

@@ -1,4 +1,5 @@
 title: Frequently Asked Questions
+description: FAQ about using ContentDB
 
 ## Users and Logins
 
@@ -11,7 +12,7 @@ be done using a GitHub account or a random string in your forum account signatur
 
 If you don't, then you can just sign up using an email address and password.
 
-GitHub can only be used to login, not to register.
+GitHub can only be used to log in, not to register.
 
 <a class="btn btn-primary" href="/user/claim/">Register</a>
 
@@ -21,16 +22,19 @@ GitHub can only be used to login, not to register.
 There are a number of reasons this may have happened:
 
 * Incorrect email address entered.
-* Temporary problem with ContentDB. 
+* Temporary problem with ContentDB.
 * Email has been unsubscribed.
 
-If the email doesn't arrive after registering by email, then you'll need to try registering again in 12 hours.
-Unconfirmed accounts are deleted after 12 hours.
+**When creating an account by email:**
+If the email doesn't arrive after registering by email, then you'll need to
+try registering again in 12 hours. Unconfirmed accounts are deleted after 12 hours.
 
-If the email verification was sent using the Email settings tab, then you can just set a new email.
+**When changing your email (or it was set after a forum-based registration)**:
+then you can just set a new email in
+[Settings > Email and Notifications](/user/settings/email/).
 
 If you have previously unsubscribed this email, then ContentDB is completely prevented from sending emails to that
-address. You'll need to use a different email address, or [contact rubenwardy](https://rubenwardy.com/contact/) to
+address. You'll need to use a different email address, or [contact the admin]({{ admin_contact_url }}) to
 remove your email from the blacklist.
 
 
@@ -40,11 +44,25 @@ remove your email from the blacklist.
 
 There are a number of methods:
 
-* [Git Update Detection](update_config): ContentDB will check your Git repo daily, and create updates or send you notifications.
-* [Webhooks](release_webhooks): you can configure your Git host to send a webhook to ContentDB, and create an update immediately.
-* the [API](api): This is especially powerful when combined with CI/CD and other API endpoints. 
+* [Git Update Detection](/help/update_config/): ContentDB will check your Git repo daily, and create updates or send you notifications.
+* [Webhooks](/help/release_webhooks/): you can configure your Git host to send a webhook to ContentDB, and create an update immediately.
+* the [API](/help/api/): This is especially powerful when combined with CI/CD and other API endpoints.
+
+### How do I learn how to make mods and games for Luanti?
+
+You should read
+[the official Luanti Modding Book](https://rubenwardy.com/minetest_modding_book/)
+for a guide to making mods and games using Luanti.
+
+### How do I install something from here?
+
+See [Installing content](/help/installing/).
+
+### How can my package get more downloads?
+
+See [Creating an appealing ContentDB page](/help/appealing_page/).
 
 
 ## How do I get help?
 
-Please [contact rubenwardy](https://rubenwardy.com/contact/).
+Please [contact rubenwardy](https://rubenwardy.com/contact/).

app/flatpages/help/featured.md

@@ -7,10 +7,10 @@ title: Featured Packages
 ## What are Featured Packages?
 
 Featured Packages are shown at the top of the ContentDB homepage. In the future,
-featured packages may be shown inside the Minetest client.
+featured packages may be shown inside the Luanti client.
 
 The purpose is to promote content that demonstrates a high quality of what is
-possible in Minetest. The selection should be varied, and should vary over time.
+possible in Luanti. The selection should be varied, and should vary over time.
 The featured content should be content that we are comfortable recommending to
 a first time player.
 
@@ -47,7 +47,7 @@ other packages to be featured, or for another reason.
 
 * MUST: Be 100% free and open source (as marked as Free on ContentDB).
 * MUST: Work out-of-the-box (no weird setup or settings required).
-* MUST: Be compatible with the latest stable Minetest release.
+* MUST: Be compatible with the latest stable Luanti release.
 * SHOULD: Use public source control (such as Git).
 * SHOULD: Have at least 3 reviews, and be largely positive.
 
@@ -94,7 +94,7 @@ is available.
 ### Usability
 
 * MUST: Unsupported mapgens are disabled in game.conf.
-* SHOULD: Passes the Beginner Test: A newbie to the game (but not Minetest) wouldn't get completely
+* SHOULD: Passes the Beginner Test: A newbie to the game (but not Luanti) wouldn't get completely
   stuck within the first 5 minutes of playing.
 * SHOULD: Have good documentation. This may include one or more of:
     * A craftguide, or other in-game learning system

app/flatpages/help/feeds.md

@@ -0,0 +1,16 @@
+title: Feeds
+
+You can follow updates from ContentDB in your RSS feed reader. If in doubt, copy the Atom URL.
+
+* All events: [Atom]({{ url_for('feeds.all_atom') }}) | [JSONFeed]({{ url_for('feeds.all_json') }})
+* New packages: [Atom]({{ url_for('feeds.packages_all_atom') }}) | [JSONFeed]({{ url_for('feeds.packages_all_json') }})
+* New releases: [Atom]({{ url_for('feeds.releases_all_atom') }}) | [JSONFeed]({{ url_for('feeds.releases_all_json') }})
+
+## Package feeds
+
+Follow new releases for a package:
+
+```
+https://content.luanti.org/packages/AUTHOR/NAME/releases_feed.atom
+https://content.luanti.org/packages/AUTHOR/NAME/releases_feed.json
+```

app/flatpages/help/game_support.md

@@ -0,0 +1,52 @@
+title: Supported Games
+
+## Why?
+
+The supported/compatible games feature allows mods to specify the games that
+they work with, which improves user experience.
+
+
+## Support sources
+
+### mod.conf / texture_pack.conf
+
+You can use `supported_games` to specify games that your mod/modpack/texture
+pack is compatible with.
+
+You can use `unsupported_games` to specify games that your package doesn't work
+with, which is useful for overriding ContentDB's automatic detection.
+
+Both of these are comma-separated lists of game technical ids. Any `_game`
+suffixes are ignored, just like in Luanti.
+
+    supported_games = minetest_game, repixture
+    unsupported_games = lordofthetest, nodecore, whynot
+
+If your package supports all games by default, you can put "*" in
+supported_games. You can still use unsupported_games to mark games as
+unsupported. You can also specify games that you've tested in supported_games.
+
+    # Should work with all games but I've only tested using Minetest Game:
+    supported_games = *, minetest_game
+
+    # But doesn't work in capturetheflag
+    unsupported_game = capturetheflag
+
+### Dependencies
+
+ContentDB will analyse hard dependencies and work out which games a mod
+supports.
+
+This uses a recursive algorithm that works out whether a dependency can be
+installed independently, or if it requires a certain game.
+
+### On ContentDB
+
+You can define supported games on ContentDB, but using .conf is recommended
+instead.
+
+
+## Combining all the sources
+
+.conf will override anything ContentDB detects. The manual override on ContentDB
+overrides .conf and dependencies.

app/flatpages/help/installing.md

@@ -0,0 +1,89 @@
+title: How to install mods, games, and texture packs
+description: A guide to installing mods, games, and texture packs in Luanti.
+
+## Installing from the main menu (recommended)
+
+### Install
+
+1. Open the mainmenu
+2. Go to the Content tab and click "Browse online content".
+   If you don't see this, then you need to update Luanti to v5.
+3. Search for the package you want to install, and click "Install".
+4. When installing a mod, you may be shown a dialog about dependencies here.
+   Make sure the base game dropdown box is correct, and then click "Install".
+
+<div class="row mt-5">
+  <div class="col-md-6">
+    <figure>
+      <a href="/static/installing_content_tab.png">
+        <img class="w-100" src="/static/installing_content_tab.png" alt="Screenshot of the content tab in Luanti">
+      </a>
+      <figcaption class="text-muted ps-1">
+        1. Click Browser Online Content in the content tab.
+      </figcaption>
+    </figure>
+  </div>
+  <div class="col-md-6">
+    <figure>
+      <a href="/static/installing_cdb_dialog.png">
+        <img class="w-100" src="/static/installing_cdb_dialog.png" alt="Screenshot of the content tab in Luanti">
+      </a>
+      <figcaption class="text-muted ps-1">
+        2. Search for the package and click "Install".
+      </figcaption>
+    </figure>
+  </div>
+</div>
+
+Troubleshooting:
+
+* I can't find it in the ContentDB dialog (Browse online content)
+    * Make sure that you're on the latest version of Luanti.
+    * Are you using Android? Packages with content warnings are hidden by default on android,
+      you can show them by removing `android_default` from the `contentdb_flag_blacklist` setting.
+    * Does the webpage show "Non-free" warnings? Non-free content is hidden by default from all clients,
+      you can show them by removing `nonfree` from the `contentdb_flag_blacklist` setting.
+* It says "required dependencies could not be found"
+    * Make sure you're using the correct "Base Game". A lot of packages only work with certain games, you can look
+      at "Compatible Games" on the web page to see which.
+
+### Enable in Select Mods
+
+1. Mods: Enable the content using "Select Mods" when selecting a world.
+2. Games: choose a game when making a world.
+3. Texture packs: Content > Select pack > Click enable.
+
+
+<div class="row mt-5">
+  <div class="col-md-6">
+    <figure>
+      <a href="/static/installing_select_mods.png">
+        <img class="w-100" src="/static/installing_select_mods.png" alt="Screenshot of Select Mods in Luanti">
+      </a>
+      <figcaption class="text-muted ps-1">
+        Enable mods using the Select Mods dialog.
+      </figcaption>
+    </figure>
+  </div>
+</div>
+
+## Installing using the command line
+
+### Git clone
+
+1. Install git
+2. Find the package on ContentDB and copy "source" link.
+3. Find the user data directory.
+   In 5.4.0 and above, you can click "Open user data directory" in the Credits tab.
+   Otherwise:
+     * Windows: wherever you extracted or installed Luanti to.
+     * Linux: usually `~/.minetest/`
+4. Open or create the folder for the type of content (`mods`, `games`, or `textures`)
+5. Git clone there
+6. For mods, make sure to install any required dependencies.
+
+### Enable
+
+* Mods: Edit world.mt in the world's folder to contain `load_file_MODNAME = true`
+* Games: Use `--game` or edit game_id in world.mt.
+* Texture packs: change the `texture_path` setting to the texture pack absolute path.

app/flatpages/help/metrics.md

@@ -9,11 +9,13 @@ and modern alerting approach".
 Prometheus Metrics can be accessed at [/metrics](/metrics), or you can view them
 on the Grafana instance below.
 
+{% if monitoring_url %}
 <p>
-    <a class="btn btn-primary" href="https://monitor.rubenwardy.com/d/3ELzFy3Wz/contentdb">
+    <a class="btn btn-primary" href="{{ monitoring_url }}">
         View ContentDB on Grafana
     </a>
 </p>
+{% endif %}
 
 ## Metrics
 

app/flatpages/help/non_free.md

@@ -13,7 +13,7 @@ and they will be subject to limited promotion.
 **ContentDB does not allow certain non-free licenses, and will limit the promotion
 of packages with non-free licenses.**
 
-Minetest is free and open source software, and is only as big as it is now
+Luanti is free and open source software, and is only as big as it is now
 because of this. It's pretty amazing you can take nearly any published mod and modify it
 to how you like - add some features, maybe fix some bugs - and then share those
 modifications without the worry of legal issues. The project, itself, relies on open
@@ -24,9 +24,9 @@ If you have played nearly any game with a large modding scene, you will find
 that most mods are legally ambiguous. A lot of them don't even provide the
 source code to allow you to bug fix or extend as you need.
 
-Limiting the promotion of problematic licenses helps Minetest avoid ending up in
+Limiting the promotion of problematic licenses helps Luanti avoid ending up in
 such a state. Licenses that prohibit redistribution or modification are
-completely banned from ContentDB and the Minetest forums. Other non-free licenses
+completely banned from ContentDB and the Luanti forums. Other non-free licenses
 will be subject to limited promotion - they won't be shown by default in
 the client.
 
@@ -37,7 +37,7 @@ you spread it.
 ## What's so bad about licenses that forbid commercial use?
 
 Please read [reasons not to use a Creative Commons -NC license](https://freedomdefined.org/Licenses/NC).
-Here's a quick summary related to Minetest content:
+Here's a quick summary related to Luanti content:
 
 1. They make your work incompatible with a growing body of free content, even if
    you do want to allow derivative works or combinations.
@@ -55,7 +55,7 @@ Here's a quick summary related to Minetest content:
 Non-free packages are hidden in the client by default, partly in order to comply
 with the rules of various Linux distributions.
 
-Users can opt-in to showing non-free software, if they wish:
+Users can opt in to showing non-free software, if they wish:
 
 1. In the main menu, go to Settings > All settings
 2. Search for "ContentDB Flag Blacklist".
@@ -66,8 +66,18 @@ Users can opt-in to showing non-free software, if they wish:
 	<figcaption class="figure-caption">Screenshot of the ContentDB Flag Blacklist setting</figcaption>
 </figure>
 
-In the future, [the `platform_default` flag](/help/content_flags/) will be used to control what content
-each platforms shows - Android is significantly stricter about mature content.
-You may wish to remove all text from that setting completely, leaving it blank,
-if you wish to view all content when this happens. Currently, [mature content is
-not permitted on ContentDB](/policy_and_guidance/).
+The [`platform_default` flag](/help/content_flags/) is used to control what content
+each platforms shows. It doesn't hide anything on Desktop, but hides all mature
+content on Android. You may wish to remove all text from that setting completely,
+leaving it blank. See [Content Warnings](/help/content_flags/#content-warnings)
+for information on mature content.
+
+## How can I hide non-free packages on the website?
+
+Clicking "Hide non-free packages" in the footer of ContentDB will hide non-free packages from search results.
+It will not hide non-free packages from user profiles.
+
+## See also
+
+* [List of non-free packages](/packages/?flag=nonfree)
+* [Copyright Guide](/help/copyright)

app/flatpages/help/oauth.md

@@ -0,0 +1,103 @@
+title: OAuth2 API
+
+<p class="alert alert-warning">
+    The OAuth2 applications API is currently experimental, and may break without notice.
+</p>
+
+ContentDB allows you to create an OAuth2 Application and obtain access tokens
+for users.
+
+
+## Scopes
+
+OAuth2 applications can currently only access public user data, using the whoami API.
+
+
+## Create an OAuth2 Client
+
+Go to Settings > [OAuth2 Applications](/user/apps/) > Create
+
+
+## Obtaining access tokens
+
+ContentDB supports the Authorization Code OAuth2 method.
+
+### Authorize
+
+Get the user to open the following URL in a web browser:
+
+```
+https://content.luanti.org/oauth/authorize/
+    ?response_type=code
+    &client_id={CLIENT_ID}
+    &redirect_uri={REDIRECT_URL}
+```
+
+The redirect_url must much the value set in your oauth client. Make sure to URL encode it.
+ContentDB also supports `state`.
+
+Afterwards, the user will be redirected to your callback URL.
+If the user accepts the authorization, you'll receive an authorization code (`code`).
+Otherwise, the redirect_url will not be modified.
+
+For example, with `REDIRECT_URL` set as `https://example.com/callback/`:
+
+* If the user accepts: `https://example.com/callback/?code=abcdef`
+* If the user cancels: `https://example.com/callback/`
+
+### Exchange auth code for access token
+
+Next, you'll need to exchange the auth for an access token.
+
+Do this by making a POST request to the `/oauth/token/` API:
+
+```bash
+curl -X POST https://content.luanti.org/oauth/token/ \
+    -F grant_type=authorization_code \
+    -F client_id="CLIENT_ID" \
+    -F client_secret="CLIENT_SECRET" \
+    -F code="abcdef" 
+```
+
+<p class="alert alert-warning">
+    <i class="fas fa-exclamation-circle me-2"></i>
+    You should make this request on a server to prevent the user
+    from getting access to your client secret.
+</p>
+
+If successful, you'll receive:
+
+```json
+{
+    "success": true,
+    "access_token": "access_token",
+    "token_type": "Bearer"
+}
+```
+
+If there's an error, you'll receive a standard API error message:
+
+```json
+{
+    "success": false,
+    "error": "The error message"
+}
+```
+
+Possible errors:
+
+* Unsupported grant_type, only authorization_code is supported
+* Missing client_id
+* Missing client_secret
+* Missing code
+* client_id and/or client_secret is incorrect
+* Incorrect code. It may have already been redeemed
+
+### Check access token
+
+Next, you should check the access token works by getting the user information:
+
+```bash
+curl https://content.luanti.org/api/whoami/ \
+    -H "Authorization: Bearer YOURTOKEN"
+```

app/flatpages/help/package_config.md

@@ -19,24 +19,37 @@ The filename of the `.conf` file depends on the content type:
 * `game.conf` for games.
 * `texture_pack.conf` for texture packs.
 
-The `.conf` uses a key-value format, separated using equals. Here's a simple example:
+The `.conf` uses a key-value format, separated using equals.
+
+Here's a simple example of `mod.conf`, `modpack.conf`, or `texture_pack.conf`:
 
 	name = mymod
+	title = My Mod
 	description = A short description to show in the client.
 
+Here's a simple example of `game.conf`:
+
+	title = My Game
+	description = A short description to show in the client.
+
+Note that you should not specify `name` in game.conf.
+
 ### Understood values
 
 ContentDB understands the following information:
 
+* `title` - A human-readable title.
 * `description` - A short description to show in the client.
 * `depends` - Comma-separated hard dependencies.
 * `optional_depends` - Comma-separated soft dependencies.
-* `min_minetest_version` - The minimum Minetest version this runs on, see [Min and Max Minetest Versions](#min_max_versions).
-* `max_minetest_version` - The maximum Minetest version this runs on, see [Min and Max Minetest Versions](#min_max_versions).
+* `min_minetest_version` - The minimum Luanti version this runs on, see [Min and Max Luanti Versions](#min_max_versions).
+* `max_minetest_version` - The maximum Luanti version this runs on, see [Min and Max Luanti Versions](#min_max_versions).
 
 and for mods only:
 
 * `name` - the mod technical name.
+* `supported_games` - List of supported game technical names.
+* `unsupported_games` - List of not supported game technical names. Useful to override game support detection.
 
 
 ## .cdb.json
@@ -55,15 +68,17 @@ It should be a JSON dictionary with one or more of the following optional keys:
 * `tags`: List of tag names, see [/api/tags/](/api/tags/).
 * `content_warnings`: List of content warning names, see [/api/content_warnings/](/api/content_warnings/).
 * `license`: A license name, see [/api/licenses/](/api/licenses/).
-* `media_license`: A license name.          
+* `media_license`: A license name.
 * `long_description`: Long markdown description.
-* `repo`: Git repo URL.
+* `repo`: Source repository (eg: Git).
 * `website`: Website URL.
 * `issue_tracker`: Issue tracker URL.
 * `forums`: forum topic ID.
 * `video_url`: URL to a video.
+* `donate_url`: URL to a donation page.
+* `translation_url`: URL to send users interested in translating your package.
 
-Use `null` to unset fields where relevant.
+Use `null` or `[]` to unset fields where relevant.
 
 Example:
 
@@ -91,11 +106,11 @@ See [Git Update Detection](/help/update_config/).
 You can also use [GitLab/GitHub webhooks](/help/release_webhooks/) or the [API](/help/api/)
 to create releases.
 
-### Min and Max Minetest Versions
+### Min and Max Luanti Versions
 
 <a name="min_max_versions" />
 
-When creating a release, the `.conf` file will be read to determine what Minetest
+When creating a release, the `.conf` file will be read to determine what Luanti
 versions the release supports. If the `.conf` doesn't specify, then it is assumed
 that it supports all versions.
 

app/flatpages/help/ranks_permissions.md

@@ -2,8 +2,8 @@ title: Ranks and Permissions
 
 ## Overview
 
-* **New Members** - mostly untrusted, cannot change package meta data or publish releases without approval.
-* **Members** - Trusted to change the meta data of their own packages', but cannot approve their own packages.
+* **New Members** - mostly untrusted, cannot change package metadata or publish releases without approval.
+* **Members** - Trusted to change the metadata of their own packages', but cannot approve their own packages.
 * **Trusted Members** - Same as above, but can approve their own releases.
 * **Approvers** - Responsible for approving new packages, screenshots, and releases.
 * **Editors** - Same as above, and can edit any package or release.
@@ -266,7 +266,7 @@ title: Ranks and Permissions
 		</tr>
 		<tr>
 			<td>Create Token</td>
-			<td></td> <!-- new -->
+			<td>✓</td> <!-- new -->
 			<td></td>
 			<td>✓</td> <!-- member -->
 			<td></td>

app/flatpages/help/release_webhooks.md

@@ -6,7 +6,7 @@ A webhook is a notification from one service to another. Put simply, a webhook
 is used to notify ContentDB that the git repository has changed.
 
 ContentDB offers the ability to automatically create releases using webhooks
-from either Github or Gitlab. If you're not using either of those services,
+from either GitHub or GitLab. If you're not using either of those services,
 you can also use the [API](../api) to create releases.
 
 ContentDB also offers the ability to poll a Git repo and check for updates
@@ -16,14 +16,21 @@ See [Git Update Detection](/help/update_config/).
 The process is as follows:
 
 1. The user creates an API Token and a webhook to use it.
-2. The user pushes a commit to the git host (Gitlab or Github).
+2. The user pushes a commit to the git host (GitLab or GitHub).
 3. The git host posts a webhook notification to ContentDB, using the API token assigned to it.
 4. ContentDB checks the API token and issues a new release.
+    * If multiple packages match, then only the first will have a release created.
+
+### Branch filtering
+
+By default, "New commit" or "push" based webhooks will only work on "master"/"main" branches.
+You can configure the branch used by changing "Branch name" in [Git update detection](update_config).
+
+For example, to support production and beta packages you can have multiple packages with the same VCS repo URL
+but different [Git update detection](update_config) branch names.
+
+Tag-based webhooks are accepted on any branch.
 
-<p class="alert alert-warning">
-    "New commit" or "push" based webhooks will currently only work on branches named `master` or
-    `main`.
-</p>
 
 ## Setting up
 
@@ -32,32 +39,33 @@ The process is as follows:
 1. Create a ContentDB API Token at [Profile > API Tokens: Manage](/user/tokens/).
 2. Copy the access token that was generated.
 3. Go to the GitLab repository's settings > Webhooks > Add Webhook.
-4. Set the payload URL to `https://content.minetest.net/github/webhook/`
+4. Set the payload URL to `https://content.luanti.org/github/webhook/`
 5. Set the content type to JSON.
 6. Set the secret to the access token that you copied.
 7. Set the events
-  * If you want a rolling release, choose "just the push event".
-  * Or if you want a stable release cycle based on tags,
-    choose "Let me select" > Branch or tag creation.
+    * If you want a rolling release, choose "just the push event".
+    * Or if you want a stable release cycle based on tags, choose "Let me select" > Branch or tag creation.
 8. Create.
+9. If desired, change [Git update detection](update_config) > Branch name to configure the [branch filtering](#branch-filtering).
 
 ### GitLab
 
 1. Create a ContentDB API Token at [Profile > API Tokens: Manage](/user/tokens/).
 2. Copy the access token that was generated.
 3. Go to the GitLab repository's settings > Webhooks.
-4. Set the URL to `https://content.minetest.net/gitlab/webhook/`
+4. Set the URL to `https://content.luanti.org/gitlab/webhook/`
 6. Set the secret token to the ContentDB access token that you copied.
 7. Set the events
     * If you want a rolling release, choose "Push events".
     * Or if you want a stable release cycle based on tags,
       choose "Tag push events".
 8. Add webhook.
+9. If desired, change [Git update detection](update_config) > Branch name to configure the [branch filtering](#branch-filtering).
 
 ## Configuring Release Creation
 
 See the [Package Configuration and Releases Guide](/help/package_config/) for
 documentation on configuring the release creation.
 
-From the Git repository, you can set the min/max Minetest versions, which files are included,
+From the Git repository, you can set the min/max Luanti versions, which files are included,
 and update the package meta.

app/flatpages/help/top_packages.md

@@ -19,7 +19,7 @@ score       = avg_downloads + reviews_sum;
 ## Pseudo rolling average of downloads
 
 Each package adds 1 to `avg_downloads` for each unique download,
-and then loses 5% (=1/20) of the value every day.
+and then loses 6.66% (=1/15) of the value every day.
 
 This is called a [Frecency](https://en.wikipedia.org/wiki/Frecency) heuristic,
 a measure which combines both frequency and recency.
@@ -33,4 +33,4 @@ downloaded from that IP.
 You can see all scores using the [scores REST API](/api/scores/), or by
 using the [Prometheus metrics](/help/metrics/) endpoint.
 
-Consider [suggesting improvements](https://github.com/minetest/contentdb/issues/new?assignees=&labels=Policy&template=policy.md&title=).
+Consider [suggesting improvements](https://github.com/luanti-org/contentdb/issues/new?assignees=&labels=Policy&template=policy.md&title=).

app/flatpages/help/update_config.md

@@ -39,5 +39,5 @@ Clicking "Save" on "Update Settings" will mark a package as up-to-date.
 See the [Package Configuration and Releases Guide](/help/package_config/) for
 documentation on configuring the release creation.
 
-From the Git repository, you can set the min/max Minetest versions, which files are included,
+From the Git repository, you can set the min/max Luanti versions, which files are included,
 and update the package meta.

app/flatpages/help/wtfpl.md

@@ -1,25 +1,6 @@
 title: WTFPL is a terrible license
 toc: False
 
-<div id="warning" class="alert alert-warning">
-	<span class="icon_message"></span>
-
-	Please reconsider the choice of WTFPL as a license.
-
-	<script src="/static/libs/jquery.min.js"></script>
-	<script>
-		// @author rubenwardy
-		// @license magnet:?xt=urn:btih:1f739d935676111cfff4b4693e3816e664797050&dn=gpl-3.0.txt GPL-v3-or-Later
-
-		var params = new URLSearchParams(location.search);
-		var r      = params.get("r");
-		if (r)
-			document.write("<a class='alert_right button' href='" + r + "'>Okay</a>");
-		else
-			$("#warning").hide();
-	</script>
-</div>
-
 The use of WTFPL as a license is discouraged for multiple reasons.
 
 * **No Warranty disclaimer:** This could open you up to being sued.<sup>[1]</sup>
@@ -37,4 +18,4 @@ license, saying:<sup>[3]</sup>
 
 1. [WTFPL is harmful to software developers](https://cubicspot.blogspot.com/2017/04/wtfpl-is-harmful-to-software-developers.html)
 2. [FSF](https://www.gnu.org/licenses/license-list.en.html)
-3. [OSI](https://opensource.org/minutes20090304)
+3. [OSI](https://opensource.org/meeting-minutes/minutes20090304)

app/flatpages/policy_and_guidance.md

@@ -1,22 +1,5 @@
 title: Package Inclusion Policy and Guidance
 
-## 0. Overview
-
-ContentDB is for the community, and as such listings should be useful to the
-community. To help with this, there are a few rules to improve the quality of
-the listings and to combat abuse.
-
-* **No inappropriate content.** <sup>2.1</sup>
-* **Content must be playable/useful, but not necessarily finished.** <sup>2.2</sup>
-* **Don't use the name of another mod unless your mod is a fork or reimplementation.** <sup>3</sup>
-* **Licenses must allow derivatives, redistribution, and must not discriminate.** <sup>4</sup>
-* **Don't put promotions or advertisements in any package metadata.** <sup>5</sup>
-* **Don't manipulate package placement using reviews or downloads.** <sup>6</sup>
-* **Screenshots must not be misleading.** <sup>7</sup>
-* **The ContentDB admin reserves the right to remove packages for any reason**,
-  including ones not covered by this document, and to ban users who abuse
-  this service. <sup>1</sup>
-
 
 ## 1. General
 
@@ -26,39 +9,53 @@ including ones not covered by this document, and to ban users who abuse this ser
 
 ## 2. Accepted Content
 
-### 2.1. Acceptable Content
+### 2.1. Mature Content
 
-Sexually-orientated content is not permitted.
-If in doubt at what this means, [contact us by raising a report](/report/).
+See the [Terms of Service](/terms/) for a full list of prohibited content.
 
-Mature content is permitted providing that it is labelled correctly.
-See [Content Flags](/help/content_flags/).
+Other mature content is permitted providing that it is labelled with the applicable
+[content warning](/help/content_flags/).
 
-The submission of malware is strictly prohibited. This includes software that
-does not do as it advertises, for example, if it posts telemetry without stating
-clearly that it does in the package meta.
+### 2.2. Useful Content / State of Completion
 
-### 2.2. State of Completion
+ContentDB is for playable and useful content - content which is sufficiently
+complete to be useful to end-users.
 
-ContentDB should only currently contain playable content - content which is
-sufficiently complete to be useful to end-users. It's fine to add stuff which
-is still a Work in Progress (WIP) as long as it adds sufficient value;
-MineClone 2 is a good example of a WIP package which may break between releases
-but still has value. Note that this doesn't mean that you should add a thing
-you started working on yesterday, it's worth adding all the basic stuff to
-make your package useful.
+It's fine to add stuff which is still a Work in Progress (WIP) as long as it
+adds sufficient value. You must make sure to mark Work in Progress stuff as
+such in the "maintenance status" dropdown, as this will help advise players.
 
-You should make sure to mark Work in Progress stuff as such in the "maintenance status" column,
-as this will help advise players.
+Adding non-player facing mods, such as libraries and server tools, is perfectly
+fine and encouraged. ContentDB isn't just for player-facing things and adding
+libraries allows Luanti to automatically install dependencies.
 
-Adding non-player facing mods, such as libraries and server tools, is perfectly fine
-and encouraged. ContentDB isn't just for player-facing things, and adding
-libraries allows them to be installed when a mod depends on it.
+### 2.3. Language
+
+We require packages to be in English with (optional) client-side translations for
+other languages. This is because Luanti currently requires English as the base language
+([Issue to change this](https://github.com/luanti-org/luanti/issues/6503)).
+
+Your package's title and short description must be in English. You can use client-side
+translations to [translate content meta](https://api.luanti.org/translations/#translating-content-meta).
+
+### 2.4. Attempt to contribute before forking
+
+You should attempt to contribute upstream before forking a package. If you choose
+to fork, you should have a justification (different objectives, maintainer is unavailable, etc).
+You should use a different title and make it clear in the long description what the
+benefit of your fork is over the original package.
+
+### 2.5. Copyright and trademarks
+
+Your package must not violate copyright or trademarks. You should avoid the use of
+trademarks in the package's title or short description. If you do use a trademark,
+ensure that you phrase it in a way that does not imply official association or
+endorsement.
 
 
 ## 3. Technical Names
 
-### 3.1 Right to a name
+### 3.1. Right to a Name
 
 A package uses a name when it has that name or contains a mod that uses that name.
 
@@ -76,23 +73,47 @@ to change the name of the package, or your package won't be accepted.
 
 We reserve the right to issue exceptions for this where we feel necessary.
 
-### 3.2. Mod Forks and Reimplementations
+### 3.2. Forks and Reimplementations
 
 An exception to the above is that mods are allowed to have the same name as a
 mod if it's a fork of that mod (or a close reimplementation). In real terms, it
-should be possible to use the new mod as a drop-in replacement.
+must be possible to use the new mod as a drop-in replacement.
 
 We reserve the right to decide whether a mod counts as a fork or
 reimplementation of the mod that owns the name.
 
+### 3.3. Game Mod Namespacing
+
+New mods introduced by a game must have a unique common prefix to avoid conflicts with
+other games and standalone mods. For example, the NodeCore game's first-party mods all
+start with `nc_`: `nc_api`, `nc_doors`.
+
+You may include existing or standard mods in your game without renaming them to use the
+namespace. For example, NodeCore could include the `awards` mod without needing to rename it.
+
+Standalone mods may not use a game's namespace unless they have been given permission by
+the game's author.
+
+The exception given by 3.2 also applies to game namespaces - you may use another game's
+prefix if your game is a fork.
+
 
 ## 4. Licenses
 
-### 4.1. Allowed Licenses
+### 4.1. License file
 
+You must have a LICENSE, LICENSE.txt, or LICENSE.md file describing the licensing of your package.
 Please ensure that you correctly credit any resources (code, assets, or otherwise)
 that you have used in your package.
 
+You may use lowercase or include a suffix in the filename (ie: `license-code.txt`). If
+you are making a game or modpack, your top level license file may just be a summary or
+refer to the license files of individual components.
+
+For help on doing copyright correctly, see the [Copyright help page](/help/copyright/).
+
+### 4.2. Allowed Licenses
+
 **The use of licenses that do not allow derivatives or redistribution is not
 permitted. This includes CC-ND (No-Derivatives) and lots of closed source licenses.
 The use of licenses that discriminate between groups of people or forbid the use
@@ -101,19 +122,19 @@ of the content on servers or singleplayer is also not permitted.**
 However, closed sourced licenses are allowed if they allow the above.
 
 If the license you use is not on the list then please select "Other", and we'll
-get around to adding it.
+get around to adding it. We reject custom/untested licenses and reserve the right
+to decide whether a license should be included.
 
 Please note that the definitions of "free" and "non-free" is the same as that
 of the [Free Software Foundation](https://www.gnu.org/philosophy/free-sw.en.html).
 
-### 4.2. Recommended Licenses
+### 4.3. Recommended Licenses
 
 It is highly recommended that you use a Free and Open Source software (FOSS)
 license. FOSS licenses result in a sharing community and will increase the
 number of potential users your package has. Using a closed source license will
-result in your package being massively penalised in the search results and
-package lists. See the help page on [non-free licenses](/help/non_free/) for more
-information.
+result in your package not being shown in Luanti by default. See the help page
+on [non-free licenses](/help/non_free/) for more information.
 
 It is recommended that you use a proper license for code with a warranty
 disclaimer, such as the (L)GPL or MIT. You should also use a proper media license
@@ -129,7 +150,7 @@ Public domain is not a valid license in many countries, please use CC0 or MIT in
 
 ## 5. Promotions and Advertisements (inc. asking for donations)
 
-You may not place any promotions or advertisements in any meta data including
+You may not place any promotions or advertisements in any metadata including
 screenshots. This includes asking for donations, promoting online shops,
 or linking to personal websites and social media. Please instead use the
 fields provided on your user profile page to place links to websites and
@@ -155,10 +176,14 @@ Doing so may result in temporary or permanent suspension from ContentDB.
 
 ## 7. Screenshots
 
-1.  **Screenshots must not violate copyright.** You should have the rights to the
-    screenshot.
+1.  We require all packages to have at least one screenshot. For packages without visual
+    content, we recommend making a symbolic image with icons, graphics, or text to depict
+    the package.
 
-2.  **Screenshots must depict the actual content of the package in some way, and
+2.  **Screenshots must not violate copyright.** This means don't just copy images
+    from Google search, see [the copyright guide](/help/copyright/).
+
+3.  **Screenshots must depict the actual content of the package in some way, and
     not be misleading.**
 
     Do not use idealized mockups or blender concept renders if they do not
@@ -174,21 +199,22 @@ Doing so may result in temporary or permanent suspension from ContentDB.
     will look like in a typical/realistic game scene, but should be "in the
     background" only as far as possible.
 
-3.  **Screenshots must only contain content appropriate for the Content Warnings of
+4.  **Screenshots must only contain content appropriate for the Content Warnings of
     the package.**
 
-4.  **Screenshots should be MOSTLY in-game screenshots, if applicable.** Some
-    alterations on in-game screenshots are okay, such as collages, added text,
-    some reasonable compositing.
 
-    Don't just use one of the textures from the package; show it in-situ as it
-    actually looks in the game.
+## 8. Security
 
-5.  **Packages should have a screenshot when reasonably applicable.**
+The submission of malware is strictly prohibited. This includes software that
+does not do as it advertises, for example, if it posts telemetry without stating
+clearly that it does in the package meta.
 
-6.  **Screenshots should be of reasonable dimensions.** We recommend using 1920x1080.
+Packages must not ask that users disable mod security (`secure.enable_security`).
+Instead, they should use the insecure environment API.
+
+Packages must not contain obfuscated code.
 
 
-## 8. Reporting Violations
+## 9. Reporting Violations
 
 Please click "Report" on the package page.

app/flatpages/privacy_policy.md

@@ -1,7 +1,8 @@
 title: Privacy Policy
+---
 
-Last Updated: 2022-01-23
-([View updates](https://github.com/minetest/contentdb/commits/master/app/flatpages/privacy_policy.md))
+Last Updated: 2024-04-30
+([View updates](https://github.com/luanti-org/contentdb/commits/master/app/flatpages/privacy_policy.md))
 
 ## What Information is Collected
 
@@ -11,8 +12,9 @@ Last Updated: 2022-01-23
 	* Time
 	* IP address
 	* Page URL
-	* Response status code
+	* Platform and Operating System
 * Preferred language/locale. This defaults to the browser's locale, but can be changed by the user
+* Whether an IP address has downloaded a particular package in the last 14 days
 
 **With an account:**
 
@@ -32,7 +34,7 @@ Please avoid giving other personal information as we do not want it.
 
 ## How this information is used
 
-* Logged HTTP requests may be used for debugging ContentDB.
+* Logged HTTP requests may be used for debugging ContentDB and combating abuse.
 * Email addresses are used to:
     * Provide essential system messages, such as password resets and privacy policy updates.
     * Send notifications - the user may configure this to their needs, including opting out.
@@ -40,13 +42,21 @@ Please avoid giving other personal information as we do not want it.
 * Passwords are used to authenticate the user.
 * The audit log is used to record actions that may be harmful.
 * Preferred language/locale is used to translate emails and the ContentDB interface.
+* Requests (such as downloads) are used for aggregated statistics and for
+  calculating the popularity of packages. For example, download counts are shown
+  for each package and release and there are also download graphs available for
+  each package.
+* Whether an IP address has downloaded a package or release is cached to prevent
+  downloads from being counted multiple times per IP address, but this
+  information is deleted after 14 days.
+* IP addresses are used to monitor and combat abuse.
 * Other information is displayed as part of ContentDB's service.
 
 ## Who has access
 
 * Only the admin has access to the HTTP requests.
   The logs may be shared with others to aid in debugging, but care will be taken to remove any personal information.
-* Encrypted backups may be shared with selected Minetest staff members (moderators + core devs).
+* Encrypted backups may be shared with selected Luanti staff members (moderators + core devs).
   The keys and the backups themselves are given to different people,
   requiring at least two staff members to read a backup.
 * Email addresses are visible to moderators and the admin.
@@ -57,44 +67,52 @@ Please avoid giving other personal information as we do not want it.
   They are either public, or visible only to the package author and editors.
 * The complete audit log is visible to moderators.
   Users may see their own audit log actions on their account settings page.
-  Owners, maintainers, and editors may be able to see the actions on a package in the future.
-* Preferred language can only be viewed by this with access to the database or a backup.
+  Owners, maintainers, and editors can see the actions on a package.
+* Preferred language can only be viewed by those with access to the database or a backup.
 * We may be required to share information with law enforcement.
 
+## Third-parties
+
+We do not share any personal information with third parties.
+
+We use <a href="https://sentry.io/">Sentry.io</a> for error logging and performance monitoring.
+
 ## Location
 
 The ContentDB production server is currently located in Germany.
 Backups are stored in the UK.
 Encrypted backups may be stored in other countries, such as the US or EU.
 
-By using this service, you give permission for the data to be moved as needed.
+By using this service, you give permission for the data to be moved within the
+United Kingdom and/or EU.
 
 ## Period of Retention
 
-The server uses log rotation, meaning that any logged HTTP requests will be
-forgotten within a few weeks.
+Logged HTTP requests are automatically deleted within 2 weeks.
 
-Usernames may be kept indefinitely, but other user information will be deleted if
-requested. See below.
+Usernames may be kept indefinitely, but other user information will be deleted
+if requested. See below.
+
+Whether an IP address has downloaded a package or release is deleted after 14 days.
 
 ## Removal Requests
 
-Please [raise a report](https://content.minetest.net/report/?anon=0) if you
-wish to remove your personal information.
+Please [raise a report](/report/?anon=0) if you wish to remove your personal
+information.
 
-ContentDB keeps a record of each username and forum topic on the forums,
-for use in indexing mod/game topics. ContentDB also requires the use of a username
-to uniquely identify a package. Therefore, an author cannot be removed completely
+ContentDB keeps a record of each username and forum topic on the forums, for use
+in indexing mod/game topics. ContentDB also requires the use of a username to
+uniquely identify a package. Therefore, an author cannot be removed completely
 from ContentDB if they have any packages or mod/game topics on the forum.
 
 If we are unable to remove your account for one of the above reasons, your user
 account will instead be wiped and deactivated, ending up exactly like an author
-who has not yet joined ContentDB. All personal information will be removed from the profile,
-and any comments or threads will be deleted.
+who has not yet joined ContentDB. All personal information will be removed from
+the profile, and any comments or threads will be deleted.
 
 ## Future Changes to Privacy Policy
 
-We will alert any future changes to the privacy policy via email and
-via notices on the ContentDB website.
+We will alert any future changes to the privacy policy via notices on the
+ContentDB website.
 
 By continuing to use this service, you agree to the privacy policy.

app/flatpages/terms.md

@@ -0,0 +1,133 @@
+title: Terms of Service
+
+Also see the [Package Inclusion Policy](/policy_and_guidance/).
+
+## Content
+
+### Prohibited content
+
+You must not post/transmit anything which is illegal under the laws in any part of the United Kingdom.
+
+You must not (or use the service to) facilitate or commit any offence under the laws in any part of the United Kingdom.
+
+This includes, in particular, terrorism content (as set out in Schedule 5, Online Safety Act 2023),
+child sexual exploitation and abuse content (as set out in Schedule 6, Online Safety Act 2023), and
+content that amounts to an offence specified in Schedule 7, Online Safety Act 2023.
+
+Prohibited content includes:
+
+* Pornographic content. This includes content of such a nature that it is reasonable to assume that it was produced
+  solely or principally for the purpose of sexual arousal.
+* Content which encourages, promotes or provides instructions for suicide
+* Content which encourages, promotes or provides instructions for an act of deliberate self-injury
+* Content which encourages, promotes or provides instructions for an eating disorder or behaviours associated with an eating disorder
+* Content which is abusive and which targets any of the following characteristics: race, religion, sex,
+  sexual orientation, disability, gender reassignment.
+* Content which incites hatred against people:
+    * of a particular race, religion, sex or sexual orientation
+    * who have a disability
+    * who have the characteristic of gender reassignment
+* Content which encourages, promotes or provides instructions for an act of serious violence against a person
+* Bullying content
+* Content which:
+    * depicts real or realistic serious violence against a person
+    * depicts the real or realistic serious injury of a person in graphic detail
+* Content which:
+    * depicts real or realistic serious violence against an animal
+    * depicts the real or realistic serious injury of an animal in graphic detail
+    * realistically depicts serious violence against a fictional creature or the serious injury of a fictional
+      creature in graphic detail
+* Content which encourages, promotes or provides instructions for a challenge or stunt highly likely to result in
+  serious injury to the person who does it or to someone else
+* Content which encourages a person to ingest, inject, inhale or in any other way self-administer:
+    * a physically harmful substance
+    * a substance in such a quantity as to be physically harmful
+
+### Protecting users from illegal content
+
+We provide this service free of charge, and on the basis that we may:
+
+* take down, or restrict access to, anything that you generate, upload or share; and
+* suspend or ban you from using all or part of the service
+
+if we think that this is reasonable to protect you, other users, the service, or us. This applies, in particular,
+to prohibited content.
+
+If we are alerted by a person to the presence of any illegal content, or we become aware of it in any other way, 
+we will swiftly take down that content.
+
+To minimise the length of time for which any illegal content within the scope of the Online Safety Act 2023 is present:
+
+* in respect of terrorism content, we offer an easy-to-access and use reporting function and will swiftly remove such content when we become aware of it.
+* in respect of child sexual exploitation or abuse content, we offer an easy-to-access and use reporting function and will swiftly remove such content when we become aware of it.
+* in respect of other content that amounts to an offence specified in Schedule 7, Online Safety Act 2023, we offer an easy-to-access and use reporting function and will swiftly remove such content when we become aware of it.
+
+### Protecting children
+
+We protect all children from the kinds of content listed in "Prohibited Content" by:
+
+* prohibiting that type of content from our service; and
+* swiftly taking down that content, if we are alerted by a person to its presence, or we become aware of it in any other way.
+
+### Proactive technology
+
+We do not use proactive technology to detect illegal content.
+
+
+## Limitation of Liability
+
+THE SERVICE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO
+THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL WE BE LIABLE
+FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
+OR IN CONNECTION WITH THE SERVICE OR THE USE OR OTHER DEALINGS IN THE SERVICE.
+
+We reserve the right to ban or suspend your account, or take down your content, for any reason.
+
+
+## Jurisdiction and Governing Law
+
+This service is subject to the jurisdiction of the United Kingdom.
+
+
+## Complaints
+
+### Reporting content
+
+You may report content by clicking the report flag next to a comment or "Report" on the page containing the content.
+
+You can also make reports by [contacting the admin]({{ admin_contact_url }}).
+
+### Complaints and Appeals
+
+You may send a complaint / request an appeal by [contacting the admin]({{ admin_contact_url }}). 
+
+### Your right to bring a claim
+
+This clause applies only to users within the United Kingdom.
+
+The Online Safety Act 2023 says that you have a right to bring a claim for breach of contract if:
+
+* anything that you generate, upload or share is taken down, or access to it is restricted, in breach of the terms of service, or
+* you are suspended or banned from using the service in breach of the terms of service.
+
+This does not apply to emails, SMS messages, MMS messages, one-to-one live aural communications,
+comments and reviews (together with any further comments on such comments or reviews), or content which identifies
+you as a user (e.g. a user name or profile picture).
+
+Whether or not a contract exists between you and us is a question of fact. If we do not have a contractual
+relationship with you in respect of the service, there can be no breach of contract and, as such, this cannot apply.
+
+It is for a court to determine:
+
+- if there is a contract between you and us and, if so, its terms
+- if there has been a breach by us of that contract
+- if that breach has caused you any recoverable loss
+- the size (e.g. value) of your loss
+
+This clause is subject to "Limitation of liability" and "Jurisdiction".
+
+
+## Acknowledgements
+    
+This terms of service was written based on [a template](https://onlinesafetyact.co.uk/online_safety_act_terms/)
+created by Neil Brown, CC BY-SA 4.0.

app/logic/approval_stats.py

@@ -0,0 +1,148 @@
+# ContentDB
+# Copyright (C) 2024  rubenwardy
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+import datetime
+from collections import namedtuple, defaultdict
+from typing import Dict, Optional
+from sqlalchemy import or_
+
+from app.models import AuditLogEntry, db, PackageState
+
+
+class PackageInfo:
+	state: Optional[PackageState]
+	first_submitted: Optional[datetime.datetime]
+	last_change: Optional[datetime.datetime]
+	approved_at: Optional[datetime.datetime]
+	wait_time: int
+	total_approval_time: int
+	is_in_range: bool
+	events: list[tuple[str, str, str]]
+
+	def __init__(self):
+		self.state = None
+		self.first_submitted = None
+		self.last_change = None
+		self.approved_at = None
+		self.wait_time = 0
+		self.total_approval_time = -1
+		self.is_in_range = False
+		self.events = []
+
+	def __lt__(self, other):
+		return self.wait_time < other.wait_time
+
+	def __dict__(self):
+		return {
+			"first_submitted": self.first_submitted.isoformat(),
+			"last_change": self.last_change.isoformat(),
+			"approved_at": self.approved_at.isoformat() if self.approved_at else None,
+			"wait_time": self.wait_time,
+			"total_approval_time": self.total_approval_time if self.total_approval_time >= 0 else None,
+			"events": [ { "date": x[0], "by": x[1], "title": x[2] } for x in self.events ],
+		}
+
+	def add_event(self, created_at: datetime.datetime, causer: str, title: str):
+		self.events.append((created_at.isoformat(), causer, title))
+
+
+def get_state(title: str):
+	if title.startswith("Approved "):
+		return PackageState.APPROVED
+
+	assert title.startswith("Marked ")
+
+	for state in PackageState:
+		if state.value in title:
+			return state
+
+	if "Work in Progress" in title:
+		return PackageState.WIP
+
+	raise Exception(f"Unable to get state for title {title}")
+
+
+Result = namedtuple("Result", "editor_approvals packages_info avg_turnaround_time max_turnaround_time")
+
+
+def _get_approval_statistics(entries: list[AuditLogEntry], start_date: Optional[datetime.datetime] = None, end_date: Optional[datetime.datetime] = None) -> Result:
+	editor_approvals = defaultdict(int)
+	package_info: Dict[str, PackageInfo] = {}
+	ignored_packages = set()
+	turnaround_times: list[int] = []
+
+	for entry in entries:
+		package_id = str(entry.package.get_id())
+		if package_id in ignored_packages:
+			continue
+
+		info = package_info.get(package_id, PackageInfo())
+		package_info[package_id] = info
+
+		is_in_range = (((start_date is None or entry.created_at >= start_date) and
+				(end_date is None or entry.created_at <= end_date)))
+		info.is_in_range = info.is_in_range or is_in_range
+
+		new_state = get_state(entry.title.replace("…", "") + (entry.description or ""))
+		if new_state == info.state:
+			continue
+
+		info.add_event(entry.created_at, entry.causer.username if entry.causer else None, new_state.value)
+
+		if info.state == PackageState.READY_FOR_REVIEW:
+			seconds = int((entry.created_at - info.last_change).total_seconds())
+			info.wait_time += seconds
+			if is_in_range:
+				turnaround_times.append(seconds)
+
+		if new_state == PackageState.APPROVED:
+			ignored_packages.add(package_id)
+			info.approved_at = entry.created_at
+			if is_in_range:
+				editor_approvals[entry.causer.username] += 1
+			if info.first_submitted is not None:
+				info.total_approval_time = int((entry.created_at - info.first_submitted).total_seconds())
+		elif new_state == PackageState.READY_FOR_REVIEW:
+			if info.first_submitted is None:
+				info.first_submitted = entry.created_at
+
+		info.state = new_state
+		info.last_change = entry.created_at
+
+	packages_info_2 = {}
+	package_count = 0
+	for package_id, info in package_info.items():
+		if info.first_submitted and info.is_in_range:
+			package_count += 1
+			packages_info_2[package_id] = info
+
+	if len(turnaround_times) > 0:
+		avg_turnaround_time = sum(turnaround_times) / len(turnaround_times)
+		max_turnaround_time = max(turnaround_times)
+	else:
+		avg_turnaround_time = 0
+		max_turnaround_time = 0
+
+	return Result(editor_approvals, packages_info_2, avg_turnaround_time, max_turnaround_time)
+
+
+def get_approval_statistics(start_date: Optional[datetime.datetime] = None, end_date: Optional[datetime.datetime] = None) -> Result:
+	entries = AuditLogEntry.query.filter(AuditLogEntry.package).filter(or_(
+		AuditLogEntry.title.like("Approved %"),
+		AuditLogEntry.title.like("Marked %"))
+	).order_by(db.asc(AuditLogEntry.created_at)).all()
+
+	return _get_approval_statistics(entries, start_date, end_date)

app/logic/game_support.py

@@ -1,5 +1,5 @@
 # ContentDB
-# Copyright (C) 2022 rubenwardy
+# Copyright (C) rubenwardy
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,30 +14,12 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
+from typing import List, Dict, Optional, Tuple
 
-import sys
+import sqlalchemy
 
-from typing import List, Dict, Optional, Iterator, Iterable
-
-from app.logic.LogicError import LogicError
-from app.models import Package, MetaPackage, PackageType, PackageState, PackageGameSupport, db
-
-"""
-get_game_support(package):
-	if package is a game:
-		return [ package ]
-
-	for all hard dependencies:
-		support = support AND get_meta_package_support(dep)
-
-	return support
-
-get_meta_package_support(meta):
-	for package implementing meta package:
-		support = support OR get_game_support(package)
-
-	return support
-"""
+from app.models import PackageType, Package, PackageState, PackageGameSupport
+from app.utils import post_bot_message
 
 
 minetest_game_mods = {
@@ -49,140 +31,331 @@ minetest_game_mods = {
 
 
 mtg_mod_blacklist = {
-	"repixture", "tutorial", "runorfall", "realtest_mt5", "mevo", "xaenvironment",
-	"survivethedays"
+	"pacman", "tutorial", "runorfall", "realtest_mt5", "mevo", "xaenvironment",
+	"survivethedays", "holidayhorrors",
 }
 
 
-class PackageSet:
-	packages: Dict[str, Package]
+class GSPackage:
+	author: str
+	name: str
+	type: PackageType
 
-	def __init__(self, packages: Optional[Iterable[Package]] = None):
+	provides: set[str]
+	depends: set[str]
+
+	user_supported_games: set[str]
+	user_unsupported_games: set[str]
+	detected_supported_games: set[str]
+	supports_all_games: bool
+
+	detection_disabled: bool
+
+	is_confirmed: bool
+	errors: set[str]
+
+	def __init__(self, author: str, name: str, type: PackageType, provides: set[str]):
+		self.author = author
+		self.name = name
+		self.type = type
+		self.provides = provides
+		self.depends = set()
+		self.user_supported_games = set()
+		self.user_unsupported_games = set()
+		self.detected_supported_games = set()
+		self.supports_all_games = False
+		self.detection_disabled = False
+		self.is_confirmed = type == PackageType.GAME
+		self.errors = set()
+
+		# For dodgy games, discard MTG mods
+		if self.type == PackageType.GAME and self.name in mtg_mod_blacklist:
+			self.provides.difference_update(minetest_game_mods)
+
+	@property
+	def id_(self) -> str:
+		return f"{self.author}/{self.name}"
+
+	@property
+	def supported_games(self) -> set[str]:
+		ret = set()
+		ret.update(self.user_supported_games)
+		if not self.detection_disabled:
+			ret.update(self.detected_supported_games)
+		ret.difference_update(self.user_unsupported_games)
+		return ret
+
+	@property
+	def unsupported_games(self) -> set[str]:
+		return self.user_unsupported_games
+
+	def add_error(self, error: str):
+		return self.errors.add(error)
+
+
+class GameSupport:
+	packages: Dict[str, GSPackage]
+	modified_packages: set[GSPackage]
+
+	def __init__(self):
 		self.packages = {}
-		if packages:
-			self.update(packages)
+		self.modified_packages = set()
 
-	def update(self, packages: Iterable[Package]):
-		for package in packages:
-			key = package.getId()
-			if key not in self.packages:
-				self.packages[key] = package
+	@property
+	def all_confirmed(self):
+		return all([x.is_confirmed for x in self.packages.values()])
 
-	def intersection_update(self, other):
-		keys = set(self.packages.keys())
-		keys.difference_update(set(other.packages.keys()))
-		for key in keys:
-			del self.packages[key]
+	@property
+	def has_errors(self):
+		return any([len(x.errors) > 0 for x in self.packages.values()])
 
-	def __len__(self):
-		return len(self.packages)
+	@property
+	def error_count(self):
+		return sum([len(x.errors) for x in self.packages.values()])
 
-	def __iter__(self):
-		return self.packages.values().__iter__()
+	@property
+	def all_errors(self) -> set[str]:
+		errors = set()
+		for package in self.packages.values():
+			for err in package.errors:
+				errors.add(package.id_ + ": " + err)
+		return errors
 
+	def add(self, package: GSPackage) -> GSPackage:
+		self.packages[package.id_] = package
+		return package
 
-class GameSupportResolver:
-	checked_packages = set()
-	checked_metapackages = set()
-	resolved_packages: Dict[str, PackageSet] = {}
-	resolved_metapackages: Dict[str, PackageSet] = {}
+	def get(self, id_: str) -> Optional[GSPackage]:
+		return self.packages.get(id_)
 
-	def resolve_for_meta_package(self, meta: MetaPackage, history: List[str]) -> PackageSet:
-		print(f"Resolving for {meta.name}", file=sys.stderr)
+	def get_all_that_provide(self, modname: str) -> List[GSPackage]:
+		return [package for package in self.packages.values() if modname in package.provides]
 
-		key = meta.name
-		if key in self.resolved_metapackages:
-			return self.resolved_metapackages.get(key)
+	def get_all_that_depend_on(self, modname: str) -> List[GSPackage]:
+		return [package for package in self.packages.values() if modname in package.depends]
 
-		if key in self.checked_metapackages:
-			print(f"Error, cycle found: {','.join(history)}", file=sys.stderr)
-			return PackageSet()
-
-		self.checked_metapackages.add(key)
-
-		retval = PackageSet()
-
-		for package in meta.packages:
-			if package.state != PackageState.APPROVED:
-				continue
-
-			if meta.name in minetest_game_mods and package.name in mtg_mod_blacklist:
-				continue
-
-			ret = self.resolve(package, history)
-			if len(ret) == 0:
-				retval = PackageSet()
+	def _get_supported_games_for_modname(self, depend: str, visited: list[str]):
+		dep_supports_all = False
+		for_dep = set()
+		for provider in self.get_all_that_provide(depend):
+			found_in = self._get_supported_games(provider, visited)
+			if found_in is None:
+				# Unsupported, keep going
+				pass
+			elif len(found_in) == 0:
+				dep_supports_all = True
 				break
+			else:
+				for_dep.update(found_in)
 
-			retval.update(ret)
+		return dep_supports_all, for_dep
 
-		self.resolved_metapackages[key] = retval
-		return retval
+	def _get_supported_games_for_deps(self, package: GSPackage, visited: list[str]) -> Optional[set[str]]:
+		ret = set()
 
-	def resolve(self, package: Package, history: List[str]) -> PackageSet:
-		db.session.merge(package)
+		for depend in package.depends:
+			dep_supports_all, for_dep = self._get_supported_games_for_modname(depend, visited)
 
-		key = package.getId()
-		print(f"Resolving for {key}", file=sys.stderr)
+			if dep_supports_all:
+				# Dep is game independent
+				pass
+			elif len(for_dep) == 0:
+				package.add_error(f"Unable to fulfill dependency {depend}")
+				return None
+			elif len(ret) == 0:
+				ret = for_dep
+			else:
+				ret.intersection_update(for_dep)
+				if len(ret) == 0:
+					package.add_error("Game support conflict, unable to install package on any games")
+					return None
 
-		history = history.copy()
-		history.append(key)
+		return ret
+
+	def _get_supported_games(self, package: GSPackage, visited: list[str]) -> Optional[set[str]]:
+		if package.id_ in visited:
+			return None
 
 		if package.type == PackageType.GAME:
-			return PackageSet([package])
+			return {package.name}
+		elif package.is_confirmed:
+			return package.supported_games
 
-		if key in self.resolved_packages:
-			return self.resolved_packages.get(key)
+		visited = visited.copy()
+		visited.append(package.id_)
 
-		if key in self.checked_packages:
-			print(f"Error, cycle found: {','.join(history)}", file=sys.stderr)
-			return PackageSet()
+		ret = self._get_supported_games_for_deps(package, visited)
+		if ret is None:
+			assert len(package.errors) > 0
+			return None
 
-		self.checked_packages.add(key)
+		ret = ret.copy()
+		ret.difference_update(package.user_unsupported_games)
+		package.detected_supported_games = ret
+		self.modified_packages.add(package)
 
-		if package.type != PackageType.MOD:
-			raise LogicError(500, "Got non-mod")
+		if len(ret) > 0:
+			for supported in package.user_supported_games:
+				if supported not in ret:
+					package.add_error(f"`{supported}` is specified in supported_games but it is impossible to run {package.name} in that game. " +
+							f"Its dependencies can only be fulfilled in {', '.join([f'`{x}`' for x in ret])}. " +
+							"Check your hard dependencies.")
 
-		retval = PackageSet()
+			if package.supports_all_games:
+				package.add_error(
+						"This package cannot support all games as some dependencies require specific game(s): " +
+						", ".join([f'`{x}`' for x in ret]))
 
-		for dep in package.dependencies.filter_by(optional=False).all():
-			ret = self.resolve_for_meta_package(dep.meta_package, history)
-			if len(ret) == 0:
-				continue
-			elif len(retval) == 0:
-				retval.update(ret)
-			else:
-				retval.intersection_update(ret)
-				if len(retval) == 0:
-					raise LogicError(500, f"Detected game support contradiction, {key} may not be compatible with any games")
+		package.is_confirmed = True
+		return package.supported_games
 
-		self.resolved_packages[key] = retval
-		return retval
+	def on_update(self, package: GSPackage, old_provides: Optional[set[str]] = None):
+		to_update = {package}
+		checked = set()
 
-	def update_all(self) -> None:
-		for package in Package.query.filter(Package.type == PackageType.MOD, Package.state != PackageState.DELETED).all():
-			retval = self.resolve(package, [])
-			for game in retval:
-				support = PackageGameSupport(package, game)
-				db.session.add(support)
+		while len(to_update) > 0:
+			current_package = to_update.pop()
+			if current_package.id_ in self.packages and current_package.type != PackageType.GAME:
+				self._get_supported_games(current_package, [])
 
-	def update(self, package: Package) -> None:
-		previous_supported: Dict[str, PackageGameSupport] = {}
-		for support in package.supported_games.all():
-			previous_supported[support.game.getId()] = support
+			provides = current_package.provides
+			if current_package == package and old_provides is not None:
+				provides = provides.union(old_provides)
 
-		retval = self.resolve(package, [])
-		for game in retval:
-			assert game
+			for modname in provides:
+				for depending_package in self.get_all_that_depend_on(modname):
+					if depending_package not in checked:
+						if depending_package.id_ in self.packages and depending_package.type != PackageType.GAME:
+							depending_package.is_confirmed = False
+							depending_package.detected_supported_games = []
 
-			lookup = previous_supported.pop(game.getId(), None)
-			if lookup is None:
-				support = PackageGameSupport(package, game)
-				db.session.add(support)
-			elif lookup.confidence == 0:
-				lookup.supports = True
-				db.session.merge(lookup)
+						to_update.add(depending_package)
+						checked.add(depending_package)
 
-		for game, support in previous_supported.items():
-			if support.confidence == 0:
-				db.session.remove(support)
+	def on_remove(self, package: GSPackage):
+		del self.packages[package.id_]
+		self.on_update(package)
+
+	def on_first_run(self):
+		for package in self.packages.values():
+			if not package.is_confirmed:
+				self.on_update(package)
+
+
+def _convert_package(support: GameSupport, package: Package) -> GSPackage:
+	# Unapproved packages shouldn't be considered to fulfill anything
+	provides = set()
+	if package.state == PackageState.APPROVED:
+		provides = set([x.name for x in package.provides])
+
+	gs_package = GSPackage(package.author.username, package.name, package.type, provides)
+	gs_package.depends = set([x.meta_package.name for x in package.dependencies if not x.optional])
+	gs_package.detection_disabled = not package.enable_game_support_detection
+	gs_package.supports_all_games = package.supports_all_games
+
+	existing_game_support = (package.supported_games
+			.filter(PackageGameSupport.game.has(state=PackageState.APPROVED),
+					PackageGameSupport.confidence > 5)
+			.all())
+	if not package.supports_all_games:
+		gs_package.user_supported_games = [x.game.name for x in existing_game_support if x.supports]
+	gs_package.user_unsupported_games = [x.game.name for x in existing_game_support if not x.supports]
+	return support.add(gs_package)
+
+
+def _create_instance(session: sqlalchemy.orm.Session) -> GameSupport:
+	support = GameSupport()
+
+	packages: List[Package] = (session.query(Package)
+			.filter(Package.state == PackageState.APPROVED, Package.type.in_([PackageType.GAME, PackageType.MOD]))
+			.all())
+
+	for package in packages:
+		_convert_package(support, package)
+
+	return support
+
+
+def _persist(session: sqlalchemy.orm.Session, support: GameSupport):
+	for gs_package in support.packages.values():
+		if len(gs_package.errors) != 0:
+			msg = "\n".join([f"- {x}" for x in gs_package.errors])
+			package = session.query(Package).filter(
+					Package.author.has(username=gs_package.author),
+					Package.name == gs_package.name).one()
+			post_bot_message(package, "Error when checking game support", msg, session)
+
+	for gs_package in support.modified_packages:
+		if not gs_package.detection_disabled:
+			package = session.query(Package).filter(
+					Package.author.has(username=gs_package.author),
+					Package.name == gs_package.name).one()
+
+			# Clear existing
+			session.query(PackageGameSupport) \
+				.filter_by(package=package, confidence=1) \
+				.delete()
+
+			# Add new
+			supported_games = gs_package.supported_games \
+				.difference(gs_package.user_supported_games)
+			for game_name in supported_games:
+				game_id = session.query(Package.id) \
+					.filter(Package.type == PackageType.GAME, Package.name == game_name, Package.state == PackageState.APPROVED) \
+					.one()[0]
+
+				new_support = PackageGameSupport()
+				new_support.package = package
+				new_support.game_id = game_id
+				new_support.confidence = 1
+				new_support.supports = True
+				session.add(new_support)
+
+
+def game_support_update(session: sqlalchemy.orm.Session, package: Package, old_provides: Optional[set[str]]) -> set[str]:
+	support = _create_instance(session)
+	gs_package = support.get(package.get_id())
+	if gs_package is None:
+		gs_package = _convert_package(support, package)
+	support.on_update(gs_package, old_provides)
+	_persist(session, support)
+	return gs_package.errors
+
+
+def game_support_update_all(session: sqlalchemy.orm.Session):
+	support = _create_instance(session)
+	support.on_first_run()
+	_persist(session, support)
+
+
+def game_support_remove(session: sqlalchemy.orm.Session, package: Package):
+	support = _create_instance(session)
+	gs_package = support.get(package.get_id())
+	if gs_package is None:
+		gs_package = _convert_package(support, package)
+	support.on_remove(gs_package)
+	_persist(session, support)
+
+
+def game_support_set(session, package: Package, game_is_supported: Dict[int, bool], confidence: int):
+	previous_supported: Dict[int, PackageGameSupport] = {}
+	for support in package.supported_games.all():
+		previous_supported[support.game.id] = support
+
+	for game_id, supports in game_is_supported.items():
+		game = session.query(Package).get(game_id)
+		lookup = previous_supported.pop(game_id, None)
+		if lookup is None:
+			support = PackageGameSupport()
+			support.package = package
+			support.game = game
+			support.confidence = confidence
+			support.supports = supports
+			session.add(support)
+		elif lookup.confidence <= confidence:
+			lookup.supports = supports
+			lookup.confidence = confidence
+
+	for game, support in previous_supported.items():
+		if support.confidence == confidence:
+			session.delete(support)

app/logic/graphs.py

@@ -0,0 +1,166 @@
+# ContentDB
+# Copyright (C) rubenwardy
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+import datetime
+from datetime import timedelta
+from typing import Optional
+
+from app.models import User, Package, PackageDailyStats, db, PackageState
+from sqlalchemy import func
+
+
+def daterange(start_date, end_date):
+	for n in range(int((end_date - start_date).days) + 1):
+		yield start_date + timedelta(n)
+
+
+keys = ["platform_minetest", "platform_other", "reason_new",
+		"reason_dependency", "reason_update"]
+
+
+def flatten_data(stats):
+	start_date = stats[0].date
+	end_date = stats[-1].date
+	result = {
+		"start": start_date.isoformat(),
+		"end": end_date.isoformat(),
+	}
+
+	for key in keys:
+		result[key] = []
+
+	i = 0
+	for date in daterange(start_date, end_date):
+		stat = stats[i]
+		if stat.date == date:
+			for key in keys:
+				result[key].append(getattr(stat, key))
+
+			i += 1
+		else:
+			for key in keys:
+				result[key].append(0)
+
+	return result
+
+
+def get_package_stats(package: Package, start_date: Optional[datetime.date], end_date: Optional[datetime.date]):
+	query = package.daily_stats.order_by(db.asc(PackageDailyStats.date))
+	if start_date:
+		query = query.filter(PackageDailyStats.date >= start_date)
+	if end_date:
+		query = query.filter(PackageDailyStats.date <= end_date)
+
+	stats = query.all()
+	if len(stats) == 0:
+		return None
+
+	return flatten_data(stats)
+
+
+def get_package_stats_for_user(user: User, start_date: Optional[datetime.date], end_date: Optional[datetime.date]):
+	query = db.session \
+		.query(PackageDailyStats.date,
+			func.sum(PackageDailyStats.platform_minetest).label("platform_minetest"),
+			func.sum(PackageDailyStats.platform_other).label("platform_other"),
+			func.sum(PackageDailyStats.reason_new).label("reason_new"),
+			func.sum(PackageDailyStats.reason_dependency).label("reason_dependency"),
+			func.sum(PackageDailyStats.reason_update).label("reason_update")) \
+		.filter(PackageDailyStats.package.has(author_id=user.id))
+
+	if start_date:
+		query = query.filter(PackageDailyStats.date >= start_date)
+	if end_date:
+		query = query.filter(PackageDailyStats.date <= end_date)
+
+	stats = query.order_by(db.asc(PackageDailyStats.date)) \
+		.group_by(PackageDailyStats.date) \
+		.all()
+	if len(stats) == 0:
+		return None
+
+	results = flatten_data(stats)
+	results["package_downloads"] = get_package_overview_for_user(user, stats[0].date, stats[-1].date)
+
+	return results
+
+
+def get_package_overview_for_user(user: Optional[User], start_date: datetime.date, end_date: datetime.date):
+	query = db.session \
+		.query(PackageDailyStats.package_id, PackageDailyStats.date,
+			(PackageDailyStats.platform_minetest + PackageDailyStats.platform_other).label("downloads"))
+
+	if user:
+		query = query.filter(PackageDailyStats.package.has(author_id=user.id))
+
+	all_stats = query \
+		.filter(PackageDailyStats.package.has(state=PackageState.APPROVED),
+				PackageDailyStats.date >= start_date, PackageDailyStats.date <= end_date) \
+		.order_by(db.asc(PackageDailyStats.package_id), db.asc(PackageDailyStats.date)) \
+		.all()
+
+	stats_by_package = {}
+	for stat in all_stats:
+		bucket = stats_by_package.get(stat.package_id, [])
+		stats_by_package[stat.package_id] = bucket
+
+		bucket.append(stat)
+
+	package_title_by_id = {}
+	pkg_query = user.packages if user else Package.query
+	for package in pkg_query.filter_by(state=PackageState.APPROVED).all():
+		if user:
+			package_title_by_id[package.id] = package.title
+		else:
+			package_title_by_id[package.id] = package.get_id()
+
+	result = {}
+
+	for package_id, stats in stats_by_package.items():
+		i = 0
+		row = []
+		result[package_title_by_id[package_id]] = row
+		for date in daterange(start_date, end_date):
+			if i >= len(stats):
+				row.append(0)
+				continue
+
+			stat = stats[i]
+			if stat.date == date:
+				row.append(stat.downloads)
+				i += 1
+			elif stat.date > date:
+				row.append(0)
+			else:
+				raise Exception(f"Invalid logic, expected stat {stat.date} to be later than {date}")
+
+	return result
+
+
+def get_all_package_stats(start_date: Optional[datetime.date] = None, end_date: Optional[datetime.date] = None):
+	now_date = datetime.datetime.utcnow().date()
+	if end_date is None or end_date > now_date:
+		end_date = now_date
+
+	min_start_date = (datetime.datetime.utcnow() - datetime.timedelta(days=29)).date()
+	if start_date is None or start_date < min_start_date:
+		start_date = min_start_date
+
+	return {
+		"start": start_date.isoformat(),
+		"end": end_date.isoformat(),
+		"package_downloads": get_package_overview_for_user(None, start_date, end_date),
+	}

app/logic/package_approval.py

@@ -0,0 +1,204 @@
+# ContentDB
+# Copyright (C) rubenwardy
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+from typing import List, Tuple, Union, Optional
+
+from flask_babel import lazy_gettext, LazyString
+from sqlalchemy import and_, or_
+
+from app.models import Package, PackageType, PackageState, PackageRelease, db, MetaPackage, ForumTopic, User, \
+	Permission, UserRank
+
+
+class PackageValidationNote:
+	# level is danger, warning, or info
+	level: str
+	message: LazyString
+	buttons: List[Tuple[str, LazyString]]
+
+	# False to prevent "Approve"
+	allow_approval: bool
+
+	# False to prevent "Submit for Approval"
+	allow_submit: bool
+
+	def __init__(self, level: str, message: LazyString, allow_approval: bool, allow_submit: bool):
+		self.level = level
+		self.message = message
+		self.buttons = []
+		self.allow_approval = allow_approval
+		self.allow_submit = allow_submit
+
+	def add_button(self, url: str, label: LazyString) -> "PackageValidationNote":
+		self.buttons.append((url, label))
+		return self
+
+	def __repr__(self):
+		return str(self.message)
+
+
+def is_package_name_taken(normalised_name: str) -> bool:
+	return Package.query.filter(
+		and_(Package.state == PackageState.APPROVED,
+			 or_(Package.name == normalised_name,
+				 Package.name == normalised_name + "_game"))).count() > 0
+
+
+def get_conflicting_mod_names(package: Package) -> set[str]:
+	conflicting_modnames = (db.session.query(MetaPackage.name)
+			.filter(MetaPackage.id.in_([mp.id for mp in package.provides]))
+			.filter(MetaPackage.packages.any(and_(Package.id != package.id, Package.state == PackageState.APPROVED)))
+			.all())
+	conflicting_modnames += (db.session.query(ForumTopic.name)
+			.filter(ForumTopic.name.in_([mp.name for mp in package.provides]))
+			.filter(ForumTopic.topic_id != package.forums)
+			.filter(~ db.exists().where(Package.forums == ForumTopic.topic_id))
+			.order_by(db.asc(ForumTopic.name), db.asc(ForumTopic.title))
+			.all())
+	return set([x[0] for x in conflicting_modnames])
+
+
+def count_packages_with_forum_topic(topic_id: int) -> int:
+	return Package.query.filter(Package.forums == topic_id, Package.state != PackageState.DELETED).count() > 1
+
+
+def get_forum_topic(topic_id: int) -> Optional[ForumTopic]:
+	return ForumTopic.query.get(topic_id)
+
+
+def validate_package_for_approval(package: Package) -> List[PackageValidationNote]:
+	retval: List[PackageValidationNote] = []
+
+	def template(level: str, allow_approval: bool, allow_submit: bool):
+		def inner(msg: LazyString):
+			note = PackageValidationNote(level, msg, allow_approval, allow_submit)
+			retval.append(note)
+			return note
+
+		return inner
+
+	danger = template("danger", allow_approval=False, allow_submit=False)
+	warning = template("warning", allow_approval=True, allow_submit=True)
+	info = template("info", allow_approval=False, allow_submit=True)
+
+	if package.type != PackageType.MOD and is_package_name_taken(package.normalised_name):
+		danger(lazy_gettext("A package already exists with this name. Please see Policy and Guidance 3"))
+
+	if package.releases.filter(PackageRelease.task_id.is_(None)).count() == 0:
+		if package.releases.count() == 0:
+			message = lazy_gettext("You need to create a release before this package can be approved.")
+		else:
+			message = lazy_gettext("Release is still importing, or has an error.")
+
+		danger(message) \
+			.add_button(package.get_url("packages.create_release"), lazy_gettext("Create release")) \
+			.add_button(package.get_url("packages.setup_releases"), lazy_gettext("Set up releases"))
+
+		# Don't bother validating any more until we have a release
+		return retval
+
+	if package.screenshots.count() == 0:
+		danger(lazy_gettext("You need to add at least one screenshot."))
+
+	missing_deps = package.get_missing_hard_dependencies_query().all()
+	if len(missing_deps) > 0:
+		missing_deps = ", ".join([ x.name for x in missing_deps])
+		danger(lazy_gettext(
+			"The following hard dependencies need to be added to ContentDB first: %(deps)s", deps=missing_deps))
+
+	if package.type != PackageType.GAME and not package.supports_all_games and package.supported_games.count() == 0:
+		danger(lazy_gettext(
+			"What games does your package support? Please specify on the supported games page", deps=missing_deps)) \
+			.add_button(package.get_url("packages.game_support"), lazy_gettext("Supported Games"))
+
+	if "Other" in package.license.name or "Other" in package.media_license.name:
+		info(lazy_gettext("Please wait for the license to be added to CDB."))
+
+	# Check similar mod name
+	conflicting_modnames = set()
+	if package.type != PackageType.TXP:
+		conflicting_modnames = get_conflicting_mod_names(package)
+
+	if len(conflicting_modnames) > 4:
+		warning(lazy_gettext("Please make sure that this package has the right to the names it uses."))
+	elif len(conflicting_modnames) > 0:
+		names_list = list(conflicting_modnames)
+		names_list.sort()
+		warning(lazy_gettext("Please make sure that this package has the right to the names %(names)s",
+				names=", ".join(names_list))) \
+			.add_button(package.get_url('packages.similar'), lazy_gettext("See more"))
+
+	# Check forum topic
+	if package.state != PackageState.APPROVED and package.forums is not None:
+		if count_packages_with_forum_topic(package.forums) > 1:
+			danger("<b>" + lazy_gettext("Error: Another package already uses this forum topic!") + "</b>")
+
+		topic = get_forum_topic(package.forums)
+		if topic is not None:
+			if topic.author != package.author:
+				danger("<b>" + lazy_gettext("Error: Forum topic author doesn't match package author.") + "</b>")
+		elif package.type != PackageType.TXP:
+			warning(lazy_gettext("Warning: Forum topic not found. The topic may have been created since the last forum crawl."))
+
+	return retval
+
+
+PACKAGE_STATE_FLOW = {
+	PackageState.WIP: {PackageState.READY_FOR_REVIEW},
+	PackageState.CHANGES_NEEDED: {PackageState.READY_FOR_REVIEW},
+	PackageState.READY_FOR_REVIEW: {PackageState.WIP, PackageState.CHANGES_NEEDED, PackageState.APPROVED},
+	PackageState.APPROVED: {PackageState.CHANGES_NEEDED},
+	PackageState.DELETED: {PackageState.READY_FOR_REVIEW},
+}
+
+
+def can_move_to_state(package: Package, user: User, new_state: Union[str, PackageState]) -> bool:
+	if not user.is_authenticated:
+		return False
+
+	if type(new_state) == str:
+		new_state = PackageState[new_state]
+	elif type(new_state) != PackageState:
+		raise Exception("Unknown state given to can_move_to_state()")
+
+	if new_state not in PACKAGE_STATE_FLOW[package.state]:
+		return False
+
+	if new_state == PackageState.READY_FOR_REVIEW or new_state == PackageState.APPROVED:
+		# Can the user approve?
+		if new_state == PackageState.APPROVED and not package.check_perm(user, Permission.APPROVE_NEW):
+			return False
+
+		# Must be able to edit or approve package to change its state
+		if not (package.check_perm(user, Permission.APPROVE_NEW) or package.check_perm(user, Permission.EDIT_PACKAGE)):
+			return False
+
+		# Are there any validation warnings?
+		validation_notes = validate_package_for_approval(package)
+		for note in validation_notes:
+			if not note.allow_submit or (new_state == PackageState.APPROVED and not note.allow_approval):
+				return False
+
+		return True
+
+	elif new_state == PackageState.CHANGES_NEEDED:
+		return package.check_perm(user, Permission.APPROVE_NEW)
+
+	elif new_state == PackageState.WIP:
+		return package.check_perm(user, Permission.EDIT_PACKAGE) and \
+			(user in package.maintainers or user.rank.at_least(UserRank.ADMIN))
+
+	return True

app/logic/packages.py

@@ -14,19 +14,21 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-
+import json
 import re
+import typing
+
 import validators
-from flask_babel import lazy_gettext
+from flask_babel import lazy_gettext, LazyString
 
 from app.logic.LogicError import LogicError
 from app.models import User, Package, PackageType, MetaPackage, Tag, ContentWarning, db, Permission, AuditSeverity, \
-	License, UserRank, PackageDevState
-from app.utils import addAuditLog
+	License, PackageDevState, PackageState
+from app.utils import add_audit_log, has_blocked_domains, diff_dictionaries, describe_difference, normalize_line_endings
 from app.utils.url import clean_youtube_url
 
 
-def check(cond: bool, msg: str):
+def check(cond: bool, msg: typing.Union[str, LazyString]):
 	if not cond:
 		raise LogicError(400, msg)
 
@@ -63,6 +65,21 @@ ALLOWED_FIELDS = {
 	"issueTracker": str,
 	"forums": int,
 	"video_url": str,
+	"donate_url": str,
+	"translation_url": str,
+}
+
+NULLABLE = {
+	"tags",
+	"content_warnings",
+	"repo",
+	"website",
+	"issue_tracker",
+	"issueTracker",
+	"forums",
+	"video_url",
+	"donate_url",
+	"translation_url",
 }
 
 ALIASES = {
@@ -82,11 +99,13 @@ def is_int(val):
 
 def validate(data: dict):
 	for key, value in data.items():
-		if value is not None:
+		if value is None:
+			check(key in NULLABLE, f"{key} must not be null")
+		else:
 			typ = ALLOWED_FIELDS.get(key)
-			check(typ is not None, key + " is not a known field")
+			check(typ is not None, f"{key} is not a known field")
 			if typ != AnyType:
-				check(isinstance(value, typ), key + " must be a " + typ.__name__)
+				check(isinstance(value, typ), f"{key} must be a " + typ.__name__)
 
 	if "name" in data:
 		name = data["name"]
@@ -98,28 +117,45 @@ def validate(data: dict):
 		value = data.get(key)
 		if value is not None:
 			check(value.startswith("http://") or value.startswith("https://"),
-					key + " must start with http:// or https://")
-
-			check(validators.url(value, public=True), key + " must be a valid URL")
+					f"{key} must start with http:// or https://")
+			check(validators.url(value), f"{key} must be a valid URL")
 
 
 def do_edit_package(user: User, package: Package, was_new: bool, was_web: bool, data: dict,
-		reason: str = None):
-	if not package.checkPerm(user, Permission.EDIT_PACKAGE):
+		reason: str = None) -> bool:
+	if not package.check_perm(user, Permission.EDIT_PACKAGE):
 		raise LogicError(403, lazy_gettext("You don't have permission to edit this package"))
 
 	if "name" in data and package.name != data["name"] and \
-			not package.checkPerm(user, Permission.CHANGE_NAME):
+			not package.check_perm(user, Permission.CHANGE_NAME):
 		raise LogicError(403, lazy_gettext("You don't have permission to change the package name"))
 
+	before_dict = None
+	if not was_new:
+		before_dict = package.as_dict("/")
+
 	for alias, to in ALIASES.items():
 		if alias in data:
+			if to in data and data[to] != data[alias]:
+				raise LogicError(403, f"Aliased field ({alias}) does not match new field ({to})")
+
 			data[to] = data[alias]
 
 	validate(data)
 
+	for field in ["short_desc", "desc", "website", "issueTracker", "repo", "video_url", "donate_url", "translation_url"]:
+		if field in data and has_blocked_domains(data[field], user.username,
+					f"{field} of {package.get_id()}"):
+			raise LogicError(403, lazy_gettext("Linking to blocked sites is not allowed"))
+
 	if "type" in data:
-		data["type"] = PackageType.coerce(data["type"])
+		new_type = PackageType.coerce(data["type"])
+		if new_type == package.type:
+			pass
+		elif package.state != PackageState.APPROVED:
+			package.type = new_type
+		else:
+			raise LogicError(403, lazy_gettext("You cannot change package type once approved"))
 
 	if "dev_state" in data:
 		data["dev_state"] = PackageDevState.coerce(data["dev_state"])
@@ -130,13 +166,16 @@ def do_edit_package(user: User, package: Package, was_new: bool, was_web: bool,
 	if "media_license" in data:
 		data["media_license"] = get_license(data["media_license"])
 
+	if "desc" in data:
+		data["desc"] = normalize_line_endings(data["desc"])
+
 	if "video_url" in data and data["video_url"] is not None:
 		data["video_url"] = clean_youtube_url(data["video_url"]) or data["video_url"]
 		if "dQw4w9WgXcQ" in data["video_url"]:
 			raise LogicError(403, "Never gonna give you up / Never gonna let you down / Never gonna run around and desert you")
 
-	for key in ["name", "title", "short_desc", "desc", "type", "dev_state", "license", "media_license",
-			"repo", "website", "issueTracker", "forums", "video_url"]:
+	for key in ["name", "title", "short_desc", "desc", "dev_state", "license", "media_license",
+			"repo", "website", "issueTracker", "forums", "video_url", "donate_url", "translation_url"]:
 		if key in data:
 			setattr(package, key, data[key])
 
@@ -148,9 +187,8 @@ def do_edit_package(user: User, package: Package, was_new: bool, was_web: bool,
 		package.provides.append(m)
 
 	if "tags" in data:
-		old_tags = list(package.tags)
 		package.tags.clear()
-		for tag_id in data["tags"]:
+		for tag_id in (data["tags"] or []):
 			if is_int(tag_id):
 				tag = Tag.query.get(tag_id)
 			else:
@@ -158,22 +196,11 @@ def do_edit_package(user: User, package: Package, was_new: bool, was_web: bool,
 				if tag is None:
 					raise LogicError(400, "Unknown tag: " + tag_id)
 
-			if not was_web and tag.is_protected:
-				continue
-
-			if tag.is_protected and tag not in old_tags and not user.rank.atLeast(UserRank.EDITOR):
-				raise LogicError(400, lazy_gettext("Unable to add protected tag %(title)s to package", title=tag.title))
-
 			package.tags.append(tag)
 
-		if not was_web:
-			for tag in old_tags:
-				if tag.is_protected:
-					package.tags.append(tag)
-
 	if "content_warnings" in data:
 		package.content_warnings.clear()
-		for warning_id in data["content_warnings"]:
+		for warning_id in (data["content_warnings"] or []):
 			if is_int(warning_id):
 				package.content_warnings.append(ContentWarning.query.get(warning_id))
 			else:
@@ -182,15 +209,28 @@ def do_edit_package(user: User, package: Package, was_new: bool, was_web: bool,
 					raise LogicError(400, "Unknown warning: " + warning_id)
 				package.content_warnings.append(warning)
 
-	if not was_new:
+	was_modified = was_new
+	if was_new:
+		msg = f"Created package {package.author.username}/{package.name}"
+		add_audit_log(AuditSeverity.NORMAL, user, msg, package.get_url("packages.view"), package)
+	else:
+		after_dict = package.as_dict("/")
+		diff = diff_dictionaries(before_dict, after_dict)
+		was_modified = len(diff) > 0
+
 		if reason is None:
 			msg = "Edited {}".format(package.title)
 		else:
 			msg = "Edited {} ({})".format(package.title, reason)
 
+		diff_desc = describe_difference(diff, 100 - len(msg) - 3) if diff else None
+		if diff_desc:
+			msg += " [" + diff_desc + "]"
+
 		severity = AuditSeverity.NORMAL if user in package.maintainers else AuditSeverity.EDITOR
-		addAuditLog(severity, user, msg, package.getURL("packages.view"), package)
+		add_audit_log(severity, user, msg, package.get_url("packages.view"), package, json.dumps(diff, indent=4))
 
-	db.session.commit()
+	if was_modified:
+		db.session.commit()
 
-	return package
+	return was_modified

app/logic/releases.py

@@ -14,36 +14,42 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-
-import datetime, re
+import datetime
+import re
+from typing import Optional
 
 from celery import uuid
 from flask_babel import lazy_gettext
 
 from app.logic.LogicError import LogicError
 from app.logic.uploads import upload_file
-from app.models import PackageRelease, db, Permission, User, Package, MinetestRelease
-from app.tasks.importtasks import makeVCSRelease, checkZipRelease
-from app.utils import AuditSeverity, addAuditLog, nonEmptyOrNone
+from app.models import PackageRelease, db, Permission, User, Package, LuantiRelease
+from app.tasks.importtasks import make_vcs_release, check_zip_release
+from app.utils import AuditSeverity, add_audit_log, nonempty_or_none, normalize_line_endings
 
 
-def check_can_create_release(user: User, package: Package):
-	if not package.checkPerm(user, Permission.MAKE_RELEASE):
+def check_can_create_release(user: User, package: Package, name: str):
+	if not package.check_perm(user, Permission.MAKE_RELEASE):
 		raise LogicError(403, lazy_gettext("You don't have permission to make releases"))
 
 	five_minutes_ago = datetime.datetime.now() - datetime.timedelta(minutes=5)
-	count = package.releases.filter(PackageRelease.releaseDate > five_minutes_ago).count()
+	count = package.releases.filter(PackageRelease.created_at > five_minutes_ago).count()
 	if count >= 5:
 		raise LogicError(429, lazy_gettext("You've created too many releases for this package in the last 5 minutes, please wait before trying again"))
 
+	if PackageRelease.query.filter_by(package_id=package.id, name=name).count() > 0:
+		raise LogicError(403, lazy_gettext("A release with this name already exists"))
 
-def do_create_vcs_release(user: User, package: Package, title: str, ref: str,
-		min_v: MinetestRelease = None, max_v: MinetestRelease = None, reason: str = None):
-	check_can_create_release(user, package)
+
+def do_create_vcs_release(user: User, package: Package, name: str, title: Optional[str], release_notes: Optional[str], ref: str,
+		min_v: LuantiRelease = None, max_v: LuantiRelease = None, reason: str = None):
+	check_can_create_release(user, package, name)
 
 	rel = PackageRelease()
 	rel.package = package
-	rel.title   = title
+	rel.name    = name
+	rel.title   = title or name
+	rel.release_notes = normalize_line_endings(release_notes)
 	rel.url     = ""
 	rel.task_id = uuid()
 	rel.min_rel = min_v
@@ -54,19 +60,19 @@ def do_create_vcs_release(user: User, package: Package, title: str, ref: str,
 		msg = "Created release {}".format(rel.title)
 	else:
 		msg = "Created release {} ({})".format(rel.title, reason)
-	addAuditLog(AuditSeverity.NORMAL, user, msg, package.getURL("packages.view"), package)
+	add_audit_log(AuditSeverity.NORMAL, user, msg, package.get_url("packages.view"), package)
 
 	db.session.commit()
 
-	makeVCSRelease.apply_async((rel.id, nonEmptyOrNone(ref)), task_id=rel.task_id)
+	make_vcs_release.apply_async((rel.id, nonempty_or_none(ref)), task_id=rel.task_id)
 
 	return rel
 
 
-def do_create_zip_release(user: User, package: Package, title: str, file,
-		min_v: MinetestRelease = None, max_v: MinetestRelease = None, reason: str = None,
+def do_create_zip_release(user: User, package: Package, name: str, title: Optional[str], release_notes: Optional[str], file,
+		min_v: LuantiRelease = None, max_v: LuantiRelease = None, reason: str = None,
 		commit_hash: str = None):
-	check_can_create_release(user, package)
+	check_can_create_release(user, package, name)
 
 	if commit_hash:
 		commit_hash = commit_hash.lower()
@@ -77,7 +83,9 @@ def do_create_zip_release(user: User, package: Package, title: str, file,
 
 	rel = PackageRelease()
 	rel.package = package
-	rel.title   = title
+	rel.name    = name
+	rel.title   = title or name
+	rel.release_notes = normalize_line_endings(release_notes)
 	rel.url     = uploaded_url
 	rel.task_id = uuid()
 	rel.commit_hash = commit_hash
@@ -89,10 +97,10 @@ def do_create_zip_release(user: User, package: Package, title: str, file,
 		msg = "Created release {}".format(rel.title)
 	else:
 		msg = "Created release {} ({})".format(rel.title, reason)
-	addAuditLog(AuditSeverity.NORMAL, user, msg, package.getURL("packages.view"), package)
+	add_audit_log(AuditSeverity.NORMAL, user, msg, package.get_url("packages.view"), package)
 
 	db.session.commit()
 
-	checkZipRelease.apply_async((rel.id, uploaded_path), task_id=rel.task_id)
+	check_zip_release.apply_async((rel.id, uploaded_path), task_id=rel.task_id)
 
 	return rel

app/logic/screenshots.py

@@ -1,3 +1,19 @@
+# ContentDB
+# Copyright (C) rubenwardy
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
 import datetime, json
 
 from flask_babel import lazy_gettext
@@ -5,7 +21,7 @@ from flask_babel import lazy_gettext
 from app.logic.LogicError import LogicError
 from app.logic.uploads import upload_file
 from app.models import User, Package, PackageScreenshot, Permission, NotificationType, db, AuditSeverity
-from app.utils import addNotification, addAuditLog
+from app.utils import add_notification, add_audit_log
 from app.utils.image import get_image_size
 
 
@@ -15,7 +31,7 @@ def do_create_screenshot(user: User, package: Package, title: str, file, is_cove
 	if count >= 20:
 		raise LogicError(429, lazy_gettext("Too many requests, please wait before trying again"))
 
-	uploaded_url, uploaded_path = upload_file(file, "image", lazy_gettext("a PNG or JPG image file"))
+	uploaded_url, uploaded_path = upload_file(file, "image", lazy_gettext("a PNG, JPEG, or WebP image file"))
 
 	counter = 1
 	for screenshot in package.screenshots.all():
@@ -26,7 +42,7 @@ def do_create_screenshot(user: User, package: Package, title: str, file, is_cove
 	ss.package  = package
 	ss.title    = title or "Untitled"
 	ss.url      = uploaded_url
-	ss.approved = package.checkPerm(user, Permission.APPROVE_SCREENSHOT)
+	ss.approved = package.check_perm(user, Permission.APPROVE_SCREENSHOT)
 	ss.order    = counter
 	ss.width, ss.height = get_image_size(uploaded_path)
 
@@ -42,8 +58,8 @@ def do_create_screenshot(user: User, package: Package, title: str, file, is_cove
 	else:
 		msg = "Created screenshot {} ({})".format(ss.title, reason)
 
-	addNotification(package.maintainers, user, NotificationType.PACKAGE_EDIT, msg, package.getURL("packages.view"), package)
-	addAuditLog(AuditSeverity.NORMAL, user, msg, package.getURL("packages.view"), package)
+	add_notification(package.maintainers, user, NotificationType.PACKAGE_EDIT, msg, package.get_url("packages.view"), package)
+	add_audit_log(AuditSeverity.NORMAL, user, msg, package.get_url("packages.view"), package)
 
 	db.session.commit()
 
@@ -64,9 +80,9 @@ def do_order_screenshots(_user: User, package: Package, order: [any]):
 		try:
 			lookup[int(ss_id)].order = counter
 			counter += 1
-		except KeyError as e:
+		except KeyError:
 			raise LogicError(400, "Unable to find screenshot with id={}".format(ss_id))
-		except (ValueError, TypeError) as e:
+		except (ValueError, TypeError):
 			raise LogicError(400, "Invalid id, not a number: {}".format(json.dumps(ss_id)))
 
 	db.session.commit()
@@ -75,7 +91,7 @@ def do_order_screenshots(_user: User, package: Package, order: [any]):
 def do_set_cover_image(_user: User, package: Package, cover_image):
 	try:
 		cover_image = int(cover_image)
-	except (ValueError, TypeError) as e:
+	except (ValueError, TypeError):
 		raise LogicError(400, "Invalid id, not a number: {}".format(json.dumps(cover_image)))
 
 	for screenshot in package.screenshots.all():

app/logic/uploads.py

@@ -14,49 +14,55 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-
 import imghdr
 import os
 
-from flask_babel import lazy_gettext
+from flask_babel import lazy_gettext, LazyString
 
+from app import app
 from app.logic.LogicError import LogicError
-from app.models import *
-from app.utils import randomString
+from app.utils import random_string
 
 
 def get_extension(filename):
 	return filename.rsplit(".", 1)[1].lower() if "." in filename else None
 
-ALLOWED_IMAGES = {"jpeg", "png"}
-def isAllowedImage(data):
+
+ALLOWED_IMAGES = {"jpeg", "png", "webp"}
+
+
+def is_allowed_image(data):
 	return imghdr.what(None, data) in ALLOWED_IMAGES
 
-def upload_file(file, fileType, fileTypeDesc):
+
+def upload_file(file, file_type: str, file_type_desc: LazyString | str, length: int=10):
 	if not file or file is None or file.filename == "":
 		raise LogicError(400, "Expected file")
 
 	assert os.path.isdir(app.config["UPLOAD_DIR"]), "UPLOAD_DIR must exist"
 
-	isImage = False
-	if fileType == "image":
-		allowedExtensions = ["jpg", "jpeg", "png"]
-		isImage = True
-	elif fileType == "zip":
-		allowedExtensions = ["zip"]
+	is_image = False
+	if file_type == "image":
+		allowed_extensions = ["jpg", "png", "webp"]
+		is_image = True
+	elif file_type == "zip":
+		allowed_extensions = ["zip"]
 	else:
 		raise Exception("Invalid fileType")
 
 	ext = get_extension(file.filename)
-	if ext is None or not ext in allowedExtensions:
-		raise LogicError(400, lazy_gettext("Please upload %(file_desc)s", file_desc=fileTypeDesc))
+	if ext == "jpeg":
+		ext = "jpg"
 
-	if isImage and not isAllowedImage(file.stream.read()):
+	if ext is None or ext not in allowed_extensions:
+		raise LogicError(400, lazy_gettext("Please upload %(file_desc)s", file_desc=file_type_desc))
+
+	if is_image and not is_allowed_image(file.stream.read()):
 		raise LogicError(400, lazy_gettext("Uploaded image isn't actually an image"))
 
 	file.stream.seek(0)
 
-	filename = randomString(10) + "." + ext
+	filename = random_string(length) + "." + ext
 	filepath = os.path.join(app.config["UPLOAD_DIR"], filename)
 	file.save(filepath)
 

app/logic/users.py

@@ -0,0 +1,60 @@
+from typing import Optional
+
+from flask import flash, redirect, url_for
+from flask_babel import gettext, get_locale
+from sqlalchemy import or_
+from werkzeug import Response
+
+from app.models import User, UserRank, PackageAlias, EmailSubscription, UserNotificationPreferences, db
+from app.utils import is_username_valid
+from app.tasks.emails import send_anon_email
+
+
+def create_user(username: str, display_name: str, email: Optional[str], oauth_provider: Optional[str] = None) -> None | Response | User:
+	if not is_username_valid(username):
+		flash(gettext("Username is invalid"))
+		return
+
+	user_by_name = User.query.filter(or_(
+			User.username == username,
+			User.username == display_name,
+			User.display_name == display_name,
+			User.forums_username == username,
+			User.github_username == username)).first()
+	if user_by_name:
+		if user_by_name.rank == UserRank.NOT_JOINED and user_by_name.forums_username:
+			flash(gettext("An account already exists for that username but hasn't been claimed yet."), "danger")
+			return redirect(url_for("users.claim_forums", username=user_by_name.forums_username))
+		elif oauth_provider:
+			flash(gettext("Unable to create an account as the username is already taken. "
+					"If you meant to log in, you need to connect %(provider)s to your account first", provider=oauth_provider), "danger")
+			return
+		else:
+			flash(gettext("That username/display name is already in use, please choose another."), "danger")
+			return
+
+	alias_by_name = (PackageAlias.query
+			.filter(or_(PackageAlias.author == username, PackageAlias.author == display_name))
+			.first())
+	if alias_by_name:
+		flash(gettext("Unable to create an account as the username was used in the past."), "danger")
+		return
+
+	if email:
+		user_by_email = User.query.filter_by(email=email).first()
+		if user_by_email:
+			send_anon_email.delay(email, get_locale().language, gettext("Email already in use"),
+				gettext("We were unable to create the account as the email is already in use by %(display_name)s. Try a different email address.",
+						display_name=user_by_email.display_name))
+			return redirect(url_for("users.email_sent"))
+		elif EmailSubscription.query.filter_by(email=email, blacklisted=True).count() > 0:
+			flash(gettext("That email address has been unsubscribed/blacklisted, and cannot be used"), "danger")
+			return
+
+	user = User(username, False, email)
+	user.notification_preferences = UserNotificationPreferences(user)
+	if display_name:
+		user.display_name = display_name
+	db.session.add(user)
+
+	return user

app/maillogger.py

@@ -1,96 +0,0 @@
-import logging
-
-from app.tasks.emails import send_user_email
-
-
-def _has_newline(line):
-	"""Used by has_bad_header to check for \\r or \\n"""
-	if line and ("\r" in line or "\n" in line):
-		return True
-	return False
-
-def _is_bad_subject(subject):
-	"""Copied from: flask_mail.py class Message def has_bad_headers"""
-	if _has_newline(subject):
-		for linenum, line in enumerate(subject.split("\r\n")):
-			if not line:
-				return True
-			if linenum > 0 and line[0] not in "\t ":
-				return True
-			if _has_newline(line):
-				return True
-			if len(line.strip()) == 0:
-				return True
-	return False
-
-
-class FlaskMailSubjectFormatter(logging.Formatter):
-	def format(self, record):
-		record.message = record.getMessage()
-		if self.usesTime():
-			record.asctime = self.formatTime(record, self.datefmt)
-		s = self.formatMessage(record)
-		return s
-
-class FlaskMailTextFormatter(logging.Formatter):
-	pass
-
-class FlaskMailHTMLFormatter(logging.Formatter):
-	def formatException(self, exc_info):
-		formatted_exception = logging.Handler.formatException(self, exc_info)
-		return "<pre>%s</pre>" % formatted_exception
-	def formatStack(self, stack_info):
-		return "<pre>%s</pre>" % stack_info
-
-
-# see: https://github.com/python/cpython/blob/3.6/Lib/logging/__init__.py (class Handler)
-
-class FlaskMailHandler(logging.Handler):
-	def __init__(self, send_to, subject_template, level=logging.NOTSET):
-		logging.Handler.__init__(self, level)
-		self.send_to = send_to
-		self.subject_template = subject_template
-
-	def setFormatter(self, text_fmt):
-		"""
-		Set the formatters for this handler. Provide at least one formatter.
-		When no text_fmt is provided, no text-part is created for the email body.
-		"""
-		assert text_fmt != None, "At least one formatter should be provided"
-		if type(text_fmt)==str:
-			text_fmt = FlaskMailTextFormatter(text_fmt)
-		self.formatter = text_fmt
-
-	def getSubject(self, record):
-		fmt = FlaskMailSubjectFormatter(self.subject_template)
-		subject = fmt.format(record)
-		# Since templates can cause header problems, and we rather have a incomplete email then an error, we fix this
-		if _is_bad_subject(subject):
-			subject="FlaskMailHandler log-entry from ContentDB [original subject is replaced, because it would result in a bad header]"
-		return subject
-
-	def emit(self, record):
-		subject = self.getSubject(record)
-		text = self.format(record)				if self.formatter	  else None
-		html = "<pre>{}</pre>".format(text)
-
-		if "The recipient has exceeded message rate limit. Try again later" in subject:
-			return
-
-		for email in self.send_to:
-			send_user_email.delay(email, "en", subject, text, html)
-
-
-def build_handler(app):
-	subject_template = "ContentDB %(message)s (%(module)s > %(funcName)s)"
-	text_template = ("Message type: %(levelname)s\n"
-		"Location: %(pathname)s:%(lineno)d\n"
-		"Module:   %(module)s\n"
-		"Function: %(funcName)s\n"
-		"Time:     %(asctime)s\n"
-		"Message: %(message)s\n\n")
-
-	mail_handler = FlaskMailHandler(app.config["MAIL_UTILS_ERROR_SEND_TO"], subject_template)
-	mail_handler.setLevel(logging.ERROR)
-	mail_handler.setFormatter(text_template)
-	return mail_handler

app/markdown.py

@@ -1,179 +0,0 @@
-from functools import partial
-
-import bleach
-from bleach import Cleaner
-from bleach.linkifier import LinkifyFilter
-from bs4 import BeautifulSoup
-from markdown import Markdown
-from flask import Markup, url_for
-from markdown.extensions import Extension
-from markdown.inlinepatterns import SimpleTagInlineProcessor
-from markdown.inlinepatterns import Pattern
-from xml.etree import ElementTree
-
-# Based on
-# https://github.com/Wenzil/mdx_bleach/blob/master/mdx_bleach/whitelist.py
-#
-# License: MIT
-
-ALLOWED_TAGS = [
-	"h1", "h2", "h3", "h4", "h5", "h6", "hr",
-	"ul", "ol", "li",
-	"p",
-	"br",
-	"pre",
-	"code",
-	"blockquote",
-	"strong",
-	"em",
-	"a",
-	"img",
-	"table", "thead", "tbody", "tr", "th", "td",
-	"div", "span", "del", "s",
-]
-
-ALLOWED_CSS = [
-	"highlight", "codehilite",
-	"hll", "c", "err", "g", "k", "l", "n", "o", "x", "p", "ch", "cm", "cp", "cpf", "c1", "cs",
-	"gd", "ge", "gr", "gh", "gi", "go", "gp", "gs", "gu", "gt", "kc", "kd", "kn", "kp", "kr",
-	"kt", "ld", "m", "s", "na", "nb", "nc", "no", "nd", "ni", "ne", "nf", "nl", "nn", "nx",
-	"py", "nt", "nv", "ow", "w", "mb", "mf", "mh", "mi", "mo", "sa", "sb", "sc", "dl", "sd",
-	"s2", "se", "sh", "si", "sx", "sr", "s1", "ss", "bp", "fm", "vc", "vg", "vi", "vm", "il",
-]
-
-
-def allow_class(_tag, name, value):
-	return name == "class" and value in ALLOWED_CSS
-
-
-ALLOWED_ATTRIBUTES = {
-	"h1": ["id"],
-	"h2": ["id"],
-	"h3": ["id"],
-	"h4": ["id"],
-	"a": ["href", "title", "data-username"],
-	"img": ["src", "title", "alt"],
-	"code": allow_class,
-	"div": allow_class,
-	"span": allow_class,
-}
-
-ALLOWED_PROTOCOLS = ["http", "https", "mailto"]
-
-md = None
-
-
-def render_markdown(source):
-	html = md.convert(source)
-
-	cleaner = Cleaner(
-		tags=ALLOWED_TAGS,
-		attributes=ALLOWED_ATTRIBUTES,
-		protocols=ALLOWED_PROTOCOLS,
-		filters=[partial(LinkifyFilter, callbacks=bleach.linkifier.DEFAULT_CALLBACKS)])
-	return cleaner.clean(html)
-
-
-class DelInsExtension(Extension):
-	def extendMarkdown(self, md):
-		del_proc = SimpleTagInlineProcessor(r"(\~\~)(.+?)(\~\~)", "del")
-		md.inlinePatterns.register(del_proc, "del", 200)
-
-		ins_proc = SimpleTagInlineProcessor(r"(\+\+)(.+?)(\+\+)", "ins")
-		md.inlinePatterns.register(ins_proc, "ins", 200)
-
-
-RE_PARTS = dict(
-	USER=r"[A-Za-z0-9._-]*\b",
-	REPO=r"[A-Za-z0-9_]+\b"
-)
-
-
-class MentionPattern(Pattern):
-	ANCESTOR_EXCLUDES = ("a",)
-
-	def __init__(self, config, md):
-		MENTION_RE = r"(@({USER})(?:\/({REPO}))?)".format(**RE_PARTS)
-		super(MentionPattern, self).__init__(MENTION_RE, md)
-		self.config = config
-
-	def handleMatch(self, m):
-		from app.models import User
-
-		label = m.group(2)
-		user = m.group(3)
-		package_name = m.group(4)
-		if package_name:
-			el = ElementTree.Element("a")
-			el.text = label
-			el.set("href", url_for("packages.view", author=user, name=package_name))
-			return el
-		else:
-			if User.query.filter_by(username=user).count() == 0:
-				return None
-
-			el = ElementTree.Element("a")
-			el.text = label
-			el.set("href", url_for("users.profile", username=user))
-			el.set("data-username", user)
-			return el
-
-
-class MentionExtension(Extension):
-	def __init__(self, *args, **kwargs):
-		super(MentionExtension, self).__init__(*args, **kwargs)
-
-	def extendMarkdown(self, md):
-		md.ESCAPED_CHARS.append("@")
-		md.inlinePatterns.register(MentionPattern(self.getConfigs(), md), "mention", 20)
-
-
-MARKDOWN_EXTENSIONS = ["fenced_code", "tables", "codehilite", "toc", DelInsExtension(), MentionExtension()]
-MARKDOWN_EXTENSION_CONFIG = {
-	"fenced_code": {},
-	"tables": {},
-	"codehilite": {
-		"guess_lang": False,
-	}
-}
-
-
-def init_markdown(app):
-	global md
-
-	md = Markdown(extensions=MARKDOWN_EXTENSIONS,
-			extension_configs=MARKDOWN_EXTENSION_CONFIG,
-			output_format="html5")
-
-	@app.template_filter()
-	def markdown(source):
-		return Markup(render_markdown(source))
-
-
-def get_headings(html: str):
-	soup = BeautifulSoup(html, "html.parser")
-	headings = soup.find_all(["h1", "h2", "h3"])
-
-	root = []
-	stack = []
-	for heading in headings:
-		this = {"link": heading.get("id") or "", "text": heading.text, "children": []}
-		this_level = int(heading.name[1:]) - 1
-
-		while this_level <= len(stack):
-			stack.pop()
-
-		if len(stack) > 0:
-			stack[-1]["children"].append(this)
-		else:
-			root.append(this)
-
-		stack.append(this)
-
-	return root
-
-
-def get_user_mentions(html: str) -> set:
-	soup = BeautifulSoup(html, "html.parser")
-	links = soup.select("a[data-username]")
-	return set([x.get("data-username") for x in links])

app/markdown/__init__.py

@@ -0,0 +1,112 @@
+# ContentDB
+# Copyright (C) rubenwardy
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+from typing import Sequence
+from bs4 import BeautifulSoup
+from jinja2.utils import markupsafe
+from markdown_it import MarkdownIt
+from markdown_it.common.utils import unescapeAll, escapeHtml
+from markdown_it.token import Token
+from markdown_it.presets import gfm_like
+from mdit_py_plugins.anchors import anchors_plugin
+from pygments import highlight
+from pygments.lexers import get_lexer_by_name
+from pygments.util import ClassNotFound
+from pygments.formatters.html import HtmlFormatter
+
+from .cleaner import clean_html
+from .mention import init_mention
+
+
+def highlight_code(code, name, attrs):
+	try:
+		lexer = get_lexer_by_name(name)
+	except ClassNotFound:
+		return None
+
+	formatter = HtmlFormatter()
+
+	return highlight(code, lexer, formatter)
+
+
+def render_code(self, tokens: Sequence[Token], idx, options, env):
+	token = tokens[idx]
+	info = unescapeAll(token.info).strip() if token.info else ""
+	langName = info.split(maxsplit=1)[0] if info else ""
+
+	if options.highlight:
+		return options.highlight(
+			token.content, langName, ""
+		) or f"<pre><code>{escapeHtml(token.content)}</code></pre>"
+
+	return f"<pre><code>{escapeHtml(token.content)}</code></pre>"
+
+
+gfm_like.make()
+md = MarkdownIt("gfm-like", {"highlight": highlight_code})
+md.use(anchors_plugin, permalink=True, permalinkSymbol="🔗", max_level=6)
+md.add_render_rule("fence", render_code)
+init_mention(md)
+
+
+def render_markdown(source, clean=True):
+	html = md.render(source)
+	if clean:
+		return clean_html(html)
+	else:
+		return html
+
+
+def init_markdown(app):
+	@app.template_filter()
+	def markdown(source):
+		return markupsafe.Markup(render_markdown(source))
+
+
+def get_headings(html: str):
+	soup = BeautifulSoup(html, "html.parser")
+	headings = soup.find_all(["h1", "h2", "h3"])
+
+	root = []
+	stack = []
+	for heading in headings:
+		text = heading.find(text=True, recursive=False)
+		this = {"link": heading.get("id") or "", "text": text, "children": []}
+		this_level = int(heading.name[1:]) - 1
+
+		while this_level <= len(stack):
+			stack.pop()
+
+		if len(stack) > 0:
+			stack[-1]["children"].append(this)
+		else:
+			root.append(this)
+
+		stack.append(this)
+
+	return root
+
+
+def get_user_mentions(html: str) -> set:
+	soup = BeautifulSoup(html, "html.parser")
+	links = soup.select("a[data-username]")
+	return set([x.get("data-username") for x in links])
+
+
+def get_links(html: str) -> set:
+	soup = BeautifulSoup(html, "html.parser")
+	links = soup.select("a[href]")
+	return set([x.get("href") for x in links])

app/markdown/cleaner.py

@@ -0,0 +1,97 @@
+# ContentDB
+# Copyright (C) rubenwardy
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+from functools import partial
+from bleach import Cleaner
+from bleach.linkifier import LinkifyFilter, DEFAULT_CALLBACKS
+
+
+# Based on
+# https://github.com/Wenzil/mdx_bleach/blob/master/mdx_bleach/whitelist.py
+#
+# License: MIT
+
+ALLOWED_TAGS = {
+	"h1", "h2", "h3", "h4", "h5", "h6", "hr",
+	"ul", "ol", "li",
+	"p",
+	"br",
+	"pre",
+	"code",
+	"blockquote",
+	"strong",
+	"em",
+	"a",
+	"img",
+	"table", "thead", "tbody", "tr", "th", "td",
+	"div", "span", "del", "s",
+	"details",
+	"summary",
+	"sup",
+}
+
+ALLOWED_CSS = [
+	"highlight", "codehilite",
+	"hll", "c", "err", "g", "k", "l", "n", "o", "x", "p", "ch", "cm", "cp", "cpf", "c1", "cs",
+	"gd", "ge", "gr", "gh", "gi", "go", "gp", "gs", "gu", "gt", "kc", "kd", "kn", "kp", "kr",
+	"kt", "ld", "m", "s", "na", "nb", "nc", "no", "nd", "ni", "ne", "nf", "nl", "nn", "nx",
+	"py", "nt", "nv", "ow", "w", "mb", "mf", "mh", "mi", "mo", "sa", "sb", "sc", "dl", "sd",
+	"s2", "se", "sh", "si", "sx", "sr", "s1", "ss", "bp", "fm", "vc", "vg", "vi", "vm", "il",
+]
+
+
+def allow_class(_tag, name, value):
+	return name == "class" and value in ALLOWED_CSS
+
+
+def allow_a(_tag, name, value):
+	return name in ["href", "title", "data-username"] or (name == "class" and value == "header-anchor")
+
+
+ALLOWED_ATTRIBUTES = {
+	"h1": ["id"],
+	"h2": ["id"],
+	"h3": ["id"],
+	"h4": ["id"],
+	"a": allow_a,
+	"img": ["src", "title", "alt"],
+	"code": allow_class,
+	"div": allow_class,
+	"span": allow_class,
+	"table": ["id"],
+}
+
+ALLOWED_PROTOCOLS = {"http", "https", "mailto"}
+
+
+def linker_callback(attrs, new=False):
+	if new:
+		text = attrs.get("_text")
+		if not (text.startswith("http://") or text.startswith("https://")):
+			return None
+
+	return attrs
+
+
+def clean_html(html: str):
+	cleaner = Cleaner(
+		tags=ALLOWED_TAGS,
+		attributes=ALLOWED_ATTRIBUTES,
+		protocols=ALLOWED_PROTOCOLS,
+		filters=[partial(LinkifyFilter,
+				callbacks=[linker_callback] + DEFAULT_CALLBACKS,
+				skip_tags={"pre", "code"})])
+	return cleaner.clean(html)

app/markdown/mention.py

@@ -0,0 +1,109 @@
+# ContentDB
+# Copyright (C) rubenwardy
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+import re
+
+from flask import url_for
+from markdown_it import MarkdownIt
+from markdown_it.token import Token
+from markdown_it.rules_core.state_core import StateCore
+from typing import Sequence, List
+
+
+def render_user_mention(self, tokens: Sequence[Token], idx, options, env):
+	token = tokens[idx]
+	username = token.content
+	url = url_for("users.profile", username=username)
+	return f"<a href=\"{url}\" data-username=\"{username}\">@{username}</a>"
+
+
+def render_package_mention(self, tokens: Sequence[Token], idx, options, env):
+	token = tokens[idx]
+	username = token.content
+	name = token.attrs["name"]
+	url = url_for("packages.view", author=username, name=name)
+	return f"<a href=\"{url}\">@{username}/{name}</a>"
+
+
+def parse_mentions(state: StateCore):
+	for block_token in state.tokens:
+		if block_token.type != "inline" or block_token.children is None:
+			continue
+
+		link_depth = 0
+		html_link_depth = 0
+
+		children = []
+		for token in block_token.children:
+			if token.type == "link_open":
+				link_depth += 1
+			elif token.type == "link_close":
+				link_depth -= 1
+			elif token.type == "html_inline":
+				# is link open / close?
+				pass
+
+			if link_depth > 0 or html_link_depth > 0 or token.type != "text":
+				children.append(token)
+			else:
+				children.extend(split_tokens(token, state))
+
+		block_token.children = children
+
+
+RE_PARTS = dict(
+	USER=r"[A-Za-z0-9._-]*\b",
+	NAME=r"[A-Za-z0-9_]+\b"
+)
+MENTION_RE = r"(@({USER})(?:\/({NAME}))?)".format(**RE_PARTS)
+
+
+def split_tokens(token: Token, state: StateCore) -> List[Token]:
+	tokens = []
+	content = token.content
+	pos = 0
+	for match in re.finditer(MENTION_RE, content):
+		username = match.group(2)
+		package_name = match.group(3)
+		(start, end) = match.span(0)
+
+		if start > pos:
+			token_text = Token("text", "", 0)
+			token_text.content = content[pos:start]
+			token_text.level = token.level
+			tokens.append(token_text)
+
+		mention = Token("package_mention" if package_name else "user_mention", "", 0)
+		mention.content = username
+		mention.attrSet("name", package_name)
+		mention.level = token.level
+		tokens.append(mention)
+
+		pos = end
+
+	if pos < len(content):
+		token_text = Token("text", "", 0)
+		token_text.content = content[pos:]
+		token_text.level = token.level
+		tokens.append(token_text)
+
+	return tokens
+
+
+def init_mention(md: MarkdownIt):
+	md.add_render_rule("user_mention", render_user_mention, "html")
+	md.add_render_rule("package_mention", render_package_mention, "html")
+	md.core.ruler.after("inline", "mention", parse_mentions)

app/models/__init__.py

@@ -13,8 +13,7 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-
-
+from flask_babel import LazyString
 from flask_migrate import Migrate
 from flask_sqlalchemy import SQLAlchemy
 from sqlalchemy_searchable import make_searchable
@@ -31,6 +30,7 @@ make_searchable(db.metadata)
 from .packages import *
 from .users import *
 from .threads import *
+from .collections import *
 
 
 class APIToken(db.Model):
@@ -47,7 +47,14 @@ class APIToken(db.Model):
 	package_id = db.Column(db.Integer, db.ForeignKey("package.id"), nullable=True)
 	package    = db.relationship("Package", foreign_keys=[package_id], back_populates="tokens")
 
-	def canOperateOnPackage(self, package):
+	client_id = db.Column(db.String(24), db.ForeignKey("oauth_client.id"), nullable=True)
+	client    = db.relationship("OAuthClient", foreign_keys=[client_id], back_populates="tokens")
+	auth_code = db.Column(db.String(34), unique=True, nullable=True)
+
+	def can_operate_on_package(self, package):
+		if self.client is not None:
+			return False
+
 		if self.package and self.package != package:
 			return False
 
@@ -63,12 +70,13 @@ class AuditSeverity(enum.Enum):
 	def __str__(self):
 		return self.name
 
-	def getTitle(self):
+	@property
+	def title(self):
 		return self.name.replace("_", " ").title()
 
 	@classmethod
 	def choices(cls):
-		return [(choice, choice.getTitle()) for choice in cls]
+		return [(choice, choice.title) for choice in cls]
 
 	@classmethod
 	def coerce(cls, item):
@@ -95,6 +103,8 @@ class AuditLogEntry(db.Model):
 
 	def __init__(self, causer, severity, title, url, package=None, description=None):
 		if len(title) > 100:
+			if description is None:
+				description = title[99:]
 			title = title[:99] + "…"
 
 		self.causer   = causer
@@ -104,9 +114,125 @@ class AuditLogEntry(db.Model):
 		self.package  = package
 		self.description = description
 
+	def check_perm(self, user, perm):
+		if not user.is_authenticated:
+			return False
+
+		if type(perm) == str:
+			perm = Permission[perm]
+		elif type(perm) != Permission:
+			raise Exception("Unknown permission given to AuditLogEntry.check_perm()")
+
+		if perm == Permission.VIEW_AUDIT_DESCRIPTION:
+			return (self.package and user in self.package.maintainers) or user.rank.at_least(UserRank.APPROVER if self.package is not None else UserRank.MODERATOR)
+		else:
+			raise Exception("Permission {} is not related to audit log entries".format(perm.name))
+
+
+class ReportCategory(enum.Enum):
+	ACCOUNT_DELETION = "account_deletion"
+	COPYRIGHT = "copyright"
+	USER_CONDUCT = "user_conduct"
+	SPAM = "spam"
+	ILLEGAL_HARMFUL = "illegal_harmful"
+	REVIEW = "review"
+	APPEAL = "appeal"
+	OTHER = "other"
+
+	def __str__(self):
+		return self.name
+
+	@property
+	def title(self) -> LazyString:
+		if self == ReportCategory.ACCOUNT_DELETION:
+			return lazy_gettext("Account deletion")
+		elif self == ReportCategory.COPYRIGHT:
+			return lazy_gettext("Copyright infringement / DMCA")
+		elif self == ReportCategory.USER_CONDUCT:
+			return lazy_gettext("User behaviour, bullying, or abuse")
+		elif self == ReportCategory.SPAM:
+			return lazy_gettext("Spam")
+		elif self == ReportCategory.ILLEGAL_HARMFUL:
+			return lazy_gettext("Illegal or harmful content")
+		elif self == ReportCategory.REVIEW:
+			return lazy_gettext("Outdated/invalid review")
+		elif self == ReportCategory.APPEAL:
+			return lazy_gettext("Appeal")
+		elif self == ReportCategory.OTHER:
+			return lazy_gettext("Other")
+		else:
+			raise Exception("Unknown report category")
+
+	@classmethod
+	def get(cls, name):
+		try:
+			return ReportCategory[name.upper()]
+		except KeyError:
+			return None
+
+	@classmethod
+	def choices(cls, with_none):
+		ret = [(choice, choice.title) for choice in cls]
+
+		if with_none:
+			ret.insert(0, (None, ""))
+
+		return ret
+
+	@classmethod
+	def coerce(cls, item):
+		if item is None or (isinstance(item, str) and item.upper() == "NONE"):
+			return None
+		return item if type(item) == ReportCategory else ReportCategory[item.upper()]
+
+
+class Report(db.Model):
+	id = db.Column(db.String(24), primary_key=True)
+
+	created_at = db.Column(db.DateTime, nullable=False, default=datetime.datetime.utcnow)
+
+	user_id  = db.Column(db.Integer, db.ForeignKey("user.id"), nullable=True)
+	user = db.relationship("User", foreign_keys=[user_id], back_populates="reports")
+
+	thread_id  = db.Column(db.Integer, db.ForeignKey("thread.id"), nullable=True)
+	thread = db.relationship("Thread", foreign_keys=[thread_id])
+
+	category = db.Column(db.Enum(ReportCategory), nullable=False)
+	url = db.Column(db.String, nullable=True)
+	title = db.Column(db.Unicode(300), nullable=False)
+	message = db.Column(db.UnicodeText, nullable=False)
+
+	is_resolved = db.Column(db.Boolean, nullable=False, default=False)
+
+	attachments = db.relationship("ReportAttachment", back_populates="report", lazy="dynamic", cascade="all, delete, delete-orphan")
+
+	def check_perm(self, user, perm):
+		if type(perm) == str:
+			perm = Permission[perm]
+		elif type(perm) != Permission:
+			raise Exception("Unknown permission given to Report.check_perm()")
+		if not user.is_authenticated:
+			return False
+
+		if perm == Permission.SEE_REPORT:
+			return user.rank.at_least(UserRank.EDITOR)
+		else:
+			raise Exception("Permission {} is not related to reports".format(perm.name))
+
+
+class ReportAttachment(db.Model):
+	id = db.Column(db.Integer, primary_key=True)
+
+	created_at = db.Column(db.DateTime, nullable=False, default=datetime.datetime.utcnow)
+
+	report_id = db.Column(db.String(24), db.ForeignKey("report.id"), nullable=False)
+	report = db.relationship("Report", foreign_keys=[report_id], back_populates="attachments")
+
+	url = db.Column(db.String(100), nullable=False)
+
 
 REPO_BLACKLIST = [".zip", "mediafire.com", "dropbox.com", "weebly.com",
-	"minetest.net", "dropboxusercontent.com", "4shared.com",
+	"minetest.net", "luanti.org", "dropboxusercontent.com", "4shared.com",
 	"digitalaudioconcepts.com", "hg.intevation.org", "www.wtfpl.net",
 	"imageshack.com", "imgur.com"]
 
@@ -118,6 +244,7 @@ class ForumTopic(db.Model):
 	author    = db.relationship("User", back_populates="forum_topics")
 
 	wip       = db.Column(db.Boolean, default=False, nullable=False)
+	# TODO: remove
 	discarded = db.Column(db.Boolean, default=False, nullable=False)
 
 	type      = db.Column(db.Enum(PackageType), nullable=False)
@@ -130,7 +257,11 @@ class ForumTopic(db.Model):
 
 	created_at = db.Column(db.DateTime, nullable=False, default=datetime.datetime.utcnow)
 
-	def getRepoURL(self):
+	@property
+	def url(self):
+		return "https://forum.luanti.org/viewtopic.php?t=" + str(self.topic_id)
+
+	def get_repo_url(self):
 		if self.link is None:
 			return None
 
@@ -140,32 +271,31 @@ class ForumTopic(db.Model):
 
 		return self.link.replace("repo.or.cz/w/", "repo.or.cz/")
 
-	def getAsDictionary(self):
+	def as_dict(self):
 		return {
 			"author": self.author.username,
 			"name":   self.name,
-			"type":   self.type.toName(),
+			"type":   self.type.to_name(),
 			"title":  self.title,
 			"id":     self.topic_id,
 			"link":   self.link,
 			"posts":  self.posts,
 			"views":  self.views,
 			"is_wip": self.wip,
-			"discarded":  self.discarded,
 			"created_at": self.created_at.isoformat(),
 		}
 
-	def checkPerm(self, user, perm):
+	def check_perm(self, user, perm):
 		if not user.is_authenticated:
 			return False
 
 		if type(perm) == str:
 			perm = Permission[perm]
 		elif type(perm) != Permission:
-			raise Exception("Unknown permission given to ForumTopic.checkPerm()")
+			raise Exception("Unknown permission given to ForumTopic.check_perm()")
 
 		if perm == Permission.TOPIC_DISCARD:
-			return self.author == user or user.rank.atLeast(UserRank.EDITOR)
+			return self.author == user or user.rank.at_least(UserRank.EDITOR)
 
 		else:
 			raise Exception("Permission {} is not related to topics".format(perm.name))

app/models/collections.py

@@ -0,0 +1,107 @@
+# ContentDB
+# Copyright (C) 2023  rubenwardy
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+import datetime
+
+from flask import url_for, current_app
+
+from . import db, Permission, User, UserRank
+
+
+class CollectionPackage(db.Model):
+	package_id = db.Column(db.Integer, db.ForeignKey("package.id"), primary_key=True)
+	package = db.relationship("Package", foreign_keys=[package_id])
+
+	collection_id = db.Column(db.Integer, db.ForeignKey("collection.id"), primary_key=True)
+	collection = db.relationship("Collection", back_populates="items", foreign_keys=[collection_id])
+
+	order = db.Column(db.Integer, nullable=False, default=0)
+	description = db.Column(db.String, nullable=True)
+	created_at = db.Column(db.DateTime, nullable=False, default=datetime.datetime.utcnow)
+
+	collection_description_nonempty = db.CheckConstraint("description = NULL OR description != ''")
+
+	def as_dict(self):
+		return {
+			"package": self.package.as_short_dict(current_app.config["BASE_URL"]),
+			"order": self.order,
+			"description": self.description,
+			"created_at": self.created_at.isoformat(),
+		}
+
+
+class Collection(db.Model):
+	id = db.Column(db.Integer, primary_key=True)
+
+	author_id = db.Column(db.Integer, db.ForeignKey("user.id"), nullable=False)
+	author = db.relationship("User", back_populates="collections", foreign_keys=[author_id])
+
+	name = db.Column(db.Unicode(100), nullable=False)
+	title = db.Column(db.Unicode(100), nullable=False)
+	short_description = db.Column(db.Unicode(200), nullable=False)
+	long_description = db.Column(db.UnicodeText, nullable=True)
+	created_at = db.Column(db.DateTime, nullable=False, default=datetime.datetime.utcnow)
+	private = db.Column(db.Boolean, nullable=False, default=False)
+	pinned = db.Column(db.Boolean, nullable=False, default=False)
+
+	packages = db.relationship("Package", secondary=CollectionPackage.__table__, backref="collections")
+	items = db.relationship("CollectionPackage", back_populates="collection", order_by=db.asc("order"),
+		cascade="all, delete, delete-orphan")
+
+	collection_name_valid = db.CheckConstraint("name ~* '^[a-z0-9_]+$' AND name != '_game'")
+	__table_args__ = (db.UniqueConstraint("author_id", "name", name="_collection_uc"),)
+
+	def get_url(self, endpoint, **kwargs):
+		return url_for(endpoint, author=self.author.username, name=self.name, **kwargs)
+
+	def as_short_dict(self):
+		return {
+			"author": self.author.username,
+			"name": self.name,
+			"title": self.title,
+			"short_description": self.short_description,
+			"created_at": self.created_at.isoformat(),
+			"private": self.private,
+			"package_count": len(self.packages)
+		}
+
+	def as_dict(self):
+		return {
+			"author": self.author.username,
+			"name": self.name,
+			"title": self.title,
+			"short_description": self.short_description,
+			"long_description": self.long_description,
+			"created_at": self.created_at.isoformat(),
+			"private": self.private,
+		}
+
+	def check_perm(self, user: User, perm):
+		if type(perm) == str:
+			perm = Permission[perm]
+		elif type(perm) != Permission:
+			raise Exception("Unknown permission given to Collection.check_perm()")
+
+		if user is None or not user.is_authenticated:
+			return perm == Permission.VIEW_COLLECTION and not self.private
+
+		can_view = not self.private or self.author == user or user.rank.at_least(UserRank.MODERATOR)
+		if perm == Permission.VIEW_COLLECTION:
+			return can_view
+		elif perm == Permission.EDIT_COLLECTION:
+			return can_view and (self.author == user or user.rank.at_least(UserRank.EDITOR))
+		else:
+			raise Exception("Permission {} is not related to collections".format(perm.name))